From 9919fe0ba57e23cfaec9dc0e9165d5b6bab56d65 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 11:41:35 +0100 Subject: [PATCH 1/8] Add application domain boundary guards --- .../test_application_domain_boundaries.py | 357 ++++++++++++++++++ .../test_repository_layer_conventions.py | 186 +++++++++ 2 files changed, 543 insertions(+) create mode 100644 ergon_core/tests/unit/architecture/test_application_domain_boundaries.py diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py new file mode 100644 index 000000000..4f462310a --- /dev/null +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -0,0 +1,357 @@ +"""Application-domain import and layout boundary guards.""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[4] +PACKAGE_ROOT = ROOT / "ergon_core" +APPLICATION_ROOT = PACKAGE_ROOT / "ergon_core" / "core" / "application" +APPLICATION_PREFIX = "ergon_core.core.application" + +PUBLIC_CROSS_DOMAIN_MODULES = {"service", "models", "errors"} +APPROVED_DOMAIN_FILES = { + "__init__.py", + "service.py", + "models.py", + "errors.py", + "repository.py", + "policy.py", + "mappers.py", + "dto_mapping.py", +} +APPROVED_DOMAIN_DIRS = {"policies", "__pycache__"} + +LAYOUT_FILE_EXCEPTIONS = { + "context": {"events.py"}, + "evaluation": {"scoring.py", "summary.py"}, + "events": {"base.py", "runtime.py"}, + "experiments": {"definition_writer.py", "handles.py", "launch.py"}, + "ports": {"dashboard.py", "resources.py"}, + "resources": {"publishing.py"}, + "runtime": { + "events.py", + "graph_lookup.py", + "graph_repository.py", + "graph_traversal.py", + "lifecycle.py", + "orchestration.py", + "resources.py", + "run_identity.py", + "run_lifecycle.py", + "run_records.py", + "status.py", + "task_cleanup.py", + "task_errors.py", + "task_execution.py", + "task_execution_repository.py", + "task_inspection.py", + "task_management.py", + "task_models.py", + "task_service.py", + "workflow_errors.py", + "workflow_models.py", + }, + "testing": {"test_harness_service.py"}, +} +LAYOUT_DIR_EXCEPTIONS: dict[str, set[str]] = {} + +_PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = ( + ( + "ergon_core.core.application.communication.service", + "ergon_core.core.application.ports.dashboard", + "PR 02: make application ports protocol-only public collaboration points.", + ), + ( + "ergon_core.core.application.experiments.launch", + "ergon_core.core.application.events.runtime", + "PR 05: route runtime event collaboration through public runtime facades.", + ), + ( + "ergon_core.core.application.experiments.launch", + "ergon_core.core.application.runtime.run_records", + "PR 05: route run-record access through public runtime facades.", + ), + ( + "ergon_core.core.application.ports.dashboard", + "ergon_core.core.application.events.base", + "PR 02: make application ports protocol-only public collaboration points.", + ), + ( + "ergon_core.core.application.resources.publishing", + "ergon_core.core.application.ports.resources", + "PR 02: make application ports protocol-only public collaboration points.", + ), + ( + "ergon_core.core.application.runtime.events", + "ergon_core.core.application.events.runtime", + "PR 05: route runtime event collaboration through public runtime facades.", + ), + ( + "ergon_core.core.application.runtime.run_lifecycle", + "ergon_core.core.application.evaluation.scoring", + "PR 03: keep evaluation scoring behind evaluation public APIs.", + ), + ( + "ergon_core.core.application.runtime.run_records", + "ergon_core.core.application.events.runtime", + "PR 05: route runtime event collaboration through public runtime facades.", + ), + ( + "ergon_core.core.application.runtime.run_records", + "ergon_core.core.application.experiments.handles", + "PR 04: decide and document experiment handle public surface.", + ), + ( + "ergon_core.core.application.runtime.task_execution", + "ergon_core.core.application.ports.dashboard", + "PR 02: make application ports protocol-only public collaboration points.", + ), + ( + "ergon_core.core.application.runtime.task_management", + "ergon_core.core.application.events.runtime", + "PR 05: route runtime event collaboration through public runtime facades.", + ), + ( + "ergon_core.core.application.runtime.task_management", + "ergon_core.core.application.ports.dashboard", + "PR 02: make application ports protocol-only public collaboration points.", + ), + ( + "ergon_core.core.application.runtime.task_models", + "ergon_core.core.application.events.runtime", + "PR 05: route runtime event collaboration through public runtime facades.", + ), +) + +PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = tuple( + pytest.param( + source_module, + target_module, + marks=pytest.mark.xfail( + strict=True, + reason=reason, + ), + id=f"{source_module} -> {target_module}", + ) + for source_module, target_module, reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER +) + + +@dataclass(frozen=True, order=True) +class ImportEdge: + source_module: str + target_module: str + + +def _module_name_for_path(path: Path) -> str: + rel = path.relative_to(PACKAGE_ROOT) + parts = list(rel.with_suffix("").parts) + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def _source_package_parts(path: Path, module_name: str) -> list[str]: + parts = module_name.split(".") + if path.name == "__init__.py": + return parts + return parts[:-1] + + +def _resolve_import_from_base( + *, + path: Path, + source_module: str, + level: int, + module: str | None, +) -> str: + if level == 0: + return module or "" + + package_parts = _source_package_parts(path, source_module) + kept_parts = package_parts[: len(package_parts) - (level - 1)] + base = ".".join(kept_parts) + if module: + return f"{base}.{module}" + return base + + +def _application_domain_and_leaf(module_name: str) -> tuple[str, str | None] | None: + if module_name == APPLICATION_PREFIX: + return None + if not module_name.startswith(f"{APPLICATION_PREFIX}."): + return None + + parts = module_name.split(".") + if len(parts) < 4: + return None + domain = parts[3] + leaf = parts[4] if len(parts) > 4 else None + return domain, leaf + + +def _application_domain_for_path(path: Path) -> str: + return path.relative_to(APPLICATION_ROOT).parts[0] + + +def _domain_child_exists(domain: str, name: str) -> bool: + domain_root = APPLICATION_ROOT / domain + return (domain_root / f"{name}.py").exists() or (domain_root / name).is_dir() + + +def _import_targets(path: Path) -> list[str]: + source_module = _module_name_for_path(path) + tree = ast.parse(path.read_text(), filename=str(path)) + targets: list[str] = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + targets.extend(alias.name for alias in node.names) + continue + + if not isinstance(node, ast.ImportFrom) or node.module == "__future__": + continue + + base = _resolve_import_from_base( + path=path, + source_module=source_module, + level=node.level, + module=node.module, + ) + if not base: + continue + + for alias in node.names: + if alias.name == "*": + targets.append(base) + continue + domain_info = _application_domain_and_leaf(base) + if domain_info is None: + targets.append(base) + continue + domain, leaf = domain_info + if leaf is None and _domain_child_exists(domain, alias.name): + targets.append(f"{base}.{alias.name}") + else: + targets.append(base) + + return targets + + +def _application_import_edges() -> set[ImportEdge]: + edges: set[ImportEdge] = set() + for path in APPLICATION_ROOT.rglob("*.py"): + if "__pycache__" in path.parts: + continue + + source_module = _module_name_for_path(path) + source_domain = _application_domain_for_path(path) + for target_module in _import_targets(path): + target_info = _application_domain_and_leaf(target_module) + if target_info is None: + continue + + target_domain, _leaf = target_info + if target_domain == source_domain: + continue + + edges.add(ImportEdge(source_module, target_module)) + + return edges + + +def _private_cross_domain_imports() -> set[ImportEdge]: + private_edges: set[ImportEdge] = set() + for edge in _application_import_edges(): + _target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( + "", + None, + ) + if leaf is None or leaf in PUBLIC_CROSS_DOMAIN_MODULES: + continue + private_edges.add(edge) + return private_edges + + +@pytest.mark.parametrize( + ("source_module", "target_module"), + PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER, +) +def test_ledgered_private_cross_domain_imports_are_removed( + source_module: str, + target_module: str, +) -> None: + """Strict xfail ledger: XPASS means a future cleanup removed the import.""" + + assert ImportEdge(source_module, target_module) not in _private_cross_domain_imports() + + +def test_no_unledgered_private_cross_domain_imports() -> None: + current = _private_cross_domain_imports() + ledgered = { + ImportEdge(source_module, target_module) + for source_module, target_module, _reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER + } + + offenders = [ + f"{edge.source_module} imports private {edge.target_module}" + for edge in sorted(current - ledgered) + ] + + assert offenders == [] + + +def test_cross_domain_imports_only_target_public_domain_modules() -> None: + offenders: list[str] = [] + for edge in sorted(_application_import_edges()): + _target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( + "", + None, + ) + if leaf is None or leaf in PUBLIC_CROSS_DOMAIN_MODULES: + continue + if any( + (edge.source_module, edge.target_module) == (source_module, target_module) + for source_module, target_module, _reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER + ): + continue + offenders.append(f"{edge.source_module} imports {edge.target_module}") + + assert offenders == [] + + +def test_application_domain_directories_keep_approved_layout() -> None: + offenders: list[str] = [] + + for domain_root in sorted(path for path in APPLICATION_ROOT.iterdir() if path.is_dir()): + if domain_root.name == "__pycache__": + continue + + allowed_files = APPROVED_DOMAIN_FILES | LAYOUT_FILE_EXCEPTIONS.get( + domain_root.name, + set(), + ) + allowed_dirs = APPROVED_DOMAIN_DIRS | LAYOUT_DIR_EXCEPTIONS.get( + domain_root.name, + set(), + ) + + for child in sorted(domain_root.iterdir()): + if child.is_file() and child.name not in allowed_files: + offenders.append( + f"{child.relative_to(ROOT)} is not an approved application " + "domain filename" + ) + elif child.is_dir() and child.name not in allowed_dirs: + offenders.append( + f"{child.relative_to(ROOT)} is not an approved application " + "domain directory" + ) + + assert offenders == [] diff --git a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py index f2e797e4d..b86371c7e 100644 --- a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py +++ b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py @@ -19,6 +19,7 @@ from __future__ import annotations +import ast import importlib import inspect from collections.abc import Iterator @@ -28,6 +29,27 @@ ROOT = Path(__file__).resolve().parents[4] PRODUCTION_ROOT = ROOT / "ergon_core" / "ergon_core" +CORE_ROOT = PRODUCTION_ROOT / "core" +APPLICATION_ROOT = CORE_ROOT / "application" +APPLICATION_PREFIX = "ergon_core.core.application" + +_REPOSITORY_REEXPORT_IMPORT_LEDGER = ( + pytest.param( + "ergon_core.core.jobs.task.worker_execute.job", + "ergon_core.core.application.resources.RunResourceRepository", + marks=pytest.mark.xfail( + strict=True, + reason=( + "PR 04: route worker output resource writes through the resources " + "facade instead of the repository re-export." + ), + ), + id=( + "ergon_core.core.jobs.task.worker_execute.job -> " + "ergon_core.core.application.resources.RunResourceRepository" + ), + ), +) _WRITE_PREFIXES = ( @@ -200,3 +222,167 @@ def test_repository_does_not_import_infrastructure(cls: type) -> None: "`ergon_core.core.infrastructure`. Repositories must stay " "framework-agnostic." ) + + +def _module_name_for_path(path: Path) -> str: + rel = path.relative_to(ROOT / "ergon_core") + parts = list(rel.with_suffix("").parts) + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def _source_package_parts(path: Path, module_name: str) -> list[str]: + parts = module_name.split(".") + if path.name == "__init__.py": + return parts + return parts[:-1] + + +def _resolve_import_from_base( + *, + path: Path, + source_module: str, + level: int, + module: str | None, +) -> str: + if level == 0: + return module or "" + + package_parts = _source_package_parts(path, source_module) + kept_parts = package_parts[: len(package_parts) - (level - 1)] + base = ".".join(kept_parts) + if module: + return f"{base}.{module}" + return base + + +def _source_application_domain(path: Path) -> str | None: + try: + return path.relative_to(APPLICATION_ROOT).parts[0] + except ValueError: + return None + + +def _repository_import_target(base: str, alias: str) -> tuple[str, str] | None: + if not base.startswith(f"{APPLICATION_PREFIX}."): + return None + + parts = base.split(".") + if len(parts) < 4: + return None + + domain = parts[3] + if len(parts) == 4 and alias == "repository": + return domain, f"{base}.repository" + if len(parts) >= 5 and parts[4] == "repository": + return domain, ".".join(parts[:5]) + if len(parts) == 4 and alias in _repository_reexports(domain): + return domain, f"{base}.{alias}" + return None + + +def _repository_reexports(domain: str) -> set[str]: + init_path = APPLICATION_ROOT / domain / "__init__.py" + if not init_path.exists(): + return set() + + tree = ast.parse(init_path.read_text(), filename=str(init_path)) + reexports: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.ImportFrom): + continue + if not node.module or not node.module.endswith(".repository"): + continue + reexports.update(alias.asname or alias.name for alias in node.names) + return reexports + + +def _application_repository_imports() -> set[tuple[str, str]]: + imports: set[tuple[str, str]] = set() + for path in CORE_ROOT.rglob("*.py"): + if "__pycache__" in path.parts: + continue + + source_module = _module_name_for_path(path) + source_domain = _source_application_domain(path) + tree = ast.parse(path.read_text(), filename=str(path)) + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(_plain_repository_imports(node, source_module, source_domain)) + continue + + if not isinstance(node, ast.ImportFrom) or node.module == "__future__": + continue + + imports.update(_from_repository_imports(path, node, source_module, source_domain)) + + return imports + + +def _plain_repository_imports( + node: ast.Import, + source_module: str, + source_domain: str | None, +) -> set[tuple[str, str]]: + imports: set[tuple[str, str]] = set() + for alias in node.names: + target = _repository_import_target(alias.name, "") + if target is None: + continue + target_domain, target_module = target + if source_domain != target_domain: + imports.add((source_module, target_module)) + return imports + + +def _from_repository_imports( + path: Path, + node: ast.ImportFrom, + source_module: str, + source_domain: str | None, +) -> set[tuple[str, str]]: + imports: set[tuple[str, str]] = set() + base = _resolve_import_from_base( + path=path, + source_module=source_module, + level=node.level, + module=node.module, + ) + for alias in node.names: + target = _repository_import_target(base, alias.name) + if target is None: + continue + target_domain, target_module = target + if source_domain != target_domain: + imports.add((source_module, target_module)) + return imports + + +@pytest.mark.parametrize( + ("source_module", "target_module"), + _REPOSITORY_REEXPORT_IMPORT_LEDGER, +) +def test_ledgered_application_repository_imports_are_removed( + source_module: str, + target_module: str, +) -> None: + """Strict xfail ledger for repository imports hidden behind package exports.""" + + assert (source_module, target_module) not in _application_repository_imports() + + +def test_application_repositories_are_imported_only_within_their_domain() -> None: + ledgered = { + (param.values[0], param.values[1]) + for param in _REPOSITORY_REEXPORT_IMPORT_LEDGER + } + offenders = [ + f"{source_module} imports repository module {target_module}" + for source_module, target_module in sorted( + _application_repository_imports() - ledgered + ) + ] + + assert offenders == [] From 0790e356d11709b64723a717009a1d5cded501cf Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 13:07:05 +0100 Subject: [PATCH 2/8] Format application boundary guards --- .../architecture/test_application_domain_boundaries.py | 6 ++---- .../architecture/test_repository_layer_conventions.py | 9 ++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index 4f462310a..304454126 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -345,13 +345,11 @@ def test_application_domain_directories_keep_approved_layout() -> None: for child in sorted(domain_root.iterdir()): if child.is_file() and child.name not in allowed_files: offenders.append( - f"{child.relative_to(ROOT)} is not an approved application " - "domain filename" + f"{child.relative_to(ROOT)} is not an approved application domain filename" ) elif child.is_dir() and child.name not in allowed_dirs: offenders.append( - f"{child.relative_to(ROOT)} is not an approved application " - "domain directory" + f"{child.relative_to(ROOT)} is not an approved application domain directory" ) assert offenders == [] diff --git a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py index b86371c7e..af5d15276 100644 --- a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py +++ b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py @@ -374,15 +374,10 @@ def test_ledgered_application_repository_imports_are_removed( def test_application_repositories_are_imported_only_within_their_domain() -> None: - ledgered = { - (param.values[0], param.values[1]) - for param in _REPOSITORY_REEXPORT_IMPORT_LEDGER - } + ledgered = {(param.values[0], param.values[1]) for param in _REPOSITORY_REEXPORT_IMPORT_LEDGER} offenders = [ f"{source_module} imports repository module {target_module}" - for source_module, target_module in sorted( - _application_repository_imports() - ledgered - ) + for source_module, target_module in sorted(_application_repository_imports() - ledgered) ] assert offenders == [] From 4ee91c7e592e94ca8c08dd98408f916721c68652 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 11:54:58 +0100 Subject: [PATCH 3/8] Standardize small application domains --- .../core/application/communication/models.py | 46 +------------------ .../core/application/communication/service.py | 10 ++-- .../core/application/context/__init__.py | 4 ++ .../context/{events.py => service.py} | 0 .../core/application/events/service.py | 24 ++++++++++ .../core/application/ports/__init__.py | 13 ++++++ .../core/application/ports/dashboard.py | 34 ++++---------- .../core/application/resources/publishing.py | 2 +- .../application/runtime/task_execution.py | 2 +- .../application/runtime/task_management.py | 6 +-- .../core/infrastructure/dashboard/emitter.py | 4 +- .../core/infrastructure/dashboard/provider.py | 2 +- .../core/infrastructure/sandbox/event_sink.py | 2 +- .../core/jobs/task/cleanup_cancelled/job.py | 2 +- .../ergon_core/core/jobs/task/evaluate/job.py | 2 +- .../core/jobs/task/worker_execute/job.py | 4 +- .../core/jobs/workflow/complete/job.py | 2 +- .../core/jobs/workflow/start/job.py | 2 +- .../core/views/dashboard_events/contracts.py | 4 +- .../ergon_core/core/views/runs/models.py | 28 ++++++++++- .../ergon_core/core/views/runs/snapshot.py | 6 +-- .../test_application_domain_boundaries.py | 26 ----------- .../test_public_api_boundaries.py | 2 +- .../dashboard/test_event_contract_types.py | 10 ++-- .../test_context_event_repository.py | 2 +- 25 files changed, 108 insertions(+), 131 deletions(-) rename ergon_core/ergon_core/core/application/context/{events.py => service.py} (100%) create mode 100644 ergon_core/ergon_core/core/application/events/service.py diff --git a/ergon_core/ergon_core/core/application/communication/models.py b/ergon_core/ergon_core/core/application/communication/models.py index 30d3f38cd..a4e82aaf9 100644 --- a/ergon_core/ergon_core/core/application/communication/models.py +++ b/ergon_core/ergon_core/core/application/communication/models.py @@ -1,51 +1,9 @@ -"""Pydantic DTOs for inter-agent communication services and read models.""" +"""Pydantic DTOs for inter-agent communication service commands and results.""" from datetime import datetime from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field - - -def _to_camel(value: str) -> str: - head, *tail = value.split("_") - return head + "".join(part.capitalize() for part in tail) - - -class CamelModel(BaseModel): - """Base model that exposes camelCase JSON to the frontend.""" - - model_config = ConfigDict( - alias_generator=_to_camel, - populate_by_name=True, - extra="forbid", - ) - - -class RunCommunicationMessageDto(CamelModel): - id: str - thread_id: str - thread_topic: str - run_id: str - task_id: str | None = None - task_execution_id: str | None = None - from_agent_id: str - to_agent_id: str - content: str - sequence_num: int - created_at: datetime - - -class RunCommunicationThreadDto(CamelModel): - id: str - run_id: str - task_id: str | None = None - topic: str - summary: str | None = None - agent_a_id: str - agent_b_id: str - created_at: datetime - updated_at: datetime - messages: list[RunCommunicationMessageDto] = Field(default_factory=list) +from pydantic import BaseModel, Field class CreateMessageRequest(BaseModel): diff --git a/ergon_core/ergon_core/core/application/communication/service.py b/ergon_core/ergon_core/core/application/communication/service.py index 589f67073..7a86d5092 100644 --- a/ergon_core/ergon_core/core/application/communication/service.py +++ b/ergon_core/ergon_core/core/application/communication/service.py @@ -3,11 +3,7 @@ import logging from uuid import UUID -from ergon_core.core.application.communication.models import ( - RunCommunicationMessageDto, - RunCommunicationThreadDto, -) -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.persistence.shared.db import get_session from ergon_core.core.persistence.telemetry.models import Thread, ThreadMessage from ergon_core.core.application.communication.models import ( @@ -18,6 +14,10 @@ ) from ergon_core.core.shared.utils import utcnow from ergon_core.core.views.dashboard_events.contracts import DashboardThreadMessageCreatedEvent +from ergon_core.core.views.runs.models import ( + RunCommunicationMessageDto, + RunCommunicationThreadDto, +) from sqlalchemy.exc import IntegrityError from sqlmodel import Session, func, select diff --git a/ergon_core/ergon_core/core/application/context/__init__.py b/ergon_core/ergon_core/core/application/context/__init__.py index b71f5b72f..c626b2aaa 100644 --- a/ergon_core/ergon_core/core/application/context/__init__.py +++ b/ergon_core/ergon_core/core/application/context/__init__.py @@ -1 +1,5 @@ """Application context event services.""" + +from ergon_core.core.application.context.service import ContextEventService + +__all__ = ["ContextEventService"] diff --git a/ergon_core/ergon_core/core/application/context/events.py b/ergon_core/ergon_core/core/application/context/service.py similarity index 100% rename from ergon_core/ergon_core/core/application/context/events.py rename to ergon_core/ergon_core/core/application/context/service.py diff --git a/ergon_core/ergon_core/core/application/events/service.py b/ergon_core/ergon_core/core/application/events/service.py new file mode 100644 index 000000000..952ab9b0f --- /dev/null +++ b/ergon_core/ergon_core/core/application/events/service.py @@ -0,0 +1,24 @@ +"""Application service locator for dashboard event publishing.""" + +from ergon_core.core.application.ports import DashboardEventPublisher + +_dashboard_event_publisher: DashboardEventPublisher | None = None + + +def set_dashboard_event_publisher( + publisher: DashboardEventPublisher, +) -> DashboardEventPublisher: + global _dashboard_event_publisher + _dashboard_event_publisher = publisher + return publisher + + +def get_dashboard_event_publisher() -> DashboardEventPublisher: + if _dashboard_event_publisher is None: + raise RuntimeError("DashboardEventPublisher has not been initialized") + return _dashboard_event_publisher + + +def reset_dashboard_event_publisher() -> None: + global _dashboard_event_publisher + _dashboard_event_publisher = None diff --git a/ergon_core/ergon_core/core/application/ports/__init__.py b/ergon_core/ergon_core/core/application/ports/__init__.py index 82dc62f26..8eb9a12ec 100644 --- a/ergon_core/ergon_core/core/application/ports/__init__.py +++ b/ergon_core/ergon_core/core/application/ports/__init__.py @@ -1 +1,14 @@ """Application-declared adapter ports.""" + +from ergon_core.core.application.ports.dashboard import ( + DashboardEventContract, + DashboardEventPublisher, +) +from ergon_core.core.application.ports.resources import ResourceBlobWriter, SandboxFileReader + +__all__ = [ + "DashboardEventContract", + "DashboardEventPublisher", + "ResourceBlobWriter", + "SandboxFileReader", +] diff --git a/ergon_core/ergon_core/core/application/ports/dashboard.py b/ergon_core/ergon_core/core/application/ports/dashboard.py index 29c14067a..460abc835 100644 --- a/ergon_core/ergon_core/core/application/ports/dashboard.py +++ b/ergon_core/ergon_core/core/application/ports/dashboard.py @@ -1,34 +1,18 @@ """Application port for publishing already-built dashboard events.""" -from typing import Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable -from ergon_core.core.application.events.base import InngestEventContract +class DashboardEventContract(Protocol): + """Structural contract for already-built dashboard events.""" -@runtime_checkable -class DashboardEventPublisher(Protocol): - """Publishes dashboard event contracts without owning their construction.""" - - async def publish(self, event: InngestEventContract) -> None: ... - - -_dashboard_event_publisher: DashboardEventPublisher | None = None + name: str + def model_dump(self, *, mode: str = "python") -> dict[str, Any]: ... -def set_dashboard_event_publisher( - publisher: DashboardEventPublisher, -) -> DashboardEventPublisher: - global _dashboard_event_publisher - _dashboard_event_publisher = publisher - return publisher - - -def get_dashboard_event_publisher() -> DashboardEventPublisher: - if _dashboard_event_publisher is None: - raise RuntimeError("DashboardEventPublisher has not been initialized") - return _dashboard_event_publisher +@runtime_checkable +class DashboardEventPublisher(Protocol): + """Publishes dashboard event contracts without owning their construction.""" -def reset_dashboard_event_publisher() -> None: - global _dashboard_event_publisher - _dashboard_event_publisher = None + async def publish(self, event: DashboardEventContract) -> None: ... diff --git a/ergon_core/ergon_core/core/application/resources/publishing.py b/ergon_core/ergon_core/core/application/resources/publishing.py index 4a09113cd..a7f2cce76 100644 --- a/ergon_core/ergon_core/core/application/resources/publishing.py +++ b/ergon_core/ergon_core/core/application/resources/publishing.py @@ -9,7 +9,7 @@ from sqlmodel import Session -from ergon_core.core.application.ports.resources import ResourceBlobWriter, SandboxFileReader +from ergon_core.core.application.ports import ResourceBlobWriter, SandboxFileReader from ergon_core.core.application.resources.models import RunResourceView from ergon_core.core.application.resources.repository import RunResourceRepository from ergon_core.core.persistence.shared.db import get_session diff --git a/ergon_core/ergon_core/core/application/runtime/task_execution.py b/ergon_core/ergon_core/core/application/runtime/task_execution.py index db0583a3a..f52e2df4f 100644 --- a/ergon_core/ergon_core/core/application/runtime/task_execution.py +++ b/ergon_core/ergon_core/core/application/runtime/task_execution.py @@ -3,7 +3,7 @@ import logging from uuid import UUID -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.persistence.definitions.models import ( ExperimentDefinition, ExperimentDefinitionWorker, diff --git a/ergon_core/ergon_core/core/application/runtime/task_management.py b/ergon_core/ergon_core/core/application/runtime/task_management.py index 9573e285f..4fa66b834 100644 --- a/ergon_core/ergon_core/core/application/runtime/task_management.py +++ b/ergon_core/ergon_core/core/application/runtime/task_management.py @@ -18,10 +18,8 @@ import inngest from ergon_core.api.benchmark.task import Task from ergon_core.api.worker.results import SpawnedTaskHandle -from ergon_core.core.application.ports.dashboard import ( - DashboardEventPublisher, - get_dashboard_event_publisher, -) +from ergon_core.core.application.events.service import get_dashboard_event_publisher +from ergon_core.core.application.ports import DashboardEventPublisher from ergon_core.core.persistence.graph.models import RunGraphNode from ergon_core.core.application.runtime.status import ( BLOCKED, diff --git a/ergon_core/ergon_core/core/infrastructure/dashboard/emitter.py b/ergon_core/ergon_core/core/infrastructure/dashboard/emitter.py index 3ca08f1b6..216d7ac65 100644 --- a/ergon_core/ergon_core/core/infrastructure/dashboard/emitter.py +++ b/ergon_core/ergon_core/core/infrastructure/dashboard/emitter.py @@ -4,7 +4,7 @@ import inngest -from ergon_core.core.application.events.base import InngestEventContract +from ergon_core.core.application.ports import DashboardEventContract from ergon_core.core.infrastructure.inngest.client import inngest_client logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def enabled(self) -> bool: def enabled(self, value: bool) -> None: self._enabled = value - async def publish(self, event: InngestEventContract) -> None: + async def publish(self, event: DashboardEventContract) -> None: if not self._enabled: return try: diff --git a/ergon_core/ergon_core/core/infrastructure/dashboard/provider.py b/ergon_core/ergon_core/core/infrastructure/dashboard/provider.py index cb08c6f01..111696616 100644 --- a/ergon_core/ergon_core/core/infrastructure/dashboard/provider.py +++ b/ergon_core/ergon_core/core/infrastructure/dashboard/provider.py @@ -5,7 +5,7 @@ """ from ergon_core.core.infrastructure.dashboard.emitter import DashboardEmitter -from ergon_core.core.application.ports.dashboard import ( +from ergon_core.core.application.events.service import ( reset_dashboard_event_publisher, set_dashboard_event_publisher, ) diff --git a/ergon_core/ergon_core/core/infrastructure/sandbox/event_sink.py b/ergon_core/ergon_core/core/infrastructure/sandbox/event_sink.py index ae5c7c4b7..94f5eed1f 100644 --- a/ergon_core/ergon_core/core/infrastructure/sandbox/event_sink.py +++ b/ergon_core/ergon_core/core/infrastructure/sandbox/event_sink.py @@ -3,7 +3,7 @@ from typing import Protocol, runtime_checkable from uuid import UUID -from ergon_core.core.application.ports.dashboard import DashboardEventPublisher +from ergon_core.core.application.ports import DashboardEventPublisher from ergon_core.core.persistence.shared.db import get_session from ergon_core.core.shared.utils import utcnow from ergon_core.core.views.dashboard_events.contracts import ( diff --git a/ergon_core/ergon_core/core/jobs/task/cleanup_cancelled/job.py b/ergon_core/ergon_core/core/jobs/task/cleanup_cancelled/job.py index fa797bfee..fa1e23ede 100644 --- a/ergon_core/ergon_core/core/jobs/task/cleanup_cancelled/job.py +++ b/ergon_core/ergon_core/core/jobs/task/cleanup_cancelled/job.py @@ -7,7 +7,7 @@ import logging -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.runtime.task_models import CleanupResult from ergon_core.core.application.runtime.task_cleanup import TaskCleanupService from ergon_core.core.jobs.sandbox._lifecycle import terminate_external_sandbox diff --git a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py index 0cb19c2fe..c4af50624 100644 --- a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py +++ b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py @@ -32,7 +32,7 @@ from ergon_core.core.application.evaluation.service import EvaluationService from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from .contract import EvaluateTaskRunResult, TaskEvaluateRequest -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.runtime.task_execution_repository import WorkerOutputRepository from ergon_core.core.infrastructure.inngest.errors import ContractViolationError from ergon_core.core.infrastructure.tracing import ( diff --git a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py index 92e8abd7e..cfea9f613 100644 --- a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py +++ b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py @@ -19,7 +19,7 @@ from ergon_core.core.jobs._events import send_job_step_event from ergon_core.core.jobs.task.execute.contract import TaskReadyEvent from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.resources import RunResourceRepository from ergon_core.core.application.runtime.task_inspection import TaskInspectionService from ergon_core.core.application.runtime.task_management import TaskManagementService @@ -29,7 +29,7 @@ ) from ergon_core.core.shared.context_parts import ContextPartChunk from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.application.context.events import ContextEventService +from ergon_core.core.application.context.service import ContextEventService from ergon_core.core.infrastructure.inngest.errors import ContractViolationError from ergon_core.core.persistence.context.models import RunContextEvent from .contract import WorkerExecuteJobRequest diff --git a/ergon_core/ergon_core/core/jobs/workflow/complete/job.py b/ergon_core/ergon_core/core/jobs/workflow/complete/job.py index c6c9d53c1..ab822fef2 100644 --- a/ergon_core/ergon_core/core/jobs/workflow/complete/job.py +++ b/ergon_core/ergon_core/core/jobs/workflow/complete/job.py @@ -7,7 +7,7 @@ from ergon_core.core.persistence.telemetry.models import RunRecord from ergon_core.core.jobs.run.cleanup.contract import RunCleanupEvent from .contract import WorkflowCompletedEvent, WorkflowCompleteResult -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.runtime.orchestration import FinalizeWorkflowCommand from ergon_core.core.application.runtime.run_lifecycle import WorkflowService from ergon_core.core.jobs._events import send_job_event diff --git a/ergon_core/ergon_core/core/jobs/workflow/start/job.py b/ergon_core/ergon_core/core/jobs/workflow/start/job.py index dd95fc0d8..55ada8f74 100644 --- a/ergon_core/ergon_core/core/jobs/workflow/start/job.py +++ b/ergon_core/ergon_core/core/jobs/workflow/start/job.py @@ -5,7 +5,7 @@ from .contract import WorkflowStartedEvent, WorkflowStartResult from ergon_core.core.jobs.task.execute.contract import TaskReadyEvent -from ergon_core.core.application.ports.dashboard import get_dashboard_event_publisher +from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.runtime.orchestration import InitializeWorkflowCommand from ergon_core.core.application.runtime.run_lifecycle import WorkflowService from ergon_core.core.jobs._events import send_job_events diff --git a/ergon_core/ergon_core/core/views/dashboard_events/contracts.py b/ergon_core/ergon_core/core/views/dashboard_events/contracts.py index a276c0bd8..f8901b5d6 100644 --- a/ergon_core/ergon_core/core/views/dashboard_events/contracts.py +++ b/ergon_core/ergon_core/core/views/dashboard_events/contracts.py @@ -11,11 +11,9 @@ from typing import ClassVar from uuid import UUID -from ergon_core.core.application.communication.models import ( +from ergon_core.core.views.runs.models import ( RunCommunicationMessageDto, RunCommunicationThreadDto, -) -from ergon_core.core.views.runs.models import ( RunSnapshotDto, RunTaskEvaluationDto, ) diff --git a/ergon_core/ergon_core/core/views/runs/models.py b/ergon_core/ergon_core/core/views/runs/models.py index d30777b1f..0c405e6f4 100644 --- a/ergon_core/ergon_core/core/views/runs/models.py +++ b/ergon_core/ergon_core/core/views/runs/models.py @@ -9,7 +9,6 @@ from typing import Any from uuid import UUID -from ergon_core.core.application.communication.models import RunCommunicationThreadDto from ergon_core.core.shared.context_parts import ContextEventType, ContextPartChunkLog from ergon_core.core.application.evaluation.summary import EvalCriterionStatus from pydantic import BaseModel, ConfigDict, Field @@ -30,6 +29,33 @@ class CamelModel(BaseModel): ) +class RunCommunicationMessageDto(CamelModel): + id: str + thread_id: str + thread_topic: str + run_id: str + task_id: str | None = None + task_execution_id: str | None = None + from_agent_id: str + to_agent_id: str + content: str + sequence_num: int + created_at: datetime + + +class RunCommunicationThreadDto(CamelModel): + id: str + run_id: str + task_id: str | None = None + topic: str + summary: str | None = None + agent_a_id: str + agent_b_id: str + created_at: datetime + updated_at: datetime + messages: list[RunCommunicationMessageDto] = Field(default_factory=list) + + class RunTaskDto(CamelModel): """REST projection of RunGraphNode for run detail pages. diff --git a/ergon_core/ergon_core/core/views/runs/snapshot.py b/ergon_core/ergon_core/core/views/runs/snapshot.py index fcf3f4273..3b09c5b02 100644 --- a/ergon_core/ergon_core/core/views/runs/snapshot.py +++ b/ergon_core/ergon_core/core/views/runs/snapshot.py @@ -5,12 +5,10 @@ from typing import cast from uuid import UUID -from ergon_core.core.application.communication.models import ( - RunCommunicationMessageDto, - RunCommunicationThreadDto, -) from ergon_core.core.application.evaluation.dto_mapping import evaluation_row_to_dto from ergon_core.core.views.runs.models import ( + RunCommunicationMessageDto, + RunCommunicationThreadDto, RunContextEventDto, RunExecutionAttemptDto, RunResourceDto, diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index 304454126..5a0ce9783 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -28,7 +28,6 @@ APPROVED_DOMAIN_DIRS = {"policies", "__pycache__"} LAYOUT_FILE_EXCEPTIONS = { - "context": {"events.py"}, "evaluation": {"scoring.py", "summary.py"}, "events": {"base.py", "runtime.py"}, "experiments": {"definition_writer.py", "handles.py", "launch.py"}, @@ -62,11 +61,6 @@ LAYOUT_DIR_EXCEPTIONS: dict[str, set[str]] = {} _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = ( - ( - "ergon_core.core.application.communication.service", - "ergon_core.core.application.ports.dashboard", - "PR 02: make application ports protocol-only public collaboration points.", - ), ( "ergon_core.core.application.experiments.launch", "ergon_core.core.application.events.runtime", @@ -77,16 +71,6 @@ "ergon_core.core.application.runtime.run_records", "PR 05: route run-record access through public runtime facades.", ), - ( - "ergon_core.core.application.ports.dashboard", - "ergon_core.core.application.events.base", - "PR 02: make application ports protocol-only public collaboration points.", - ), - ( - "ergon_core.core.application.resources.publishing", - "ergon_core.core.application.ports.resources", - "PR 02: make application ports protocol-only public collaboration points.", - ), ( "ergon_core.core.application.runtime.events", "ergon_core.core.application.events.runtime", @@ -107,21 +91,11 @@ "ergon_core.core.application.experiments.handles", "PR 04: decide and document experiment handle public surface.", ), - ( - "ergon_core.core.application.runtime.task_execution", - "ergon_core.core.application.ports.dashboard", - "PR 02: make application ports protocol-only public collaboration points.", - ), ( "ergon_core.core.application.runtime.task_management", "ergon_core.core.application.events.runtime", "PR 05: route runtime event collaboration through public runtime facades.", ), - ( - "ergon_core.core.application.runtime.task_management", - "ergon_core.core.application.ports.dashboard", - "PR 02: make application ports protocol-only public collaboration points.", - ), ( "ergon_core.core.application.runtime.task_models", "ergon_core.core.application.events.runtime", diff --git a/ergon_core/tests/unit/architecture/test_public_api_boundaries.py b/ergon_core/tests/unit/architecture/test_public_api_boundaries.py index c33af4e06..e60e834e7 100644 --- a/ergon_core/tests/unit/architecture/test_public_api_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_public_api_boundaries.py @@ -340,7 +340,7 @@ def test_read_context_and_resource_modules_stay_in_application_and_views_layout( core_root / "application" / "communication" / "models.py", core_root / "application" / "communication" / "errors.py", core_root / "application" / "context" / "__init__.py", - core_root / "application" / "context" / "events.py", + core_root / "application" / "context" / "service.py", core_root / "application" / "resources" / "__init__.py", core_root / "application" / "resources" / "models.py", core_root / "application" / "resources" / "repository.py", diff --git a/ergon_core/tests/unit/dashboard/test_event_contract_types.py b/ergon_core/tests/unit/dashboard/test_event_contract_types.py index 6aedbf48d..9f5b23587 100644 --- a/ergon_core/tests/unit/dashboard/test_event_contract_types.py +++ b/ergon_core/tests/unit/dashboard/test_event_contract_types.py @@ -6,10 +6,6 @@ from uuid import uuid4 import pytest -from ergon_core.core.application.communication.models import ( - RunCommunicationMessageDto, - RunCommunicationThreadDto, -) from ergon_core.core.application.events.base import InngestEventContract from ergon_core.core.infrastructure.dashboard import emitter as dashboard_emitter_module from ergon_core.core.infrastructure.dashboard.emitter import DashboardEmitter @@ -19,7 +15,11 @@ DashboardThreadMessageCreatedEvent, DashboardWorkflowStartedEvent, ) -from ergon_core.core.views.runs.models import RunSnapshotDto +from ergon_core.core.views.runs.models import ( + RunCommunicationMessageDto, + RunCommunicationThreadDto, + RunSnapshotDto, +) REPO_ROOT = Path(__file__).resolve().parents[4] diff --git a/ergon_core/tests/unit/persistence/test_context_event_repository.py b/ergon_core/tests/unit/persistence/test_context_event_repository.py index f8ae778e1..df31c8b65 100644 --- a/ergon_core/tests/unit/persistence/test_context_event_repository.py +++ b/ergon_core/tests/unit/persistence/test_context_event_repository.py @@ -11,7 +11,7 @@ UserMessagePart, ) from ergon_core.core.persistence.context.models import RunContextEvent -from ergon_core.core.application.context.events import ContextEventService +from ergon_core.core.application.context.service import ContextEventService from ergon_core.core.persistence.definitions.models import ExperimentDefinition from ergon_core.core.persistence.graph.models import RunGraphNode from ergon_core.core.persistence.shared.enums import RunStatus, TaskExecutionStatus From 175e860c47f6c9104b79b0dd99a023db61c0f3f6 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 12:06:58 +0100 Subject: [PATCH 4/8] Standardize evaluation application domain --- .../core/application/evaluation/mappers.py | 95 +++++++++++++ .../core/application/evaluation/service.py | 126 ++++-------------- .../core/application/runtime/run_lifecycle.py | 4 +- .../ergon_core/core/jobs/task/evaluate/job.py | 10 +- .../runs/evaluation_mapping.py} | 4 +- .../ergon_core/core/views/runs/models.py | 2 +- .../ergon_core/core/views/runs/service.py | 13 +- .../ergon_core/core/views/runs/snapshot.py | 2 +- .../test_application_domain_boundaries.py | 5 - .../evaluation/test_evaluation_dto_mapping.py | 2 +- .../test_evaluation_score_aggregation.py | 6 +- .../test_evaluation_summary_contracts.py | 6 +- 12 files changed, 153 insertions(+), 122 deletions(-) create mode 100644 ergon_core/ergon_core/core/application/evaluation/mappers.py rename ergon_core/ergon_core/core/{application/evaluation/dto_mapping.py => views/runs/evaluation_mapping.py} (97%) diff --git a/ergon_core/ergon_core/core/application/evaluation/mappers.py b/ergon_core/ergon_core/core/application/evaluation/mappers.py new file mode 100644 index 000000000..bb9203b75 --- /dev/null +++ b/ergon_core/ergon_core/core/application/evaluation/mappers.py @@ -0,0 +1,95 @@ +"""Internal mapping helpers for evaluation-domain summaries.""" + +from typing import Protocol + +from ergon_core.api.rubric import TaskEvaluationResult +from ergon_core.core.application.evaluation.models import CriterionSpec +from ergon_core.core.application.evaluation.summary import ( + CriterionOutcomeEntry, + EvaluationSummary, +) +from ergon_core.core.infrastructure.inngest.errors import ContractViolationError + + +class EvaluationSummarySource(Protocol): + result: TaskEvaluationResult + specs: list[CriterionSpec] + + +def _criterion_status(*, passed: bool, error: dict | None, skipped_reason: str | None) -> str: + if error is not None: + return "errored" + if skipped_reason is not None: + return "skipped" + return "passed" if passed else "failed" + + +def _summary_max_score( + result: TaskEvaluationResult, + specs: list[CriterionSpec], +) -> float: + if result.metadata.get("score_scale") == "normalized_0_1": + return 1.0 + return sum(s.max_score for s in specs) if specs else 1.0 + + +def build_evaluation_summary( + service_result: EvaluationSummarySource, + evaluation_input: str | None, +) -> EvaluationSummary: + result = service_result.result + specs = service_result.specs + spec_by_idx = {s.criterion_idx: s for s in specs} + max_score_total = _summary_max_score(result, specs) + entries: list[CriterionOutcomeEntry] = [] + for i, cr in enumerate(result.criterion_results): + spec = spec_by_idx.get(i) + if spec is None: + raise ContractViolationError( + f"Criterion result at index {i} ({cr.slug!r}) has no matching " + "CriterionSpec - specs and results are out of sync", + ) + entries.append( + CriterionOutcomeEntry( + criterion_slug=cr.slug, + criterion_name=cr.name or cr.slug, + criterion_type=spec.criterion.type_slug, + criterion_description=spec.criterion.description, + stage_num=spec.stage_idx, + stage_name=spec.stage_name, + criterion_num=spec.criterion_idx, + status=_criterion_status( + passed=cr.passed, + error=cr.error, + skipped_reason=cr.skipped_reason, + ), + score=cr.score, + max_score=spec.max_score, + passed=cr.passed, + weight=cr.weight, + contribution=cr.score, + feedback=cr.feedback, + model_reasoning=cr.model_reasoning, + skipped_reason=cr.skipped_reason, + evaluation_input=cr.evaluation_input or evaluation_input, + evaluated_action_ids=cr.evaluated_action_ids, + evaluated_resource_ids=cr.evaluated_resource_ids, + observation=cr.observation, + error=cr.error, + ) + ) + stage_names = {s.stage_name for s in specs} + stages_passed = sum( + 1 + for stage_name in stage_names + if all(e.passed for e in entries if e.stage_name == stage_name) + ) + return EvaluationSummary( + evaluator_name=result.evaluator_name, + max_score=max_score_total, + normalized_score=result.score, + stages_evaluated=len(stage_names), + stages_passed=stages_passed, + metadata=result.metadata, + criterion_results=entries, + ) diff --git a/ergon_core/ergon_core/core/application/evaluation/service.py b/ergon_core/ergon_core/core/application/evaluation/service.py index 9d30e9d03..f16a13ad8 100644 --- a/ergon_core/ergon_core/core/application/evaluation/service.py +++ b/ergon_core/ergon_core/core/application/evaluation/service.py @@ -1,22 +1,19 @@ """Single front-door service for task evaluation workflow.""" +from collections.abc import Iterable +from datetime import datetime from uuid import UUID -from ergon_core.api.benchmark import Task from ergon_core.api.criterion.context import CriterionContext from ergon_core.api.criterion.outcome import CriterionOutcome from ergon_core.api.rubric import Evaluator, TaskEvaluationResult -from ergon_core.core.application.evaluation.dto_mapping import ( - build_dashboard_evaluation_dto, - evaluation_row_to_dto, -) from ergon_core.core.application.evaluation.models import CriterionSpec -from ergon_core.core.application.evaluation.scoring import aggregate_evaluation_scores -from ergon_core.core.application.evaluation.summary import ( - CriterionOutcomeEntry, - EvaluationSummary, +from ergon_core.core.application.evaluation.scoring import ( + EvaluationScoreSummary, + ScoredEvaluation, + aggregate_evaluation_scores, ) -from ergon_core.core.views.runs.models import RunTaskEvaluationDto +from ergon_core.core.application.evaluation.summary import EvaluationSummary from ergon_core.core.infrastructure.inngest.errors import ContractViolationError from ergon_core.core.persistence.definitions.models import ExperimentDefinitionEvaluator from ergon_core.core.persistence.shared.db import get_session @@ -28,6 +25,8 @@ from pydantic import BaseModel from sqlmodel import Session, select +from .mappers import build_evaluation_summary + class EvaluationServiceResult(BaseModel): """Internal result carrying both the public evaluation + spec metadata.""" @@ -37,17 +36,29 @@ class EvaluationServiceResult(BaseModel): class PersistedEvaluation(BaseModel): - """Evaluation row and dashboard DTO produced by persistence.""" + """Public fields produced after persisting one task evaluation.""" model_config = {"frozen": True} summary: EvaluationSummary - dashboard_dto: RunTaskEvaluationDto + evaluation_id: UUID + run_id: UUID + task_id: UUID + total_score: float + created_at: datetime class EvaluationService: """Execute and persist task evaluations.""" + @staticmethod + def summarize_scores( + evaluations: Iterable[ScoredEvaluation], + ) -> EvaluationScoreSummary: + """Aggregate run-level evaluation scores through the evaluation facade.""" + + return aggregate_evaluation_scores(evaluations) + async def evaluate( self, *, @@ -120,7 +131,11 @@ async def persist_success( session.refresh(evaluation) return PersistedEvaluation( summary=summary, - dashboard_dto=evaluation_row_to_dto(evaluation), + evaluation_id=evaluation.id, + run_id=evaluation.run_id, + task_id=evaluation.task_id, + total_score=0.0 if evaluation.score is None else evaluation.score, + created_at=evaluation.created_at, ) finally: session.close() @@ -212,7 +227,7 @@ def _refresh_run_evaluation_summary(self, session: Session, run_id: UUID) -> Non if run is None: return evaluations = _list_task_evaluations(session, run_id) - score_summary = aggregate_evaluation_scores(evaluations) + score_summary = self.summarize_scores(evaluations) existing_summary = dict({} if run.summary_json is None else run.summary_json) existing_summary.update( { @@ -257,86 +272,3 @@ async def _create_task_evaluation( session.add(evaluation) session.flush() return evaluation - - -def _criterion_status(*, passed: bool, error: dict | None, skipped_reason: str | None) -> str: - # TODO: inline to fix Locality of behavior violation - # also investigate if this is even needed, seems messy. - if error is not None: - return "errored" - if skipped_reason is not None: - return "skipped" - return "passed" if passed else "failed" - - -def _summary_max_score( - result: TaskEvaluationResult, - specs: list[CriterionSpec], -) -> float: - # TODO: inline to fix Locality of behavior violation - # also investigate if this is even needed, seems messy. - if result.metadata.get("score_scale") == "normalized_0_1": - return 1.0 - return sum(s.max_score for s in specs) if specs else 1.0 - - -def build_evaluation_summary( - service_result: EvaluationServiceResult, - evaluation_input: str | None, -) -> EvaluationSummary: - result = service_result.result - specs = service_result.specs - spec_by_idx = {s.criterion_idx: s for s in specs} - max_score_total = _summary_max_score(result, specs) - entries: list[CriterionOutcomeEntry] = [] - for i, cr in enumerate(result.criterion_results): - spec = spec_by_idx.get(i) - if spec is None: - raise ContractViolationError( - f"Criterion result at index {i} ({cr.slug!r}) has no matching " - "CriterionSpec - specs and results are out of sync", - ) - entries.append( - CriterionOutcomeEntry( - criterion_slug=cr.slug, - criterion_name=cr.name or cr.slug, - criterion_type=spec.criterion.type_slug, - criterion_description=spec.criterion.description, - stage_num=spec.stage_idx, - stage_name=spec.stage_name, - criterion_num=spec.criterion_idx, - status=_criterion_status( - passed=cr.passed, - error=cr.error, - skipped_reason=cr.skipped_reason, - ), - score=cr.score, - max_score=spec.max_score, - passed=cr.passed, - weight=cr.weight, - contribution=cr.score, - feedback=cr.feedback, - model_reasoning=cr.model_reasoning, - skipped_reason=cr.skipped_reason, - evaluation_input=cr.evaluation_input or evaluation_input, - evaluated_action_ids=cr.evaluated_action_ids, - evaluated_resource_ids=cr.evaluated_resource_ids, - observation=cr.observation, - error=cr.error, - ) - ) - stage_names = {s.stage_name for s in specs} - stages_passed = sum( - 1 - for stage_name in stage_names - if all(e.passed for e in entries if e.stage_name == stage_name) - ) - return EvaluationSummary( - evaluator_name=result.evaluator_name, - max_score=max_score_total, - normalized_score=result.score, - stages_evaluated=len(stage_names), - stages_passed=stages_passed, - metadata=result.metadata, - criterion_results=entries, - ) diff --git a/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py b/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py index e4b673093..340c9304f 100644 --- a/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py +++ b/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py @@ -17,7 +17,7 @@ RunTaskEvaluation, RunTaskExecution, ) -from ergon_core.core.application.evaluation.scoring import aggregate_evaluation_scores +from ergon_core.core.application.evaluation.service import EvaluationService from ergon_core.core.application.runtime.events import ( RuntimeEventDispatcher, TaskReadyDispatcher, @@ -157,7 +157,7 @@ def finalize(self, command: FinalizeWorkflowCommand) -> FinalizedWorkflowResult: select(RunTaskEvaluation).where(RunTaskEvaluation.run_id == command.run_id) ).all() ) - score_summary = aggregate_evaluation_scores(evaluations) + score_summary = EvaluationService.summarize_scores(evaluations) completion = RunCompletionData( completed_at=utcnow(), final_score=score_summary.final_score, diff --git a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py index c4af50624..5207cba86 100644 --- a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py +++ b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py @@ -43,6 +43,7 @@ from ergon_core.core.persistence.shared.db import get_session from ergon_core.core.persistence.telemetry.models import RunTaskExecution from ergon_core.core.views.dashboard_events.contracts import DashboardTaskEvaluationUpdatedEvent +from ergon_core.core.views.runs.evaluation_mapping import build_dashboard_evaluation_dto if TYPE_CHECKING: from ergon_core.api.rubric import Evaluator @@ -184,7 +185,14 @@ async def _run_evaluation( DashboardTaskEvaluationUpdatedEvent( run_id=run_id, task_id=view.task_id, - evaluation=persisted.dashboard_dto, + evaluation=build_dashboard_evaluation_dto( + evaluation_id=persisted.evaluation_id, + run_id=persisted.run_id, + task_id=persisted.task_id, + total_score=persisted.total_score, + created_at=persisted.created_at, + summary=persisted.summary, + ), ) ) diff --git a/ergon_core/ergon_core/core/application/evaluation/dto_mapping.py b/ergon_core/ergon_core/core/views/runs/evaluation_mapping.py similarity index 97% rename from ergon_core/ergon_core/core/application/evaluation/dto_mapping.py rename to ergon_core/ergon_core/core/views/runs/evaluation_mapping.py index 558d17a4c..2af8dd91d 100644 --- a/ergon_core/ergon_core/core/application/evaluation/dto_mapping.py +++ b/ergon_core/ergon_core/core/views/runs/evaluation_mapping.py @@ -1,14 +1,14 @@ -"""Mapping helpers for persisted evaluation rows.""" +"""Dashboard DTO mapping for persisted evaluation rows.""" from datetime import datetime from uuid import UUID from ergon_core.core.application.evaluation.summary import EvaluationSummary +from ergon_core.core.persistence.telemetry.models import RunTaskEvaluation from ergon_core.core.views.runs.models import ( RunEvaluationCriterionDto, RunTaskEvaluationDto, ) -from ergon_core.core.persistence.telemetry.models import RunTaskEvaluation def build_dashboard_evaluation_dto( diff --git a/ergon_core/ergon_core/core/views/runs/models.py b/ergon_core/ergon_core/core/views/runs/models.py index 0c405e6f4..22127cbe7 100644 --- a/ergon_core/ergon_core/core/views/runs/models.py +++ b/ergon_core/ergon_core/core/views/runs/models.py @@ -9,8 +9,8 @@ from typing import Any from uuid import UUID -from ergon_core.core.shared.context_parts import ContextEventType, ContextPartChunkLog from ergon_core.core.application.evaluation.summary import EvalCriterionStatus +from ergon_core.core.shared.context_parts import ContextEventType, ContextPartChunkLog from pydantic import BaseModel, ConfigDict, Field diff --git a/ergon_core/ergon_core/core/views/runs/service.py b/ergon_core/ergon_core/core/views/runs/service.py index 8d26b1d11..2665e247c 100644 --- a/ergon_core/ergon_core/core/views/runs/service.py +++ b/ergon_core/ergon_core/core/views/runs/service.py @@ -28,11 +28,11 @@ Thread, ThreadMessage, ) -from ergon_core.core.application.runtime.models import GraphMutationRecordDto -from ergon_core.core.application.evaluation.scoring import ( +from ergon_core.core.application.evaluation.service import ( EvaluationScoreSummary, - aggregate_evaluation_scores, + EvaluationService, ) +from ergon_core.core.application.runtime.models import GraphMutationRecordDto from ergon_core.core.views.runs.snapshot import ( _build_communication_threads, _build_task_map, @@ -160,7 +160,7 @@ def build_run_snapshot(self, run_id: UUID) -> RunSnapshotDto | None: execution_task_map, ) - score_summary = aggregate_evaluation_scores(evaluations) + score_summary = EvaluationService.summarize_scores(evaluations) duration_seconds: float | None = None if run.started_at and run.completed_at: @@ -248,7 +248,10 @@ def get_resource_blob(self, run_id: UUID, resource_id: UUID) -> RunResourceBlob ) -def _display_run_score(score_summary: EvaluationScoreSummary, run_status: str) -> float | None: +def _display_run_score( + score_summary: EvaluationScoreSummary, + run_status: str, +) -> float | None: if run_status != RunStatus.COMPLETED: return None # TODO: this is a hack, we need to fix the calculation / rename variables to make clear that the output score should be normalised by here. diff --git a/ergon_core/ergon_core/core/views/runs/snapshot.py b/ergon_core/ergon_core/core/views/runs/snapshot.py index 3b09c5b02..44379b88b 100644 --- a/ergon_core/ergon_core/core/views/runs/snapshot.py +++ b/ergon_core/ergon_core/core/views/runs/snapshot.py @@ -5,7 +5,7 @@ from typing import cast from uuid import UUID -from ergon_core.core.application.evaluation.dto_mapping import evaluation_row_to_dto +from ergon_core.core.views.runs.evaluation_mapping import evaluation_row_to_dto from ergon_core.core.views.runs.models import ( RunCommunicationMessageDto, RunCommunicationThreadDto, diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index 5a0ce9783..cd4331b52 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -76,11 +76,6 @@ "ergon_core.core.application.events.runtime", "PR 05: route runtime event collaboration through public runtime facades.", ), - ( - "ergon_core.core.application.runtime.run_lifecycle", - "ergon_core.core.application.evaluation.scoring", - "PR 03: keep evaluation scoring behind evaluation public APIs.", - ), ( "ergon_core.core.application.runtime.run_records", "ergon_core.core.application.events.runtime", diff --git a/ergon_core/tests/unit/evaluation/test_evaluation_dto_mapping.py b/ergon_core/tests/unit/evaluation/test_evaluation_dto_mapping.py index 759483f44..b8072f4b7 100644 --- a/ergon_core/tests/unit/evaluation/test_evaluation_dto_mapping.py +++ b/ergon_core/tests/unit/evaluation/test_evaluation_dto_mapping.py @@ -1,8 +1,8 @@ from uuid import uuid4 -from ergon_core.core.application.evaluation.dto_mapping import evaluation_row_to_dto from ergon_core.core.application.evaluation.summary import EvaluationSummary from ergon_core.core.persistence.telemetry.models import RunTaskEvaluation +from ergon_core.core.views.runs.evaluation_mapping import evaluation_row_to_dto def test_evaluation_row_to_dto_maps_multiple_criterion_outcomes() -> None: diff --git a/ergon_core/tests/unit/runtime/test_evaluation_score_aggregation.py b/ergon_core/tests/unit/runtime/test_evaluation_score_aggregation.py index 8826c1fd7..2ef78d7b4 100644 --- a/ergon_core/tests/unit/runtime/test_evaluation_score_aggregation.py +++ b/ergon_core/tests/unit/runtime/test_evaluation_score_aggregation.py @@ -1,10 +1,10 @@ from types import SimpleNamespace -from ergon_core.core.application.evaluation.scoring import aggregate_evaluation_scores +from ergon_core.core.application.evaluation.service import EvaluationService def test_aggregate_evaluation_scores_counts_all_evaluators_and_averages_scored_rows() -> None: - summary = aggregate_evaluation_scores( + summary = EvaluationService.summarize_scores( [ SimpleNamespace(score=2.0), SimpleNamespace(score=None), @@ -18,7 +18,7 @@ def test_aggregate_evaluation_scores_counts_all_evaluators_and_averages_scored_r def test_aggregate_evaluation_scores_returns_none_scores_when_nothing_scored() -> None: - summary = aggregate_evaluation_scores([SimpleNamespace(score=None)]) + summary = EvaluationService.summarize_scores([SimpleNamespace(score=None)]) assert summary.final_score is None assert summary.normalized_score is None diff --git a/ergon_core/tests/unit/runtime/test_evaluation_summary_contracts.py b/ergon_core/tests/unit/runtime/test_evaluation_summary_contracts.py index f9b671457..963b2d10b 100644 --- a/ergon_core/tests/unit/runtime/test_evaluation_summary_contracts.py +++ b/ergon_core/tests/unit/runtime/test_evaluation_summary_contracts.py @@ -15,11 +15,9 @@ from ergon_core.api.rubric import TaskEvaluationResult from ergon_core.core.application.evaluation.summary import CriterionOutcomeEntry from ergon_core.core.application.evaluation.models import CriterionSpec -from ergon_core.core.application.evaluation.service import ( - build_dashboard_evaluation_dto, - build_evaluation_summary, -) +from ergon_core.core.application.evaluation.mappers import build_evaluation_summary from ergon_core.core.application.evaluation.service import EvaluationServiceResult +from ergon_core.core.views.runs.evaluation_mapping import build_dashboard_evaluation_dto from pydantic import ValidationError From e68094c1b0c1c0736de94e78c9b1052ef6f120b8 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 12:21:07 +0100 Subject: [PATCH 5/8] Standardize experiments and resources domains --- ergon_core/ergon_core/api/__init__.py | 2 +- ergon_core/ergon_core/api/worker/context.py | 47 ++++++------- .../experiments/definition_writer.py | 2 +- .../core/application/experiments/handles.py | 24 +------ .../core/application/experiments/launch.py | 3 +- .../core/application/experiments/models.py | 20 +++++- .../core/application/experiments/service.py | 15 ++++ .../core/application/resources/__init__.py | 4 +- .../core/application/resources/service.py | 69 +++++++++++++++++++ .../core/application/runtime/run_records.py | 2 +- .../http/routes/test_harness.py | 2 +- .../core/jobs/task/worker_execute/job.py | 4 +- .../api/worker/test_worker_context_facade.py | 37 +++++----- .../test_application_domain_boundaries.py | 7 +- .../test_repository_layer_conventions.py | 18 +---- ...test_worker_execute_live_sandbox_attach.py | 4 +- .../test_experiment_definition_service.py | 49 +++++++++++++ .../unit/runtime/test_identity_invariants.py | 4 +- .../runtime/test_walkthrough_smoketest.py | 2 +- .../test_worker_context_containment.py | 2 +- 20 files changed, 214 insertions(+), 103 deletions(-) create mode 100644 ergon_core/ergon_core/core/application/resources/service.py diff --git a/ergon_core/ergon_core/api/__init__.py b/ergon_core/ergon_core/api/__init__.py index a7f82ecfc..2f71a8af6 100644 --- a/ergon_core/ergon_core/api/__init__.py +++ b/ergon_core/ergon_core/api/__init__.py @@ -21,7 +21,7 @@ SandboxKindMismatch, SandboxNotLiveError, ) -from ergon_core.core.application.experiments.definition_writer import persist_benchmark +from ergon_core.core.application.experiments.service import persist_benchmark from ergon_core.api.rubric import Evaluator, Rubric, TaskEvaluationResult from ergon_core.api.sandbox import Sandbox, SandboxRuntime from ergon_core.api.worker import ( diff --git a/ergon_core/ergon_core/api/worker/context.py b/ergon_core/ergon_core/api/worker/context.py index 46d2cf241..3c63a5cbb 100644 --- a/ergon_core/ergon_core/api/worker/context.py +++ b/ergon_core/ergon_core/api/worker/context.py @@ -1,7 +1,6 @@ """Per-execution runtime state passed to Worker.execute().""" from collections.abc import Callable -from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, ContextManager, TypeAlias from uuid import UUID @@ -17,18 +16,18 @@ if TYPE_CHECKING: from sqlmodel import Session - from ergon_core.core.application.resources.repository import RunResourceRepository + from ergon_core.core.application.resources.service import RunResourceReadService from ergon_core.core.application.runtime.task_inspection import TaskInspectionService from ergon_core.core.application.runtime.task_management import TaskManagementService TaskManagementServiceAlias: TypeAlias = TaskManagementService TaskInspectionServiceAlias: TypeAlias = TaskInspectionService - RunResourceRepositoryAlias: TypeAlias = RunResourceRepository + RunResourceReadServiceAlias: TypeAlias = RunResourceReadService SessionFactory: TypeAlias = Callable[[], ContextManager[Session]] else: TaskManagementServiceAlias: TypeAlias = Any TaskInspectionServiceAlias: TypeAlias = Any - RunResourceRepositoryAlias: TypeAlias = Any + RunResourceReadServiceAlias: TypeAlias = Any SessionFactory: TypeAlias = Callable[[], ContextManager[Any]] @@ -46,8 +45,8 @@ def _require_injected_dependency(value: object | None) -> object: TaskInspectionServiceAlias, AfterValidator(_require_injected_dependency), ] -RunResourceRepositoryDependency: TypeAlias = Annotated[ - RunResourceRepositoryAlias, +RunResourceReadServiceDependency: TypeAlias = Annotated[ + RunResourceReadServiceAlias, AfterValidator(_require_injected_dependency), ] SessionFactoryDependency: TypeAlias = Annotated[ @@ -97,7 +96,7 @@ class WorkerContext(BaseModel): exclude=True, repr=False, ) - resource_repo: RunResourceRepositoryDependency = Field( + resource_service: RunResourceReadServiceDependency = Field( exclude=True, repr=False, ) @@ -114,7 +113,7 @@ def _for_job( sandbox_id: str, task_mgmt: TaskManagementServiceAlias, task_inspect: TaskInspectionServiceAlias, - resource_repo: RunResourceRepositoryAlias, + resource_service: RunResourceReadServiceAlias, session_factory: SessionFactory, ) -> "WorkerContext": """Construct the job runtime ``WorkerContext``. @@ -132,7 +131,7 @@ def _for_job( sandbox_id=sandbox_id, task_mgmt=task_mgmt, task_inspect=task_inspect, - resource_repo=resource_repo, + resource_service=resource_service, session_factory=session_factory, ) @@ -257,28 +256,22 @@ async def resources( run. Lifecycle methods remain descendant-contained. """ - with self.session_factory() as session: - rows = self.resource_repo.list_for_run( - session, - run_id=self.run_id, - task_id=task_id, - task_execution_id=execution_id, - kind=kind, - name=name, - ) - return tuple(rows) + return self.resource_service.list_for_run( + run_id=self.run_id, + task_id=task_id, + task_execution_id=execution_id, + kind=kind, + name=name, + ) async def read_resource(self, resource_id: UUID) -> bytes: """Read a visible resource blob from this run.""" - with self.session_factory() as session: - resource = self.resource_repo.get(session, resource_id) - if resource.run_id != self.run_id: - raise ContainmentViolation( - parent_task_id=self.task_id, - target_task_id=resource_id, - ) - return Path(resource.file_path).read_bytes() + return self.resource_service.read_bytes( + run_id=self.run_id, + current_task_id=self.task_id, + resource_id=resource_id, + ) async def _assert_descendant(self, task_id: UUID) -> None: """Raise ``ContainmentViolation`` if ``task_id`` is not self.task_id or a descendant.""" diff --git a/ergon_core/ergon_core/core/application/experiments/definition_writer.py b/ergon_core/ergon_core/core/application/experiments/definition_writer.py index 9de2eab0d..d6cb1b7df 100644 --- a/ergon_core/ergon_core/core/application/experiments/definition_writer.py +++ b/ergon_core/ergon_core/core/application/experiments/definition_writer.py @@ -9,7 +9,7 @@ from ergon_core.api.benchmark import Benchmark from ergon_core.api.errors import SandboxKindMismatch -from ergon_core.core.application.experiments.handles import DefinitionHandle +from ergon_core.core.application.experiments.models import DefinitionHandle from ergon_core.core.persistence.definitions.models import ( ExperimentDefinition, ExperimentDefinitionEvaluator, diff --git a/ergon_core/ergon_core/core/application/experiments/handles.py b/ergon_core/ergon_core/core/application/experiments/handles.py index 24fbe03a5..00ec41c43 100644 --- a/ergon_core/ergon_core/core/application/experiments/handles.py +++ b/ergon_core/ergon_core/core/application/experiments/handles.py @@ -1,23 +1,5 @@ -"""Core lifecycle handles for persisted workflow definitions.""" +"""Compatibility import for experiment handle contracts.""" -from datetime import datetime -from typing import Any -from uuid import UUID +from ergon_core.core.application.experiments.models import DefinitionHandle -from ergon_core.core.shared.utils import utcnow -from pydantic import BaseModel, Field - - -class DefinitionHandle(BaseModel): - """Rich handle returned after a benchmark definition is persisted.""" - - model_config = {"frozen": True} - - definition_id: UUID - benchmark_type: str - worker_bindings: dict[str, str] = Field(default_factory=dict) - evaluator_bindings: dict[str, str] = Field(default_factory=dict) - instance_count: int = 0 - task_count: int = 0 - created_at: datetime = Field(default_factory=utcnow) - metadata: dict[str, Any] = Field(default_factory=dict) # slopcop: ignore[no-typing-any] +__all__ = ["DefinitionHandle"] diff --git a/ergon_core/ergon_core/core/application/experiments/launch.py b/ergon_core/ergon_core/core/application/experiments/launch.py index 324227cea..d440ec903 100644 --- a/ergon_core/ergon_core/core/application/experiments/launch.py +++ b/ergon_core/ergon_core/core/application/experiments/launch.py @@ -6,9 +6,8 @@ import inngest from ergon_core.core.application.events.runtime import WorkflowStartedEvent from ergon_core.core.application.experiments.errors import DefinitionNotFoundError -from ergon_core.core.application.experiments.models import ExperimentRunResult +from ergon_core.core.application.experiments.models import DefinitionHandle, ExperimentRunResult from ergon_core.core.application.runtime.run_records import create_run -from ergon_core.core.application.experiments.handles import DefinitionHandle from ergon_core.core.infrastructure.inngest.client import inngest_client from ergon_core.core.persistence.definitions.models import ExperimentDefinition from ergon_core.core.persistence.shared.db import get_session diff --git a/ergon_core/ergon_core/core/application/experiments/models.py b/ergon_core/ergon_core/core/application/experiments/models.py index e8efac127..2dae77a66 100644 --- a/ergon_core/ergon_core/core/application/experiments/models.py +++ b/ergon_core/ergon_core/core/application/experiments/models.py @@ -1,12 +1,30 @@ -"""DTOs for experiment launch services.""" +"""DTOs and public contracts for experiment services.""" +from datetime import datetime +from typing import Any from typing import Self from uuid import UUID from ergon_core.core.shared.json_types import JsonObject +from ergon_core.core.shared.utils import utcnow from pydantic import BaseModel, Field, model_validator +class DefinitionHandle(BaseModel): + """Rich handle returned after a benchmark definition is persisted.""" + + model_config = {"frozen": True} + + definition_id: UUID + benchmark_type: str + worker_bindings: dict[str, str] = Field(default_factory=dict) + evaluator_bindings: dict[str, str] = Field(default_factory=dict) + instance_count: int = 0 + task_count: int = 0 + created_at: datetime = Field(default_factory=utcnow) + metadata: dict[str, Any] = Field(default_factory=dict) # slopcop: ignore[no-typing-any] + + class ExperimentRunRequest(BaseModel): definition_id: UUID timeout_seconds: int | None = None diff --git a/ergon_core/ergon_core/core/application/experiments/service.py b/ergon_core/ergon_core/core/application/experiments/service.py index 4fa356773..b94601c52 100644 --- a/ergon_core/ergon_core/core/application/experiments/service.py +++ b/ergon_core/ergon_core/core/application/experiments/service.py @@ -1,17 +1,32 @@ """Module-level launch for persisted object-bound benchmark definitions.""" from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING from uuid import UUID from ergon_core.core.application.experiments.launch import launch_run from ergon_core.core.application.experiments.models import ( + DefinitionHandle, ExperimentRunRequest, ExperimentRunResult, ) +if TYPE_CHECKING: + from ergon_core.api.benchmark import Benchmark + WorkflowStartedEmitter = Callable[[UUID, UUID], Awaitable[None]] +def persist_benchmark(benchmark: "Benchmark") -> DefinitionHandle: + """Persist a configured object-bound Benchmark as an experiment definition.""" + + from ergon_core.core.application.experiments.definition_writer import ( + persist_benchmark as _persist_benchmark, + ) + + return _persist_benchmark(benchmark) + + async def run_experiment( request: ExperimentRunRequest, *, diff --git a/ergon_core/ergon_core/core/application/resources/__init__.py b/ergon_core/ergon_core/core/application/resources/__init__.py index 85eafeec4..b67ff24fb 100644 --- a/ergon_core/ergon_core/core/application/resources/__init__.py +++ b/ergon_core/ergon_core/core/application/resources/__init__.py @@ -1,5 +1,5 @@ from ergon_core.core.application.resources.errors import RunResourceNotFoundError from ergon_core.core.application.resources.models import RunResourceView -from ergon_core.core.application.resources.repository import RunResourceRepository +from ergon_core.core.application.resources.service import RunResourceReadService -__all__ = ["RunResourceNotFoundError", "RunResourceRepository", "RunResourceView"] +__all__ = ["RunResourceNotFoundError", "RunResourceReadService", "RunResourceView"] diff --git a/ergon_core/ergon_core/core/application/resources/service.py b/ergon_core/ergon_core/core/application/resources/service.py new file mode 100644 index 000000000..8d1c99c1d --- /dev/null +++ b/ergon_core/ergon_core/core/application/resources/service.py @@ -0,0 +1,69 @@ +"""Application service for reading run resources.""" + +from collections.abc import Callable +from contextlib import AbstractContextManager +from pathlib import Path +from typing import TypeAlias +from uuid import UUID + +from sqlmodel import Session + +from ergon_core.api.errors import ContainmentViolation +from ergon_core.core.application.resources.models import RunResourceView +from ergon_core.core.application.resources.repository import RunResourceRepository +from ergon_core.core.persistence.shared.db import get_session + +SessionFactory: TypeAlias = Callable[[], AbstractContextManager[Session]] + + +class RunResourceReadService: + """Owns worker-facing read policy for run resources.""" + + def __init__( + self, + *, + repository: RunResourceRepository | None = None, + session_factory: SessionFactory = get_session, + ) -> None: + self._resource_repo = repository or RunResourceRepository() + self._session_factory = session_factory + + def list_for_run( + self, + *, + run_id: UUID, + task_id: UUID | None = None, + task_execution_id: UUID | None = None, + kind: str | None = None, + name: str | None = None, + ) -> tuple[RunResourceView, ...]: + """List resources visible inside a run.""" + + with self._session_factory() as session: + rows = self._resource_repo.list_for_run( + session, + run_id=run_id, + task_id=task_id, + task_execution_id=task_execution_id, + kind=kind, + name=name, + ) + return tuple(rows) + + def read_bytes( + self, + *, + run_id: UUID, + current_task_id: UUID, + resource_id: UUID, + ) -> bytes: + """Read a resource blob after enforcing run containment.""" + + with self._session_factory() as session: + resource = self._resource_repo.get(session, resource_id) + if resource.run_id != run_id: + raise ContainmentViolation( + parent_task_id=current_task_id, + target_task_id=resource_id, + ) + return Path(resource.file_path).read_bytes() diff --git a/ergon_core/ergon_core/core/application/runtime/run_records.py b/ergon_core/ergon_core/core/application/runtime/run_records.py index 9713c2d79..ed3396cf4 100644 --- a/ergon_core/ergon_core/core/application/runtime/run_records.py +++ b/ergon_core/ergon_core/core/application/runtime/run_records.py @@ -4,7 +4,7 @@ from uuid import UUID import inngest -from ergon_core.core.application.experiments.handles import DefinitionHandle +from ergon_core.core.application.experiments.models import DefinitionHandle from ergon_core.core.application.events.runtime import RunCancelledEvent, RunCleanupEvent from ergon_core.core.shared.json_types import JsonObject from ergon_core.core.persistence.shared.db import get_session diff --git a/ergon_core/ergon_core/core/infrastructure/http/routes/test_harness.py b/ergon_core/ergon_core/core/infrastructure/http/routes/test_harness.py index f23c69e2c..0fe2e18fb 100644 --- a/ergon_core/ergon_core/core/infrastructure/http/routes/test_harness.py +++ b/ergon_core/ergon_core/core/infrastructure/http/routes/test_harness.py @@ -234,7 +234,7 @@ async def submit_experiment_runs( by launch. Slots submit sequentially — typical N ≤ 3, so the parallel-gather savings are negligible. """ - from ergon_core.core.application.experiments.definition_writer import persist_benchmark + from ergon_core.core.application.experiments.service import persist_benchmark run_ids: list[UUID] = [] for slot in body.slots: diff --git a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py index cfea9f613..c8f90b75f 100644 --- a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py +++ b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py @@ -20,7 +20,7 @@ from ergon_core.core.jobs.task.execute.contract import TaskReadyEvent from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from ergon_core.core.application.events.service import get_dashboard_event_publisher -from ergon_core.core.application.resources import RunResourceRepository +from ergon_core.core.application.resources.service import RunResourceReadService from ergon_core.core.application.runtime.task_inspection import TaskInspectionService from ergon_core.core.application.runtime.task_management import TaskManagementService from ergon_core.core.application.runtime.task_execution_repository import ( @@ -92,7 +92,7 @@ async def run_worker_execute_job( sandbox_id=payload.sandbox_id, task_mgmt=_task_management_service_for_context(ctx), task_inspect=TaskInspectionService(), - resource_repo=RunResourceRepository(), + resource_service=RunResourceReadService(), session_factory=get_session, ) diff --git a/ergon_core/tests/unit/api/worker/test_worker_context_facade.py b/ergon_core/tests/unit/api/worker/test_worker_context_facade.py index e1460f6a1..9b2940604 100644 --- a/ergon_core/tests/unit/api/worker/test_worker_context_facade.py +++ b/ergon_core/tests/unit/api/worker/test_worker_context_facade.py @@ -13,7 +13,7 @@ from ergon_core.api.worker.context import WorkerContext from ergon_core.api.worker.results import AwaitCompletionNotSupportedError, SpawnedTaskHandle from ergon_core.core.application.resources.models import RunResourceView -from ergon_core.core.application.resources.repository import RunResourceRepository +from ergon_core.core.application.resources.service import RunResourceReadService from ergon_core.core.application.runtime.task_models import ( CancelTaskCommand, RefineTaskCommand, @@ -118,8 +118,8 @@ def __init__(self, *, run_id, other_run_id, blob_path: Path) -> None: self.blob_path = blob_path self.calls: list[tuple[str, object]] = [] - def list_for_run(self, session, **kwargs): - self.calls.append(("list_for_run", session, kwargs)) + def list_for_run(self, **kwargs): + self.calls.append(("list_for_run", kwargs)) return [ RunResourceView( id=uuid4(), @@ -137,13 +137,18 @@ def list_for_run(self, session, **kwargs): ) ] - def get(self, session, resource_id): - self.calls.append(("get", session, resource_id)) + def read_bytes(self, *, run_id, current_task_id, resource_id): + self.calls.append(("read_bytes", run_id, current_task_id, resource_id)) run_id = self.other_run_id if str(resource_id).endswith("ffff") else self.run_id - return SimpleNamespace(run_id=run_id, file_path=str(self.blob_path)) + if run_id != self.run_id: + raise ContainmentViolation( + parent_task_id=current_task_id, + target_task_id=resource_id, + ) + return self.blob_path.read_bytes() -def _context(*, run_id, task_id, inspect=None, resource_repo=None) -> WorkerContext: +def _context(*, run_id, task_id, inspect=None, resource_service=None) -> WorkerContext: return WorkerContext._for_job( run_id=run_id, task_id=task_id, @@ -152,7 +157,7 @@ def _context(*, run_id, task_id, inspect=None, resource_repo=None) -> WorkerCont sandbox_id="sbx", task_mgmt=_FakeTaskManagement(), task_inspect=inspect or _FakeInspection(), - resource_repo=resource_repo or SimpleNamespace(), + resource_service=resource_service or SimpleNamespace(), session_factory=_session_factory, ) @@ -172,7 +177,7 @@ async def test_facade_mutations_call_current_service_command_signatures() -> Non sandbox_id="sbx", task_mgmt=mgmt, task_inspect=inspect, - resource_repo=SimpleNamespace(), + resource_service=SimpleNamespace(), session_factory=_session_factory, ) @@ -201,7 +206,7 @@ async def test_spawn_task_uses_empty_tuple_dependency_default() -> None: sandbox_id="sbx", task_mgmt=mgmt, task_inspect=_FakeInspection(), - resource_repo=SimpleNamespace(), + resource_service=SimpleNamespace(), session_factory=_session_factory, ) @@ -267,14 +272,14 @@ async def test_resources_are_run_scoped_not_descendant_scoped(tmp_path: Path) -> blob = tmp_path / "blob.txt" blob.write_bytes(b"ok") repo = _FakeResources(run_id=run_id, other_run_id=other_run_id, blob_path=blob) - context = _context(run_id=run_id, task_id=root_id, resource_repo=repo) + context = _context(run_id=run_id, task_id=root_id, resource_service=repo) resources = await context.resources(task_id=sibling_task_id, execution_id=execution_id) data = await context.read_resource(resources[0].id) assert data == b"ok" - assert repo.calls[0][2]["task_id"] == sibling_task_id - assert repo.calls[0][2]["task_execution_id"] == execution_id + assert repo.calls[0][1]["task_id"] == sibling_task_id + assert repo.calls[0][1]["task_execution_id"] == execution_id @pytest.mark.asyncio @@ -284,7 +289,7 @@ async def test_read_resource_rejects_cross_run_rows(tmp_path: Path) -> None: blob = tmp_path / "blob.txt" blob.write_bytes(b"ok") repo = _FakeResources(run_id=run_id, other_run_id=other_run_id, blob_path=blob) - context = _context(run_id=run_id, task_id=uuid4(), resource_repo=repo) + context = _context(run_id=run_id, task_id=uuid4(), resource_service=repo) cross_run_resource_id = uuid4() cross_run_resource_id = type(cross_run_resource_id)(f"{str(cross_run_resource_id)[:-4]}ffff") @@ -381,7 +386,7 @@ def session_factory(): sandbox_id="sbx", task_mgmt=_FakeTaskManagement(), task_inspect=_FakeInspection(), - resource_repo=RunResourceRepository(), + resource_service=RunResourceReadService(session_factory=session_factory), session_factory=session_factory, ) @@ -408,7 +413,7 @@ def test_context_requires_facade_services_at_construction() -> None: sandbox_id="sbx", task_mgmt=None, task_inspect=_FakeInspection(), - resource_repo=SimpleNamespace(), + resource_service=SimpleNamespace(), session_factory=_session_factory, ) diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index cd4331b52..8d18e60b0 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -30,6 +30,8 @@ LAYOUT_FILE_EXCEPTIONS = { "evaluation": {"scoring.py", "summary.py"}, "events": {"base.py", "runtime.py"}, + # Experiments exposes cross-domain application behavior through service.py. + # These files are domain-internal implementation modules, not public subfacades. "experiments": {"definition_writer.py", "handles.py", "launch.py"}, "ports": {"dashboard.py", "resources.py"}, "resources": {"publishing.py"}, @@ -81,11 +83,6 @@ "ergon_core.core.application.events.runtime", "PR 05: route runtime event collaboration through public runtime facades.", ), - ( - "ergon_core.core.application.runtime.run_records", - "ergon_core.core.application.experiments.handles", - "PR 04: decide and document experiment handle public surface.", - ), ( "ergon_core.core.application.runtime.task_management", "ergon_core.core.application.events.runtime", diff --git a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py index af5d15276..972190f7d 100644 --- a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py +++ b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py @@ -33,23 +33,7 @@ APPLICATION_ROOT = CORE_ROOT / "application" APPLICATION_PREFIX = "ergon_core.core.application" -_REPOSITORY_REEXPORT_IMPORT_LEDGER = ( - pytest.param( - "ergon_core.core.jobs.task.worker_execute.job", - "ergon_core.core.application.resources.RunResourceRepository", - marks=pytest.mark.xfail( - strict=True, - reason=( - "PR 04: route worker output resource writes through the resources " - "facade instead of the repository re-export." - ), - ), - id=( - "ergon_core.core.jobs.task.worker_execute.job -> " - "ergon_core.core.application.resources.RunResourceRepository" - ), - ), -) +_REPOSITORY_REEXPORT_IMPORT_LEDGER = () _WRITE_PREFIXES = ( diff --git a/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py b/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py index a1d8644ec..d4b688ec2 100644 --- a/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py +++ b/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py @@ -55,7 +55,7 @@ async def _publish(_event): monkeypatch.setattr(module.ContextEventService, "persist_chunk", _persist) monkeypatch.setattr(module, "TaskManagementService", lambda **kwargs: object()) monkeypatch.setattr(module, "TaskInspectionService", lambda: object()) - monkeypatch.setattr(module, "RunResourceRepository", lambda: object()) + monkeypatch.setattr(module, "RunResourceReadService", lambda: object()) monkeypatch.setattr( module, "get_dashboard_event_publisher", @@ -102,7 +102,7 @@ async def node(self, _session, *, run_id, task_id, sandbox_id=None): monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _NonLiveRepo()) monkeypatch.setattr(module, "TaskManagementService", lambda: object()) monkeypatch.setattr(module, "TaskInspectionService", lambda: object()) - monkeypatch.setattr(module, "RunResourceRepository", lambda: object()) + monkeypatch.setattr(module, "RunResourceReadService", lambda: object()) with pytest.raises(Exception, match="live sandbox"): await run_worker_execute_job( diff --git a/ergon_core/tests/unit/runtime/test_experiment_definition_service.py b/ergon_core/tests/unit/runtime/test_experiment_definition_service.py index 4b84fd361..086bee848 100644 --- a/ergon_core/tests/unit/runtime/test_experiment_definition_service.py +++ b/ergon_core/tests/unit/runtime/test_experiment_definition_service.py @@ -6,3 +6,52 @@ ``run_experiment`` (``service``); coverage lives in ``test_walkthrough_smoketest.py`` and ``test_experiment_launch_service.py``. """ + +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from ergon_core.core.application.experiments import definition_writer +from ergon_core.core.application.experiments import service +from ergon_core.core.application.experiments.models import ( + DefinitionHandle, + ExperimentRunRequest, + ExperimentRunResult, +) + + +def test_persist_benchmark_uses_experiments_service_public_facade(monkeypatch) -> None: + benchmark = SimpleNamespace(type_slug="ci-benchmark") + handle = DefinitionHandle(definition_id=uuid4(), benchmark_type="ci-benchmark") + seen: list[object] = [] + + def fake_persist(candidate: object) -> DefinitionHandle: + seen.append(candidate) + return handle + + monkeypatch.setattr(definition_writer, "persist_benchmark", fake_persist) + + assert service.persist_benchmark(benchmark) == handle + assert seen == [benchmark] + + +@pytest.mark.asyncio +async def test_run_experiment_uses_experiments_service_public_facade(monkeypatch) -> None: + definition_id = uuid4() + run_id = uuid4() + result = ExperimentRunResult( + definition_id=definition_id, + run_ids=[run_id], + definition_ids=[definition_id], + ) + seen: list[tuple[object, object]] = [] + + async def fake_launch_run(candidate_definition_id, *, emit_workflow_started=None): + seen.append((candidate_definition_id, emit_workflow_started)) + return result + + monkeypatch.setattr(service, "launch_run", fake_launch_run) + + assert await service.run_experiment(ExperimentRunRequest(definition_id=definition_id)) == result + assert seen == [(definition_id, None)] diff --git a/ergon_core/tests/unit/runtime/test_identity_invariants.py b/ergon_core/tests/unit/runtime/test_identity_invariants.py index ca357cc08..ec79dec3e 100644 --- a/ergon_core/tests/unit/runtime/test_identity_invariants.py +++ b/ergon_core/tests/unit/runtime/test_identity_invariants.py @@ -358,7 +358,7 @@ async def test_dynamic_task_id_has_no_definition_row( sandbox_id="sandbox-identity", task_mgmt=task_mgmt, task_inspect=task_inspect, - resource_repo=object(), + resource_service=object(), session_factory=management_module.get_session, ) @@ -398,7 +398,7 @@ async def test_dynamic_task_id_has_no_definition_row( sandbox_id="sandbox-child", task_mgmt=task_mgmt, task_inspect=task_inspect, - resource_repo=object(), + resource_service=object(), session_factory=management_module.get_session, ) assert child_context.task_id == handle.task_id diff --git a/ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py b/ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py index f659c2352..b7ddeb763 100644 --- a/ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py +++ b/ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py @@ -426,7 +426,7 @@ async def test_dynamic_spawn_writes_only_to_run_graph_nodes( sandbox_id="sandbox-smoke", task_mgmt=task_mgmt, task_inspect=task_inspect, - resource_repo=object(), + resource_service=object(), session_factory=management_module.get_session, ) diff --git a/ergon_core/tests/unit/runtime/test_worker_context_containment.py b/ergon_core/tests/unit/runtime/test_worker_context_containment.py index aab8311ee..b8780f031 100644 --- a/ergon_core/tests/unit/runtime/test_worker_context_containment.py +++ b/ergon_core/tests/unit/runtime/test_worker_context_containment.py @@ -139,7 +139,7 @@ def _build_context( sandbox_id="sandbox-test", task_mgmt=task_mgmt, task_inspect=task_inspect, - resource_repo=object(), + resource_service=object(), session_factory=management_module.get_session, ) From b427fd3afa3628c1ff7510bf8c0a81937ac5a7e1 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 12:42:55 +0100 Subject: [PATCH 6/8] Expose runtime public facades --- .../core/application/experiments/launch.py | 4 +- .../core/application/runtime/events.py | 2 +- .../core/application/runtime/run_lifecycle.py | 19 ++- .../core/application/runtime/run_records.py | 2 +- .../application/runtime/task_execution.py | 80 +++++++++++- .../application/runtime/task_management.py | 2 +- .../core/application/runtime/task_models.py | 2 +- .../jobs/resources/persist_outputs/job.py | 5 +- .../ergon_core/core/jobs/sandbox/setup/job.py | 5 +- .../ergon_core/core/jobs/task/evaluate/job.py | 12 +- .../ergon_core/core/jobs/task/execute/job.py | 6 +- .../core/jobs/task/worker_execute/job.py | 19 ++- .../test_application_domain_boundaries.py | 62 ++++------ .../test_runtime_read_boundaries.py | 6 +- .../test_execute_task_object_bound_fanout.py | 16 +-- .../test_persist_outputs_public_sandbox.py | 10 +- .../jobs/test_sandbox_setup_public_sandbox.py | 6 +- ...test_worker_execute_live_sandbox_attach.py | 24 ++-- .../test_execute_task_evaluator_fanout.py | 20 ++- .../unit/runtime/test_failure_error_json.py | 4 +- .../runtime/test_graph_worker_identity.py | 8 +- .../unit/runtime/test_identity_invariants.py | 19 +-- .../unit/runtime/test_persist_outputs.py | 10 +- .../test_task_execution_public_facade.py | 117 ++++++++++++++++++ 24 files changed, 330 insertions(+), 130 deletions(-) create mode 100644 ergon_core/tests/unit/runtime/test_task_execution_public_facade.py diff --git a/ergon_core/ergon_core/core/application/experiments/launch.py b/ergon_core/ergon_core/core/application/experiments/launch.py index d440ec903..871e11b42 100644 --- a/ergon_core/ergon_core/core/application/experiments/launch.py +++ b/ergon_core/ergon_core/core/application/experiments/launch.py @@ -4,10 +4,10 @@ from uuid import UUID import inngest -from ergon_core.core.application.events.runtime import WorkflowStartedEvent +from ergon_core.core.application.events import WorkflowStartedEvent from ergon_core.core.application.experiments.errors import DefinitionNotFoundError from ergon_core.core.application.experiments.models import DefinitionHandle, ExperimentRunResult -from ergon_core.core.application.runtime.run_records import create_run +from ergon_core.core.application.runtime.run_lifecycle import create_run from ergon_core.core.infrastructure.inngest.client import inngest_client from ergon_core.core.persistence.definitions.models import ExperimentDefinition from ergon_core.core.persistence.shared.db import get_session diff --git a/ergon_core/ergon_core/core/application/runtime/events.py b/ergon_core/ergon_core/core/application/runtime/events.py index 964225faa..040819f8f 100644 --- a/ergon_core/ergon_core/core/application/runtime/events.py +++ b/ergon_core/ergon_core/core/application/runtime/events.py @@ -14,7 +14,7 @@ import inngest from ergon_core.core.infrastructure.inngest.client import inngest_client -from ergon_core.core.application.events.runtime import TaskReadyEvent +from ergon_core.core.application.events import TaskReadyEvent logger = logging.getLogger(__name__) diff --git a/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py b/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py index 340c9304f..95c12c341 100644 --- a/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py +++ b/ergon_core/ergon_core/core/application/runtime/run_lifecycle.py @@ -17,13 +17,16 @@ RunTaskEvaluation, RunTaskExecution, ) -from ergon_core.core.application.evaluation.service import EvaluationService from ergon_core.core.application.runtime.events import ( RuntimeEventDispatcher, TaskReadyDispatcher, ) +from ergon_core.core.application.runtime.run_records import ( + cancel_run, + create_run, + latest_run_for_definition, +) from ergon_core.core.application.runtime.run_identity import definition_id_for_run -from ergon_core.core.infrastructure.sandbox.manager import BaseSandboxManager, DefaultSandboxManager from ergon_core.core.application.runtime.graph_lookup import GraphNodeLookup from ergon_core.core.application.runtime.lifecycle import ( get_initial_ready_tasks, @@ -76,7 +79,7 @@ class WorkflowService: def __init__( self, *, - sandbox_manager_factory: Callable[[str], BaseSandboxManager] | None = None, + sandbox_manager_factory: Callable[[str], "BaseSandboxManager"] | None = None, graph_repository: RuntimeGraphRepository | None = None, task_ready_dispatcher: TaskReadyDispatcher | None = None, ) -> None: @@ -151,6 +154,10 @@ async def initialize(self, command: InitializeWorkflowCommand) -> InitializedWor def finalize(self, command: FinalizeWorkflowCommand) -> FinalizedWorkflowResult: """Aggregate evaluations and close the run.""" + # reason: importing EvaluationService at module load creates a cycle via + # ergon_core.api -> experiments.service -> experiments.launch -> run_lifecycle. + from ergon_core.core.application.evaluation.service import EvaluationService + with get_session() as session: evaluations = list( session.exec( @@ -651,7 +658,11 @@ async def materialize_resource( # slopcop: ignore[max-function-params] -- mirro return result.model_copy(update={"copied_resource_id": copy.id}) @staticmethod - def _sandbox_manager_for(benchmark_type: str) -> BaseSandboxManager: + def _sandbox_manager_for(benchmark_type: str) -> "BaseSandboxManager": + # reason: application runtime should not import the E2B adapter at module load time; + # this default factory is only needed when materializing resources into a sandbox. + from ergon_core.core.infrastructure.sandbox.manager import DefaultSandboxManager + _ = benchmark_type return DefaultSandboxManager() diff --git a/ergon_core/ergon_core/core/application/runtime/run_records.py b/ergon_core/ergon_core/core/application/runtime/run_records.py index ed3396cf4..9d6419bfa 100644 --- a/ergon_core/ergon_core/core/application/runtime/run_records.py +++ b/ergon_core/ergon_core/core/application/runtime/run_records.py @@ -5,7 +5,7 @@ import inngest from ergon_core.core.application.experiments.models import DefinitionHandle -from ergon_core.core.application.events.runtime import RunCancelledEvent, RunCleanupEvent +from ergon_core.core.application.events import RunCancelledEvent, RunCleanupEvent from ergon_core.core.shared.json_types import JsonObject from ergon_core.core.persistence.shared.db import get_session from ergon_core.core.persistence.shared.enums import TERMINAL_RUN_STATUSES, RunStatus diff --git a/ergon_core/ergon_core/core/application/runtime/task_execution.py b/ergon_core/ergon_core/core/application/runtime/task_execution.py index f52e2df4f..cc267ead1 100644 --- a/ergon_core/ergon_core/core/application/runtime/task_execution.py +++ b/ergon_core/ergon_core/core/application/runtime/task_execution.py @@ -14,7 +14,8 @@ from ergon_core.core.persistence.shared.enums import TaskExecutionStatus from ergon_core.core.persistence.telemetry.models import RunRecord, RunTaskExecution from ergon_core.core.infrastructure.inngest.errors import ConfigurationError -from ergon_core.core.application.runtime.models import MutationMeta +from ergon_core.api.worker.results import WorkerOutput +from ergon_core.core.application.runtime.models import MutationMeta, RunGraphNodeView from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from ergon_core.core.application.runtime.orchestration import ( FailTaskExecutionCommand, @@ -25,7 +26,10 @@ from ergon_core.core.application.runtime.lifecycle import ( mark_task_failed_by_node, ) -from ergon_core.core.application.runtime.task_execution_repository import TaskExecutionRepository +from ergon_core.core.application.runtime.task_execution_repository import ( + TaskExecutionRepository, + WorkerOutputRepository, +) from ergon_core.core.shared.utils import require_not_none, utcnow from ergon_core.core.views.dashboard_events.contracts import DashboardTaskStatusChangedEvent from sqlmodel import Session, select @@ -63,9 +67,75 @@ async def _emit_task_status( class TaskExecutionService: - def __init__(self) -> None: - self._graph_repo = RuntimeGraphRepository() - self._task_execution_repo = TaskExecutionRepository() + """Public facade for task execution reads and execution-row writes. + + Jobs use this boundary for task view hydration, worker output persistence, + and sandbox identity stamping instead of importing runtime repositories. + """ + + def __init__( + self, + *, + graph_repo: RuntimeGraphRepository | None = None, + task_execution_repo: TaskExecutionRepository | None = None, + worker_output_repo: WorkerOutputRepository | None = None, + ) -> None: + self._graph_repo = graph_repo or RuntimeGraphRepository() + self._task_execution_repo = task_execution_repo or TaskExecutionRepository() + self._worker_output_repo = worker_output_repo or WorkerOutputRepository() + + async def load_task_view( + self, + session: Session, + *, + run_id: UUID, + task_id: UUID, + sandbox_id: str | None = None, + ) -> RunGraphNodeView: + """Load the object-bound runtime task view by public task_id.""" + return await self._graph_repo.node( + session, + run_id=run_id, + task_id=task_id, + sandbox_id=sandbox_id, + ) + + async def persist_worker_output( + self, + session: Session, + *, + execution_id: UUID, + output: WorkerOutput, + ) -> None: + """Persist terminal worker output for evaluator fanout.""" + await self._worker_output_repo.persist( + session, + execution_id=execution_id, + output=output, + ) + + async def load_worker_output( + self, + session: Session, + *, + execution_id: UUID, + ) -> WorkerOutput: + """Load worker output persisted for an execution.""" + return await self._worker_output_repo.load(session, execution_id=execution_id) + + async def attach_sandbox_to_execution( + self, + session: Session, + *, + execution_id: UUID, + sandbox_id: str, + ) -> None: + """Stamp the live sandbox id on an execution row.""" + await self._task_execution_repo.set_sandbox_id( + session, + execution_id=execution_id, + sandbox_id=sandbox_id, + ) async def prepare(self, command: PrepareTaskExecutionCommand) -> PreparedTaskExecution: return await self._prepare_run_node(command) diff --git a/ergon_core/ergon_core/core/application/runtime/task_management.py b/ergon_core/ergon_core/core/application/runtime/task_management.py index 4fa66b834..405f9c9f9 100644 --- a/ergon_core/ergon_core/core/application/runtime/task_management.py +++ b/ergon_core/ergon_core/core/application/runtime/task_management.py @@ -41,7 +41,7 @@ TaskNotTerminalError, TaskRunningError, ) -from ergon_core.core.application.events.runtime import ( +from ergon_core.core.application.events import ( CancelCause, PropagationCancelCause, TaskCancelledEvent, diff --git a/ergon_core/ergon_core/core/application/runtime/task_models.py b/ergon_core/ergon_core/core/application/runtime/task_models.py index 5c1e21bf8..1ad50705d 100644 --- a/ergon_core/ergon_core/core/application/runtime/task_models.py +++ b/ergon_core/ergon_core/core/application/runtime/task_models.py @@ -2,7 +2,7 @@ from uuid import UUID -from ergon_core.core.application.events.runtime import TaskCancelledEvent +from ergon_core.core.application.events import TaskCancelledEvent from ergon_core.core.application.runtime.status import NodeStatus from ergon_core.core.persistence.shared.types import NodeId, RunId from pydantic import BaseModel, Field diff --git a/ergon_core/ergon_core/core/jobs/resources/persist_outputs/job.py b/ergon_core/ergon_core/core/jobs/resources/persist_outputs/job.py index 5683a714e..c3de7834e 100644 --- a/ergon_core/ergon_core/core/jobs/resources/persist_outputs/job.py +++ b/ergon_core/ergon_core/core/jobs/resources/persist_outputs/job.py @@ -9,7 +9,7 @@ import logging from datetime import UTC, datetime -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository +from ergon_core.core.application.runtime.task_execution import TaskExecutionService from ergon_core.core.infrastructure.inngest.errors import ContractViolationError from .contract import PersistOutputsRequest, PersistOutputsResult from .composition import publish_public_sandbox_resources @@ -45,8 +45,9 @@ async def run_persist_outputs_job(payload: PersistOutputsRequest) -> PersistOutp task_id=task_id, ) + task_execution = TaskExecutionService() with get_session() as session: - view = await RuntimeGraphRepository().node( + view = await task_execution.load_task_view( session, run_id=payload.run_id, task_id=payload.task_id, diff --git a/ergon_core/ergon_core/core/jobs/sandbox/setup/job.py b/ergon_core/ergon_core/core/jobs/sandbox/setup/job.py index f6158db31..3e2928b15 100644 --- a/ergon_core/ergon_core/core/jobs/sandbox/setup/job.py +++ b/ergon_core/ergon_core/core/jobs/sandbox/setup/job.py @@ -4,7 +4,7 @@ from datetime import UTC, datetime from functools import partial -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository +from ergon_core.core.application.runtime.task_execution import TaskExecutionService from ergon_core.core.persistence.shared.db import get_session from .contract import SandboxReadyResult, SandboxSetupRequest from ergon_core.core.infrastructure.tracing import ( @@ -31,8 +31,9 @@ async def run_sandbox_setup_job(ctx: Any, payload: SandboxSetupRequest) -> Sandb benchmark_type, ) + task_execution = TaskExecutionService() with get_session() as session: - view = await RuntimeGraphRepository().node( + view = await task_execution.load_task_view( session, run_id=run_id, task_id=task_id, diff --git a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py index 5207cba86..32ab7b9ab 100644 --- a/ergon_core/ergon_core/core/jobs/task/evaluate/job.py +++ b/ergon_core/ergon_core/core/jobs/task/evaluate/job.py @@ -7,8 +7,8 @@ reconstructed locally from the run-tier read boundary: - execution row + stamped ``sandbox_id`` ← ``session.get(RunTaskExecution)`` -- typed Task view ← ``RuntimeGraphRepository.node(..., sandbox_id=...)`` -- persisted ``WorkerOutput`` ← ``WorkerOutputRepository.load`` +- typed Task view ← ``TaskExecutionService.load_task_view(..., sandbox_id=...)`` +- persisted ``WorkerOutput`` ← ``TaskExecutionService.load_worker_output`` - evaluator instance ← ``task.evaluators[payload.evaluator_index]`` Criteria run inline via ``EvaluationService.evaluate``: no @@ -30,10 +30,9 @@ from ergon_core.api.criterion.context import CriterionContext from ergon_core.core.application.evaluation.service import EvaluationService -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from .contract import EvaluateTaskRunResult, TaskEvaluateRequest from ergon_core.core.application.events.service import get_dashboard_event_publisher -from ergon_core.core.application.runtime.task_execution_repository import WorkerOutputRepository +from ergon_core.core.application.runtime.task_execution import TaskExecutionService from ergon_core.core.infrastructure.inngest.errors import ContractViolationError from ergon_core.core.infrastructure.tracing import ( CompletedSpan, @@ -51,6 +50,7 @@ logger = logging.getLogger(__name__) _evaluation_persistence = EvaluationService() +_task_execution = TaskExecutionService() def _evaluator_binding_key(evaluator: "Evaluator", evaluator_index: int) -> str: @@ -80,13 +80,13 @@ async def run_evaluate_task_run_job( task_id=task_id, execution_id=execution_id, ) - view = await RuntimeGraphRepository().node( + view = await _task_execution.load_task_view( session, run_id=run_id, task_id=task_id, sandbox_id=execution.sandbox_id, ) - worker_output = await WorkerOutputRepository().load( + worker_output = await _task_execution.load_worker_output( session, execution_id=execution_id, ) diff --git a/ergon_core/ergon_core/core/jobs/task/execute/job.py b/ergon_core/ergon_core/core/jobs/task/execute/job.py index 1375b836c..bee5ec326 100644 --- a/ergon_core/ergon_core/core/jobs/task/execute/job.py +++ b/ergon_core/ergon_core/core/jobs/task/execute/job.py @@ -53,7 +53,6 @@ from typing import Any from uuid import UUID -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from .contract import TaskExecuteResult, TaskReadyEvent from ergon_core.core.jobs.resources.persist_outputs.contract import ( PersistOutputsRequest, @@ -173,6 +172,7 @@ async def _invoke_worker_execute( async def _fan_out_evaluators( ctx: Any, + svc: TaskExecutionService, payload: TaskReadyEvent, prepared: PreparedTaskExecution, evaluate_task_run_function: Any, @@ -193,7 +193,7 @@ async def _fan_out_evaluators( """ with get_session() as session: - view = await RuntimeGraphRepository().node( + view = await svc.load_task_view( session, run_id=payload.run_id, task_id=payload.task_id, @@ -360,7 +360,7 @@ async def run_execute_task_job( # orchestrator emits `task/completed` only after all evaluators # return, and the sibling sandbox_cleanup function terminates # the external sandbox from that terminal event. - await _fan_out_evaluators(ctx, payload, prepared, evaluate_task_run_function) + await _fan_out_evaluators(ctx, svc, payload, prepared, evaluate_task_run_function) await svc.finalize_success( FinalizeTaskExecutionCommand( diff --git a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py index c8f90b75f..2da5aa313 100644 --- a/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py +++ b/ergon_core/ergon_core/core/jobs/task/worker_execute/job.py @@ -18,15 +18,11 @@ from ergon_core.api.worker.results import SpawnedTaskHandle from ergon_core.core.jobs._events import send_job_step_event from ergon_core.core.jobs.task.execute.contract import TaskReadyEvent -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository from ergon_core.core.application.events.service import get_dashboard_event_publisher from ergon_core.core.application.resources.service import RunResourceReadService +from ergon_core.core.application.runtime.task_execution import TaskExecutionService from ergon_core.core.application.runtime.task_inspection import TaskInspectionService from ergon_core.core.application.runtime.task_management import TaskManagementService -from ergon_core.core.application.runtime.task_execution_repository import ( - TaskExecutionRepository, - WorkerOutputRepository, -) from ergon_core.core.shared.context_parts import ContextPartChunk from ergon_core.core.persistence.shared.db import get_session from ergon_core.core.application.context.service import ContextEventService @@ -63,9 +59,10 @@ async def run_worker_execute_job( # Read the typed run-tier view instead of rebuilding Task # from definition rows. No definition-tier repository, no component # catalog, no raw graph row read in this job — all of that lives - # inside `graph_repo.node`. + # behind TaskExecutionService. + task_execution = TaskExecutionService() with get_session() as session: - view = await RuntimeGraphRepository().node( + view = await task_execution.load_task_view( session, run_id=payload.run_id, task_id=payload.task_id, @@ -150,19 +147,19 @@ async def _publish_context_event(event: RunContextEvent) -> None: # receives only a thin `TaskEvaluateRequest` and reloads everything # else from the run-tier read boundary: # - # WorkerOutput ← WorkerOutputRepository.load(execution_id) + # WorkerOutput ← TaskExecutionService.load_worker_output(execution_id) # live sandbox_id ← session.get(RunTaskExecution, ...).sandbox_id - # (then fed to graph_repo.node(..., sandbox_id=)) + # (then fed to load_task_view(..., sandbox_id=)) # # Both reads happen *after* the orchestrator's gather starts, so # both writes have to commit before this function returns. with get_session() as session: - await WorkerOutputRepository().persist( + await task_execution.persist_worker_output( session, execution_id=payload.execution_id, output=output, ) - await TaskExecutionRepository().set_sandbox_id( + await task_execution.attach_sandbox_to_execution( session, execution_id=payload.execution_id, sandbox_id=payload.sandbox_id, diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index 8d18e60b0..909f6b8e0 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -15,6 +15,20 @@ APPLICATION_PREFIX = "ergon_core.core.application" PUBLIC_CROSS_DOMAIN_MODULES = {"service", "models", "errors"} +# Runtime is intentionally split into small public subfacades rather than a +# single god service: run lifecycle, task execution, task management, resource +# views, read-only task inspection, and orchestration command/result DTOs each +# have distinct collaborators. +PUBLIC_CROSS_DOMAIN_MODULES_BY_DOMAIN = { + "runtime": { + "orchestration", + "resources", + "run_lifecycle", + "task_execution", + "task_inspection", + "task_management", + } +} APPROVED_DOMAIN_FILES = { "__init__.py", "service.py", @@ -62,38 +76,7 @@ } LAYOUT_DIR_EXCEPTIONS: dict[str, set[str]] = {} -_PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = ( - ( - "ergon_core.core.application.experiments.launch", - "ergon_core.core.application.events.runtime", - "PR 05: route runtime event collaboration through public runtime facades.", - ), - ( - "ergon_core.core.application.experiments.launch", - "ergon_core.core.application.runtime.run_records", - "PR 05: route run-record access through public runtime facades.", - ), - ( - "ergon_core.core.application.runtime.events", - "ergon_core.core.application.events.runtime", - "PR 05: route runtime event collaboration through public runtime facades.", - ), - ( - "ergon_core.core.application.runtime.run_records", - "ergon_core.core.application.events.runtime", - "PR 05: route runtime event collaboration through public runtime facades.", - ), - ( - "ergon_core.core.application.runtime.task_management", - "ergon_core.core.application.events.runtime", - "PR 05: route runtime event collaboration through public runtime facades.", - ), - ( - "ergon_core.core.application.runtime.task_models", - "ergon_core.core.application.events.runtime", - "PR 05: route runtime event collaboration through public runtime facades.", - ), -) +_PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = () PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = tuple( pytest.param( @@ -235,16 +218,23 @@ def _application_import_edges() -> set[ImportEdge]: def _private_cross_domain_imports() -> set[ImportEdge]: private_edges: set[ImportEdge] = set() for edge in _application_import_edges(): - _target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( + target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( "", None, ) - if leaf is None or leaf in PUBLIC_CROSS_DOMAIN_MODULES: + if leaf is None or leaf in _public_cross_domain_modules(target_domain): continue private_edges.add(edge) return private_edges +def _public_cross_domain_modules(target_domain: str) -> set[str]: + return PUBLIC_CROSS_DOMAIN_MODULES | PUBLIC_CROSS_DOMAIN_MODULES_BY_DOMAIN.get( + target_domain, + set(), + ) + + @pytest.mark.parametrize( ("source_module", "target_module"), PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER, @@ -276,11 +266,11 @@ def test_no_unledgered_private_cross_domain_imports() -> None: def test_cross_domain_imports_only_target_public_domain_modules() -> None: offenders: list[str] = [] for edge in sorted(_application_import_edges()): - _target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( + target_domain, leaf = _application_domain_and_leaf(edge.target_module) or ( "", None, ) - if leaf is None or leaf in PUBLIC_CROSS_DOMAIN_MODULES: + if leaf is None or leaf in _public_cross_domain_modules(target_domain): continue if any( (edge.source_module, edge.target_module) == (source_module, target_module) diff --git a/ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py b/ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py index ef686c6b3..d78eee6c5 100644 --- a/ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py @@ -82,9 +82,9 @@ def test_evaluate_task_run_uses_thin_payload_and_run_tier_read() -> None: # No registry-based evaluator resolution inside the body. assert "ComponentCatalogService" not in body - # Uses the same run-tier loader the orchestrator uses. - assert "RuntimeGraphRepository" in body - assert ".node(" in body + # Uses the public runtime execution facade for run-tier loading. + assert "TaskExecutionService" in body + assert "load_task_view(" in body def test_evaluate_task_run_uses_object_bound_evaluators() -> None: diff --git a/ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py b/ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py index 1e32cfb60..dc868679b 100644 --- a/ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py +++ b/ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py @@ -31,11 +31,11 @@ def __init__(self) -> None: self.group = _FakeGroup() -class _FakeGraphRepo: +class _FakeTaskExecutionService: def __init__(self, task: SimpleNamespace) -> None: self._task = task - async def node(self, _session: object, *, run_id, task_id, sandbox_id=None): + async def load_task_view(self, _session: object, *, run_id, task_id, sandbox_id=None): del run_id, task_id, sandbox_id return SimpleNamespace(task=self._task) @@ -54,8 +54,6 @@ def _prepared(run_id, definition_id, task_id, execution_id) -> PreparedTaskExecu @pytest.mark.asyncio async def test_fanout_uses_object_bound_evaluator_count(monkeypatch) -> None: - from ergon_core.core.jobs.task.execute import job as module - run_id = uuid4() definition_id = uuid4() task_id = uuid4() @@ -65,11 +63,13 @@ async def test_fanout_uses_object_bound_evaluator_count(monkeypatch) -> None: evaluators=(object(), object()), ) + from ergon_core.core.jobs.task.execute import job as module + monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(task)) await _fan_out_evaluators( ctx, + _FakeTaskExecutionService(task), TaskReadyEvent( run_id=run_id, definition_id=definition_id, @@ -84,8 +84,6 @@ async def test_fanout_uses_object_bound_evaluator_count(monkeypatch) -> None: @pytest.mark.asyncio async def test_fanout_emits_no_jobs_without_inline_evaluators(monkeypatch) -> None: - from ergon_core.core.jobs.task.execute import job as module - run_id = uuid4() definition_id = uuid4() task_id = uuid4() @@ -95,11 +93,13 @@ async def test_fanout_emits_no_jobs_without_inline_evaluators(monkeypatch) -> No evaluators=(), ) + from ergon_core.core.jobs.task.execute import job as module + monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(task)) await _fan_out_evaluators( ctx, + _FakeTaskExecutionService(task), TaskReadyEvent( run_id=run_id, definition_id=definition_id, diff --git a/ergon_core/tests/unit/core/application/jobs/test_persist_outputs_public_sandbox.py b/ergon_core/tests/unit/core/application/jobs/test_persist_outputs_public_sandbox.py index 3df380f96..d12b18ae4 100644 --- a/ergon_core/tests/unit/core/application/jobs/test_persist_outputs_public_sandbox.py +++ b/ergon_core/tests/unit/core/application/jobs/test_persist_outputs_public_sandbox.py @@ -8,11 +8,11 @@ from ergon_core.core.jobs.resources.persist_outputs.job import run_persist_outputs_job -class _FakeGraphRepo: +class _FakeTaskExecutionService: def __init__(self, seen: list[str | None]) -> None: self._seen = seen - async def node(self, _session, *, run_id, task_id, sandbox_id=None): + async def load_task_view(self, _session, *, run_id, task_id, sandbox_id=None): del run_id, task_id self._seen.append(sandbox_id) sandbox = SimpleNamespace( @@ -48,7 +48,11 @@ async def test_persist_outputs_publishes_from_public_sandbox_output_path(monkeyp seen_sandbox_ids: list[str | None] = [] monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(seen_sandbox_ids)) + monkeypatch.setattr( + module, + "TaskExecutionService", + lambda: _FakeTaskExecutionService(seen_sandbox_ids), + ) monkeypatch.setattr(composition, "SandboxResourcePublisher", _FakePublisher) monkeypatch.setattr(composition, "RunResourcePublishService", _FakePublishService) diff --git a/ergon_core/tests/unit/core/application/jobs/test_sandbox_setup_public_sandbox.py b/ergon_core/tests/unit/core/application/jobs/test_sandbox_setup_public_sandbox.py index 10a62f673..edc4b88b5 100644 --- a/ergon_core/tests/unit/core/application/jobs/test_sandbox_setup_public_sandbox.py +++ b/ergon_core/tests/unit/core/application/jobs/test_sandbox_setup_public_sandbox.py @@ -34,11 +34,11 @@ def sandbox_id(self) -> str: return "sbx-public" -class _FakeGraphRepo: +class _FakeTaskExecutionService: def __init__(self, sandbox: _PublicSandbox) -> None: self._sandbox = sandbox - async def node(self, _session, *, run_id, task_id, sandbox_id=None): + async def load_task_view(self, _session, *, run_id, task_id, sandbox_id=None): del run_id, task_id, sandbox_id return SimpleNamespace(task=SimpleNamespace(sandbox=self._sandbox)) @@ -49,7 +49,7 @@ async def test_sandbox_setup_provisions_public_sandbox(monkeypatch) -> None: sandbox = _PublicSandbox() monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(sandbox)) + monkeypatch.setattr(module, "TaskExecutionService", lambda: _FakeTaskExecutionService(sandbox)) result = await run_sandbox_setup_job( _FakeCtx(), diff --git a/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py b/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py index d4b688ec2..0c81c6d84 100644 --- a/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py +++ b/ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py @@ -20,16 +20,24 @@ async def execute(self, task, *, context): yield WorkerOutput(output="ok") -class _FakeGraphRepo: +class _FakeTaskExecutionService: def __init__(self, seen: list[str | None]) -> None: self._seen = seen + self.persisted_outputs = [] + self.attached_sandboxes = [] - async def node(self, _session, *, run_id, task_id, sandbox_id=None): + async def load_task_view(self, _session, *, run_id, task_id, sandbox_id=None): del run_id, task_id self._seen.append(sandbox_id) sandbox = SimpleNamespace(is_live=sandbox_id == "sbx-live") return SimpleNamespace(task=SimpleNamespace(worker=_FakeWorker(), sandbox=sandbox)) + async def persist_worker_output(self, _session, *, execution_id, output): + self.persisted_outputs.append((execution_id, output)) + + async def attach_sandbox_to_execution(self, _session, *, execution_id, sandbox_id): + self.attached_sandboxes.append((execution_id, sandbox_id)) + class _FakeSession: def commit(self) -> None: @@ -48,10 +56,10 @@ async def _persist(*args, **kwargs): async def _publish(_event): return None + task_execution = _FakeTaskExecutionService(seen_sandbox_ids) + monkeypatch.setattr(module, "get_session", lambda: nullcontext(_FakeSession())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(seen_sandbox_ids)) - monkeypatch.setattr(module.WorkerOutputRepository, "persist", _persist) - monkeypatch.setattr(module.TaskExecutionRepository, "set_sandbox_id", _persist) + monkeypatch.setattr(module, "TaskExecutionService", lambda: task_execution) monkeypatch.setattr(module.ContextEventService, "persist_chunk", _persist) monkeypatch.setattr(module, "TaskManagementService", lambda **kwargs: object()) monkeypatch.setattr(module, "TaskInspectionService", lambda: object()) @@ -88,8 +96,8 @@ async def test_worker_execute_rejects_object_bound_worker_without_live_sandbox( ) -> None: from ergon_core.core.jobs.task.worker_execute import job as module - class _NonLiveRepo: - async def node(self, _session, *, run_id, task_id, sandbox_id=None): + class _NonLiveTaskExecutionService: + async def load_task_view(self, _session, *, run_id, task_id, sandbox_id=None): del run_id, task_id, sandbox_id return SimpleNamespace( task=SimpleNamespace( @@ -99,7 +107,7 @@ async def node(self, _session, *, run_id, task_id, sandbox_id=None): ) monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _NonLiveRepo()) + monkeypatch.setattr(module, "TaskExecutionService", lambda: _NonLiveTaskExecutionService()) monkeypatch.setattr(module, "TaskManagementService", lambda: object()) monkeypatch.setattr(module, "TaskInspectionService", lambda: object()) monkeypatch.setattr(module, "RunResourceReadService", lambda: object()) diff --git a/ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py b/ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py index 998629628..e0de8bf83 100644 --- a/ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py +++ b/ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py @@ -73,11 +73,11 @@ async def _run( return await fn() -def _prepared(execution_id, node_id) -> PreparedTaskExecution: +def _prepared(execution_id, task_id) -> PreparedTaskExecution: return PreparedTaskExecution( run_id=uuid4(), definition_id=uuid4(), - task_id=node_id, + task_id=task_id, execution_id=execution_id, task_slug="t", task_description="d", @@ -89,8 +89,7 @@ def _prepared(execution_id, node_id) -> PreparedTaskExecution: ) -def _ready_event(run_id, definition_id, task_id, node_id) -> TaskReadyEvent: - del node_id +def _ready_event(run_id, definition_id, task_id) -> TaskReadyEvent: return TaskReadyEvent( run_id=run_id, definition_id=definition_id, @@ -122,11 +121,10 @@ async def test_execute_task_emits_completed_strictly_after_eval_gather( run_id = uuid4() definition_id = uuid4() task_id = uuid4() - node_id = uuid4() execution_id = uuid4() - prepared = _prepared(execution_id, node_id) - payload = _ready_event(run_id, definition_id, task_id, node_id) + prepared = _prepared(execution_id, task_id) + payload = _ready_event(run_id, definition_id, task_id) async def fake_prepare( _ctx: inngest.Context, @@ -179,16 +177,15 @@ async def fake_emit_completed( # Two evaluators bound on the task so the parallel fanout sees real work. view_task = SimpleNamespace(evaluators=(object(), object())) - view = SimpleNamespace(task=view_task) - repo = SimpleNamespace(node=AsyncMock(return_value=view)) async def fake_fanout( ctx: inngest.Context, + svc: TaskExecutionService, payload: TaskReadyEvent, prepared: PreparedTaskExecution, eval_fn: inngest.Function, ) -> None: - del payload, prepared + del svc, payload, prepared # Use the real call shape but bypass the session machinery so we # don't need a populated database — just count the invokes. for i in range(len(view_task.evaluators)): @@ -211,7 +208,6 @@ async def fake_fanout( monkeypatch.setattr(execute_task_module, "_emit_task_completed", fake_emit_completed) monkeypatch.setattr(execute_task_module, "_fan_out_evaluators", fake_fanout) monkeypatch.setattr(execute_task_module, "TaskExecutionService", lambda: svc) - monkeypatch.setattr(execute_task_module.RuntimeGraphRepository, "node", repo.node) result = await execute_task_module.run_execute_task_job( ctx, @@ -250,7 +246,7 @@ async def test_execute_task_emits_failed_when_worker_fails( run_id = uuid4() prepared = _prepared(uuid4(), uuid4()) - payload = _ready_event(run_id, uuid4(), uuid4(), prepared.task_id) + payload = _ready_event(run_id, uuid4(), prepared.task_id) async def fake_prepare( _ctx: inngest.Context, diff --git a/ergon_core/tests/unit/runtime/test_failure_error_json.py b/ergon_core/tests/unit/runtime/test_failure_error_json.py index 823c4e276..c1cfc703b 100644 --- a/ergon_core/tests/unit/runtime/test_failure_error_json.py +++ b/ergon_core/tests/unit/runtime/test_failure_error_json.py @@ -13,11 +13,11 @@ async def test_finalize_failure_preserves_structured_error_json(monkeypatch) -> execution_id = uuid4() run_id = uuid4() - node_id = uuid4() + task_id = uuid4() execution = SimpleNamespace( id=execution_id, run_id=run_id, - task_id=node_id, + task_id=task_id, ) class Session: diff --git a/ergon_core/tests/unit/runtime/test_graph_worker_identity.py b/ergon_core/tests/unit/runtime/test_graph_worker_identity.py index 488e8e1af..abea584a1 100644 --- a/ergon_core/tests/unit/runtime/test_graph_worker_identity.py +++ b/ergon_core/tests/unit/runtime/test_graph_worker_identity.py @@ -133,7 +133,7 @@ def test_graph_initialization_writes_concrete_worker_slug_from_definition_bindin @pytest.mark.asyncio -async def test_workflow_initialization_returns_node_ids_for_initial_ready_static_tasks( +async def test_workflow_initialization_returns_task_ids_for_initial_ready_static_tasks( monkeypatch: pytest.MonkeyPatch, ) -> None: session = _session() @@ -170,15 +170,15 @@ async def test_dynamic_prepare_uses_node_worker_slug_and_run_model_without_defin session = _session() definition_id = _definition_with_worker(session, worker_type="minif2f-react") run_id = _run(session, definition_id=definition_id, model_target="stub:constant") - node_id = uuid4() + task_id = uuid4() task = task_with_id( - node_id, + task_id, task_slug="dynamic-leaf", instance_key="sample-1", description="Dynamic specialist task", ) node = RunGraphNode( - task_id=node_id, + task_id=task_id, run_id=run_id, instance_key="sample-1", task_slug="dynamic-leaf", diff --git a/ergon_core/tests/unit/runtime/test_identity_invariants.py b/ergon_core/tests/unit/runtime/test_identity_invariants.py index ec79dec3e..2175b725c 100644 --- a/ergon_core/tests/unit/runtime/test_identity_invariants.py +++ b/ergon_core/tests/unit/runtime/test_identity_invariants.py @@ -188,10 +188,10 @@ def test_sandbox_identity_is_preserved_across_worker_to_evaluate_boundary() -> N """Δ.5: the sandbox acquired in worker_execute is the one each evaluate_task_run invocation attaches to via sandbox_id. - PR 4 makes this concrete: ``worker_execute`` stamps the live - ``sandbox_id`` onto the execution row, and ``evaluate_task_run`` - reads ``execution.sandbox_id`` and passes it to - ``graph_repo.node(..., sandbox_id=...)`` to reattach. The + PR 5 keeps this concrete behind the public task execution facade: + ``worker_execute`` stamps the live ``sandbox_id`` onto the execution + row, and ``evaluate_task_run`` reads ``execution.sandbox_id`` and + passes it to ``load_task_view(..., sandbox_id=...)`` to reattach. The ``sandbox_id`` field on ``RunTaskExecution`` is the carrier; the structural guard checks both halves of the contract. """ @@ -205,20 +205,21 @@ def test_sandbox_identity_is_preserved_across_worker_to_evaluate_boundary() -> N # Carrier: the execution row owns the sandbox_id. assert "sandbox_id" in RunTaskExecution.model_fields - # Writer: TaskExecutionRepository.set_sandbox_id is the only - # writer of that column on the runtime path. + # Internal writer still owns the column mutation; jobs reach it through + # TaskExecutionService.attach_sandbox_to_execution. assert hasattr(TaskExecutionRepository, "set_sandbox_id") root = Path(__file__).resolve().parents[4] worker_text = (root / "ergon_core/ergon_core/core/jobs/task/worker_execute/job.py").read_text() eval_text = (root / "ergon_core/ergon_core/core/jobs/task/evaluate/job.py").read_text() - # Producer side: worker_execute stamps sandbox_id on the row. - assert "set_sandbox_id(" in worker_text + # Producer side: worker_execute stamps sandbox_id through the facade. + assert "attach_sandbox_to_execution(" in worker_text assert "sandbox_id=payload.sandbox_id" in worker_text # Consumer side: evaluate_task_run reads sandbox_id from the # execution row and forwards it to the run-tier loader. + assert "load_task_view(" in eval_text assert "execution.sandbox_id" in eval_text assert "sandbox_id=execution.sandbox_id" in eval_text @@ -228,7 +229,7 @@ def test_execution_id_is_unique_per_attempt_and_shared_across_evaluators() -> No execution_id; a retry mints a new one. PR 4 makes execution_id the join key linking - ``RunTaskExecution`` ⇄ ``WorkerOutputRepository`` ⇄ each + ``RunTaskExecution`` ⇄ ``TaskExecutionService`` ⇄ each ``TaskEvaluateRequest`` invocation. The structural guard checks that (a) ``TaskEvaluateRequest`` carries ``execution_id``, (b) the orchestrator's fanout reuses ``prepared.execution_id`` for every diff --git a/ergon_core/tests/unit/runtime/test_persist_outputs.py b/ergon_core/tests/unit/runtime/test_persist_outputs.py index b165727b1..8e6310c78 100644 --- a/ergon_core/tests/unit/runtime/test_persist_outputs.py +++ b/ergon_core/tests/unit/runtime/test_persist_outputs.py @@ -8,11 +8,11 @@ from ergon_core.core.jobs.resources.persist_outputs.job import run_persist_outputs_job -class _FakeGraphRepo: +class _FakeTaskExecutionService: def __init__(self, seen: list[str | None]) -> None: self._seen = seen - async def node(self, _session, *, run_id, task_id, sandbox_id=None): + async def load_task_view(self, _session, *, run_id, task_id, sandbox_id=None): del run_id, task_id self._seen.append(sandbox_id) sandbox = SimpleNamespace(output_path="/workspace/public-output/") @@ -50,7 +50,11 @@ async def test_persist_outputs_publishes_public_sandbox_through_resource_service seen_sandbox_ids: list[str | None] = [] monkeypatch.setattr(module, "get_session", lambda: nullcontext(object())) - monkeypatch.setattr(module, "RuntimeGraphRepository", lambda: _FakeGraphRepo(seen_sandbox_ids)) + monkeypatch.setattr( + module, + "TaskExecutionService", + lambda: _FakeTaskExecutionService(seen_sandbox_ids), + ) monkeypatch.setattr(composition, "SandboxResourcePublisher", _FakePublisher) monkeypatch.setattr(composition, "RunResourcePublishService", _FakePublishService) diff --git a/ergon_core/tests/unit/runtime/test_task_execution_public_facade.py b/ergon_core/tests/unit/runtime/test_task_execution_public_facade.py new file mode 100644 index 000000000..d1394a7df --- /dev/null +++ b/ergon_core/tests/unit/runtime/test_task_execution_public_facade.py @@ -0,0 +1,117 @@ +from uuid import uuid4 + +import pytest + +from ergon_core.api.worker.results import WorkerOutput +from ergon_core.core.application.runtime.task_execution import TaskExecutionService + + +class _GraphRepo: + def __init__(self) -> None: + self.calls = [] + + async def node(self, session, *, run_id, task_id, sandbox_id=None): + self.calls.append( + { + "session": session, + "run_id": run_id, + "task_id": task_id, + "sandbox_id": sandbox_id, + } + ) + return object() + + +class _TaskExecutionRepo: + def __init__(self) -> None: + self.sandbox_calls = [] + + async def set_sandbox_id(self, session, *, execution_id, sandbox_id): + self.sandbox_calls.append( + { + "session": session, + "execution_id": execution_id, + "sandbox_id": sandbox_id, + } + ) + + +class _WorkerOutputRepo: + def __init__(self) -> None: + self.persist_calls = [] + self.loaded = WorkerOutput(success=True, output="ok") + + async def persist(self, session, *, execution_id, output): + self.persist_calls.append( + { + "session": session, + "execution_id": execution_id, + "output": output, + } + ) + + async def load(self, session, *, execution_id): + return self.loaded + + +@pytest.mark.asyncio +async def test_task_execution_facade_loads_task_view_by_task_id() -> None: + graph_repo = _GraphRepo() + service = TaskExecutionService(graph_repo=graph_repo) + session = object() + run_id = uuid4() + task_id = uuid4() + + view = await service.load_task_view( + session, + run_id=run_id, + task_id=task_id, + sandbox_id="sandbox-1", + ) + + assert view is not None + assert graph_repo.calls == [ + { + "session": session, + "run_id": run_id, + "task_id": task_id, + "sandbox_id": "sandbox-1", + } + ] + + +@pytest.mark.asyncio +async def test_task_execution_facade_owns_worker_output_and_sandbox_writes() -> None: + task_execution_repo = _TaskExecutionRepo() + worker_output_repo = _WorkerOutputRepo() + service = TaskExecutionService( + task_execution_repo=task_execution_repo, + worker_output_repo=worker_output_repo, + ) + session = object() + execution_id = uuid4() + output = WorkerOutput(success=True, output="hello") + + await service.persist_worker_output(session, execution_id=execution_id, output=output) + await service.attach_sandbox_to_execution( + session, + execution_id=execution_id, + sandbox_id="sandbox-2", + ) + loaded = await service.load_worker_output(session, execution_id=execution_id) + + assert worker_output_repo.persist_calls == [ + { + "session": session, + "execution_id": execution_id, + "output": output, + } + ] + assert task_execution_repo.sandbox_calls == [ + { + "session": session, + "execution_id": execution_id, + "sandbox_id": "sandbox-2", + } + ] + assert loaded is worker_output_repo.loaded From ccad27ecbf1b46eeafefe0945575057d8b933473 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 12:50:16 +0100 Subject: [PATCH 7/8] Finalize application domain guards --- .../test_application_domain_boundaries.py | 70 ++++++++----------- .../test_repository_layer_conventions.py | 18 +---- 2 files changed, 31 insertions(+), 57 deletions(-) diff --git a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py index 909f6b8e0..c75dd2f92 100644 --- a/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py +++ b/ergon_core/tests/unit/architecture/test_application_domain_boundaries.py @@ -6,11 +6,10 @@ from dataclasses import dataclass from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parents[4] PACKAGE_ROOT = ROOT / "ergon_core" +CORE_ROOT = PACKAGE_ROOT / "ergon_core" / "core" APPLICATION_ROOT = PACKAGE_ROOT / "ergon_core" / "core" / "application" APPLICATION_PREFIX = "ergon_core.core.application" @@ -76,19 +75,13 @@ } LAYOUT_DIR_EXCEPTIONS: dict[str, set[str]] = {} -_PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = () - -PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER = tuple( - pytest.param( - source_module, - target_module, - marks=pytest.mark.xfail( - strict=True, - reason=reason, - ), - id=f"{source_module} -> {target_module}", - ) - for source_module, target_module, reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER +RETIRED_APPLICATION_MODULES = ( + "ergon_core.core.application.context.events", + "ergon_core.core.application.graph", + "ergon_core.core.application.tasks", + "ergon_core.core.application.workflows", + "ergon_core.core.application.jobs", + "ergon_core.core.application.read_models", ) @@ -235,29 +228,10 @@ def _public_cross_domain_modules(target_domain: str) -> set[str]: ) -@pytest.mark.parametrize( - ("source_module", "target_module"), - PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER, -) -def test_ledgered_private_cross_domain_imports_are_removed( - source_module: str, - target_module: str, -) -> None: - """Strict xfail ledger: XPASS means a future cleanup removed the import.""" - - assert ImportEdge(source_module, target_module) not in _private_cross_domain_imports() - - def test_no_unledgered_private_cross_domain_imports() -> None: - current = _private_cross_domain_imports() - ledgered = { - ImportEdge(source_module, target_module) - for source_module, target_module, _reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER - } - offenders = [ f"{edge.source_module} imports private {edge.target_module}" - for edge in sorted(current - ledgered) + for edge in sorted(_private_cross_domain_imports()) ] assert offenders == [] @@ -272,16 +246,32 @@ def test_cross_domain_imports_only_target_public_domain_modules() -> None: ) if leaf is None or leaf in _public_cross_domain_modules(target_domain): continue - if any( - (edge.source_module, edge.target_module) == (source_module, target_module) - for source_module, target_module, _reason in _PRIVATE_CROSS_DOMAIN_IMPORT_LEDGER - ): - continue offenders.append(f"{edge.source_module} imports {edge.target_module}") assert offenders == [] +def test_retired_application_paths_are_absent_from_source_and_filesystem() -> None: + offenders: list[str] = [] + + for module_name in RETIRED_APPLICATION_MODULES: + module_path = PACKAGE_ROOT.joinpath(*module_name.split(".")) + for path in (module_path.with_suffix(".py"), module_path): + if path.exists(): + offenders.append(f"{path.relative_to(ROOT)} still exists") + + for path in CORE_ROOT.rglob("*.py"): + if "__pycache__" in path.parts: + continue + + source = path.read_text() + for module_name in RETIRED_APPLICATION_MODULES: + if module_name in source: + offenders.append(f"{path.relative_to(ROOT)} still references {module_name}") + + assert offenders == [] + + def test_application_domain_directories_keep_approved_layout() -> None: offenders: list[str] = [] diff --git a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py index 972190f7d..211cee630 100644 --- a/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py +++ b/ergon_core/tests/unit/architecture/test_repository_layer_conventions.py @@ -33,8 +33,6 @@ APPLICATION_ROOT = CORE_ROOT / "application" APPLICATION_PREFIX = "ergon_core.core.application" -_REPOSITORY_REEXPORT_IMPORT_LEDGER = () - _WRITE_PREFIXES = ( "add_", @@ -344,24 +342,10 @@ def _from_repository_imports( return imports -@pytest.mark.parametrize( - ("source_module", "target_module"), - _REPOSITORY_REEXPORT_IMPORT_LEDGER, -) -def test_ledgered_application_repository_imports_are_removed( - source_module: str, - target_module: str, -) -> None: - """Strict xfail ledger for repository imports hidden behind package exports.""" - - assert (source_module, target_module) not in _application_repository_imports() - - def test_application_repositories_are_imported_only_within_their_domain() -> None: - ledgered = {(param.values[0], param.values[1]) for param in _REPOSITORY_REEXPORT_IMPORT_LEDGER} offenders = [ f"{source_module} imports repository module {target_module}" - for source_module, target_module in sorted(_application_repository_imports() - ledgered) + for source_module, target_module in sorted(_application_repository_imports()) ] assert offenders == [] From 8fe9ad6b9b4c1c104b303a5834702b3361bd6d51 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 20 May 2026 13:06:07 +0100 Subject: [PATCH 8/8] Remove in-repo planning docs --- docs/TY_PASS_PLAN.md | 496 --- docs/architecture/01_public_api.md | 373 --- docs/architecture/02_runtime_lifecycle.md | 172 -- docs/architecture/03_providers.md | 197 -- docs/architecture/04_persistence.md | 223 -- docs/architecture/05_dashboard.md | 174 -- docs/architecture/06_builtins.md | 297 -- docs/architecture/07_testing.md | 195 -- docs/architecture/08_rl_loop.md | 181 -- docs/architecture/README.md | 139 - docs/architecture/cross_cutting/artifacts.md | 179 -- .../cross_cutting/error_propagation.md | 186 -- .../cross_cutting/sandbox_lifecycle.md | 94 - docs/bugs/TEMPLATE.md | 44 - ...26-04-17-sandbox-event-sink-unactivated.md | 110 - .../fixed/2026-04-18-ci-docker-caching.md | 88 - ...-18-dashboard-emitter-methods-not-wired.md | 100 - ...04-18-swebench-criterion-spawns-sandbox.md | 81 - ...026-04-17-dashboard-process-local-state.md | 94 - docs/bugs/open/2026-04-18-blob-store-no-gc.md | 65 - ...2026-04-18-dashboard-reconnect-stale-ui.md | 81 - ...04-18-sandbox-manager-shared-state-race.md | 108 - ...026-04-22-edge-added-mutation-unhandled.md | 114 - .../2026-04-23-inngest-function-failures.md | 458 --- ...pr-3-prepare-run-node-detached-instance.md | 170 -- docs/dashboard/snapshot-parity-fix.md | 359 --- docs/dead-code-audit-2026-04-25.md | 184 -- docs/event-wal/01_AUDIT.md | 419 --- docs/event-wal/02_INCREMENTAL_PERSISTENCE.md | 930 ------ docs/event-wal/03_WORKFLOW_RESUMPTION.md | 668 ---- .../rq1-cli-specialism/changelog.md | 295 -- docs/features/context-event-log.md | 1396 --------- docs/integration-spec/0-task-status-design.md | 717 ----- docs/integration-spec/1-audit.md | 105 - docs/integration-spec/2-control-flow-spec.md | 717 ----- docs/integration-spec/3-structural-checks.md | 312 -- .../4-violated-assumptions.md | 212 -- docs/integration-spec/5-test-harness.md | 102 - .../integration-spec/6-implementation-plan.md | 712 ----- docs/integration-spec/README.md | 52 - docs/integration-test-audit-2026-04-23.md | 1112 ------- docs/karnas-experiment-log-2026-04-25.md | 304 -- docs/real-llm-rollout-harness.md | 374 --- docs/release.md | 36 - docs/rfcs/TEMPLATE.md | 47 - ...26-04-17-criterion-runtime-di-container.md | 1416 --------- .../2026-04-17-delete-run-task-state-event.md | 861 ------ ...026-04-17-sandbox-event-sink-activation.md | 881 ------ ...4-18-dashboard-event-wiring-enforcement.md | 761 ----- ...-04-18-onboarding-deps-on-benchmark-abc.md | 928 ------ ...2-worker-interface-and-artifact-routing.md | 735 ----- ...-cleanup-cancelled-task-release-sandbox.md | 921 ------ ...-04-17-sandbox-lifetime-covers-criteria.md | 1193 -------- ...-04-17-static-sibling-failure-semantics.md | 1005 ------ ...2026-04-18-dashboard-paginated-runs-api.md | 1163 ------- .../2026-04-18-sandbox-manager-key-cleanup.md | 747 ----- ...026-04-18-sandbox-manager-process-state.md | 927 ------ .../2026-04-18-template-spec-public-api.md | 1049 ------- .../2026-04-21-real-llm-debug-harness.md | 363 --- .../00-readme.md | 155 - .../01-api-surface.md | 1219 -------- .../02-persistence-layer.md | 524 ---- .../03-runtime.md | 745 ----- .../04-walkthrough.md | 750 ----- .../05-cli-authoring-interface.md | 247 -- .../06-inngest-event-contracts.md | 473 --- .../07-test-strategy.md | 583 ---- .../08-decisions-log.md | 611 ---- .../09-implementation-plan.md | 101 - .../09-implementation-plan/00-program.md | 275 -- .../01-pr-00-transition-ledger.md | 986 ------ .../01b-pr-0-5-repository-standard.md | 669 ---- .../02-pr-01-run-tier-task-snapshot.md | 709 ----- .../03-pr-02-typed-run-node-boundary.md | 440 --- .../04-pr-03-worker-execute-typed-node.md | 346 --- .../05-pr-04-inline-criteria.md | 884 ------ .../06-pr-05-object-bound-api.md | 1182 -------- .../07-pr-06-minif2f-vertical.md | 404 --- .../07b-pr-6-5-domain-colocation.md | 1402 --------- .../08-pr-07-persistence-collapse.md | 491 --- .../09-pr-08-cli-composition.md | 487 --- .../10-pr-09-dynamic-subtasks.md | 603 ---- .../11-pr-10a-swebench.md | 586 ---- .../11b-pr-10b-researchrubrics.md | 453 --- .../11c-pr-10c-gdpeval.md | 486 --- .../12-pr-11-deletion-final-schema.md | 1196 -------- .../13-pr-12-walkthrough-ci.md | 701 ----- .../_post-pr8-drift-audit.md | 281 -- .../_public-api-lifecycle-audit.md | 238 -- .../reconciliation_plan/README.md | 96 - .../reconciliation_plan/dead_code.MD | 136 - .../reconciliation_plan/duplication.mD | 359 --- .../post_pr16_flow_digest.md | 356 --- .../reconciliation_plan/pr11_gap_register.md | 143 - ...e-identity-and-dynamic-task-correctness.md | 183 -- .../pr13-evaluation-cleanup.md | 147 - .../pr14-public-api-registry-surface.md | 155 - .../pr15-dashboard-event-contracts.md | 158 - .../pr16-core-debt-sweep.md | 180 -- .../pr17-remove-legacy-compatibility-notes.md | 51 - .../reconciliation_plan/pr_stack_recut.md | 147 - .../README.md | 192 -- .../audits/current-structure.md | 375 --- ...frastructure-application-boundary-audit.md | 360 --- .../audits/persistence-boundary-audit.md | 367 --- .../audits/runtime-domain-merge-audit.md | 409 --- .../audits/schema-concept-debt-audit.md | 626 ---- .../implementation-plan/00-program.md | 118 - .../01-persistence-evaluation-and-status.md | 79 - .../02-persistence-schema-concept-cleanup.md | 74 - .../03-shared-context-and-domain-deletion.md | 63 - .../04-views-package-foundation.md | 68 - ...-dashboard-contracts-and-publisher-port.md | 72 - .../06-rest-http-adapter-boundary.md | 69 - ...07-sandbox-resource-publishing-boundary.md | 62 - ...-legacy-experiment-and-cohort-isolation.md | 65 - ...9-legacy-cohort-and-experiment-deletion.md | 62 - .../10-job-composition-modules.md | 69 - .../11-application-runtime-restructure.md | 78 - ...nal-folder-state-and-architecture-gates.md | 79 - .../prds/00-move-vs-delete-decisions.md | 42 - .../prds/01-persistence-schema-boundary.md | 168 - .../prds/02-shared-context-contract.md | 98 - .../03-infrastructure-adapter-boundary.md | 138 - .../prds/04-rest-inbound-adapter-boundary.md | 96 - .../05-application-runtime-restructure.md | 124 - .../prds/06-views-dashboard-contracts.md | 166 - ...7-deprecated-cohorts-legacy-experiments.md | 124 - .../prds/08-job-composition-modules.md | 291 -- .../prds/09-final-core-folder-state.md | 376 --- ...10-application-domain-layout-convention.md | 667 ---- .../01-smell-remediation-map.md | 847 ------ .../02-target-folder-shape.md | 411 --- .../README.md | 447 --- .../pr-plans/00-cli-hygiene-cleanup.md | 300 -- .../pr-plans/00-program.md | 165 - .../01-dynamic-subtask-toolkit-ownership.md | 304 -- ...2-cli-composition-root-and-parser-split.md | 199 -- .../03-shared-output-errors-command-models.md | 188 -- .../04-runs-experiments-service-boundary.md | 218 -- ...enchmarks-onboarding-metadata-ownership.md | 223 -- .../06-final-domain-folder-migration.md | 198 -- .../prds/00-cli-hygiene-audit.md | 363 --- .../01-dynamic-subtask-authoring-toolkit.md | 602 ---- .../01-logic-smell-audit.md | 885 ------ .../02-implementation-stack.md | 378 --- .../README.md | 482 --- .../01-dependency-inversion.md | 418 --- .../02-test-brittleness-and-gaps.md | 441 --- .../03-code-quality.md | 642 ---- .../architecture-refactor-audit/README.md | 142 - .../final-worker-output-source-of-truth.md | 177 -- ...026-04-21-agent-framework-adapter-layer.md | 205 -- ...026-04-21-projection-operator-call-tree.md | 191 -- .../2026-04-21-projection-operator-mcts.md | 219 -- ...04-21-projection-operator-option-tagged.md | 172 -- ...2026-04-21-rl-trainer-adapter-expansion.md | 210 -- docs/rfcs/paper-nice-to-haves/README.md | 77 - .../2026-04-21-real-llm-debug-harness.md | 156 - .../brainstorms/2026-05-13-unit-test-audit.md | 113 - .../2026-05-15-kill-experiment-class.md | 417 --- ...-16-restart-and-downstream-invalidation.md | 232 -- .../plans/2026-04-16-subtask-lifecycle.md | 2094 ------------- .../2026-04-17-swebench-verified-benchmark.md | 1810 ----------- .../2026-04-21-real-llm-debug-harness.md | 1292 -------- ...2-worker-interface-and-artifact-routing.md | 2696 ----------------- .../2026-04-23-code-smell-and-ci-cleanup.md | 749 ----- .../2026-04-23-test-quality-improvements.md | 2089 ------------- ...04-25-e2e-contract-playwright-hardening.md | 410 --- .../plans/2026-04-25-slopcop-0-1-1-cleanup.md | 584 ---- .../2026-04-25-slopcop-ignore-cleanup.md | 336 -- ...26-04-26-communication-thread-workspace.md | 558 ---- .../plans/2026-04-26-core-test-logic-audit.md | 1113 ------- .../2026-04-26-finish-agent-workflow-cli.md | 99 - .../plans/2026-04-26-mas-navigation-cli.md | 1439 --------- ...6-run-workspace-interaction-corrections.md | 932 ------ .../2026-04-26-trace-spans-ux-refinements.md | 116 - ...26-04-27-frontend-evaluation-visibility.md | 1390 --------- ...2026-04-27-react-worker-context-capture.md | 1116 ------- ...27-react-worker-failure-context-capture.md | 650 ---- .../2026-04-28-agent-tool-budget-harness.md | 810 ----- .../2026-04-28-context-part-chunk-stream.md | 1359 --------- ...ore-hybrid-domain-layout-implementation.md | 1259 -------- .../2026-04-28-core-hybrid-domain-layout.md | 584 ---- .../2026-04-28-core-schema-deduplication.md | 1178 ------- ...-04-28-ergon-builtins-rebuild-structure.md | 709 ----- ...2026-04-28-ergon-cli-refactor-structure.md | 772 ----- ...2026-04-28-ergon-e2e-refactor-test-plan.md | 831 ----- ...evaluation-resource-context-and-scoring.md | 908 ------ ...26-04-28-mas-rebase-regression-recovery.md | 386 --- ...6-04-28-public-api-audit-and-ergonomics.md | 1556 ---------- .../2026-04-28-public-api-folder-plan.md | 413 --- ...026-04-28-runtime-services-layout-audit.md | 665 ---- ...-04-29-core-component-registry-refactor.md | 1229 -------- .../2026-04-29-dashboard-boundary-refactor.md | 1224 -------- ...-04-29-finish-builtins-cli-e2e-refactor.md | 841 ----- ...stent-component-catalog-and-test-layout.md | 1944 ------------ ...ded-export-and-deadline-dataset-release.md | 420 --- ...pr11-local-smoke-debugging-and-coverage.md | 313 -- .../2026-05-19-builtins-pr04-todo-cleanup.md | 1583 ---------- .../mas-run-visual-debugger/00-program.md | 110 - .../01-contracts-and-state.md | 232 -- .../02-frontend-implementation.md | 321 -- .../03-tests-and-e2e.md | 275 -- .../mas-run-visual-debugger/04-phases.md | 230 -- .../05-implementation-shape.md | 302 -- .../06-fast-feedback-and-visual-review.md | 309 -- .../plans/mas-run-visual-debugger/README.md | 22 - .../plans/test-refactor/00-program.md | 163 - .../plans/test-refactor/01-fixtures.md | 1191 -------- .../test-refactor/02-drivers-and-asserts.md | 829 ----- .../03-dashboard-and-playwright.md | 278 -- .../test-refactor/04-ci-and-workflows.md | 321 -- .../plans/test-refactor/05-deletions.md | 210 -- .../plans/test-refactor/06-phases.md | 286 -- .../superpowers/plans/test-refactor/README.md | 27 - .../plans/workflow-agent-cli/00-program.md | 131 - .../01-resource-semantics.md | 84 - .../02-schema-and-services.md | 159 - .../03-cli-command-surface.md | 242 -- .../04-agent-tool-and-worker.md | 127 - .../05-tests-and-acceptance.md | 268 -- .../plans/workflow-agent-cli/06-phases.md | 143 - .../plans/workflow-agent-cli/README.md | 23 - 224 files changed, 107099 deletions(-) delete mode 100644 docs/TY_PASS_PLAN.md delete mode 100644 docs/architecture/01_public_api.md delete mode 100644 docs/architecture/02_runtime_lifecycle.md delete mode 100644 docs/architecture/03_providers.md delete mode 100644 docs/architecture/04_persistence.md delete mode 100644 docs/architecture/05_dashboard.md delete mode 100644 docs/architecture/06_builtins.md delete mode 100644 docs/architecture/07_testing.md delete mode 100644 docs/architecture/08_rl_loop.md delete mode 100644 docs/architecture/README.md delete mode 100644 docs/architecture/cross_cutting/artifacts.md delete mode 100644 docs/architecture/cross_cutting/error_propagation.md delete mode 100644 docs/architecture/cross_cutting/sandbox_lifecycle.md delete mode 100644 docs/bugs/TEMPLATE.md delete mode 100644 docs/bugs/fixed/2026-04-17-sandbox-event-sink-unactivated.md delete mode 100644 docs/bugs/fixed/2026-04-18-ci-docker-caching.md delete mode 100644 docs/bugs/fixed/2026-04-18-dashboard-emitter-methods-not-wired.md delete mode 100644 docs/bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md delete mode 100644 docs/bugs/open/2026-04-17-dashboard-process-local-state.md delete mode 100644 docs/bugs/open/2026-04-18-blob-store-no-gc.md delete mode 100644 docs/bugs/open/2026-04-18-dashboard-reconnect-stale-ui.md delete mode 100644 docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md delete mode 100644 docs/bugs/open/2026-04-22-edge-added-mutation-unhandled.md delete mode 100644 docs/bugs/open/2026-04-23-inngest-function-failures.md delete mode 100644 docs/bugs/open/2026-05-14-pr-3-prepare-run-node-detached-instance.md delete mode 100644 docs/dashboard/snapshot-parity-fix.md delete mode 100644 docs/dead-code-audit-2026-04-25.md delete mode 100644 docs/event-wal/01_AUDIT.md delete mode 100644 docs/event-wal/02_INCREMENTAL_PERSISTENCE.md delete mode 100644 docs/event-wal/03_WORKFLOW_RESUMPTION.md delete mode 100644 docs/experiments/rq1-cli-specialism/changelog.md delete mode 100644 docs/features/context-event-log.md delete mode 100644 docs/integration-spec/0-task-status-design.md delete mode 100644 docs/integration-spec/1-audit.md delete mode 100644 docs/integration-spec/2-control-flow-spec.md delete mode 100644 docs/integration-spec/3-structural-checks.md delete mode 100644 docs/integration-spec/4-violated-assumptions.md delete mode 100644 docs/integration-spec/5-test-harness.md delete mode 100644 docs/integration-spec/6-implementation-plan.md delete mode 100644 docs/integration-spec/README.md delete mode 100644 docs/integration-test-audit-2026-04-23.md delete mode 100644 docs/karnas-experiment-log-2026-04-25.md delete mode 100644 docs/real-llm-rollout-harness.md delete mode 100644 docs/release.md delete mode 100644 docs/rfcs/TEMPLATE.md delete mode 100644 docs/rfcs/accepted/2026-04-17-criterion-runtime-di-container.md delete mode 100644 docs/rfcs/accepted/2026-04-17-delete-run-task-state-event.md delete mode 100644 docs/rfcs/accepted/2026-04-17-sandbox-event-sink-activation.md delete mode 100644 docs/rfcs/accepted/2026-04-18-dashboard-event-wiring-enforcement.md delete mode 100644 docs/rfcs/accepted/2026-04-18-onboarding-deps-on-benchmark-abc.md delete mode 100644 docs/rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md delete mode 100644 docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md delete mode 100644 docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md delete mode 100644 docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md delete mode 100644 docs/rfcs/active/2026-04-18-dashboard-paginated-runs-api.md delete mode 100644 docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md delete mode 100644 docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md delete mode 100644 docs/rfcs/active/2026-04-18-template-spec-public-api.md delete mode 100644 docs/rfcs/active/2026-04-21-real-llm-debug-harness.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/02-persistence-layer.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/03-runtime.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/04-walkthrough.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/05-cli-authoring-interface.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/06-inngest-event-contracts.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/08-decisions-log.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01-pr-00-transition-ledger.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01b-pr-0-5-repository-standard.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/02-pr-01-run-tier-task-snapshot.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/03-pr-02-typed-run-node-boundary.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/04-pr-03-worker-execute-typed-node.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/05-pr-04-inline-criteria.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/06-pr-05-object-bound-api.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07-pr-06-minif2f-vertical.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07b-pr-6-5-domain-colocation.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/08-pr-07-persistence-collapse.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/09-pr-08-cli-composition.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/10-pr-09-dynamic-subtasks.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11-pr-10a-swebench.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11b-pr-10b-researchrubrics.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11c-pr-10c-gdpeval.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/12-pr-11-deletion-final-schema.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/13-pr-12-walkthrough-ci.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_post-pr8-drift-audit.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_public-api-lifecycle-audit.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/README.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/dead_code.MD delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/duplication.mD delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/post_pr16_flow_digest.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr11_gap_register.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr12-runtime-identity-and-dynamic-task-correctness.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr13-evaluation-cleanup.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr14-public-api-registry-surface.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr15-dashboard-event-contracts.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr16-core-debt-sweep.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr17-remove-legacy-compatibility-notes.md delete mode 100644 docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr_stack_recut.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/README.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/current-structure.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/infrastructure-application-boundary-audit.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/persistence-boundary-audit.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/runtime-domain-merge-audit.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/schema-concept-debt-audit.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/00-program.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/01-persistence-evaluation-and-status.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/02-persistence-schema-concept-cleanup.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/03-shared-context-and-domain-deletion.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/04-views-package-foundation.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/05-dashboard-contracts-and-publisher-port.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/06-rest-http-adapter-boundary.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/07-sandbox-resource-publishing-boundary.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/08-legacy-experiment-and-cohort-isolation.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/09-legacy-cohort-and-experiment-deletion.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/10-job-composition-modules.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/11-application-runtime-restructure.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/12-final-folder-state-and-architecture-gates.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/00-move-vs-delete-decisions.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/01-persistence-schema-boundary.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/02-shared-context-contract.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/03-infrastructure-adapter-boundary.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/04-rest-inbound-adapter-boundary.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/05-application-runtime-restructure.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/06-views-dashboard-contracts.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/07-deprecated-cohorts-legacy-experiments.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/08-job-composition-modules.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/09-final-core-folder-state.md delete mode 100644 docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/10-application-domain-layout-convention.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/01-smell-remediation-map.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/02-target-folder-shape.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/README.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-cli-hygiene-cleanup.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-program.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/01-dynamic-subtask-toolkit-ownership.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/02-cli-composition-root-and-parser-split.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/03-shared-output-errors-command-models.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/04-runs-experiments-service-boundary.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/05-benchmarks-onboarding-metadata-ownership.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/06-final-domain-folder-migration.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/00-cli-hygiene-audit.md delete mode 100644 docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/01-dynamic-subtask-authoring-toolkit.md delete mode 100644 docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/01-logic-smell-audit.md delete mode 100644 docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/02-implementation-stack.md delete mode 100644 docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/README.md delete mode 100644 docs/rfcs/active/architecture-refactor-audit/01-dependency-inversion.md delete mode 100644 docs/rfcs/active/architecture-refactor-audit/02-test-brittleness-and-gaps.md delete mode 100644 docs/rfcs/active/architecture-refactor-audit/03-code-quality.md delete mode 100644 docs/rfcs/active/architecture-refactor-audit/README.md delete mode 100644 docs/rfcs/active/final-worker-output-source-of-truth.md delete mode 100644 docs/rfcs/paper-nice-to-haves/2026-04-21-agent-framework-adapter-layer.md delete mode 100644 docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-call-tree.md delete mode 100644 docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-mcts.md delete mode 100644 docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-option-tagged.md delete mode 100644 docs/rfcs/paper-nice-to-haves/2026-04-21-rl-trainer-adapter-expansion.md delete mode 100644 docs/rfcs/paper-nice-to-haves/README.md delete mode 100644 docs/superpowers/brainstorms/2026-04-21-real-llm-debug-harness.md delete mode 100644 docs/superpowers/brainstorms/2026-05-13-unit-test-audit.md delete mode 100644 docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md delete mode 100644 docs/superpowers/plans/2026-04-16-restart-and-downstream-invalidation.md delete mode 100644 docs/superpowers/plans/2026-04-16-subtask-lifecycle.md delete mode 100644 docs/superpowers/plans/2026-04-17-swebench-verified-benchmark.md delete mode 100644 docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md delete mode 100644 docs/superpowers/plans/2026-04-22-worker-interface-and-artifact-routing.md delete mode 100644 docs/superpowers/plans/2026-04-23-code-smell-and-ci-cleanup.md delete mode 100644 docs/superpowers/plans/2026-04-23-test-quality-improvements.md delete mode 100644 docs/superpowers/plans/2026-04-25-e2e-contract-playwright-hardening.md delete mode 100644 docs/superpowers/plans/2026-04-25-slopcop-0-1-1-cleanup.md delete mode 100644 docs/superpowers/plans/2026-04-25-slopcop-ignore-cleanup.md delete mode 100644 docs/superpowers/plans/2026-04-26-communication-thread-workspace.md delete mode 100644 docs/superpowers/plans/2026-04-26-core-test-logic-audit.md delete mode 100644 docs/superpowers/plans/2026-04-26-finish-agent-workflow-cli.md delete mode 100644 docs/superpowers/plans/2026-04-26-mas-navigation-cli.md delete mode 100644 docs/superpowers/plans/2026-04-26-run-workspace-interaction-corrections.md delete mode 100644 docs/superpowers/plans/2026-04-26-trace-spans-ux-refinements.md delete mode 100644 docs/superpowers/plans/2026-04-27-frontend-evaluation-visibility.md delete mode 100644 docs/superpowers/plans/2026-04-27-react-worker-context-capture.md delete mode 100644 docs/superpowers/plans/2026-04-27-react-worker-failure-context-capture.md delete mode 100644 docs/superpowers/plans/2026-04-28-agent-tool-budget-harness.md delete mode 100644 docs/superpowers/plans/2026-04-28-context-part-chunk-stream.md delete mode 100644 docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout-implementation.md delete mode 100644 docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md delete mode 100644 docs/superpowers/plans/2026-04-28-core-schema-deduplication.md delete mode 100644 docs/superpowers/plans/2026-04-28-ergon-builtins-rebuild-structure.md delete mode 100644 docs/superpowers/plans/2026-04-28-ergon-cli-refactor-structure.md delete mode 100644 docs/superpowers/plans/2026-04-28-ergon-e2e-refactor-test-plan.md delete mode 100644 docs/superpowers/plans/2026-04-28-evaluation-resource-context-and-scoring.md delete mode 100644 docs/superpowers/plans/2026-04-28-mas-rebase-regression-recovery.md delete mode 100644 docs/superpowers/plans/2026-04-28-public-api-audit-and-ergonomics.md delete mode 100644 docs/superpowers/plans/2026-04-28-public-api-folder-plan.md delete mode 100644 docs/superpowers/plans/2026-04-28-runtime-services-layout-audit.md delete mode 100644 docs/superpowers/plans/2026-04-29-core-component-registry-refactor.md delete mode 100644 docs/superpowers/plans/2026-04-29-dashboard-boundary-refactor.md delete mode 100644 docs/superpowers/plans/2026-04-29-finish-builtins-cli-e2e-refactor.md delete mode 100644 docs/superpowers/plans/2026-04-29-persistent-component-catalog-and-test-layout.md delete mode 100644 docs/superpowers/plans/2026-05-05-sharded-export-and-deadline-dataset-release.md delete mode 100644 docs/superpowers/plans/2026-05-18-pr11-local-smoke-debugging-and-coverage.md delete mode 100644 docs/superpowers/plans/2026-05-19-builtins-pr04-todo-cleanup.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/00-program.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/01-contracts-and-state.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/02-frontend-implementation.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/03-tests-and-e2e.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/04-phases.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/05-implementation-shape.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/06-fast-feedback-and-visual-review.md delete mode 100644 docs/superpowers/plans/mas-run-visual-debugger/README.md delete mode 100644 docs/superpowers/plans/test-refactor/00-program.md delete mode 100644 docs/superpowers/plans/test-refactor/01-fixtures.md delete mode 100644 docs/superpowers/plans/test-refactor/02-drivers-and-asserts.md delete mode 100644 docs/superpowers/plans/test-refactor/03-dashboard-and-playwright.md delete mode 100644 docs/superpowers/plans/test-refactor/04-ci-and-workflows.md delete mode 100644 docs/superpowers/plans/test-refactor/05-deletions.md delete mode 100644 docs/superpowers/plans/test-refactor/06-phases.md delete mode 100644 docs/superpowers/plans/test-refactor/README.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/00-program.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/01-resource-semantics.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/02-schema-and-services.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/03-cli-command-surface.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/04-agent-tool-and-worker.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/05-tests-and-acceptance.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/06-phases.md delete mode 100644 docs/superpowers/plans/workflow-agent-cli/README.md diff --git a/docs/TY_PASS_PLAN.md b/docs/TY_PASS_PLAN.md deleted file mode 100644 index fcbc31a72..000000000 --- a/docs/TY_PASS_PLAN.md +++ /dev/null @@ -1,496 +0,0 @@ -# Make `ty` Pass in Ergon - -**Date:** 2026-04-13 -**Current state:** 36 errors, 177 warnings across 213 diagnostics. Warnings are -already suppressed via `[tool.ty]` overrides in `pyproject.toml`. The 36 errors -prevent `ty check` from exiting 0. - ---- - -## Error Inventory (36 total) - -| Cat | Category | Errors | Fix Strategy | -|-----|---------------------------------------|--------|------------------------------------------------| -| F | `**` unpacking → `.model_validate()` | 10 | Rewrite construction sites | -| A | Missing/wrong ty override rules | 6 | Add rules to `pyproject.toml` | -| B | Real bugs in production code | 2 | Fix the code | -| C | Unresolved imports (optional deps) | 2 | Config + inline comment | -| D | Test narrowing issues | 14 | Assert narrowing + override | -| E | Test API mismatch | 2 | Add to test override | - ---- - -## Monorepo Note - -ty properly discovers the `.venv` and all editable-installed sub-packages: - -``` -info: 1. ergon_core (extra search path) -info: 2. ergon_builtins (extra search path) -info: 3. ergon_cli (extra search path) -info: 4. ergon_infra (extra search path) -info: 7. .venv/lib/python3.13/site-packages (site-packages) -``` - -Per-package ty configs would **not** help — every override targets the boundary -between first-party code and third-party library types (SQLModel, pydantic-ai, -e2b, inngest, transformers), not between ergon sub-packages. The existing -overrides are surgical (per-file pattern + per-rule) with explanatory comments. -Alternatives (inline `# type: ignore` everywhere, or `exclude` patterns) would -be worse. - ---- - -## Category F: Convert `**` Unpacking → `.model_validate()` (3 sites, eliminates 10 errors) - -Only 3 Pydantic `**` unpacking sites exist in the codebase. Converting them to -`.model_validate()` is both a type-safety win (returns the correct type without -`**` inference issues) and a correctness win (handles aliases like -`provider_details`/`vendor_details`, runs validators, applies defaults). - -### F.1 — `react_worker.py:225` (eliminates 7 errors) - -**File:** `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -```python -# Before -def _reconstruct_response(raw: dict[str, object]) -> ModelResponse: - return ModelResponse(**raw) - -# After -def _reconstruct_response(raw: dict[str, object]) -> ModelResponse: - return ModelResponse.model_validate(raw) -``` - -### F.2 — `react_worker.py:230` (eliminates 3 errors) - -**File:** `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -```python -# Before -def _reconstruct_request(raw: dict[str, object]) -> ModelRequest: - return ModelRequest(**raw) - -# After -def _reconstruct_request(raw: dict[str, object]) -> ModelRequest: - return ModelRequest.model_validate(raw) -``` - -### F.3 — `saved_specs/repositories.py:105` (no current errors, pattern alignment) - -**File:** `ergon_core/ergon_core/core/persistence/saved_specs/repositories.py` - -```python -# Before -row = model_class(**kwargs) - -# After -row = model_class.model_validate(kwargs) -``` - ---- - -## Category A: Missing ty Override Rules (6 errors) - -Third-party type-system limitations that remain after the `.model_validate()` -migration. Each needs a targeted rule addition to `pyproject.toml`. - -### A.1 — `transformers_backend.py` (2 errors: `unknown-argument`) - -**File:** `ergon_builtins/ergon_builtins/models/transformers_backend.py:115,128` -**Issue:** `provider_details` kwarg on `ModelResponse`. ty can't resolve Pydantic -`AliasChoices` (`provider_details` / `vendor_details`). -**Fix:** Add `unknown-argument = "warn"` to the existing transformers_backend -override. - -```toml -# Existing override — add unknown-argument -[[tool.ty.overrides]] -include = ["**/models/transformers_backend.py"] -[tool.ty.overrides.rules] -invalid-argument-type = "warn" -unresolved-attribute = "warn" -call-non-callable = "warn" -invalid-return-type = "warn" -unknown-argument = "warn" # ← ADD -``` - -### A.2 — `worker.py` (2 errors: `invalid-yield`, `invalid-argument-type`) - -**File:** `ergon_core/ergon_core/api/worker.py:54,83` -**Issue:** Abstract async generator body needs a `yield` (ty flags it as -`invalid-yield`). `response_text` on `GenerationTurn` resolves to -`Unknown | str | None` from the turn repo. -**Fix:** New override. - -```toml -[[tool.ty.overrides]] -include = ["ergon_core/ergon_core/api/worker.py"] -[tool.ty.overrides.rules] -invalid-yield = "warn" -invalid-argument-type = "warn" -``` - -### A.3 — `app.py` (1 error: `invalid-argument-type`) - -**File:** `ergon_core/ergon_core/core/api/app.py:28` -**Issue:** `inngest_client.send_sync` has signature -`(events: Event | list[Event], *, skip_middleware: bool) -> list[str]` but -`RolloutService.__init__` declares `inngest_send: Callable[[Event], None]`. -**Fix:** New override (the mismatch is harmless — we only ever pass a single -Event and ignore the return). - -```toml -[[tool.ty.overrides]] -include = ["**/core/api/app.py"] -[tool.ty.overrides.rules] -invalid-argument-type = "warn" -``` - -### A.4 — `rollout_service.py` (1 error: `invalid-return-type`) - -**File:** `ergon_core/ergon_core/core/rl/rollout_service.py:74` -**Issue:** `AutoTokenizer.from_pretrained()` returns `Unknown` (transformers is -in `allowed-unresolved-imports`), so ty can't verify the return satisfies the -`Tokenizer` protocol. -**Fix:** Add `invalid-return-type` to the existing rl override. - -```toml -# Existing override — add invalid-return-type -[[tool.ty.overrides]] -include = ["**/core/rl/**"] -[tool.ty.overrides.rules] -invalid-argument-type = "warn" -unresolved-attribute = "warn" -invalid-return-type = "warn" # ← ADD -``` - ---- - -## Category B: Real Bugs in Production Code (2 errors) - -### B.1 — `emitter.py:156` — `task_id` should be non-nullable - -**File:** `ergon_core/ergon_core/core/dashboard/emitter.py:153-167` -**Issue:** Method signature declares `task_id: UUID | None` but the event -contract `DashboardTaskEvaluationUpdatedEvent` requires `task_id: UUID`. No -callers pass `None`. -**Fix:** Tighten the method signature. - -```python -# Before -async def task_evaluation_updated( - self, - run_id: UUID, - task_id: UUID | None, # ← wrong - evaluation: dict[str, Any], -) -> None: - -# After -async def task_evaluation_updated( - self, - run_id: UUID, - task_id: UUID, # ← fixed - evaluation: dict[str, Any], -) -> None: -``` - -### B.2 — `emitter.py:317` — `sandbox_command()` missing `run_id` - -**File:** `ergon_core/ergon_core/core/dashboard/emitter.py:304-326` -**Issue:** `DashboardSandboxCommandEvent` requires `run_id: UUID` but -`sandbox_command()` never accepts or passes it. -**Fix:** Add `run_id: UUID` parameter and pass it through. - -```python -# Before -async def sandbox_command( - self, - task_id: UUID, - sandbox_id: str, - command: str, - ... -) -> None: - ... - evt = DashboardSandboxCommandEvent( - task_id=task_id, - ... - ) - -# After -async def sandbox_command( - self, - run_id: UUID, # ← ADD - task_id: UUID, - sandbox_id: str, - command: str, - ... -) -> None: - ... - evt = DashboardSandboxCommandEvent( - run_id=run_id, # ← ADD - task_id=task_id, - ... - ) -``` - -Also update all callers of `sandbox_command()` to pass `run_id`. - ---- - -## Category C: Unresolved Imports (2 errors) - -### C.1 — `vllm_model.py:12` — pydantic-ai re-export - -**File:** `ergon_core/ergon_core/core/providers/generation/vllm_model.py` -**Issue:** `from pydantic_ai.models.openai import OpenAIChatModel` — works at -runtime (verified) but ty can't resolve the re-export through the module's -`__all__`. -**Fix:** Inline comment. - -```python -from pydantic_ai.models.openai import OpenAIChatModel # type: ignore[unresolved-import] -``` - -### C.2 — `verl_http.py:21` — optional `verl` dependency - -**File:** `ergon_infra/ergon_infra/adapters/verl_http.py` -**Issue:** `from verl.experimental.agent_loop import ...` is inside try/except -(optional dep). ty doesn't recognize the try/except pattern for optional imports. -**Fix:** Add to `allowed-unresolved-imports` in `pyproject.toml`. - -```toml -[tool.ty.analysis] -allowed-unresolved-imports = [ - ... - "verl", "verl.**", # ← ADD -] -``` - ---- - -## Category D: Test Narrowing Issues (14 errors) - -### D.1 — `test_full_lifecycle.py` + `test_full_lifecycle_with_eval.py` (6 errors) - -**Files:** -- `tests/integration/test_full_lifecycle.py:109,201,202` -- `tests/integration/test_full_lifecycle_with_eval.py:248,248,249` - -**Issue:** `session.get(RunRecord, id)` returns `RunRecord | None`, then -`.status` is accessed without narrowing. -**Fix:** Add `assert` before access. - -```python -# Before -final_run = session.get(RunRecord, run.id) -assert final_run.status == RunStatus.COMPLETED - -# After -final_run = session.get(RunRecord, run.id) -assert final_run is not None -assert final_run.status == RunStatus.COMPLETED -``` - -### D.2 — `test_full_lifecycle_with_eval.py` (3 errors) - -**File:** `tests/integration/test_full_lifecycle_with_eval.py:120,133,139` -**Issue:** `prepared.execution_id` is `UUID | None` and `prepared.worker_type` -is `str | None`, used where non-nullable is required. -**Fix:** Add assertions after `prepare()`. - -```python -prepared = exec_svc.prepare(...) -assert not prepared.skipped -assert prepared.worker_type is not None -assert prepared.execution_id is not None # already exists at line 127 -# ... existing code that uses prepared.worker_type and prepared.execution_id -``` - -### D.3 — `test_benchmarks_stubbed.py` + `test_thread_execution_link.py` (5 errors) - -**Files:** -- `tests/e2e/test_benchmarks_stubbed.py:78,103,125,169` -- `tests/state/test_thread_execution_link.py:134` - -**Issue:** SQLModel column expressions (`.desc()`, `.is_(None)`) on -Python-typed fields. ty sees `datetime` / `UUID | None` instead of SQLAlchemy -column proxies. -**Fix:** New test override in `pyproject.toml`. - -```toml -[[tool.ty.overrides]] -include = ["tests/**"] -[tool.ty.overrides.rules] -unresolved-attribute = "warn" -``` - ---- - -## Category E: Test API Mismatch (2 errors) - -**Files:** -- `tests/integration/test_full_lifecycle.py:152` -- `tests/integration/test_full_lifecycle_with_eval.py:136` - -**Issue:** `asyncio.run(live_worker.execute(task_data, context=ctx))` where -`execute()` now returns `AsyncGenerator[GenerationTurn, None]`, not a -`Coroutine`. The tests use the pre-refactor API. -**Fix:** Covered by the test override from D.3 (`invalid-argument-type` as -`"warn"`). A proper fix would rewrite the tests to consume the async generator, -but that's a larger change. - -```toml -# Extend the test override from D.3 -[[tool.ty.overrides]] -include = ["tests/**"] -[tool.ty.overrides.rules] -unresolved-attribute = "warn" -invalid-argument-type = "warn" # ← covers E.1/E.2 -``` - ---- - -## Execution Order - -1. **G** — Tighten nullable types (see below) -2. **F** — `.model_validate()` migration (3 sites in react_worker + saved_specs) -3. **B** — Fix `emitter.py` bugs (tighten `task_id`, thread `run_id`) -4. **D.1, D.2** — Add `assert is not None` guards in test files -5. **A, C.2, D.3, E** — Update `pyproject.toml` (all config changes in one edit) -6. **C.1** — Add inline `# type: ignore` for `vllm_model.py` -7. **Verify** — Run `ty check` and confirm 0 errors - -**Estimated effort:** ~45 minutes across ~12 files. - ---- - -## Category G: Nullable Type Audit - -Full audit of all `UUID | None` fields across the codebase. Classified as -**tighten** (should be non-nullable), **discriminated union** (nullable is -correct but the grouping should be expressed in the type system), or -**correct** (legitimately optional). - -### G.1 — Tighten: `DashboardEmitter.task_evaluation_updated.task_id` - -Already covered in B.1. - -### G.2 — Discriminated Union: `PreparedTaskExecution` - -**File:** `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py:61-75` - -`execution_id`, `worker_type`, and `model_target` are all `UUID | None` / -`str | None`, but they form a correlated group: - -- When `skipped=False`: all three should always be present -- When `skipped=True`: all three are absent - -The runtime already encodes this invariant defensively: - -```python -# execute_task.py:76-77 -if prepared.execution_id is None and not prepared.skipped: - raise RuntimeError(...) -``` - -**Recommended fix:** Split into two return types. This eliminates downstream -null-checks and the 3 ty errors in `test_full_lifecycle_with_eval.py`. - -```python -class SkippedTaskExecution(BaseModel): - model_config = {"frozen": True} - - run_id: UUID - definition_id: UUID - task_id: UUID - task_key: str - task_description: str - benchmark_type: str - skipped: Literal[True] = True - skip_reason: str - - -class PreparedTaskExecution(BaseModel): - model_config = {"frozen": True} - - run_id: UUID - definition_id: UUID - task_id: UUID - task_key: str - task_description: str - benchmark_type: str - worker_binding_key: str - worker_type: str - model_target: str | None = None # model_target CAN be None (default model) - execution_id: UUID - skipped: Literal[False] = False -``` - -Callers use `isinstance` or check `.skipped`: - -```python -prepared = svc.prepare(...) -if isinstance(prepared, SkippedTaskExecution): - return TaskExecuteResult(skipped=True, ...) -# prepared.execution_id is now UUID, not UUID | None -``` - -**Impact:** -- `orchestration_dto.py` — split class -- `task_execution_service.py` — return type becomes union -- `execute_task.py` — simplify guard logic -- `test_full_lifecycle.py`, `test_full_lifecycle_with_eval.py` — remove null - assertions, use isinstance -- Eliminates 3 ty errors in test_full_lifecycle_with_eval.py (D.2) - -### G.3 — Discriminated Union: `TaskExecuteResult.execution_id` - -**File:** `ergon_core/ergon_core/core/runtime/services/inngest_function_results.py:20-30` - -Same pattern: `execution_id: UUID | None = None` is `None` only when -`skipped=True`. Same fix approach as G.2 (split into skipped/success/failure -variants) or just tighten the type and make `execution_id: UUID` required, -since even the skip path currently passes `prepared.execution_id` (which -after G.2 would always be `UUID`). - -After G.2, the skip path would not reach `TaskExecuteResult` construction -(it returns early), so `execution_id` can just be `UUID`. - -### G.4 — Correct (Legitimately Optional) - -These are all correctly `UUID | None`: - -| Field | Reason | -|-------|--------| -| `RunRecord.cohort_id` | Runs can exist outside cohorts | -| `ExperimentDefinitionTask.parent_task_id` | Root tasks have no parent | -| `TaskDescriptor.parent_task_id` | Same | -| `DashboardTaskStatusChangedEvent.parent_task_id` | Same | -| `DashboardTaskStatusChangedEvent.assigned_worker_id` | Task may not be assigned yet | -| `TraceContext.run_id/task_id/execution_id/evaluator_id` | Contexts start empty, narrow as they descend | -| `CreateMessageRequest.task_execution_id` | Messages can exist outside executions | -| `MessageResponse.task_execution_id` | Same | -| `RunGraphNode.definition_task_id` | Dynamic nodes may not map to definitions | -| `RunGraphEdge.definition_dependency_id` | Same | -| `GraphNodeDto.definition_task_id` | DTO mirrors the model | -| `GraphEdgeDto.definition_dependency_id` | Same | -| `RunResource.task_execution_id` | Resources can exist at run level | -| `ThreadMessage.task_execution_id` | Messages can exist outside executions | -| `RunTaskExecution.definition_worker_id` | Worker lookup can fail (no assignment) | -| `mark_task_failed.execution_id` | Task can fail before execution starts | -| `BaseSandboxManager._emit_wal_entry.task_id` | Sandbox can be at run level | -| `BaseSandboxManager.create.display_task_id` | Display only, optional | -| `GraphNodeLookup.node_id()` return | Lookup may not find a match | -| `GraphNodeLookup.edge_id*()` returns | Same | -| API query params (`definition_id`, `cohort_id`) | Optional filters | - -### G.5 — `sandbox_command` missing `run_id` (already B.2, wider scope) - -The `run_id` is missing from the **entire call chain**, not just `emitter.py`. -The fix needs to thread `run_id` through: - -1. `BaseSandboxManager._log_command()` / callers → pass `run_id` -2. `SandboxEventSink.sandbox_command()` → add `run_id` param -3. `SandboxInstrumentation.__init__` → store `run_id` -4. `DashboardEmitter.sandbox_command()` → add `run_id` param (B.2) diff --git a/docs/architecture/01_public_api.md b/docs/architecture/01_public_api.md deleted file mode 100644 index ccda20e24..000000000 --- a/docs/architecture/01_public_api.md +++ /dev/null @@ -1,373 +0,0 @@ -# 01 — public api - -## purpose - -This layer defines the types a contributor touches to add a benchmark, worker, -evaluator, or criterion to Ergon. It is the narrow seam between user-authored -research code and the runtime: a handful of frozen Pydantic models, two abstract -bases, one Protocol, and one declarative binding. Everything below this layer -(Inngest functions, RL adapters, DB writes, provider clients) is implementation -detail the public API deliberately hides. If a type appears in this document it -is part of the contract; if it does not, it is internal and may move without -notice. - -## core abstractions - -All public-surface Pydantic models are `frozen=True`; mutation is done through -`model_copy(update=...)`. The one deliberate exception is `WorkerContext` -(unfrozen since PR 9) so the runtime can inject `TaskManagementService`, -`TaskInspectionService`, and `RunResourceRepository` into `PrivateAttr` slots -via `WorkerContext._for_job`; the facade methods (`spawn_task`, `cancel_task`, -`refine_task`, `restart_task`, `subtasks`, `descendants`, `get_task`) then -delegate to those services. Every type below lives under `ergon_core/api/` -and is owned by that module. - -- **`Benchmark`** — abstract base. Produces work units via `build_instances()` - (a mapping from `instance_key` to a sequence of `TaskSpec` or `Task`) and declares - the evaluator binding keys it expects via `evaluator_requirements()`. Carries - a `type_slug: ClassVar[str]` that is a keyed identifier across the CLI, the - onboarding profile, and the benchmark registry — renames are breaking. - `Benchmark.__init__` accepts four keyword-only identity kwargs with sensible - defaults: `name: str | None = None` (collapses to the subclass name via - `or self.__class__.__name__`), `description: str | None = None` (collapses to - `""`), `metadata: Mapping[str, Any] | None = None` (copied into a fresh dict), - and `created_by: str | None = None` (PR 7; preserved as `None` when unset — - see the asymmetry note in `benchmark.py`). These four fields are exactly what - `persist_benchmark` reads off the instance to populate the corresponding - `experiment_definitions` columns; there is no separate config object. - Two authoring shapes are supported: - - **v1 (legacy):** `build_instances()` returns `TaskSpec` objects; the evaluator - slot is declared via `evaluator_requirements()` and bound by the `Experiment`. - - **v2 (object-bound):** `build_instances()` returns `Task` objects with inline - `worker`, `sandbox`, and `evaluators` fields. `evaluator_requirements()` returns - `()`. The task carries its own execution contract and round-trips through - persisted JSON via `_type` discriminators on every sub-component. -- **`BenchmarkTask`** — frozen Pydantic model describing a single unit of work. - It has two textual surfaces with very different audiences: `description` is - the only free-text the worker ever sees; `task_payload` is a dict the - evaluator (and benchmark itself) may read but the worker never does. The - split is load-bearing — see invariants. Intra-instance task graph edges - (`parent_task_key`, `dependency_task_keys`) are validated by - `Experiment.validate()`. -- **`Worker`** — abstract base. `execute()` is an async generator that MUST - yield a `GenerationTurn` per LLM call (see invariants). The runtime uses each - yield as both an RL observation point and a cancellation checkpoint. Workers - are serializable Pydantic `BaseModel` subclasses and carry a `_type` - discriminator so they round-trip through `Task` JSON snapshots in the v2 - authoring path. `Worker.__init__` takes `name: str` and `model: str | None`; - concrete workers may add further fields. `ReActWorker` adds an optional - `toolkit` field (a serializable `BaseModel` holding tool configuration) that - is bound to the live sandbox at `execute()` time — the toolkit calls - `toolkit.tools(sandbox, task)` to produce live `pydantic_ai.Tool` instances. - Workers MUST NOT own per-task environment setup — setup belongs to the sandbox - manager (see `BaseSandboxManager._install_dependencies`). Workers MUST NOT - return files or blobs through `WorkerOutput` — the non-durable `artifacts` - field was removed (RFC 2026-04-22); nothing crosses the Inngest - `worker_execute` boundary except the persisted `WorkerOutput` (`output`, - `success`, `metadata`). Files → write to `/workspace/final_output/` - (auto-published as `RunResource` rows). -- **`Sandbox`** — abstract Pydantic `BaseModel`. Carries serializable - configuration; live runtime state is held in a `PrivateAttr`. Two lifecycle - methods: `provision()` creates a new sandbox and sets `_runtime`; the runtime - calls `Sandbox.from_definition(json, sandbox_id=...)` on the eval side to - reconnect. Sandbox subclasses inject a `_type` discriminator via - `model_serializer` so they round-trip through persisted task JSON. -- **`WorkerSpec`** — frozen Pydantic model, the config-time descriptor of a - worker binding. Fields: `worker_slug: str`, `name: str`, `model: str | None`. - An `Experiment` holds `Mapping[str, WorkerSpec]`, not live `Worker` - instances — that keeps runtime identity (`task_id`, `sandbox_id`) out of - the declarative binding and avoids placeholder-sentinel leakage. The - registry factory for `worker_slug` is invoked exactly once per task - execution, inside `worker_execute`, with the prepared identity — the - resulting `Worker` lives only for that execution. `WorkerSpec.validate_spec()` - checks `worker_slug` against `WORKERS` and rejects empty names; it is - called from `Experiment.validate()` for every binding. -- **`Tool`** — public alias re-exported from `ergon_core.api` for the toolkit - primitive concrete workers consume. `ReActWorker`'s `tools: list[Tool]` - kwarg uses this alias; registry factories build the list and pass it - explicitly per benchmark (no toolkit defaults inside the worker). -- **`GenerationTurn`** — frozen model holding the per-turn LLM trace (input - messages, response parts, tool results, token IDs, logprobs, policy version, - timing). Not persisted directly; the runtime decomposes each turn into one - `RunContextEvent` per message at the event sink. -- **`Evaluator`** — abstract base. Scores a completed task by examining - recorded turns and published artifacts, delegating per-aspect scoring to - `Criterion` implementations and aggregating the results. The provided - `Rubric` subclass aggregates by weighted average over a fixed criteria list. -- **`Criterion`** — abstract base. Scores a single aspect of a completed task. - Criteria MAY be agentic — they may themselves call an LLM, issue tool calls, - or read sandbox resources. This is intentional: rubric-style benchmarks want - LLM-judge criteria, correctness-style benchmarks want criteria that exec - tests inside the task sandbox. -- **`CriterionRuntime`** — Protocol. The execution context an agentic - criterion uses to reach into its environment. **Surface-area constraint:** - this Protocol is narrowly scoped to sandbox lifecycle and resource I/O; it - should not grow into a generic service locator. The current surface is 12 - methods: sandbox lifecycle (7), resource I/O (3 — `list_resources`, - `read_resource`, `get_all_files_for_task`), DB read (1), and event - emission (1). `get_all_files_for_task()` is the materializing helper that - returns `{name: bytes}` for every resource produced by the task; criteria - use it when they want the whole output bundle rather than iterating - `list_resources()` + `read_resource()` themselves. The one method that is - not about sandbox/I/O is a candidate for extraction if the surface - continues to accumulate capabilities. -- **`persist_benchmark(benchmark: Benchmark) -> DefinitionHandle`** — - the v2 authoring entrypoint. Replaces the deleted `Experiment` class - (PR 6.5 hard-deleted it; see - `docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md`). - Walks `benchmark.build_instances()` to validate sandbox/worker - compatibility, then persists the configured benchmark as a definition - row with a UUID handle. Identity (`name`, `description`, `metadata`, - `created_by`) is read directly off the `Benchmark` instance — the - former `*, name=..., description=..., metadata=..., created_by=..., - experiment=...` kwargs were dropped in PR 6.5's cleanup once the - `Experiment` wrapper went away. Callers configure those fields by - passing them to `Benchmark.__init__` (see above); there is no - parallel config-object surface. - -## control flow - -``` -Benchmark.build_instances() - | - v -dict[instance_key, Sequence[BenchmarkTask]] - | - v -Experiment binding resolves (worker, evaluator) per instance_key - | - v -runtime fans out one execution per (task, worker_binding) - | - v -Worker.execute() ---- async generator ----> GenerationTurn (n times) - | - v - decomposed into - RunContextEvent rows - (one per message) - | - v - persisted via event sink - | - v (on completion) -Evaluator runs against recorded turns + artifacts - | - v -Evaluator invokes its Criteria - | | - v v - scalar criterion agentic criterion - \ / - \ CriterionRuntime / - \ (for agentic) / - v v - scores land in RunTaskEvaluation -``` - -Every arrow crosses a process or await boundary; there is no synchronous path -from `build_instances()` to `RunTaskEvaluation`. The runtime is free to -reorder, retry, and parallelise everything downstream of `build_instances()` -within the constraints the invariants below impose. - -## invariants - -- **Public-API models are frozen.** Every Pydantic model under - `ergon_core/api/` sets `frozen=True`. Mutation is done by `model_copy`. This - lets the runtime cache, hash, and cross process boundaries without defensive - copies. The deliberate exception is `WorkerContext` (PR 9): it is non-frozen - so the runtime can inject service instances into `PrivateAttr` fields via - `_for_job`. The injected services are not part of the public contract; user - code only sees the facade methods. -- **Worker spawn containment (PR 9).** `WorkerContext.spawn_task(task, *, - depends_on=())` writes a `run_graph_nodes` row with `is_dynamic=True` and - the full Task JSON snapshot — no synthetic `experiment_definition_tasks` - row. Lifecycle facade methods (`cancel_task`, `refine_task`, `restart_task`, - `get_task`) enforce containment: they raise `ContainmentViolation` if the - target task is not the worker's own task or a descendant. Containment is - evaluated against the live graph via `TaskInspectionService.descendant_ids`, - which walks `parent_node_id` with a recursive CTE. -- **Workers MUST yield.** `Worker.execute()` yields at least one - `GenerationTurn` per invocation, including stubs. The runtime uses turns as - the unit of RL observation and of cancellation checkpointing; a silent - worker breaks both. -- **`description` is the worker's only window.** `BenchmarkTask.description` - is the single free-text field passed to the worker. `task_payload` is for - the evaluator and the benchmark itself. Never leak evaluator-only fields - (hidden tests, reference patches, rubric answer keys) into `description`. - Canonical pattern: - `ergon_builtins/benchmarks/swebench_verified/task_schemas.py:76`. -- **`Experiment` stays in `ergon_core/api/`.** It is the declarative binding - and the import contributors rely on. Do not move it into the runtime, the - registry, or a plugin package. -- **`type_slug` is a keyed identifier.** The slug on a `Benchmark` subclass is - referenced by the CLI, the onboarding profile, and the benchmark registry. - All three must agree. A mismatch silently strands a benchmark: the registry - has it, the onboarding flow does not prompt for its credentials, and users - hit opaque errors at run time. -- **Evaluator binding keys form a typed contract.** The keys a benchmark - declares in `evaluator_requirements()` must be a superset of the keys its - `BenchmarkTask.evaluator_binding_keys` emit; evaluators select bindings by - matching these keys. `Experiment.validate()` enforces coverage. -- **RL-signal fields are nullable, not empty-filled.** - `GenerationTurn.turn_token_ids` and `turn_logprobs` are `| None`. Backends - that do not support logprobs MUST return `None`, not `[]` or a list of - zeros. The RL extractor treats `None` as "no logprobs collected" and pads; - `[]` would be interpreted as "zero tokens generated" and silently corrupt - token budget math (`ergon_core/core/rl/extraction.py:173`). -- **`final_assistant_message` is the canonical agent-text field name.** The - field that carries the agent's final assistant-text message is named - `final_assistant_message` end-to-end — from the Inngest - `WorkerExecuteResult` through `FinalizeTaskExecutionCommand` to the - `RunTaskExecution.final_assistant_message` column. Future work MUST NOT - reintroduce `output_text` as a synonym; the rename propagates into - dashboard readers and CLI surfaces in the same PR that renames the column. - (The code-side rename lands later in this same PR.) -- **`WorkerOutput` carries `output`, `success`, and `metadata` only.** The - former `artifacts: dict[str, Any]` field was removed in RFC 2026-04-22: - nothing carried it across the Inngest `worker_execute` serialization - boundary, and every live criterion now reads file-shaped artifacts via - the sandbox → `RunResource` path (see `cross_cutting/artifacts.md`) or - produces computed artifacts via `CriterionRuntime.run_command`. Adding a - new mutable-bag-of-bytes field to `WorkerOutput` is a regression. - -## extension points - -### add a new benchmark - -**v2 object-bound path (preferred for new benchmarks):** - -1. Subclass `Benchmark` under `ergon_builtins/benchmarks//`. -2. Set `type_slug` to the stable identifier. -3. Implement `build_instances()` returning `Task` objects with inline `worker`, - `sandbox`, and `evaluators`. Use factory helpers (`make__worker()`, - `make__rubric()`) so the task JSON is fully deterministic and round- - trippable. Pattern: `ergon_builtins/benchmarks/minif2f/benchmark.py`. -4. Implement `evaluator_requirements()` returning `()` — all binding is inline. -5. Implement a `Sandbox` subclass if the benchmark needs a custom sandbox; - colocate it with the benchmark at - `ergon_builtins/benchmarks//sandbox.py`. Pattern: - `ergon_builtins/benchmarks/minif2f/sandbox.py`. (The top-level - `ergon_builtins/sandbox/` package is for cross-cutting adapter infra - reused across benchmarks — see `06_builtins.md` § cardinality matrix.) -6. Register in the benchmark registry. -7. Add an entry to `ergon_cli/onboarding/profile.py::BENCHMARK_DEPS`. Skipping - this is the most common onboarding regression. -8. If the benchmark needs a custom sandbox Docker template, add an `ergon - benchmark setup ` subcommand. Do not bake template builds into - `build_instances()`. - -**v1 legacy path (existing benchmarks; migrate to v2 over time):** - -1–4. Same as before except `build_instances()` returns `TaskSpec` objects and - `evaluator_requirements()` declares the binding key set. -5. The worker and sandbox are referenced by slug strings and resolved by the - registry at run time. - -### add a new worker - -1. Subclass `Worker`. `__init__` takes `name: str` and `model: str | None` - as required keyword-only kwargs; do NOT add nullable-with-default - fallbacks. -2. Implement `execute()` as an async generator. Yield a `GenerationTurn` for - every LLM call; stubs must yield at least once. -3. Resolve LLM clients via `resolve_model_target(...)` from - `ergon_core/core/providers/generation/model_resolution.py`. Never import a - provider SDK directly — the resolver presents a uniform cross-provider - interface. -4. For benchmark-specific glue (tools, system prompts, iteration budget), - do NOT subclass — build a factory closure in the registry that - instantiates `ReActWorker(tools=..., system_prompt=..., max_iterations=..., - name=..., model=...)` with every kwarg explicit. -5. Per-task environment setup (clone a repo, install deps) belongs in - `BaseSandboxManager._install_dependencies`, not in the worker. The - sandbox manager reads the per-task payload via - `queries.task_executions.get_task_payload(task_id)`. -6. Register per the worker registry conventions. - -### add a new evaluator or criterion - -1. Subclass `Evaluator`. Select tasks by matching binding keys against - `BenchmarkTask.evaluator_binding_keys`. -2. For per-aspect scoring, implement `Criterion`. Plain criteria can be pure - functions over the recorded turns and artifacts. -3. For an agentic criterion, implement against the `CriterionRuntime` - Protocol. A criterion runs in the task's existing sandbox; it does not - allocate its own. -4. Ensure the benchmark's `evaluator_requirements()` keys match the keys the - evaluator consumes. - -## anti-patterns - -- **Importing an LLM SDK directly in a worker.** All model clients come from - `resolve_model_target` in - `ergon_core/core/providers/generation/model_resolution.py`. The pattern is - currently clean across `ergon_builtins/workers/`; keep it that way. -- **Mutating a frozen model in place.** Pydantic will raise at runtime. Use - `model_copy(update={...})`. If you find yourself reaching for a setter, the - abstraction is wrong. -- **Stub workers that `return` without yielding.** Zero RL observations, zero - cancellation checkpoints; the runtime treats it as a no-op and the evaluator - sees an empty turn list. -- **Evaluator-only fields in `BenchmarkTask.description`.** Route hidden - tests, reference patches, and rubric answers through `task_payload` and - build a worker-safe `description` explicitly. -- **Leaving `type_slug` out of the onboarding registry.** Users running - `ergon onboard` will not be prompted for the benchmark's API keys and the - benchmark fails opaquely at run time. -- **Emitting empty RL-signal lists from a non-logprob backend.** A - logprob-less backend MUST set `turn_logprobs=None`. `[]` or a list of zeros - is read as "zero tokens generated" and silently corrupts downstream token - accounting. -- **Criteria that allocate their own sandbox.** Agentic criteria must run in - the task's existing sandbox via the `CriterionRuntime` seam. Enforced by - `tests/state/test_criteria_do_not_spawn_sandboxes.py`. -- **Nullable-with-default kwargs on concrete Worker `__init__`.** - `tools: list[Tool] | None = None`, `max_iterations: int = 10`, etc. hide - sizing decisions in a shared default and mask per-benchmark intent. - Concrete workers declare their required construction contract; factories - pass every kwarg explicitly. -- **Reading or writing `WorkerOutput.artifacts`.** The field no longer - exists (removed in RFC 2026-04-22). Criteria read files via - `CriterionRuntime.read_resource(name)` or - `CriterionRuntime.get_all_files_for_task()`; workers publish by writing - to `/workspace/final_output/`. Reintroducing an ad-hoc dict-typed bag - on `WorkerOutput` is an anti-pattern — it can't cross the Inngest - serialization boundary and hides the real publish/read contract. - -## follow-ups - -- **`CriterionRuntime` DI container expansion** — RFC at - `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`. Adds - resource and event-sink accessors to the Protocol; explicitly does not add - sandbox-allocation methods (criteria share the task sandbox). Every current - agentic criterion is in scope for migration. Open direction: if the surface - keeps growing, the LLM-judge helper is a candidate for extraction into a - mixin so the Protocol stays focused on sandbox and resource I/O. -- **Artifact handoff between worker and evaluator** — see - `docs/architecture/cross_cutting/artifacts.md`. The canonical path is - `SandboxResourcePublisher` on the worker side (triggered by writing to - `/workspace/final_output/`) and `CriterionRuntime.read_resource` / - `get_all_files_for_task` on the evaluator side; the old `WorkerOutput. - artifacts` escape hatch was removed in RFC 2026-04-22. Follow-up work: - inventory any remaining benchmark-local retrieval code and move it behind - the runtime's read surface. - -## code map - -Compact onboarding reference. The architectural argument above stands without -this table; it is here only so contributors can find the files fast. - -| Type | File | -|------|------| -| `Benchmark` | `ergon_core/api/benchmark.py` | -| `BenchmarkTask` | `ergon_core/api/task_types.py` | -| `Worker` | `ergon_core/api/worker.py` | -| `GenerationTurn` | `ergon_core/api/generation.py` | -| `Evaluator`, `Rubric` | `ergon_core/api/evaluator.py` | -| `Criterion` | `ergon_core/api/criterion.py` | -| `CriterionRuntime` | `ergon_core/api/criterion_runtime.py` | -| `persist_benchmark` | `ergon_core/core/application/experiments/definition_writer.py` (re-exported from `ergon_core.api`) | -| Composition examples | `ergon_cli/composition/__init__.py` | -| Onboarding deps registry | `ergon_cli/onboarding/profile.py` | -| Per-benchmark Python authoring example | `ergon_builtins/benchmarks/README.md` | diff --git a/docs/architecture/02_runtime_lifecycle.md b/docs/architecture/02_runtime_lifecycle.md deleted file mode 100644 index 6655b1362..000000000 --- a/docs/architecture/02_runtime_lifecycle.md +++ /dev/null @@ -1,172 +0,0 @@ -# 02 — Runtime Lifecycle - -## 1. Purpose - -The runtime layer turns a persisted `ExperimentDefinition` into a durable, observable execution of a DAG of tasks. It owns three jobs: fanning Inngest functions across the run→task→evaluator levels, driving each task through its state machine with every transition recorded as an append-only mutation, and finalizing the run — success, failure, or cancel — while keeping sandboxes and database rows in a consistent terminal state. It does not know about benchmarks, workers, or LLMs; those live at the providers and builtins layers. Its upward contract is narrow: the caller supplies a persisted definition and a run id, and the runtime guarantees the run reaches a terminal status and that history is reconstructable from storage alone. - -## 2. Core abstractions - -State at this layer is split between mutable graph rows and an immutable audit log. Runtime code treats the log as ground truth; the rows are a cache of the latest `new_value` per target. - -### Graph state - -- **`RunGraphNode`** — one row per task per run. Carries a free-form `status` string; the runtime only branches on "terminal vs not" (see `status_conventions.TERMINAL_STATUSES`). -- **`RunGraphEdge`** — one row per dependency. Transitions: `pending → satisfied | invalidated`. A dependent advances only when every incoming edge is `satisfied`. -- **`RunGraphMutation`** — append-only log of every node/edge transition. Monotonic `sequence`, `old_value`/`new_value`, `actor`, `reason`. This is the single source of truth for run history. -- **`WorkflowGraphRepository`** — the *only* legal path for mutating node or edge state. Every write takes a `MutationMeta(actor, reason)` and inserts the `RunGraphMutation` row inside the same transaction as the state change. -- **`GraphNodeLookup`** — read-side mediator indexing `definition_task_id → node_id` and `(src, tgt) → edge_id` inside a session. - -### Service layer (pure business logic, no Inngest imports) - -Each Inngest function is a thin wrapper around one service. The services are unit-testable without an event loop: - -- `WorkflowInitializationService` — builds the graph from a definition, marks roots PENDING. -- `TaskExecutionService` — prepares a `RunTaskExecution` row, marks the node RUNNING, and finalizes success or failure. -- `TaskPropagationService` — on terminal, updates outgoing edge status and either activates or cancels downstream candidates (see Section 4). -- `SubtaskCancellationService` — BFS over `parent_node_id` to cancel every non-terminal descendant in one transaction. -- `EvaluatorDispatchService` / `RubricEvaluationService` — fan criteria out per evaluator binding. -- `WorkflowFinalizationService` — aggregates scores, closes the `RunRecord`. -- `TaskCleanupService` — marks a cancelled execution row CANCELLED (idempotent). Sandbox release is still a stub (`task_cleanup_service.py:53-54`). - -### Inngest fabric - -- **`core/jobs/**/{contract,job,inngest}.py`** — every durable job owns its orchestration body and Inngest adapter together. `contract.py` is a compatibility export for the canonical application event DTOs. `job.py` owns business orchestration. `inngest.py` owns framework registration and decorator metadata. -- **`inngest_client`** (`core/infrastructure/inngest/client.py`) — singleton app. Job-local `inngest.py` modules register functions against it and `core/infrastructure/inngest/registry.py::ALL_FUNCTIONS` lists the serving order. -- **`RUN_CANCEL`** / **`TASK_CANCEL`** matchers — declarative cancel predicates attached to long-running function decorators. When a `run/cancelled` event arrives, Inngest kills every in-flight function whose trigger payload carries the matching `run_id`. -- **Runtime event contracts** under `core/application/events/runtime.py` — `WorkflowStartedEvent`, `TaskReadyEvent`, `TaskCompletedEvent`, `TaskFailedEvent`, `TaskCancelledEvent`, `WorkflowCompletedEvent`, `WorkflowFailedEvent`, and `RunCleanupEvent`. Each defines its own Inngest event name and Pydantic payload; job-local `contract.py` files re-export those application-owned DTOs so existing imports stay narrow while runtime services can depend on event shapes without importing jobs. -- **Composition boundary** — concrete Inngest client sends and sandbox adapter wiring live in `inngest.py` or job-local composition helpers such as `core/jobs/_events.py`, not in `job.py`. Architecture tests enforce this narrower PR10 boundary while PR11 finishes moving runtime data access behind services. - -### Freeze status - -- `RunGraphNode` / `RunGraphEdge` / `RunGraphMutation` table shapes: **frozen** for existing runs. Schema changes require a migration plan. -- `TaskExecutionStatus`: **frozen** set. Adding a status requires coordinated changes in propagation, cascade cancellation, evaluator gating, and workflow-terminal checks. -- Inngest event payloads: **stable**. Adding optional fields is safe; renaming is a breaking change across the Inngest queue. - -## 3. Control flow - -``` -CLI / API run/cancelled ──────────┐ - │ │ (kills any - ▼ │ in-flight -benchmark-run-start ─► workflow/started │ function via - │ │ RUN_CANCEL - ▼ │ matcher) - workflow-start - • builds graph from definition - • marks roots PENDING - │ - ▼ - task/ready (fan-out, one per root) - │ - ▼ - task-execute ◄── concurrency=15, retries=0 - ├─ prepare-execution (insert RunTaskExecution, RUNNING) - ├─ invoke sandbox-setup (provision via manager) - ├─ invoke worker-execute (yield GenerationTurns, - │ persist RunContextEvents) - ├─ invoke persist-outputs (download + blob-publish) - └─ finalize + emit task/completed (or task/failed on exc) - │ - ┌──────────────────┼──────────────────────┐ - ▼ ▼ ▼ - task-propagate sandbox-cleanup cancel-orphans-on-cancelled - • edges→SAT • terminal task event • BFS subtree → - • deps-sat releases sandbox CANCELLED - node→PENDING (emits task/cancelled - • workflow per transitioned node) - terminal? - │ - ▼ - workflow/completed - or workflow/failed - │ - ▼ - workflow-complete / workflow-failed - • close RunRecord - • emit run/cleanup - │ - ▼ - run-cleanup - • terminate external sandbox if present - • reconcile RunRecord.status -``` - -Three fan-out levels remain, but PR10 colocates them under `core/jobs`: `workflow-start` fans out ready tasks, `task-execute` invokes `evaluate-task-run` once per evaluator binding after worker output persistence, and each evaluator's own criteria are executed inside `evaluate-task-run`. The workflow synchronization point lives in propagation services that `task-propagate` consults on every terminal to decide whether to emit `workflow/completed` or `workflow/failed`. - -Dashboard delivery hangs off state mutation (see `05_dashboard.md`); it is not a gating concern for runtime correctness — if the dashboard is down, runs still finish. - -## 4. Invariants - -1. **All state writes go through `WorkflowGraphRepository`.** Every node or edge transition is paired with a `RunGraphMutation` row in the same transaction, so the log is a complete replay. Enforced by convention and by the repository refusing a write without `MutationMeta`. Raw `session.add(RunGraphNode(...))` on live state is an anti-pattern (see Section 6). - -2. **Terminal statuses are terminal.** `COMPLETED`, `FAILED`, and `CANCELLED` are absorbing. The one deliberate exception is re-activating a `CANCELLED` managed subtask (`parent_node_id is not None`) when its dependencies re-satisfy — see the `is_reactivatable_cancelled` guard in `ergon_core/core/runtime/execution/propagation.py:546-583`. New exceptions must live in that guard, not in an ad-hoc branch. - -3. **Side-effectful tasks do not retry.** `task-execute` and `worker-execute` carry `retries=0` because they create sandboxes, call provider APIs, and write execution rows — replaying those would duplicate side effects and desynchronize the mutation log from Inngest's durable position. Orchestration and cleanup functions that are idempotent (`task-propagate`, `workflow-complete`, `cleanup-cancelled-task`) carry modest retries (1 or 3). Retry policy is owned by the decorator, never by inner code; do not wrap a worker call in a retry loop. - -4. **Cancellation is event-driven, not write-driven.** A `TaskCancelledEvent` fires three consumers in parallel: `cancel-orphans-on-cancelled` (recurse through children), `cleanup-cancelled-task` (mark the execution row, release resources), and `execute-task` cancellation via the `TASK_CANCEL` matcher (drop any queued or still-running invocation). Writing `CANCELLED` to a node row without emitting the event skips all three and leaves the subtree live. - -5. **Run-level cancellation uses a declarative matcher, not a side channel.** `ergon run cancel` (`ergon_cli/commands/run.py:50`, backed by `run_service.cancel_run`) does three things: marks the `RunRecord` CANCELLED, sends `run/cancelled`, sends `run/cleanup`. The `RUN_CANCEL` matcher on every long-running function — keyed on `event.data.run_id == async.data.run_id` — is what actually kills in-flight work; Inngest enforces it, not application code. - -6. **Cascade cancellation is one transaction, not an event chain.** `SubtaskCancellationService.cancel_orphans` walks the entire descendant subtree via BFS on `parent_node_id` in a single DB transaction (`subtask_cancellation_service.py:66-111`). A dropped or delayed Inngest event cannot leave a grandchild running under a cancelled parent. The subsequent `task/cancelled` events are for per-node cleanup, not recursion. - -7. **Sandbox lifecycle is per-task, teardown happens after evaluators.** `task-execute` invokes `sandbox-setup`, then keeps that sandbox id alive through worker execution, output persistence, evaluator fan-out, and every criterion run. Only after those steps finish does it emit `task/completed`; `sandbox-cleanup-on-completed` and `sandbox-cleanup-on-failed` own teardown from terminal task events. `run-cleanup` remains a run-level reconciliation leg for residual sandbox ids recorded on the `RunRecord.summary_json`. See `03_providers.md` §4 for the definitive treatment. - -8. **Workflow finalization is replay-safe.** `workflow-complete` and `workflow-failed` re-read the current `RunRecord` and evaluation rows each invocation; repeated delivery writes the same terminal status with the same completion timestamp logic. `run-cleanup` checks the status before overwriting. - -### 4.1 Known limits - -- **Static-sibling failure auto-cancels today.** When a static task (no `parent_node_id`) fails, `propagation.on_task_completed_or_failed` marks its siblings CANCELLED (`execution/propagation.py:515-526`). The intended fractal-OS semantic is that static siblings stay PENDING so a higher-level manager can adapt — matching managed-subtask behavior. Changing this also requires teaching `is_workflow_complete_v2` to terminate on blocked-by-failed chains, otherwise workflows hang. Tracked in `docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md`. -- **Cancellation cleanup is still being consolidated.** `cleanup-cancelled-task` releases a sandbox when the cleanup service can identify one, but cancel payloads still do not carry first-class sandbox/benchmark identity. PR11 should finish this handoff so cancellation cleanup no longer depends on execution-row lookup. -- **`RunTaskStateEvent` is deprecated and unread.** Propagation no longer writes to it (`propagation.py:7-8`). `StateEventsQueries` is the last reader and goes away with the table in `docs/rfcs/active/2026-04-17-delete-run-task-state-event.md`. New code must read state from `RunGraphNode` via `GraphNodeLookup`. - -## 5. Extension points - -- **Adding a new Inngest function.** Add a package under `core/jobs///` with `contract.py`, `job.py`, and `inngest.py`. Define new runtime event payloads in `core/application/events/runtime.py` with a `ClassVar[str] name`; keep job-local `contract.py` as a DTO-only re-export when callers need the old job-local import path. Put framework registration, trigger, retry, cancel, concurrency, and output metadata in `inngest.py`. Put orchestration in `job.py`; if it must send Inngest events or touch concrete sandbox adapters, route that through job-local composition helpers rather than importing concrete infrastructure in the job body. Import the function in `core/infrastructure/inngest/registry.py` and append it to `ALL_FUNCTIONS`. Pick `retries` by blast radius: 0 for side-effect-bearing, 1 for idempotent orchestration, 3 for best-effort cleanup. - -- **Adding a new task status.** Add the value to `TaskExecutionStatus` and the graph `status_conventions`. Update `TERMINAL_STATUSES` if terminal. Teach `on_task_completed_or_failed` the new transition, update `is_workflow_complete_v2` / `is_workflow_failed_v2`, and update the `is_reactivatable_cancelled` guard if the new status should be re-activatable. Every write goes through `WorkflowGraphRepository.update_node_status` with a `MutationMeta`. - -- **Dynamic subtask creation.** The graph is eagerly built for static DAGs at `workflow-start`. After launch, the sanctioned worker-authored path is `WorkerContext.spawn_task(task: Task, *, depends_on=())` → `TaskManagementService.spawn_dynamic_task(*, run_id, parent_task_id, task, depends_on)` → `WorkflowGraphRepository.add_node` with `task_json=task.model_dump(mode="json")` and `is_dynamic=True` (mutation kind `node.added`, actor `worker-context`). No `experiment_definition_tasks` row is written; the full Task snapshot lives in `run_graph_nodes.task_json` and re-inflates through `graph_repo.node` → `Task.from_definition`. Returns a `SpawnedTaskHandle(task_id=node.id)`. Workers must not touch the repository or a session directly, and only that worker's descendants are reachable via `cancel_task` / `refine_task` / `restart_task` / `get_task` (containment is enforced by `WorkerContext._assert_descendant`, raising `ContainmentViolation`). Slug-based manager toolkit creation has been removed from the public surface; `SubtaskLifecycleToolkit` is now a worker-context wrapper rather than an alternate graph writer. - -## 6. Anti-patterns - -- **Direct DB writes to `RunGraphNode.status`.** Skips the mutation log, breaks audit-replay, and is invisible to the dashboard emitter that listens on the repository. Always go through `WorkflowGraphRepository.update_node_status`. - -- **Cancelling by writing `CANCELLED` to a row.** Must emit `TaskCancelledEvent` instead. The three consumers (`cancel-orphans`, `cleanup-cancelled-task`, `TASK_CANCEL` matcher) only fire on the event. A direct write leaves the subtree live and blocks finalization. - -- **Importing a provider SDK (E2B, OpenAI, Anthropic) into a runtime function.** Runtime functions see sandboxes through `BaseSandboxManager` and models through `resolve_model_target` — both are providers-layer concerns. A runtime function reaching for `AsyncSandbox(...)` or `AsyncOpenAI(...)` directly couples orchestration to a specific substrate and breaks the substitution boundary described in `03_providers.md`. - -- **Internal retry loops inside workers or services.** The `retries=` decorator argument is the only legal knob. Wrapping a provider call in a `for i in range(3):` loop multiplies sandbox cost, blurs the durable replay semantics, and desynchronizes execution-row state from Inngest's view of the function. - -- **Branching inside an Inngest function on "should I also do X".** Emit a second event and add a second function instead. Conditional side-effect paths inside one function fight the single-responsibility rule and the retry contract — the replay semantics of each branch diverge. - -- **Swallowing exceptions in a function body.** Each function must succeed, fail loudly (so Inngest retries per decorator), or emit a specific terminal event. Swallowing makes Inngest see "succeeded" while the mutation log and downstream events disagree with reality. - -- **Using `ergon experiment cancel`.** That CLI does not exist. The entrypoint is `ergon run cancel `. - -## 7. Follow-ups - -Active RFCs touching this layer: - -- `docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md` — stop auto-cancelling static siblings on upstream failure; teach `is_workflow_complete_v2` about blocked-by-failed. -- `docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md` — land the `release-sandbox` step by extending `TaskCancelledEvent` with `sandbox_id` and `benchmark_slug`. -- `docs/rfcs/active/2026-04-17-delete-run-task-state-event.md` — drop the deprecated `RunTaskStateEvent` table and the last reader in `StateEventsQueries`. -- `docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md` — formalize the invariant that sandbox timeout >= task timeout + max criterion timeout, so the evaluator fan-out cannot outlive the sandbox. - -## 8. Code map - -A brief index of where runtime functions live. The architectural claims above stand without this table; it is onboarding reference material. - -| Concern | File | -| --- | --- | -| Entry + init | `core/jobs/workflow/start/` | -| Task orchestration | `core/jobs/task/execute/` | -| Task child steps | `core/jobs/sandbox/setup/`, `core/jobs/task/worker_execute/`, `core/jobs/resources/persist_outputs/` | -| Propagation | `core/jobs/task/propagate/` | -| Evaluator execution | `core/jobs/task/evaluate/` | -| Cancellation cascade | `core/jobs/task/cancel_orphans/`, `core/jobs/task/cleanup_cancelled/` | -| Finalization | `core/jobs/workflow/complete/`, `core/jobs/workflow/fail/`, `core/jobs/run/cleanup/` | -| Registry | `core/infrastructure/inngest/registry.py` | -| Client + cancel matchers | `core/infrastructure/inngest/client.py` | -| Runtime event contracts | `core/application/events/runtime.py` | -| State-machine core | `runtime/execution/propagation.py` | -| Services | `core/application/**` | diff --git a/docs/architecture/03_providers.md b/docs/architecture/03_providers.md deleted file mode 100644 index 7b1d900eb..000000000 --- a/docs/architecture/03_providers.md +++ /dev/null @@ -1,197 +0,0 @@ -# 03 — Providers - -## 1. Purpose - -The provider-style boundaries are Ergon's adapters between runtime code and external execution substrates. Model resolution lives in the generation registry, while sandbox infrastructure now lives under `ergon_core.core.sandbox` because it owns lifecycle, instrumentation, event emission, and artifact publishing rather than just a third-party provider adapter. - -## 2. Core abstractions - -| Name | Kind | Location | Freeze status | Owner | -| --- | --- | --- | --- | --- | -| `_BACKEND_REGISTRY` | module-level dict | `ergon_core/core/providers/generation/model_resolution.py` | Frozen shape; entries grow via registration. | Providers layer. | -| `resolve_model_target` | function | `ergon_core/core/providers/generation/model_resolution.py` | Public, frozen signature. Returns `ResolvedModel`. | Providers layer. | -| `register_model_backend` | function | `ergon_core/core/providers/generation/model_resolution.py` | Public, frozen signature. | Providers layer; callers are backend modules executing at import time. | -| `BaseSandboxManager` | abstract class + singleton | `ergon_core/core/sandbox/manager.py` | Shape stable; `event_sink` activation path in flux. | Sandbox domain. | -| `DefaultSandboxManager` | concrete class | `ergon_core/core/sandbox/manager.py` | Frozen. | Sandbox domain. | -| `SWEBenchSandboxManager`, `MiniF2FSandboxManager`, `ResearchRubricsSandboxManager` | concrete subclasses | `ergon_builtins/` | Owned per benchmark; singletons. | Benchmark authors. | -| `SandboxEventSink` | `typing.Protocol` | `ergon_core/core/sandbox/event_sink.py` | Frozen protocol; activation path in flux. | Sandbox domain. | -| `NoopSandboxEventSink`, `DashboardEmitterSandboxEventSink` | implementations | `ergon_core/core/sandbox/event_sink.py` | Frozen. | Sandbox domain. | -| `SandboxResourcePublisher` | class | `ergon_core/core/sandbox/resource_publisher.py` | Frozen API; storage backend swappable via `ERGON_BLOB_ROOT`. | Sandbox domain. | -| `TransformersModel` | `pydantic_ai.models.Model` subclass | `ergon_builtins/ergon_builtins/models/transformers_backend.py` | Frozen. | ML team (TRL training loop callers). | - -### 2.1 Generation registry - -`_BACKEND_REGISTRY` is a prefix-keyed dispatch table of resolver callables. `resolve_model_target` splits the target on its first colon, dispatches to the resolver, and returns a `ResolvedModel` wrapping either a `pydantic_ai.models.Model` instance or a passthrough string. Unknown prefixes fall through to a passthrough `ResolvedModel` — PydanticAI's own `infer_model` is invoked on use. Backends mutate the registry at import time; the builtins pack registers all four in a single loop at `ergon_builtins/ergon_builtins/registry.py:81`. - -The four prefixes registered today are `vllm:*` (local vLLM server via PydanticAI's `OpenAIChatModel`), `openai:*` / `anthropic:*` / `google:*` (passthrough to `infer_model`), and `transformers:*` (custom `TransformersModel` for TRL-trained checkpoints not served over vLLM). - -Workers are expected to hold no hardcoded SDK client constructions (`AsyncOpenAI`, `anthropic.Client`, `genai.Client`). This is an invariant (Section 4), not a coincidence, and is currently honored — enforcement is grep discipline. - -### 2.2 Sandbox managers - -`BaseSandboxManager` is both abstract and a **singleton per subclass**. The singleton is load-bearing: criteria and workers reconnect to a running sandbox by re-instantiating the subclass and calling `get_sandbox(task_id)`, which reads shared class-level state. This works only because all actors run inside the same Python process; see Section 4, invariant 3. - -A `template: ClassVar[str] | None` names the E2B template. Subclasses usually set the class attribute; `SWEBenchSandboxManager` instead assigns `self.template` in `__init__` from a pinned-template-id file written by `ergon benchmark setup swebench-verified`, because its template is rebuilt periodically. - -Three lifecycle hooks are defined for subclasses: - -- `_create_directory_structure` — concrete on the base class; lays out `/inputs`, `/workspace`, `/skills`, `/tools` and smoke-tests writability. Override only if the default set is wrong. -- `_install_dependencies` — abstract today; expected to become optional with a concrete no-op default (tracked by the process-state RFC below). -- `_verify_setup` — concrete no-op; override to raise on template/env mismatch. - -`create()` takes a sandbox key, run id, timeout, envs, and display-task id. The `sandbox_key` / `task_id` / `display_task_id` triplet is **debt**: in every current call site they collapse to one value, except the SWE-Bench evaluator criterion which spawns a fresh `uuid4()`. The three-parameter shape is kept because it appears in every caller; collapse is scoped in `docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md`. - -`event_sink` is a **constructor** parameter, not a `create()` kwarg. Because the subclass is a singleton, the first construction wins; subsequent constructions with a non-None `event_sink` silently overwrite the shared instance's sink. That stomp is a latent race tracked in `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` (P3); the fix direction is a class-level `set_event_sink()` setter called once at app init. - -**Sandbox lifecycle is per-task.** One sandbox per task per run, kept alive across all of `Worker.execute()`'s generator turns, across the task-completion boundary, and through the full fan-out of evaluator criteria. Teardown runs only after every criterion terminates, via the static `BaseSandboxManager.terminate_by_sandbox_id(sandbox_id)` invoked from `ergon_core/core/runtime/inngest/check_evaluators.py:82`. The static method is the sole cross-process teardown path — it does not touch class-level state, which is why Inngest steps can call it without sharing process identity with the worker. Teardown does NOT run during `finalize_success`. See `cross_cutting/sandbox_lifecycle.md` for the definitive treatment. - -There is no `reconnect()` method. In-process criteria reconnect via `get_sandbox(task_id)` reading shared class state. Cross-process criteria (production case: SWE-Bench) spawn a fresh sandbox of their own rather than reconnecting. Moving "reconnect" onto a real cross-process primitive is the subject of `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md`. - -### 2.3 DefaultSandboxManager - -`DefaultSandboxManager` is the documented default for benchmarks whose only requirement is the stock E2B image: no pinned template, no install step, no verify step. - -When `E2B_API_KEY` is absent, `DefaultSandboxManager.create()` transparently delegates to `StubSandboxManager`, which returns a structurally-identifiable stub sandbox id (`stub-sandbox-`). Downstream teardown code skips stub ids via `is_stub_sandbox_id(sandbox_id)` rather than branching on a magic string constant. This is what makes stub-mode CI runs work without an E2B key — the worker runs, the criterion evaluator runs, and no real sandbox is provisioned. - -### 2.4 SandboxEventSink - -`SandboxEventSink` is a `typing.Protocol` with async hooks for sandbox create, per-command execution, and close. Two implementations ship: `NoopSandboxEventSink` (default) and `DashboardEmitterSandboxEventSink` (forwards to the dashboard emitter). - -**Status as of 2026-04-18: unwired on the live path.** Every `BaseSandboxManager` construction site omits `event_sink=`, so every manager runs with the noop default (grep `SandboxManager(` for the call sites). There is no second, inline path — `dashboard_emitter.sandbox_*` has zero callers anywhere in the tree. Consequence: the dashboard's sandbox view populates only on cold-start via the REST snapshot in `build_run_snapshot()`. Symptom tracked in `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md`. - -The broader `DashboardEmitter` surface is similarly partial: most of its typed methods (including the sandbox trio and `resource_published`) are defined but unwired. Tracked in `docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md`. - -### 2.5 SandboxResourcePublisher - -`SandboxResourcePublisher` is a content-addressed blob store writer. The blob root is `ERGON_BLOB_ROOT` (default `/var/ergon/blob`). Its contract: - -1. Workers write files under a configured sandbox directory (default `/workspace/final_output/` as `RunResourceKind.REPORT`; managers override `publish_dirs` to add more). -2. `publisher.sync()` lists each directory, reads bytes, SHA-256-hashes, and writes to `//`. -3. Writes are atomic on POSIX (`.tmp` then rename); hash collisions short-circuit. -4. One `run_resources` row is appended per new hash; content-hash dedup keeps repeated `sync()` calls idempotent. -5. Non-filesystem artifacts (e.g. `WorkerOutput` fields) go through `publish_value(kind, name, content, ...)`, which writes through the same blob path without listing a directory. - -Workers never write to `` directly; the publisher is the single writer (invariant 6). - -There is no `dashboard_emitter.resource_published` call from `publish()` today; the dashboard picks up resources only via the cold-start REST snapshot. Evaluator retrieval is ad-hoc — each criterion reads `run_resources` directly and re-reads blob bytes. Unification behind `CriterionRuntime.read_resource(name)` is scoped in `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`. - -### 2.6 Template registry / benchmark setup - -There is no centralized template registry. Each benchmark that needs a custom Docker template owns its own Dockerfile, inside-image setup script, and pinned-template-id file under its benchmark directory; SWE-Bench (`ergon_builtins/benchmarks/swebench_verified/sandbox/`) is the reference. A CLI subcommand `ergon benchmark setup ` builds the template and writes the pin. - -The decentralized shape means `ergon benchmark setup` iterates over whatever subcommands happen to be registered — a new benchmark shipping with a required template but forgetting the subcommand registration will be silently skipped. A `TEMPLATE_REGISTRY` closing this hole is scoped in `docs/rfcs/active/2026-04-18-template-spec-public-api.md`. - -## 3. Control flow - -``` -Worker.execute() - | - +-> resolve_model_target(self.model) --> ResolvedModel - | (prefix dispatch; 4 backends + fallthrough to infer_model) - | - +-> ManagerClass() (singleton; returns cached instance) - | ManagerClass().create(sandbox_key=task_id, run_id=run_id, ...) - | +-> per-key asyncio.Lock, idempotent on existing sandbox - | +-> AsyncSandbox.create(template=, ...) - | +-> register in class-level state - | +-> event_sink.sandbox_created (noop today) - | +-> _create_directory_structure / _install_dependencies / _verify_setup - | - +-> yield GenerationTurn per LLM call - | sandbox.commands.run(...) for tool calls - | resource_publisher.sync() on final outputs - | - +-> worker returns -> task COMPLETED event - | - v - check_evaluators fans out criteria - criteria reconnect via ManagerClass().get_sandbox(task_id) - (works because singleton + shared class state; - cross-process criteria spawn a fresh sandbox instead) - criteria read resources via direct run_resources DB read - | - v - all criteria done -> BaseSandboxManager.terminate_by_sandbox_id(sandbox_id) - (static, cross-process) - -> finalize_success -``` - -Movement of data across this diagram: - -- **Model id (string) flows down** into `resolve_model_target` and emerges as a `ResolvedModel`. No model object escapes back up — the worker owns it for its lifetime. -- **`sandbox_key` flows into `create`**; the returned E2B `sandbox_id` is persisted on the execution row. That id is the durable link criteria use — for in-process reconnect or cross-process teardown. -- **Sandbox bytes flow out** via the resource publisher, which writes once to the content-addressed blob store and records a `run_resources` row. The blob store is the single source of truth for cross-process data; the DB row is an index. -- **Events flow out** via `SandboxEventSink` — currently unwired on the live path (Section 2.4). - -## 4. Invariants - -1. **One entry point to LLM resolution.** Every model reference goes through `resolve_model_target`. Enforced by grep discipline and review; no runtime check. -2. **Backends register at import time.** `register_model_backend` must be called before any caller hits `resolve_model_target`. Enforced by the builtins pack running its registration loop at import, before any worker module imports. -3. **Singleton managers hold authoritative sandbox state.** A subclass's class-level state is the only source of truth for in-process reconnect. Enforced by `__new__` caching the instance and `get_sandbox` reading the class dict. Applies only within a single Python process; cross-process actors must use `terminate_by_sandbox_id` or provision their own sandbox. -4. **Sandbox lifecycle is per-task.** Enforced by `create` accepting `sandbox_key` and by the worker runtime persisting `sandbox_id` on the execution row. -5. **Sandbox lives across evaluator fan-out.** Teardown runs at the end of `check_evaluators`, not at worker completion, not in `finalize_success`. Enforced by the evaluator harness, not by the manager itself. -6. **Resource publication is content-addressed.** The publisher hashes before storing and is the single writer to ``. Repeated `sync()` calls against unchanged bytes are no-ops. - -### 4.1 Known limits - -- **Sandbox event sink unwired.** All construction sites omit `event_sink=`; the singleton holds the noop default, so sandbox lifecycle is not visible on the live dashboard path. Tracked in `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md`. -- **Constructor `event_sink` stomp.** Because subclasses are singletons and `__init__` conditionally overwrites the sink, any late re-construction with a non-None sink silently replaces the active sink on every in-flight task. Filed at `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` (P3). -- **On-crash sandbox leak.** Sandbox cleanup is best-effort. If `check_evaluators` crashes before teardown, the remote E2B sandbox leaks until E2B's 30-minute idle timeout fires. Acceptable: the E2B timeout is the canonical safety net. -- **Class-dict unbounded growth.** Manager class-level state is cleared only inside `terminate()`. Any task that never reaches `terminate()` leaks its entries for the process lifetime. Acceptable for research workloads with bounded process lifetimes. -- **Blob store has no GC.** The publisher writes under the blob root and never deletes. Tracked as P4 in `docs/bugs/open/2026-04-18-blob-store-no-gc.md`. -- **Key-triplet debt.** `sandbox_key` / `task_id` / `display_task_id` collapse to one value in every production call site. -- **No `reconnect()` method.** Cross-process criteria must spawn their own sandbox (see `ergon_builtins/benchmarks/swebench_verified/criterion.py:66`). - -## 5. Extension points - -### 5.1 Add a new LLM backend - -1. Write a resolver that maps `"myprefix:foo"` to a `pydantic_ai.models.Model` instance wrapped in `ResolvedModel`. -2. Register it in the builtins-pack registration loop so `register_model_backend` is called at import time. -3. Ensure the builtins pack is imported before any worker that references `myprefix:*` model ids. -4. Add an entry to `LLMProvider` and `PROVIDER_KEY_MAP` in `ergon_cli/onboarding/profile.py` so onboarding prompts for the key or server URL. - -### 5.2 Add a new sandbox manager - -1. Subclass `BaseSandboxManager`. -2. Set `template: ClassVar[str]` on the class, or override `__init__` to assign `self.template` from a pinned-id file if the template is rebuilt periodically (`SWEBenchSandboxManager` is the reference). -3. Implement `_install_dependencies` (required today; expected to become optional). Override `_verify_setup` if you need a smoke check. `_create_directory_structure` is concrete — override only if the default set is wrong. -4. Use the subclass at call sites as `ManagerClass().create(...)`. Treat the class as a singleton — re-instantiation is how callers acquire the cached instance. -5. Do NOT pass `event_sink=` at construction today; the stomp described in Section 2.2 makes that unsafe. The eventual path is a class-level setter called once at app init. -6. Register no registry entry — managers are discovered by the benchmarks that import them directly. - -### 5.3 Add a new sandbox Docker template - -1. Create the Dockerfile and inside-image setup script under `ergon_builtins/benchmarks//sandbox/`. -2. Add a pinned-template-id file populated by the setup subcommand. -3. Add a CLI subcommand `ergon benchmark setup ` that builds the template and writes the pin. -4. Use `ergon_builtins/benchmarks/swebench_verified/sandbox/` as the reference. - -## 6. Anti-patterns - -- **Importing an LLM SDK directly in a worker.** Bypasses backend registration, onboarding key flow, and future instrumentation. No current offenders — keep it that way. -- **Constructing an `AsyncSandbox` directly from worker or criterion code.** Go through a manager subclass; the manager owns template pinning, directory scaffolding, singleton state, event emission, and the teardown contract. -- **Passing `event_sink=` at manager construction.** The constructor stomps the sink on the shared singleton. Today: do nothing; leave the noop default. Eventual path: the class-level setter. -- **Reaching for `dashboard_emitter.sandbox_*()` as a workaround for the unwired sink.** Adding inline emitter calls trades one wiring bug for two. -- **Hardcoding a template string in a manager.** E2B templates are rebuilt when the Dockerfile changes; a literal id in code breaks on CI rebuild. Use a pin file, as `SWEBenchSandboxManager` does. -- **Reading `run_resources` rows directly from a criterion.** Works today but couples every criterion to the blob store schema. The unified entry point will be `CriterionRuntime.read_resource(name)`. -- **Calling instance `terminate(task_id)` from a cross-process actor.** The instance method relies on the singleton's class-level state, which is not shared across processes. Use the static `terminate_by_sandbox_id(sandbox_id)` for cross-process teardown. - -## 7. Follow-ups - -Active RFCs relevant to the providers layer: - -- `docs/rfcs/active/2026-04-17-sandbox-event-sink-activation.md` — install the event sink via a class-level setter at app init. -- `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md` — reform class-level state so rollout paths stop depending on a single Python process; owns the class-dict growth limit, criterion reconnect, and the shared-state race. -- `docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md` — collapse the `sandbox_key` / `task_id` / `display_task_id` triplet. -- `docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md` — wire the remaining `DashboardEmitter` methods or delete them, with a lint guard. -- `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` — expose `get_sandbox()` and `read_resource(name)` through `CriterionRuntime`. -- `docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md` — formalize the sandbox-timeout invariant (sandbox timeout >= `task_timeout + max_criterion_timeout`). -- `docs/rfcs/active/2026-04-18-template-spec-public-api.md` — centralized `TEMPLATE_REGISTRY` with `TemplateSpec` entries. - -Open bugs: - -- `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md` — sink never fires on the live path. -- `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` — P3. -- `docs/bugs/open/2026-04-18-blob-store-no-gc.md` — P4. -- `docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md` — most emitter methods have no callers. diff --git a/docs/architecture/04_persistence.md b/docs/architecture/04_persistence.md deleted file mode 100644 index d9e9bca7c..000000000 --- a/docs/architecture/04_persistence.md +++ /dev/null @@ -1,223 +0,0 @@ -# 04 — Persistence - -## 1. Purpose - -The persistence layer owns durable state for runs, tasks, graph mutations, -context events, resources, evaluations, and telemetry. Postgres is the -single source of truth; all in-memory structures (the dashboard store, -runtime caches) are derived views that can be rebuilt from the database at -any time. Disaster recovery is Postgres backups — not log replay. - -The layer is intentionally split into two shapes. **Mutable tables** carry -the current state of a run and serve fast indexed reads for scheduling and -rehydration. **Append-only logs** carry the history of how the run reached -that state, and exist for audit, debugging, and RL trajectory extraction. -The mutable tables are the operational truth; the mutation log is a diary, -not a recovery source. - -## 2. Core abstractions - -The graph repository (`WorkflowGraphRepository`) is the only writer for -graph state. Everything else — runtime workers, services, dashboards — -either reads the mutable rows or submits mutations through the repository. - -Design-load-bearing facts: - -- **Mutable state** lives in the run graph node/edge tables plus the - per-attempt execution, evaluation, and resource tables. These are the - scheduling read path. -- **Append-only logs** are the mutation log (every graph state change) and - the context event log (one row per message in a generation turn). These - are audit / RL material. -- **Annotations** are a namespaced metadata table attached to nodes and - edges. Core reserves the `"payload"` namespace; every other namespace is - an extension seam (see §5). -- **Definition-side rows** (`ExperimentDefinitionTask` and its dependency - table) are immutable after creation. Runtime reads them to seed the - initial graph and never mutates them. -- **`RunTaskStateEvent` is legacy.** The table is frozen. New code MUST - NOT read or write it; rehydration of legacy runs is the only permitted - read. - -See the Code map below for where each table lives. - -## 3. Control flow - -### Writes - -All graph state changes go through `WorkflowGraphRepository`: - -``` -runtime / worker - | - v -WorkflowGraphRepository.apply_mutation(meta=MutationMeta(actor, reason), ...) - | - | 1. append new row to the mutation log (sequence += 1) - | 2. update the corresponding node / edge row - | 3. return the applied mutation - v -Postgres (single transaction) - | - v -DashboardEmitter.graph_mutation(...) (out-of-transaction, best effort) -``` - -### Initial graph construction - -At run-creation time, the initialization service reads the definition-side -rows and eagerly creates every statically-declared node and edge. Task -payloads are attached as annotations in the core-reserved `"payload"` -namespace. - -### Manager-spawned subtasks (dynamic graph growth) - -The graph is append-only at the row level: new nodes and edges can enter -after initialization without slot reservation. Manager workers grow the -graph by submitting mutations through `WorkflowGraphRepository` under a -`MutationMeta(actor="manager-worker", reason="manager_decision")`. The -runtime then fires Inngest `TaskReadyEvent`s for any no-deps roots of the -new subgraph; those nodes are routed through the same execution path as -statically-declared nodes. - -There is no pre-allocation of capacity or reservation of slots. Every -dynamic node is a normal node row committed inside the same mutation-log -transaction as the originating `node.added` entry. - -### Reads - -- Runtime reads the mutable node/edge rows for scheduling decisions. -- Dashboard rehydration reads a full run snapshot composed from the - mutable tables directly. It does NOT replay the mutation log; the - mutable tables are the read path. - ([`ergon_core/core/api/runs.py:343`](../../ergon_core/ergon_core/core/api/runs.py).) -- RL extraction consumes the mutation log and context event log in - sequence order to reconstruct trajectories. - -## 4. Invariants - -- **Mutation log is append-only with dense, monotonic sequence.** Every - DAG state change writes a new row; no gaps, no updates, no deletes. Old - and new values round-trip so the diary is reconstructible, but - consumers treat this as audit / RL material rather than the recovery - source of truth. Enforced by `WorkflowGraphRepository`. -- **All graph state writes go through `WorkflowGraphRepository`** with a - `MutationMeta(actor=..., reason=...)`. Raw `session.add` on live state - is an anti-pattern (see §6). -- **`RunTaskStateEvent` is frozen.** No new writes. Reads permitted only - for legacy-run rehydration. New code uses the mutation log plus node - status instead. -- **Alembic revisions are the only schema-change path.** Every migration - ships in a reviewed PR. The revision chain is preserved and NOT - squashed: each revision documents the intent of its PR, and the chain - is the audit trail for schema evolution. Migrations are LLM-generated - (autogen or authored by hand) and reviewed like any other code. -- **Context events are forever-append today.** No pruning job exists and - none is planned in the near term; retention is "keep everything". Per-run - TTL is an open question (see §7). -- **Event ordering within a turn is deterministic.** The tuple - (run_id, task_id, turn_id, event_index) is unique for context events. -- **Namespace `"payload"` on annotations is core-reserved.** All other - namespaces are available for extension code. - -## 5. Extension points - -- **Graph annotations — user metadata seam.** The generic - `WorkflowGraphRepository.set_annotation(target_id, namespace, value)` - entry point is a first-class public extension surface for user-written - experiment code. Attach arbitrary structured metadata to nodes and - edges — manager/worker debate transcripts, refinement history, - experiment-specific scheduler hints, anything that benefits from being - co-located with the graph but does not belong in the mutation log. The - pattern is: - - ```python - repo.set_annotation( - target_id=node_id, - namespace="my_experiment.debate", - value={"round": 2, "transcript": [...]}, - ) - ``` - - Claim a unique namespace per extension. Core reserves `"payload"`; - everything else is yours. This is NOT an internal helper — third-party - experiment code is expected to write annotations. - -- **Dynamic subgraph injection via the subtask lifecycle toolkit.** Any - worker (including custom managers supplied by an experiment) can grow - the live DAG. The contract: the toolkit translates worker intent into - `WorkflowGraphRepository` mutations under a - `MutationMeta(actor="manager-worker", reason="manager_decision")`, and - the runtime fires Inngest `TaskReadyEvent`s for new roots. Extensions - that need to steer scheduling at runtime should prefer this path over - hand-rolled graph writes. - -- **New mutation kinds.** Extend the mutation payload discriminator with - a new variant and a corresponding method on `WorkflowGraphRepository`. - Do not inline ad-hoc JSON blobs into existing mutation kinds. - -- **New tables.** Add an Alembic revision under the migrations tree. - Downstream consumers (dashboard, RL extraction) must be updated in the - same PR or behind a feature flag. - -- **Custom snapshot views.** Compose on top of the run-snapshot builder; - do not duplicate the rehydration logic. - -## 6. Anti-patterns - -- **Writing to `RunTaskStateEvent`.** The table is frozen legacy. Any new - write is a regression; use the mutation log plus node status. -- **Bypassing `WorkflowGraphRepository` for graph writes.** All writes - MUST append to the mutation log; raw `session.add` on a graph node or - `UPDATE run_graph_node SET status = ...` breaks the audit trail and - desynchronises consumers. -- **Reading state by replaying the mutation log in the runtime hot path.** - The mutation log is audit and RL material. Scheduling and rehydration - read the mutable tables. Replay-as-read is slow and defeats the split. -- **Schema changes without an Alembic revision.** Even additive changes - must have a migration so environments converge. Squashing the existing - revision chain is also forbidden — each revision is documentation for - the PR that introduced it. -- **Overloading annotation namespace `"payload"` from user code.** Pick a - unique namespace; `"payload"` is reserved for task payloads written by - the initialization service. -- **Emitting a dashboard event from inside the DB transaction.** The - emitter is best-effort and out-of-band; coupling it to commit semantics - risks transaction rollback leaving the dashboard with phantom state. - -## 7. Follow-ups - -Open questions and in-flight work that touch this layer: - -- **Context event retention shape.** Forever-append is the intentional - default today, but a per-run user-configurable TTL is expected - eventually. No RFC exists yet; when drafted it will cover retention - policy, optional tiered storage, and interaction with RL trajectory - extraction. Track as a future RFC rather than a bug. -- **`RunTaskStateEvent` deletion.** An RFC is in flight to remove the - legacy table outright. Until it lands, the frozen-table invariant - stands; on merge, drop the legacy entry from §2 and the matching - anti-pattern bullet. -- **Production migration policy.** The system is local-only today, so - there is no formal rule on who runs Alembic against a shared - environment. When a hosted deployment appears, this section will gain - a deployment-time policy (who runs `alembic upgrade head`, rollback - expectations, staging parity). - -## Code map - -Compact reference for where the tables and services live. Not load-bearing -for the design argument above — here as onboarding material. - -| Concern | Location | -|---|---| -| Graph node / edge / annotation / mutation models | `ergon_core/core/persistence/graph/models.py` | -| Run / execution / evaluation / resource / thread models | `ergon_core/core/persistence/telemetry/models.py` | -| Context event model | `ergon_core/core/persistence/context/models.py` | -| Experiment definition models | `ergon_core/core/persistence/definitions/` | -| Graph repository (sole writer) | `ergon_core/core/persistence/graph/` (`WorkflowGraphRepository`) | -| Initial graph construction | `ergon_core/core/runtime/services/workflow_initialization_service.py` | -| Subtask lifecycle toolkit | `ergon_builtins/tools/subtask_lifecycle_toolkit.py` | -| Task management service (dynamic subtask entry) | `ergon_core/core/runtime/services/task_management_service.py` | -| Run snapshot builder | `ergon_core/core/api/runs.py` (`build_run_snapshot`) | -| Alembic revisions | `ergon_core/migrations/versions/` | diff --git a/docs/architecture/05_dashboard.md b/docs/architecture/05_dashboard.md deleted file mode 100644 index c273e7e45..000000000 --- a/docs/architecture/05_dashboard.md +++ /dev/null @@ -1,174 +0,0 @@ -# 05 — Dashboard - -## 1. Purpose - -The dashboard is a Next.js app that renders run state. It consumes a narrow -set of Inngest events from the Python runtime, maintains an in-memory -`DashboardStore`, and pushes updates to connected browsers via Socket.io. -The durable source of truth is Postgres; the in-memory store is a cache -that can always be rebuilt from the database via `build_run_snapshot`. - -Current live-update surface is narrow: only graph mutations and context -events flow through the delta stream. All other dashboard data (task -statuses rendered outside graph nodes, sandbox activity, resources, -evaluations, thread messages) is populated by a cold-start REST snapshot -and does not update live today. See Follow-ups for tracked gaps. - -## 2. Core abstractions - -- `DashboardEmitter` — Python-side fire-and-forget sender. Exposes one - typed async method per `dashboard/*` event; errors are caught and - logged and the emitter never blocks callers. Most methods are defined - but unwired today (see Follow-ups). Lives in - `ergon_core/ergon_core/core/dashboard/emitter.py`. - -- `DashboardXxxEvent` contracts — typed Pydantic models that define the - wire format, in - `ergon_core/ergon_core/core/dashboard/event_contracts.py`. A matching - Zod schema lives at `ergon-dashboard/src/lib/contracts/events.ts`; - whether the mirror is codegen or hand-maintained has drifted (see - Follow-ups). - -- `DashboardEmitterSandboxEventSink` — sink adapter intended to route - E2B sandbox lifecycle into the emitter. Defined but has no constructor - site today (see Follow-ups). - -- Next.js Inngest handlers — one registration per event type that - updates `DashboardStore` and broadcasts via Socket.io. Live under - `ergon-dashboard/src/inngest/functions/`. - -- `DashboardStore` — process-local in-memory cache - (`ergon-dashboard/src/lib/state/store.ts`) with a bounded retention - policy: oldest runs beyond a cap are pruned on new-run arrival. The - cap is configurable via env; the default lives in - `ergon-dashboard/src/lib/config.ts:29`. Pruned runs become - inaccessible because there is no historical REST endpoint today (see - Follow-ups). - -- Socket.io server (`ergon-dashboard/src/lib/socket/server.ts`) — - broadcasts to per-run rooms (`run:`) and answers cold-start - snapshot requests. Clients explicitly subscribe/unsubscribe per run. - -- `SocketProvider` — browser-side socket.io client - (`ergon-dashboard/src/providers/SocketProvider.tsx`). Owns subscription - state for the current view. - -- `useRunState` — per-run hydration + live-update hook - (`ergon-dashboard/src/hooks/useRunState.ts`). Requests a fresh snapshot - when `runId` changes. - -- `RunEvent` union — chronological shape consumed by - `UnifiedEventStream` and `RunTimeline`, defined in - `ergon-dashboard/src/lib/runEvents.ts`. - -## 3. Control flow - -Cold-start (authoritative, works today): - -``` -browser opens /runs/ - | - v -useRunState -> socket.emit("request:run", runId) - | - v -Socket.io server -> build_run_snapshot(run_id) via REST to Python - | - v -DashboardStore populated + sync:run emitted to this socket -``` - -Live delta stream (what actually flows today): - -``` -Python runtime Next.js --------------- ------- -WorkflowGraphRepository mutation listener --> dashboard/graph_mutation -ContextEventRepository listener --> dashboard/context_event - | - v - DashboardStore reducer - | - v - Socket.io room run: -``` - -The broader set of `DashboardEmitter` methods exists for the target -pipeline shape but has no live call sites yet; see Follow-ups. - -## 4. Invariants - -- The dashboard is event-driven end-to-end for the surfaces that are - wired. No polling from the backend. No SSE. -- Every persistent backend state change on a wired surface must have a - corresponding `dashboard/*` event; a state change without an emit is - a bug in the emitter layer. Enforcement is not automated today (see - Follow-ups). -- `DashboardStore` is a cache. The durable source of truth is Postgres. - On cold start, the browser sends `request:run` and the store is - rebuilt via `build_run_snapshot`. -- Browser clients subscribe to specific run rooms; they do NOT receive - events for runs they have not subscribed to. -- Emitter calls are best-effort. The backend never blocks on dashboard - delivery, and no business logic depends on emission success. - -## 5. Extension points - -- New event type: add a Pydantic contract in `event_contracts.py`, a - method on `DashboardEmitter`, the matching handler under - `ergon-dashboard/src/inngest/functions/`, a reducer in - `DashboardStore`, and a call site in the runtime. All five are - required to complete the pipeline. -- New graph mutation kind: extend `RunGraphMutation.kind`; the Python - listener forwards automatically. Add the corresponding TS reducer - under `ergon-dashboard/src/inngest/functions/` or - `features/graph/contracts/graphMutations.ts`. -- New store slice: extend `DashboardStore` and expose a selector. Do - not read the raw store from JSX. - -## 6. Anti-patterns - -- Adding a backend state change without a corresponding - `DashboardEmitter.*` call. Already the dominant failure mode — most - emitter methods are currently dead (see Follow-ups). -- Reading `DashboardStore` as if it were durable. It is a cache; use - `build_run_snapshot` for authoritative state. -- Polling the dashboard REST API for live state. REST is for - rehydration; live state is Socket.io. -- Emitting from inside a DB transaction. Emission is best-effort and - must not gate commit. -- Assuming reconnect rehydrates. `SocketProvider` does not re-subscribe - on reconnect; `useRunState` does not re-fetch the snapshot; the - server does not replay. Tracked in Follow-ups. -- Hand-editing the Zod contracts without updating the Pydantic source - (or vice-versa). - -## 7. Follow-ups - -Known limitations tracked as bugs or RFCs: - -- `docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md` — - most emitter methods are defined but never invoked. -- `docs/bugs/open/2026-04-18-dashboard-reconnect-stale-ui.md` — no - re-subscribe, no snapshot re-fetch, no replay on Socket.io reconnect. -- `docs/bugs/open/2026-04-17-dashboard-process-local-state.md` — - `global.__socketIO` / `global.__dashboardStore` are process-local, so - multi-replica deployments diverge. Treated as intentional research- - tool simplification for now; long-term direction undecided. -- `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md` — the - `DashboardEmitterSandboxEventSink` has no constructor site. -- `docs/rfcs/active/2026-04-18-dashboard-paginated-runs-api.md` — - proposes replacing the in-memory cap + `request:runs` with a - paginated REST endpoint. -- `docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md` — - proposes a contract test that fails CI when an emitter method has - zero call sites, and when a `RunGraphMutation.kind` has no TS - reducer. - -Open questions: - -- Contract versioning: whether the Zod schema is meant to be generated - from the Pydantic contracts, and if so how to wire the generator into - `pnpm run check:fe`. -- HA posture: whether to keep process-local state indefinitely or adopt - Redis / session affinity is unresolved. diff --git a/docs/architecture/06_builtins.md b/docs/architecture/06_builtins.md deleted file mode 100644 index 28623fcf4..000000000 --- a/docs/architecture/06_builtins.md +++ /dev/null @@ -1,297 +0,0 @@ -# 06 — Builtins - -## 1. Purpose - -`ergon_builtins/` is the library of first-party benchmarks, workers, -evaluators, criteria, rubrics, and model backends shipped with the runtime. -This document describes the registration pattern, the file-layout convention -per benchmark, and the stub-worker contract. It is a statement of how the -layer is organized today and what must remain true for a benchmark to be -runnable — not a catalog of registered implementations. - -## 2. Core abstractions - -- Benchmark registry. - - Plain `dict[str, type[Benchmark]]` populated by eager imports at module - load, split between an always-available core set and a data-extra-gated - set. No decorators, no entry points. - - Surface-area constraint: registration is a dict entry keyed by - `type_slug`. External packages cannot register without editing the - registry module; this is the intentional short-term choice for monorepo - development. - - Freeze status: layout is conventional, not enforced at import time. - Divergences are debt. - - Owner: benchmark author. - -- Worker registry. - - Reference workers that run across benchmarks live under - `ergon_builtins/workers/baselines/`; benchmark-specific worker variants - live alongside their benchmark. - - Registry shape: `WORKERS: dict[str, WorkerFactory]` where - `WorkerFactory = Callable[..., Worker]` and every entry is called with - `(name=..., model=..., task_id=..., sandbox_id=...)`. Plain subclasses - are referenced directly (`"stub-worker": StubWorker`) because base - `Worker.__init__` requires `task_id` / `sandbox_id` and every concrete - subclass forwards them through to `super().__init__`. Benchmark-specific - entries (`"minif2f-react"`, `"swebench-react"`) are small closures that - build a live toolkit from the sandbox and pass every ctor kwarg — - including the runtime identity kwargs — through to `ReActWorker(...)`. - The previous `_plain(cls)` shim has been removed (RFC 2026-04-22 §1). - - Freeze status: additive. - - Owner: worker author. - -- Evaluator, Criterion, and Rubric layout. - - Reusable **Criterion** primitives live under - `ergon_builtins/evaluators/criteria/`; reusable **Rubric** composites - live under `ergon_builtins/evaluators/rubrics/`. Benchmark-specific - Criteria and Rubrics live alongside the benchmark subpackage. - - Semantics: a Criterion is a primitive scoring rule; a Rubric bundles - Criteria into an Evaluator; an Evaluator is the top-level scoring unit - bound to the benchmark via `evaluator_requirements()`. - - Surface-area constraint: Rubrics bundle Criteria, not other Rubrics. - Rubric nesting is not supported and there are no plans to change that. - - Third-party users primarily extend at the Criterion layer. - -- Model target resolution. - - Builtins do not register cloud model backends. Model target strings are - resolved centrally by `resolve_model_target` in `ergon_core`. - - Freeze status: stable API; adding a backend is additive inside the - providers layer. - -- ReAct toolkit composition. - - There is one concrete ReAct worker class — `ReActWorker` — with a - serializable Pydantic construction contract. **v2 (object-bound):** - `ReActWorker(name=..., model=..., system_prompt=..., max_iterations=..., - toolkit=)`. The `toolkit` field is a serializable - `Toolkit` subclass (e.g. `MiniF2FToolkit`, `SWEBenchToolkit`, - `ResearchRubricsToolkit`, `GDPEvalToolkit`) that carries only config; live - `pydantic_ai.Tool` instances are built lazily at `execute()` time via - `toolkit.tools(sandbox, task)`. This makes `ReActWorker` fully - round-trippable through task JSON without holding non-serializable state. - **v1 (registry-closure):** `ReActWorker(name=..., model=..., task_id=..., - sandbox_id=..., tools=[...], system_prompt=..., max_iterations=...)` — - constructed by a factory closure at run time, not stored in task JSON. - - Per-task environment setup (clone a repo, install deps, apply a - harness spec) lives in `BaseSandboxManager._install_dependencies`, not - in the worker or an adapter. The sandbox manager reads the per-task - payload via `queries.task_executions.get_task_payload(task_id)`. - - Freeze status: adding a v2 benchmark that needs ReAct means a - `make__worker()` factory that returns a serializable - `ReActWorker(toolkit=...)` instance, not a new registry closure. - -- Onboarding profile. - - Today a hand-maintained `BENCHMARK_DEPS` dict in - `ergon_cli/onboarding/profile.py` declares each benchmark's E2B - requirement, extras, and optional API keys so `ergon onboard` can prompt - correctly. It is a parallel registry hand-synced with the benchmark - registry — see invariant in section 4 and follow-up in section 7. - -## 3. Control flow — adding a benchmark - -At concept level, adding a benchmark means: - -1. Create a subpackage under `ergon_builtins/benchmarks//` that - provides the canonical pieces (benchmark class, task schemas, optional - sandbox + manager, criteria/rubric/evaluator, stub worker). -2. Register the benchmark in the appropriate registry module (core vs. - data-extra). -3. Declare the benchmark's onboarding deps so `ergon onboard` prompts - correctly (see invariants). -4. If the benchmark needs a custom sandbox template, wire a - `ergon benchmark setup ` path so users can build it. -5. Ship a stub worker so CI can exercise graph propagation and the eval - pipeline without external LLM or sandbox dependencies. - -Runtime data flow when a benchmark runs: - -``` -Benchmark loader → Task instances → Worker - │ - ▼ - SandboxManager (optional sandbox) - │ - ▼ - Evaluator → Rubric → Criteria (scoring) - │ - ▼ - Persisted evaluation rows -``` - -## 4. Invariants - -- `type_slug` is the stable identifier used by CLI, onboarding, and - registry. It matches the directory name by convention and MUST NOT change - after a benchmark has run in any persisted dataset. Renaming orphans - persisted runs. -- A custom sandbox template implies a matching - `ergon benchmark setup ` code path. No silent template dependencies. -- Every benchmark MUST ship a stub worker that exercises the graph - propagation and eval pipeline without external LLM or sandbox - dependencies. Enforcement is weak today: stub-worker coverage lags the - registered benchmark set. -- Every registered benchmark MUST have a matching onboarding deps entry. - The two registries being separate means they can drift, and have: the - `BENCHMARK_DEPS` dict regressed on `swebench-verified` as recently as - 2026-04-17. -- Criteria MUST NOT spawn their own sandboxes. A Criterion that - instantiates a `SandboxManager` directly bypasses the runtime's sandbox - lifecycle and resource accounting. Enforced by - `tests/state/test_criteria_do_not_spawn_sandboxes.py`. - -## 5. Extension points - -- **New benchmark.** Follow the concept-level recipe in section 3. Do not - invent new file names for the canonical pieces — the layout convention is - what makes the subpackage legible without reading every file. -- **New worker.** Add under `ergon_builtins/workers/baselines/` if it is - cross-benchmark; alongside the benchmark otherwise. The contract is which - task schemas it supports. -- **New model backend.** Add an explicit `resolve_model_target` branch in - `ergon_core/core/providers/generation/`; prefer short, stable prefixes. -- **New Criterion.** Place in `ergon_builtins/evaluators/criteria/` if - reusable, alongside the benchmark if benchmark-specific. This is the - layer third-party users most often extend. -- **New Rubric.** Place in `ergon_builtins/evaluators/rubrics/` when - reusable; alongside the benchmark otherwise. Rubrics bundle Criteria, - never other Rubrics. -- **Template setup.** Today the pattern is implicit — some benchmarks ship - a pre-built E2B template ID, others install dependencies at sandbox - startup, others have no template at all. See section 7. -- **External registration.** Not supported today. A third-party package - cannot register a benchmark without editing the registry module. - -## 6. Anti-patterns - -- **Adding a benchmark without a stub worker.** CI then cannot prove the - pipeline still works end-to-end after refactors; the graph-propagation - and eval paths become untested for that benchmark. -- **Inlining evaluator logic into `benchmark.py`.** The per-concern file - split is what makes the subpackage navigable; collapsing it defeats the - layout convention. -- **Forgetting the onboarding deps entry.** `ergon onboard` then fails to - prompt for required API keys or extras, and the benchmark is unrunnable - out of the box. -- **Custom sandbox template without a `ergon benchmark setup ` - path.** Users cannot build the template and the benchmark is - non-runnable. -- **Benchmark code importing worker-specific internals.** Benchmarks - describe tasks; they must be worker-agnostic so multiple workers can - target the same benchmark. -- **Renaming `type_slug` after the benchmark has run in a shared - environment.** Persisted runs become orphaned (see invariant). -- **A Criterion spawning its own sandbox.** Enforced by - `tests/state/test_criteria_do_not_spawn_sandboxes.py`. -- **Worker subclasses for per-benchmark glue.** Benchmark-specific wiring - is a factory-closure concern (registry), not a class hierarchy. The - worker `__init__` contract is `tools: list[Tool]` + prompt only; a new - benchmark that reuses `ReActWorker` means a new registry factory, not a - new `ReActWorker` subclass. -- **Per-task setup inside workers.** Setup scripts (clone, install deps, - environment bootstrap) belong to `BaseSandboxManager._install_dependencies` - — sandbox lifecycle, not worker lifecycle. The manager reads the - per-task payload via `queries.task_executions.get_task_payload(task_id)`. -- **Nullable-with-default kwargs on concrete Worker `__init__`.** - `tools: list[Tool] | None = None`, `max_iterations: int = 10`, etc. hide - sizing decisions in a shared default and mask per-benchmark intent. - Concrete workers declare their required construction contract; factories - pass every kwarg explicitly. -- **Per-benchmark code under top-level `sandboxes/` or `toolkits/`.** - A `LeanSandbox` is only useful inside MiniF2F; a `MiniF2FToolkit` - assumes `LeanSandbox`. These belong in `benchmarks/minif2f/`, not in - cross-cutting top-level dirs that imply reuse that doesn't exist. The - cardinality test: a file is cross-cutting only if N different benchmarks - would import it. PR 6 introduced top-level `sandboxes/` / `toolkits/` - dirs; PR 6.5 deleted them and moved their contents into per-benchmark - subpackages. -- **`ReActWorker(ReActWorker)` subclasses.** Making - per-benchmark worker subclasses costs N×M classes (N strategies × - M benchmarks) and forces the agentic-loop logic to live in N subclasses - that all `super().execute()`. The v2 pattern is: one worker class per - strategy in `workers/baselines/`, plus a per-benchmark factory function - in `benchmarks//workers.py` that binds the strategy to the local - sandbox + toolkit + prompt. Cost is N + M, not N × M. - -## 7. CLI observation commands - -After kicking off a run from Python, observe it via the CLI: - -- `ergon run status ` — current state of one run -- `ergon run list [--status=S] [--experiment=]` — list runs, optionally filtered -- `ergon experiment show ` — full experiment detail (UUID-based) -- `ergon experiment list` — list recent experiments -- `ergon experiment tags` — list distinct experiment-tag strings -- `ergon experiment by-tag ` — list definitions sharing a tag, with latest run status - -The `--experiment=` filter operates on `RunRecord.experiment`. The `tags` -and `by-tag` commands operate on v2 `ExperimentDefinition.metadata_json["experiment"]`. -Launch and test-harness paths copy that metadata tag onto `RunRecord.experiment` -so run lists and experiment-definition views group by the same stable string. -If no tags are set, `ergon experiment tags` returns an empty list with a message -pointing at v2 experiment definitions. - -All CLI commands are read-only against persisted state. Authoring (defining a benchmark, -persisting it, launching a run) is Python-only; see `ergon_core.api`. - -## 8. Follow-ups - -Known limits and open questions touching this layer: - -- The `BENCHMARK_DEPS` dict is a parallel registry that can drift from the - benchmark registry; the drift invariant (section 4) is enforced only by - vigilance today. -- Stub-worker coverage lags the registered benchmark set; the per-benchmark - stub pattern (stub worker plus stub sandbox manager plus smoke test) is - not yet uniform. -- Template setup is implicit — there is no declared shape for "this - benchmark needs no template" vs. "this benchmark ships a template ID" - vs. "this benchmark installs deps at sandbox startup". -- External benchmark registration has no supported path. Revisit if a - concrete external use case appears. - -Active RFCs and bugs in this area live under `docs/rfcs/active/` and -`docs/bugs/open/`; grep for `benchmark`, `criterion`, or `onboarding` in -those trees for the current set. This doc describes how the layer works -today and will be updated when an RFC lands and changes an invariant. - -## Code map - -| Concern | Location | -|---|---| -| Public benchmark classes | `ergon_builtins/benchmarks//benchmark.py` | -| Public rubric classes | `ergon_builtins/benchmarks//rubric.py` | -| Benchmark subpackages | `ergon_builtins/benchmarks//` | -| Per-benchmark Sandbox subclass | `ergon_builtins/benchmarks//sandbox.py` | -| Per-benchmark Toolkit config | `ergon_builtins/benchmarks//toolkit.py` | -| Per-benchmark worker factories | `ergon_builtins/benchmarks//workers.py` | -| Cross-benchmark reference workers | `ergon_builtins/workers/baselines/` | -| Cross-cutting sandbox-adapter infra | `ergon_builtins/sandbox/` | -| Reusable Criterion primitives | `ergon_builtins/evaluators/criteria/` | -| Reusable Rubric composites | `ergon_builtins/evaluators/rubrics/` | -| Model backends | `ergon_builtins/models/` | -| Onboarding deps dict | `ergon_cli/onboarding/profile.py` | - -## Cardinality and Colocation - -The file layout follows the actual coupling cardinalities between -`Sandbox`, `Toolkit`, `Worker`, `Evaluator`, `Criterion`, and `Benchmark`: - -| Pair | Cardinality | Lives where | -|------|-------------|-------------| -| Benchmark ↔ Sandbox | 1↔1 (in practice) | `benchmarks//sandbox.py` | -| Sandbox ↔ Toolkit | 1↔N possible, 1↔1 in practice | `benchmarks//toolkit.py` | -| Toolkit ↔ Worker class | N↔M (loose) | Worker class in `workers/baselines/`; binding in `benchmarks//workers.py` | -| Worker class ↔ Benchmark | N↔1 (one ReActWorker, N benchmarks) | Worker class in `workers/baselines/` | -| Evaluator ↔ Benchmark | 1↔1 (rubric); N↔M (reusable criteria) | Rubric in `benchmarks//rubric.py`; benchmark-specific criteria in `benchmarks//criteria/`; reusable criteria in top-level `evaluators/criteria/` | -| Evaluator ↔ Sandbox | 1↔1 if agentic, 0 otherwise | Same as Evaluator ↔ Benchmark | - -**The test for "where does X live?":** - -- 1:1 with a benchmark → `benchmarks//` -- N:1 across benchmarks → top-level under its category - -`Sandbox` and `Toolkit` fail the test for cross-cutting (every concrete -class is 1:1 with a benchmark); they live per-benchmark. `ReActWorker` -passes (one class powers every benchmark); it stays under -`workers/baselines/`. The shared manager-backed sandbox adapter passes -(one adapter wraps every benchmark's `BaseSandboxManager` subclass until -PR 11); it gets its own top-level home in `sandbox/` (singular). diff --git a/docs/architecture/07_testing.md b/docs/architecture/07_testing.md deleted file mode 100644 index ae2784a2b..000000000 --- a/docs/architecture/07_testing.md +++ /dev/null @@ -1,195 +0,0 @@ -# 07 — Testing - -## 1. Purpose - -Describe Ergon's standing testing posture: four tiers with non-overlapping roles, a canonical smoke program that every PR runs against real E2B + real Postgres, and the handoff contracts between layers (criteria, runtime, dashboard). A green CI run implies production correctness for everything except non-deterministic model behaviour, which lives in its own tier. - -Transition history + rationale behind this posture live in [`docs/superpowers/plans/test-refactor/`](../superpowers/plans/test-refactor/) (delete that folder after the standing system has been running for a week). - -## 2. Tiers - -Path-based, not marker-based. The local gate and the CI workflow both dispatch by directory. - -| Tier | Path | Infra | CI trigger | Proves | -|------|------|-------|------------|--------| -| **Unit** | `tests/unit/` | None — no I/O, no fixtures | every PR (`ci-fast.yml`) | Pure logic: Pydantic, validators, registry wiring, pure functions, static lints | -| **Integration** | `tests/integration/` | Real Postgres 15 + real Inngest dev server (docker-compose.ci.yml) | every PR (`ci-fast.yml`) | Graph / service / persistence semantics; API boundaries; harness round-trips | -| **E2E smoke** | `tests/e2e/` | Full Docker stack + **real E2B** + dashboard + Playwright | every PR (`e2e-benchmarks.yml` matrix) | Cross-service + cross-process + UI truth; sandbox provisioning at volume; experiment-group parallel scheduling; partial-work persistence on FAILED tasks | -| **Real-LLM** | `tests/real_llm/` | As e2e + real model calls, budget-gated | on demand + nightly | Non-deterministic model behaviour; RL trajectory extraction | - -`@pytest.mark.slow` is available for local dev ergonomics only; CI runs everything in-tier. - -**Decision rule.** Pure function / validator / pydantic model / registry key → unit. Exercises graph, persistence, or an HTTP boundary → integration. Needs sandbox + dashboard + UI → e2e smoke. Needs LLM → real-llm. - -## 3. Canonical smoke program - -Every PR runs three benchmark legs in parallel via `.github/workflows/e2e-benchmarks.yml`: - -| Leg | Slot 1 | Slot 2 | -|---|---|---| -| `researchrubrics` | happy | **sad** — `l_2` forced FAIL | -| `minif2f` | happy | **sad** — `l_2` forced FAIL | -| `swebench-verified` | happy | **sad** — `l_2` forced FAIL | - -**6 top-level runs per PR; 57 dynamic child sandbox acquisitions** (3 happy × 11 child tasks + 3 sad × 8 child tasks — `l_3` never provisions on sad runs because its dependency failed). - -### 3.1 Smoke DAG - -Every smoke run starts with the same 9 direct children: - -``` -Diamond (4): Line (3): Singletons (2): - d_root l_1 → l_2 → l_3 s_a s_b - ↙ ↘ -d_left d_right - ↘ ↙ - d_join -``` - -Happy-path runs route top-level `l_2` to `{env}-smoke-recursive-worker`, which plans a nested two-node line under `l_2`: - -```text -l_2 -└─ l_2_a → l_2_b -``` - -Top-level `l_3` depends on `l_2`, so the smoke proves dependency propagation waits for a non-leaf dynamic task before releasing downstream work. Sad-path runs route `l_2` to the failing leaf instead, so `l_3` remains blocked. - -Topology is enforced by `ergon_core/test_support/smoke_fixtures/smoke_base/worker_base.py::SmokeWorkerBase.execute` being decorated `@typing.final`. Subclasses supply the leaf slug via `leaf_slug` and override `_spec_for(slug, deps, desc)` only to route specific slugs elsewhere. They cannot change the direct-child DAG itself. - -The single source of truth for the direct-child topology is [`ergon_core/test_support/smoke_fixtures/smoke_base/constants.py`](../../ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/constants.py): - -```python -EXPECTED_SUBTASK_SLUGS = ( - "d_root", "d_left", "d_right", "d_join", - "l_1", "l_2", "l_3", - "s_a", "s_b", -) -``` - -### 3.2 Fixture residency — test-only, out of `ergon_builtins` - -`ergon_builtins/` contains only production baselines (ReActWorker, TrainingStubWorker). All smoke workers, leaves, and criteria live under [`tests/fixtures/smoke_components/`](../../tests/fixtures/smoke_components/) and register into the process-level core component registry through `register_smoke_fixtures()`. - -19 registry rows total — none production: - -| Slug | Kind | -|---|---| -| `{env}-smoke-worker` × 3 | Worker (parent) — inherits `SmokeWorkerBase` | -| `{env}-smoke-leaf` × 3 | Worker (leaf) — inherits `BaseSmokeLeafWorker` | -| `{env}-smoke-recursive-worker` × 3 | Worker (nested `l_2` parent) — inherits `RecursiveSmokeWorkerBase` | -| `{env}-sadpath-smoke-worker` × 3 | Worker (sad-path parent) | -| `{env}-smoke-leaf-failing` × 3 | Worker (sad-path failing leaf) | -| `{env}-smoke-criterion` × 3 | Criterion — inherits `SmokeCriterionBase` | -| `smoke-post-root-timing-criterion` | Criterion — second root evaluator used for timing assertions | - -where `{env} ∈ {researchrubrics, minif2f, swebench}`. - -### 3.3 Turn persistence - -- Parent `SmokeWorkerBase.execute` yields **3** `GenerationTurn`s (planning → planned → awaiting) so incremental turn persistence is exercised on every run. -- Happy-path recursive `l_2` yields **3** `GenerationTurn`s. -- Each leaf `BaseSmokeLeafWorker.execute` yields **2** turns (attaching → done). -- Total per happy run: **3 + 3 + 10 × 2 = 26** `GenerationTurn` rows; driver asserts on this. - -### 3.4 Inter-agent messaging - -Each happy-path leaf calls `CommunicationService.save_message` once on the `smoke-completion` thread (first production caller of that service). The recursive `l_2` worker also sends one completion message after nested children finish. Happy runs emit 11 `ThreadMessage` rows (`9` direct slugs + `l_2_a`, `l_2_b`), sequence_num 1..11 per thread. Sad-path `l_2` raises before reaching this call and `l_3` blocks — 7 messages on a sad run, with `l_2` and `l_3` missing. - -### 3.5 Sandbox-side checks - -The criterion's `_verify_sandbox_setup` hook runs a trivial env-specific command in the parent task's live sandbox via `context.runtime.run_command(...)` (per RFC `2026-04-17-criterion-runtime-di-container`, accepted). The sandbox is kept alive through criterion execution per RFC `2026-04-17-sandbox-lifetime-covers-criteria`. Per env: - -- **researchrubrics** — bash + coreutils + `/tmp` writability (echo → wc -l → OK marker). -- **minif2f** — `lean --check` of `theorem health_check : True := trivial`. No `|| true`; toolchain breakage fails loudly. -- **swebench** — `python` runs a HEALTH_OK marker + `import pytest` resolves. - -## 4. Per-run assertion surface - -For each run in an experiment group, the pytest driver asserts: - -| Channel | What it checks | -|---|---| -| `RunGraphNode` | Happy: 12 nodes (1 root + 9 direct children + 2 nested children), all COMPLETED; sad: cascade pattern with `l_2` FAILED and `l_3` BLOCKED | -| `RunGraphEdge` | Expected dependency edges (diamond, top-level line, nested `l_2_a → l_2_b`) | -| `RunResource` | Happy: 20 rows (10 outputs + 10 probes); all with non-empty `content_hash` | -| `GenerationTurn` | Exactly 26 rows per happy run | -| `ThreadMessage` (topic `smoke-completion`) | 11 messages per happy run / 7 per sad; `sequence_num` strictly 1..N | -| Blob store round-trip | Re-read of one probe JSON is byte-stable + parses | -| Temporal ordering | `RunTaskExecution.started_at` of children ≥ `completed_at` of parents | -| `RunTaskEvaluation` | Happy: 2 root rows, both score 1.0 and created after root execution completion; sad: no successful final score | - -Sad-path adds: partial artifact persisted (partial_*.md exists as RunResource), pre-failure WAL entry present, `l_3` status BLOCKED/CANCELLED per RFC `static-sibling-failure-semantics`. - -## 5. Harness - -`/api/test/*` FastAPI router at [`ergon_core/core/api/test_harness.py`](../../ergon_core/ergon_core/core/api/test_harness.py). Mounted only when `ENABLE_TEST_HARNESS=1`; write endpoints additionally gated by `X-Test-Secret: ${TEST_HARNESS_SECRET}`. - -Read endpoints (Playwright + pytest consume): - -| Endpoint | Shape | -|---|---| -| `GET /api/test/read/run/{run_id}/state` | `TestRunStateDto` — graph nodes, mutations, evaluations, resource count | -| `GET /api/__danger__/test-harness/read/experiment/{experiment}/runs` | `[{run_id, status}]` — returns empty list on miss (not 404) for cheap polling | - -Write endpoints (`POST /write/run/seed`, `POST /write/reset`) are dashboard-fixture scaffolding; smoke does not use them. - -`tests/e2e/_asserts.py::wait_for_terminal` polls the read endpoint every 2s until `status ∈ {completed, failed, cancelled}`. - -## 6. Dashboard + Playwright - -Per leg, the pytest driver subprocesses `pnpm --dir ergon-dashboard exec playwright test tests/e2e/{env}.smoke.spec.ts`, passing experiment-group state via env vars: - -- `EXPERIMENT_KEY`, `SCREENSHOT_DIR`, `TEST_HARNESS_SECRET`, `ERGON_API_BASE_URL` -- `SMOKE_EXPERIMENT_JSON` — JSON array of `[{run_id, kind}]` for one experiment group, enabling per-kind dispatch in the Playwright spec - -Per-env spec is a 3-line file that delegates to the shared factory at `ergon-dashboard/tests/e2e/_shared/smoke.ts`. The factory iterates the experiment run array, asserts against the backend harness DTO + the dashboard UI (keyed on `data-testid`), and captures screenshots per-run. The harness access goes through `ergon-dashboard/tests/helpers/backendHarnessClient.ts`. - -Required `data-testid` attributes: `run-status`, `task-node-{slug}` (one per `EXPECTED_SUBTASK_SLUGS`), `graph-canvas`, `experiment-run-row`, `experiment-env-label`. - -### 6.1 Dashboard harness job (`ci-fast.yml` → `frontend-e2e`) - -This job runs `docker compose up -d --wait postgres api inngest-dev`, then `pnpm -C ergon-dashboard run e2e` (Playwright starts `pnpm dev:test` locally). The dashboard route `GET /api/health` probes the Ergon API (`GET /experiments?limit=1`), so Compose `--wait` must not return until the API process is actually serving HTTP. The **`api`** service therefore carries a **Docker `healthcheck`** that curls `http://127.0.0.1:9000/health` inside the container; without it, only Postgres had a healthcheck and CI could hit `/api/health` while Uvicorn was still importing, yielding **503**. Playwright specs that drag `react-resizable-panels` separators poll for non-null `boundingBox()` after `toBeVisible` because layout geometry can trail visibility in headless Chromium. - -## 7. CI workflow - -[`.github/workflows/e2e-benchmarks.yml`](../../.github/workflows/e2e-benchmarks.yml): - -- Trigger: `pull_request` + `workflow_dispatch`. -- Matrix: `env ∈ {researchrubrics, minif2f, swebench-verified}`. -- Per leg: `docker compose up --build --wait` with BuildKit GHA cache (`cache_from/cache_to: type=gha`) → `ci/wait_for_stack.sh` → dashboard prod build + start → `ci/wait_for_dashboard.sh` → Playwright install (cached) → `uv run pytest tests/e2e/test_{env}_smoke.py --timeout=270`. -- Per leg: 10-min job timeout; pytest hard ceiling 5 min (RFC). -- Screenshot upload runs `if: always()` so dashboard state is captured even on failure: `ci/push_screenshots.sh` pushes PNGs to the orphan branch `screenshots/pr-{N}`; `ci/pr_comment_screenshots.sh` posts a markdown comment linking them. -- `cleanup-screenshots.yml` deletes the branch on PR close. - -## 8. Invariants - -1. **Topology is identical across all envs.** Enforced by `@final` on `SmokeWorkerBase.execute`. Tested by `tests/unit/smoke_base/test_smoke_worker_base_final.py`. -2. **No LLM calls on the smoke path.** Enforced by convention + grep: `rg 'OPENROUTER|anthropic|openai|pydantic_ai' tests/e2e/` must return zero. -3. **Test stubs live in test fixture packages, not `ergon_builtins/`.** Smoke fixtures import their object-bound benchmark and worker classes explicitly; production builtins expose authoring classes directly instead of mutating a process-local registry. Exception: `training_stub_worker.py` — it's a real RL-trajectory baseline, not test scaffolding; operators invoke it via CLI. -4. **Criteria reconnect via the CriterionRuntime DI container, never via `AsyncSandbox.connect` directly.** Enforced by code inspection; the anti-pattern previously fixed by `bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md`. -5. **Sandbox outlives the task until all criteria finish.** RFC `sandbox-lifetime-covers-criteria`. Smoke is the living regression test for this. -6. **Experiment grouping parallelism exercised on every PR.** 2-run happy/sad experiment groups prove concurrent workflow submission and run aggregation at the scale smoke uses. -7. **Partial work persists on FAILED leaves.** Sad-path `AlwaysFailSubworker` writes a file + runs a probe command, then raises. Driver asserts the partial artifact and pre-failure WAL entry survive. -8. **Job-module boundaries are architecture-tested.** `tests/unit/architecture/test_job_composition_modules.py` enforces the PR10 `core/jobs` ownership split: `contract.py` stays DTO-only, `job.py` may orchestrate application services and current persistence reads/writes but may not import concrete Inngest clients or sandbox adapters, and `inngest.py` stays a framework adapter without SQLModel queries or business service construction. Direct job persistence is a documented PR10 limit until PR11 runtime consolidation. -9. **Inngest serving order and metadata are pinned.** `tests/unit/registry/test_inngest_job_registry.py` checks decorator triggers plus `ALL_FUNCTIONS` membership/order and SDK-exposed retry, cancel, concurrency, and output metadata. Changing any of those is an architecture change, not a harmless import shuffle. - -## 9. Budget - -| Measure | Value | -|---|---| -| Per matrix leg | 10-min job timeout; 5-min pytest timeout | -| Dynamic child sandbox acquisitions per leg | 19 (1 happy × 11 child tasks + 1 sad × 8 child tasks) | -| Dynamic child sandbox acquisitions per PR | 57 across 3 sandbox images | -| Parent-task sandbox per run | 1 (used by parent worker + attached to by the criterion). Not additional at evaluation time. | -| Parallel workflow runs per PR | 6 (3 legs × 2-run experiment group) | -| Warm wall-clock per leg | 1–3 min (post-Docker cache) | -| Cold wall-clock per leg | up to 5 min | - -E2B API key required on every PR — accepted for private-repo phase; revisit before open-sourcing. - -## 10. Known follow-ups - -- **`BaseSandboxManager.reconnect`** shipped in `manager.py` (see `bugs/fixed/2026-04-17-sandbox-lifetime-covers-criteria*` / RFC) but `CriterionRuntime.ensure_sandbox` still uses the in-process `get_sandbox(task_id)` path. Cross-process criterion reconnect (Phase G of the test-refactor program) wires `ensure_sandbox` through `reconnect` so the path is actually exercised. -- **Sandbox command WAL / lifecycle event persistence.** `SandboxEventSink.sandbox_command` + `sandbox_closed` fire reliably into the dashboard event sink, but no dedicated Postgres persistence exists yet. The corresponding driver assertions in `tests/e2e/_asserts.py` soft-skip when the tables are absent; land persistence to turn those into hard assertions. -- **Static-sibling-failure-semantics RFC** (`docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md`) is still active. The sad-path driver accepts `l_3 status ∈ {blocked, cancelled}`; tighten once the RFC pins the exact value. diff --git a/docs/architecture/08_rl_loop.md b/docs/architecture/08_rl_loop.md deleted file mode 100644 index 553bf88c9..000000000 --- a/docs/architecture/08_rl_loop.md +++ /dev/null @@ -1,181 +0,0 @@ -# 08 — RL Loop - -## Purpose - -Generation turns produced by workers during experiment runs are the training -signal for TRL/GRPO. This layer describes how turns are persisted, how the -trainer pulls them, and how rewards from evaluators are joined at training -time. The loop is deliberately asymmetric: workers emit turns into a durable -event log, evaluators write scores into a separate table, and a server-side -extractor pure-functionally combines the two into immutable `Trajectory` -records that the trainer consumes over HTTP. The RL layer therefore sits on -top of the persistence and evaluation layers rather than beside them, and -never touches the live worker loop. - -## Core abstractions - -| Type | Location | Freeze | Owner | -|------|----------|--------|-------| -| `GenerationTurn` | `ergon_core/api/generation.py:97-118` | Unstable; worker-internal | Worker layer | -| `RunContextEvent` | `ergon_core/core/persistence/context/models.py:25` | Stable wire format (table `run_context_events`) | Persistence layer | -| `ContextEventType` | enum in context persistence module | Stable; additions only | Persistence layer | -| `RunTaskEvaluation` | evaluation ORM row | Stable | Evaluation layer | -| `Trajectory` | `ergon_core/core/rl/rollout_types.py:38-51` | Stable HTTP contract | RL layer | -| `AgentTrajectory` | intermediate in `extract_agent_trajectories()` | Unstable; extraction-internal | RL layer | -| `reward_strategy.assign()` | `ergon_core/core/rl/extraction.py` | Single seam for reward joining | RL layer | - -`GenerationTurn` is the in-memory, worker-side abstraction yielded from the -worker's async generator. It carries `messages_in`, `response_parts`, -`tool_results`, `turn_token_ids`, `turn_logprobs`, `policy_version`, -`started_at`, and `completed_at`. It is never serialized outside the worker -process. - -`RunContextEvent` is the persisted unit. A single `GenerationTurn` is -DECOMPOSED into multiple context-event rows — one per message -(`system_prompt`, `user_message`, `assistant_text`, `tool_call`, `thinking`, -`tool_result`). The row shape is `id`, `run_id`, `task_execution_id`, -`worker_binding_key`, `sequence`, `event_type`, `payload` (JSON), -`started_at`, `completed_at`, `created_at`, `policy_version`. The -`sequence` field preserves intra-turn ordering; the tuple -`(task_execution_id, sequence)` is monotonic. - -`RunTaskEvaluation` is where reward scores live. It is keyed by -`execution_id` and produced by the evaluator layer, not the worker. Rewards -are NEVER stored on the turn. - -`Trajectory` is the trainer-facing dataclass with fields `prompt_ids`, -`completion_ids`, `logprobs`, `completion_reward`, and `env_mask`. It is -what crosses the HTTP boundary to TRL. `AgentTrajectory` is the -intermediate produced by `extract_agent_trajectories()` before HTTP -serialization. - -## Control flow - -``` -Worker.execute() yields GenerationTurn - | - v -worker_execute.py:81 -> _persist_context_events() (lines 150-174) - | decomposes the turn into RunContextEvent rows, one per message - v -turn stored in run_context_events; also emits DashboardContextEventEvent (Inngest) - | - v -task COMPLETED -> evaluators run -> RunTaskEvaluation.score populated - | - v -TRL trainer (ergon_infra/adapters/trl_http.py): - 1. POST /rollouts/submit with (definition_id, num_episodes) [lines 44-89] - 2. poll GET /rollouts/{batch_id} until status="complete" [lines 57-86] - 3. receive list[Trajectory] with (prompt_ids, completion_ids, logprobs, reward) - | - v -server-side (ergon_core/core/rl/rollout_service.py): - _extract_trajectories() (lines 243-311) - reads RunContextEvent rows (line 247) - reads RunTaskEvaluation rows (line 259) - joins by execution_id (lines 272-286) - calls extract_agent_trajectories() in core/rl/extraction.py (lines 49-117) - -> converts events + scores into (prompt_ids, completion_ids, logprobs, env_mask, reward) - -> wraps in Trajectory for HTTP response -``` - -Three data movements matter: - -1. Worker -> persistence. The yield-site in the worker's async generator is - the persistence hook; nothing else writes `run_context_events`. -2. Evaluator -> evaluation table. Independent of the worker loop. Runs - after task COMPLETED. -3. Trainer -> runtime. Pull-only, over HTTP. The trainer drives cadence; - the runtime never pushes. - -`_extract_trajectories()` is the join site. It reads both stores, joins by -`execution_id`, and hands the merged record to `extract_agent_trajectories` -which performs tokenization bookkeeping (prompt vs completion spans, -`env_mask` for multi-turn credit assignment) before returning an -`AgentTrajectory`. That is finally converted to `Trajectory` for the -response. - -## Invariants - -- `RunContextEvent` is the persistence unit; `GenerationTurn` is only - in-memory. Adding a new kind of event on the worker side requires adding a - matching `ContextEventType` and teaching `extract_agent_trajectories` how - to encode it. Enforced by: the extractor switch statement in - `ergon_core/core/rl/extraction.py` will raise on an unknown event type. -- Reward is NEVER stored on the turn. It lives in `RunTaskEvaluation.score` - and is joined at extraction time via `reward_strategy.assign()` - (`extraction.py`). Enforced by: `RunContextEvent.payload` schema has no - reward field; there is no code path that writes one. -- The trainer pulls. The runtime does not push to the trainer. Enforced by: - there is no client for the trainer host; only the HTTP endpoints in - `rollout_service.py` exist. -- Trajectories returned to the trainer are immutable. Extraction is pure on - top of persisted rows; running `_extract_trajectories` twice with the - same inputs returns identical output. Enforced by: no mutation of - `RunContextEvent` rows during extraction; `AgentTrajectory` is a frozen - dataclass. -- `(task_execution_id, sequence)` ordering is monotonic per task. Enforced - at insert time in `_persist_context_events`. - -## Extension points - -- **Add a new event type** (multimodal, structured tool result, etc.): add - a variant to `ContextEventType`; extend `_persist_context_events` - (`worker_execute.py:150-174`) to emit it; extend - `extract_agent_trajectories` (`extraction.py:49-117`) to encode it into - `prompt_ids`/`completion_ids`. Tokenization belongs in the extractor, not - the worker. -- **Change reward computation:** `reward_strategy.assign()` is the single - seam. Change the strategy, not the extraction logic. Strategies are - composable: outcome-based, dense, stepwise, etc., all plug in here. -- **Add a new trainer backend:** follow the pattern of `trl_http.py` — - POST `/rollouts/submit`, poll `/rollouts/{batch_id}`, consume - `Trajectory`. The backend owns batching and lr schedule; the runtime - owns supply. -- **Add a new rollout filter** (reject short episodes, dedupe by prompt, - etc.): filter at the server-side extractor rather than the trainer. - Keeps the trainer contract narrow. - -## Anti-patterns - -- **Writing reward onto a `RunContextEvent`.** Reward joins at extraction. - Don't mix the two stores. This would couple the evaluator cadence to the - worker cadence, which is exactly what the decomposition is designed to - avoid. -- **Trainer-side direct Postgres reads.** The trainer is decoupled via HTTP - for good reasons — model-provider isolation, multi-replica extractor, - independent scaling. Any trainer talking to Postgres directly defeats - those. -- **Producing a `GenerationTurn` without yielding through the worker's - async generator.** The persistence hook is attached to the yield. A turn - returned any other way is invisible to the RL loop. -- **Mutating a `RunContextEvent` after insert.** The append-only shape is - what makes extraction pure. Corrections should be a new event, never an - update. -- **Assembling a `Trajectory` on the worker side and shipping it anywhere.** - The worker has no reward, no global view, and no stable tokenization - contract. Trajectories are a trainer-facing type; they are built by the - extractor, not the worker. - -## Follow-ups - -- **Silent trajectory drop bug.** If a criterion fails to produce a - `RunTaskEvaluation.score` row, `extract_agent_trajectories` drops the - trajectory or assigns the default reward. Need to document this as a - stronger invariant: criteria MUST produce a score row (even if null) for - every evaluator binding. Consider filing as an RFC. Today the failure - mode is silent and looks like "training didn't improve" rather than - "data was missing." -- **Reward-strategy documentation.** `reward_strategy.assign()` currently - has minimal docs. Needs an architecture note describing the available - strategies and how to add one. Pair with the cross_cutting/artifacts - doc, since some strategies read from the blob store. -- **Extraction replay tool.** Because extraction is pure, we should expose - a CLI that re-runs `_extract_trajectories` for a finished run without - touching the trainer — useful for debugging reward shaping and - regression testing extractor changes. -- **Tokenization contract versioning.** `extract_agent_trajectories` - implicitly assumes a tokenizer shape; changes here silently break - previously captured runs. Pin a contract version into `Trajectory` and - refuse mismatches at the trainer. diff --git a/docs/architecture/README.md b/docs/architecture/README.md deleted file mode 100644 index 7be955574..000000000 --- a/docs/architecture/README.md +++ /dev/null @@ -1,139 +0,0 @@ -# Ergon Architecture - -Canonical reference for how the Ergon runtime works. Single source of truth. -Parallel work streams coordinate through this doc — if you're adding a feature, -this is what you read first, and this is what you update if your change alters -an invariant. - -## How to read these docs - -Each architecture doc follows a fixed seven-section contract: - -1. **Purpose** — one paragraph on what the layer is for. -2. **Core abstractions** — named types, their freeze status, who owns each. -3. **Control flow** — how data moves; a diagram where it clarifies. -4. **Invariants** — things that must be true, enforced by what. -5. **Extension points** — how to add a new thing at this layer. -6. **Anti-patterns** — what NOT to do, with current offenders cited. -7. **Follow-ups** — pending refactors or open questions touching this layer. - -If you find a contradiction between these docs and the code, the **code is the -ground truth** and the doc is the bug — file a PR updating the doc. - -## What these docs are (and aren't) - -These are **design docs**, not a map of the code. Agents and humans can read -the code to learn what a function is called or where a class lives; they come -here to learn what the system is *supposed to do* and why. - -**Write about:** -- **Invariants** — things that must be true, and what enforces them. -- **Control flow at concept level** — "benchmark fans out to workers, workers - fan in to an aggregator", not the literal call sequence. -- **Anti-patterns with reasoning** — what not to do, and why. -- **Extension points and their contracts** — the shape of the seam, what the - runtime guarantees. -- **Surface-area constraints** — "`Experiment` should not expose persistence - internals" (the *bound*, not the method list). -- **Known limits and open questions** — invariants that don't hold yet, - pointing at the tracking RFC/bug. - -**Avoid in narrative prose:** -- **Function-name drift fixes.** "This function is called `complete_workflow_fn` - not `finalize_success_fn`" — code is truth. -- **Implementation details re-specified.** Paraphrasing what the code already - says clearly ("takes `task_id: str`, returns `TaskResult`"). -- **Method-list inventories inside design arguments.** Keep surface-area - constraints; drop the catalog. -- **Forward-referencing unaccepted RFCs as load-bearing.** The doc should - describe how things work today. -- **Test/fixture name duplication.** - -### Inventory vs. cross-reference - -A single `path:line` pointer after an architectural claim is a -**cross-reference** — it helps the reader find the implementation. A bulleted -list of five `path:line` entries is **inventory** — the doc is leaning on the -map to make the argument. - -- ✅ "`dashboard_emitter` is a process-level singleton - ([`emitter.py:451`](...))." — claim first, pointer as aside. -- ❌ "Offenders: `foo.py:23`, `bar.py:45`, `baz.py:67`, `qux.py:89`, - `quux.py:101`." — delete the list, say "five call sites omit this kwarg - (grep `SandboxManager(`)". - -**A dedicated "Code map" section is fine** — a compact table of "where the -Inngest functions live", "where the criterion implementations are" — as -onboarding reference material, kept separate from the design narrative. Test: -if you removed the code map, would the architectural argument still hold? If -yes, it's doing its job. If no, the doc is leaning on inventory and needs a -rewrite. - -## Layer map - -| File | Scope | -|------|-------| -| [`01_public_api.md`](01_public_api.md) | The types contributors touch: `Benchmark`, `Worker`, `Evaluator`, `Criterion`, `Experiment`, `BenchmarkTask`. | -| [`02_runtime_lifecycle.md`](02_runtime_lifecycle.md) | Inngest fan-out, task state machine, cancellation, finalization. | -| [`03_providers.md`](03_providers.md) | Sandbox managers, generation registry, event sinks, resource publisher. | -| [`04_persistence.md`](04_persistence.md) | Graph WAL, run snapshots, Alembic policy. | -| [`05_dashboard.md`](05_dashboard.md) | Inngest→Next.js→Socket.io pipeline; HA constraints. | -| [`06_builtins.md`](06_builtins.md) | Benchmark registration, template setup, stub worker pattern. | -| [`07_testing.md`](07_testing.md) | Fast / state / e2e tiers; what each layer tests. | -| [`08_rl_loop.md`](08_rl_loop.md) | Rollout service, reward plumbing, TRL HTTP adapter. | - -Cross-cutting concerns that span layers live in [`cross_cutting/`](cross_cutting/): - -| File | Scope | -|------|-------| -| [`cross_cutting/artifacts.md`](cross_cutting/artifacts.md) | Worker→criterion artifact handoff via `SandboxResourcePublisher` + `CriterionRuntime.read_resource`. | -| [`cross_cutting/sandbox_lifecycle.md`](cross_cutting/sandbox_lifecycle.md) | Per-task default, reconnect, teardown timing. | -| [`cross_cutting/error_propagation.md`](cross_cutting/error_propagation.md) | Failure semantics, cancel cascade, fractal-OS semantics. | - -## Status: landed - -This doc tree was bootstrapped in a Q&A session with the system owner on -2026-04-17. The core-domain structure standardization stack landed on -2026-05-19; these docs now describe the accepted `ergon_core.core` layout. - -Final top-level packages: - -| Package | Owner | -|---------|-------| -| `core/application` | Use cases, runtime lifecycle, command-side services, ports, and application-owned events. | -| `core/infrastructure` | HTTP, Inngest, dashboard, sandbox, tracing, and other external adapters. | -| `core/jobs` | Durable job composition modules with colocated `contract.py`, `job.py`, and `inngest.py` files. | -| `core/persistence` | SQLModel tables, session setup, storage-safe types, and low-level row validation. | -| `core/rl` | RL rollout and reward-loop orchestration. | -| `core/shared` | Cross-layer value contracts such as context stream parts and JSON helpers. | -| `core/views` | Read-only DTOs and view builders for dashboard/API hydration. | - -Retired roots stay deleted: `core/domain`, `core/rest_api`, -`core/application/jobs`, `core/application/read_models`, -`core/application/graph`, `core/application/tasks`, -`core/application/workflows`, and `core/infrastructure/inngest/handlers`. -The architecture tests enforce these names, the import boundaries between -layers, and the deleted compatibility symbols from the refactor PRDs. - -PR10 note: durable runtime jobs now live under `core/jobs/**` as colocated -`contract.py`, `job.py`, and `inngest.py` modules. `02_runtime_lifecycle.md` -documents the ownership split and the temporary direct-persistence exception -that remains until PR11 runtime consolidation. - -## Related trees - -- [`../rfcs/`](../rfcs/) — feature proposals and fix designs, grouped by status. -- [`../bugs/`](../bugs/) — triaged bug backlog, grouped by status. -- [`../superpowers/plans/`](../superpowers/plans/) — executable implementation plans. -- [`../superpowers/brainstorms/`](../superpowers/brainstorms/) — pre-RFC problem framing. - -## The rule - -Every feature PR either: -- **cites** the architecture section(s) it relies on (OK for mechanical changes - inside an established pattern), or -- **updates** those sections (required if the change alters an invariant, adds - an extension point, or removes an anti-pattern offender). - -Cross-cutting changes must update `cross_cutting/` explicitly. PRs that break -an invariant without updating the doc are NAK'd regardless of test state. diff --git a/docs/architecture/cross_cutting/artifacts.md b/docs/architecture/cross_cutting/artifacts.md deleted file mode 100644 index 89e1f87ad..000000000 --- a/docs/architecture/cross_cutting/artifacts.md +++ /dev/null @@ -1,179 +0,0 @@ -# Cross-cutting — Artifacts - -## Purpose - -Workers produce artifacts (files, diffs, reports) that evaluators need to -read. The canonical path for this handoff is `SandboxResourcePublisher`, a -content-addressed blob store. The legacy `WorkerOutput.artifacts: dict` -escape-hatch was removed in RFC 2026-04-22 — it never survived the Inngest -`worker_execute` serialization boundary, and its presence masked the real -publish/read contract. Today every live criterion reads files via -`CriterionRuntime.read_resource(name)` / `get_all_files_for_task()` or -produces computed artifacts through `CriterionRuntime.run_command(...)`. - -## Core abstractions - -| Type | Location | Freeze | Owner | -|------|----------|--------|-------| -| `SandboxResourcePublisher` | `ergon_core/core/sandbox/resource_publisher.py` | Stable | Sandbox domain | -| `RunResource` | ORM row; table `run_resources` | Stable wire shape | Persistence layer | -| `dashboard/resource.published` | Inngest event | Stable | Dashboard lane | -| `CriterionRuntime.read_resource(name)` | Proposed per RFC | Pending | Evaluator layer | -| `CriterionRuntime.list_resources()` | Proposed per RFC | Pending | Evaluator layer | -| `ERGON_BLOB_ROOT` | env var; filesystem root | Ops-owned | Runtime ops | - -`SandboxResourcePublisher` is configured via the `ERGON_BLOB_ROOT` env var. -On `publish(name, path)` it reads the file from the sandbox, sha256-hashes -the content, writes to `ERGON_BLOB_ROOT/`, and inserts a `RunResource` -row for lookup by name and task id. Content addressing makes the store -deduplicating and replay-safe. The `dashboard/resource.published` event -emitter exists on `DashboardEmitter` but is not called from `publish()` -today — the live resource lane is therefore unpopulated and the dashboard -sees resources only via the cold-start REST snapshot. Wiring this is part -of the dashboard-event enforcement follow-up. - -`RunResource` is the DB-side handle. The lookup key is -`(run_id, task_id, name)`. The content hash is recorded but is not part of -the lookup key — callers ask for the name, and the layer maps name -> -hash -> bytes. Names are stable per task; the hash is an implementation -detail. - -`CriterionRuntime.read_resource(name)` and `list_resources()` are the -intended evaluator-side read path. They are pending per the RFC listed in -follow-ups; today evaluators reach through lower layers. - -## Control flow (intended, once RFC lands) - -``` -Worker writes file in sandbox: /workspace/final_output/fix.patch - | - v -Worker calls publisher.publish(name="fix.patch", path="/workspace/final_output/fix.patch") - | - +--> publisher reads the file from the sandbox - +--> hashes contents (sha256) - +--> writes to ERGON_BLOB_ROOT/ - +--> inserts RunResource(run_id, task_id, name="fix.patch", hash=...) row - +--> emits dashboard/resource.published event - | - v -Task completes; check_evaluators fan out criteria. - | - v -Criterion calls runtime.read_resource("fix.patch") - -> looks up RunResource by (run_id, task_id, name) - -> reads ERGON_BLOB_ROOT/ - -> returns bytes -``` - -Two independent movements: the publish (worker writes durable bytes + row) -and the read (evaluator does a keyed lookup). Neither depends on the -sandbox still existing, and neither depends on the in-process memory of the -worker. That is the whole point. - -## Current state - -- `WorkerOutput.artifacts` no longer exists on the public surface; the - serializable worker result carries `output`, `success`, and `metadata` - only (see `ergon_core/api/results.py`). Workers that need to hand files - off to an evaluator MUST write to `/workspace/final_output/` and let - `SandboxResourcePublisher.sync()` do the work. -- Legacy fallback paths that evaluators used to take (direct - `sandbox.files.read(...)` or DB-inline reads of `RunResource`) are - closed. Every in-tree criterion now reads via - `CriterionRuntime.read_resource(name)` / - `CriterionRuntime.get_all_files_for_task()` or computes artifacts in - the sandbox via `CriterionRuntime.run_command(...)`. -- Enforcement is architectural: there is no `.artifacts` field to - populate, so the failure mode is a type error at construction, not a - silent data loss at the Inngest seam. -- Imported public artifact releases are exported by Ergon itself through - `ergon_ingestion.exports`. The exporter paginates imported `RunRecord` - rows, writes sharded Parquet for runs/reducers/drops, copies - content-addressed `RunResource` blobs into `resources//`, - writes `manifest.json`, `checksums.json`, and `state.json`, and verifies - resource hashes before publication. Artifact repositories may invoke this - exporter, but they must not reimplement pagination, sharding, resource - materialization, checksums, or manifest semantics. - -## Example of the correct pattern (reference) - -See `ergon_builtins/benchmarks/swebench_verified/` — workers publish -`fix.patch` via the publisher; the evaluator reads by name. This is the -pattern the rest of the codebase should converge on, and is what the RFC -promotes into the runtime's public contract. - -## Invariants (intended) - -- `WorkerOutput` MUST NOT grow a dict-typed bag-of-bytes field like the - old `artifacts` escape hatch. File-shaped artifacts are published via - `SandboxResourcePublisher.sync()` from `/workspace/final_output/`; - criteria read them via `CriterionRuntime.read_resource(name)` or - `get_all_files_for_task()`. Computed artifacts (e.g. `git diff`) are - produced by the criterion itself via - `CriterionRuntime.run_command(...)`. -- Artifacts from a worker that an evaluator needs MUST go through - `SandboxResourcePublisher`. Enforced (once RFC lands) by: the only - evaluator read path is `runtime.read_resource(name)`, which only returns - rows backed by a publish. -- Lookups happen by `(run_id, task_id, name)`. Names are stable per task; - the content hash is an implementation detail. -- The publisher is the intended producer of the `dashboard/resource.published` - event so the resource lane in `UnifiedEventStream` stays in sync — partially - wired today: the emitter method exists but `publish()` does not call it, so - the lane is populated only on cold-start snapshot. Enforcement tracked under - the dashboard-event wiring follow-up. -- `RunResource` rows are append-only within a task. A second publish under - the same name yields a new row (or a replace, per the publisher's - policy); readers always see a consistent snapshot. -- The blob store is content-addressed and immutable. A given hash always - dereferences to the same bytes. - -## Extension points - -- **Add a new artifact type:** pick a stable `name`; publish via - `resource_publisher.publish(name=..., path=...)` from the worker. The - evaluator reads by the same name via `runtime.read_resource(name)`. No - new types, no schema changes. -- **Add new metadata alongside a blob** (content-type, producer, size): - extend `RunResource` columns; keep the blob bytes unchanged. Do not - encode metadata into the blob name. -- **Swap the blob backend** (S3, GCS, local FS): the - `SandboxResourcePublisher` is the only write-site; a new backend - implements the same interface and the `ERGON_BLOB_ROOT` config becomes a - URI. - -## Anti-patterns (with offenders) - -- **Reintroducing an `artifacts: dict`-style field on `WorkerOutput`.** - Removed in RFC 2026-04-22 for a reason: the dict did not cross the - Inngest `worker_execute` serialization boundary, so the data loss was - silent. Adding a new bag-of-bytes field relitigates that regression. - Publish via `SandboxResourcePublisher` and read via - `CriterionRuntime.read_resource` — that is the contract. -- **Evaluator reading directly from the sandbox.** Only safe while the - sandbox is alive. Criteria timing makes this a race. Use the publisher. - Any call to `sandbox.files.read(...)` from inside an evaluator is suspect. -- **Publishing without a name.** Blob is unreachable by the evaluator. - Names must be stable per task. An auto-generated name (uuid, timestamp) - defeats the lookup key. -- **Publishing the same name from two different workers in the same task.** - Last-writer-wins, which is rarely what you want. Scope names by the - worker's role. -- **Reading by hash rather than by name.** The hash is not part of the - public key; it can change if the content changes. Name-based reads are - the contract. - -## Follow-ups - -- `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` — adds - `read_resource(name)` / `list_resources()` to `CriterionRuntime`. This is - the RFC that closes the seam described above; once it lands, the - "intended" control flow is the actual one. -- Proposed future: lint rule forbidding new `dict[str, Any]` bag-of-bytes - fields on `WorkerOutput` (or sibling result models). The field removal - closes the current offenders but nothing stops a future PR from - reintroducing the pattern under a new name. -- Retention policy. `ERGON_BLOB_ROOT` has no documented GC. Decide on a - per-run TTL and whether blobs survive task deletion; wire into the - RunResource lifecycle. diff --git a/docs/architecture/cross_cutting/error_propagation.md b/docs/architecture/cross_cutting/error_propagation.md deleted file mode 100644 index d88ac13ba..000000000 --- a/docs/architecture/cross_cutting/error_propagation.md +++ /dev/null @@ -1,186 +0,0 @@ -# Cross-cutting — Error Propagation - -## Purpose - -Captures how failure and cancellation propagate through the DAG. This is a -cross-cutting concern spanning runtime, persistence, and worker layers; it -is also an area where current behavior diverges from the intended -fractal-OS semantics — documented here so every PR touching propagation -knows the target. The central claim: failure must be treated as a signal -for an adaptive planner, not as a reason to cascade-cancel the entire -workflow. Today we partially honor that for managed subtasks and violate -it for static workflow siblings. - -## Core abstractions - -| Type | Location | Freeze | Owner | -|------|----------|--------|-------| -| `TaskExecutionStatus` | `ergon_core/core/persistence/...` | Stable enum: PENDING / RUNNING / COMPLETED / FAILED / CANCELLED | Persistence layer | -| `on_task_completed_or_failed` | `ergon_core/core/runtime/execution/propagation.py:438-586` | Central propagation seam | Runtime layer | -| `CancelCause` | `runtime/events/task_events.py` | Literal type: `"parent_terminal" \| "run_cancel"` | Runtime layer | -| `cancel_orphans_on_*_fn` | `runtime/inngest/cancel_orphan_subtasks.py` | Three Inngest functions | Runtime layer | -| `RunGraphEdge` | persistence row | Stable; edge state: PENDING / SATISFIED / INVALIDATED | Persistence layer | -| `is_workflow_complete_v2` | runtime finalization check | Stable; depends on propagation semantics | Runtime layer | -| `TaskCancelledEvent` | `runtime/events/task_events.py` | The ONLY cancellation trigger for a live task | Runtime layer | - -`on_task_completed_or_failed` is the central propagation function. It walks -`RunGraphEdge` outward from the terminal node, updates edge statuses, and -decides target node fates. All failure/cancellation outcomes for -non-terminal nodes are settled here; no other code writes edge statuses. - -`CancelCause` records why a task was cancelled. Two values today: -`"parent_terminal"` (cascade from a terminal parent) and `"run_cancel"` -(explicit run-level cancel). The value drives which dashboard events fire -and how the UI renders the cause. - -`cancel_orphans_on_*_fn` is a set of three Inngest functions that -cascade-cancel the entire subtask subtree when a parent reaches terminal. -They are the enforcement point for the "cancellation flows strictly -downward along parent->subtask links" invariant. - -## Current behavior (as of 2026-04-17) - -1. Task COMPLETED, dependents with all deps satisfied -> PENDING. - Correct. -2. Task FAILED or CANCELLED, MANAGED subtask dependents - (`parent_node_id` is not None) -> target stays PENDING, edge becomes - INVALIDATED. Correct; matches fractal-OS semantics. The manager - observes the failure and adapts. -3. Task FAILED or CANCELLED, STATIC workflow dependents - (`parent_node_id is None`) -> target AUTO-CANCELLED. Diverges from - intent — see below. -4. Parent reaches terminal -> `cancel_orphans_on_*_fn` cancels the entire - subtask subtree. Correct. -5. CANCELLED managed subtask with re-satisfied deps -> CAN re-activate to - PENDING. Guarded at `propagation.py:546-583`; static workflow - CANCELLED nodes do NOT re-activate. Correct at the seam level; the - asymmetry is intentional because static nodes have no adaptive planner - above them. - -## Intended behavior (system owner steer, 2026-04-17) - -- Static workflow sibling dependents on upstream failure -> should stay - PENDING, not auto-cancel. The model is: "B depends on A; A fails; B - stays PENDING forever unless an adaptive planner unblocks it." -- This makes failure propagation UNIFORM across static and managed nodes. - The only asymmetry becomes re-activation, which is still justified - because static nodes lack a manager to re-plan around them. -- Consequence: `is_workflow_complete_v2` will hang on a - failed-node-with-dependents. Finalization needs a new rule: - "terminate when every non-terminal node is blocked by a failed-dependency - chain." That rule is the companion change to the semantic flip. - -## Control flow - -``` -task reaches terminal (COMPLETED | FAILED | CANCELLED) - | - +--> propagate_execution fires - | | - | +--> on_task_completed_or_failed - | | walks outgoing edges - | | | - | | +--> COMPLETED: edge -> SATISFIED; - | | | if target has all deps satisfied -> PENDING - | | | - | | +--> FAILED/CANCELLED: - | | edge -> INVALIDATED - | | | - | | +--> managed subtask target: stay PENDING (manager adapts) - | | +--> static sibling target: AUTO-CANCELLED (CURRENT) - | | STAY PENDING (INTENDED; RFC in flight) - | | - | +--> cancel_orphans_on_* (if parent terminal) - | cascade-cancel the full subtask subtree - | - v -check_evaluators if COMPLETED; finalize_{success,failure} otherwise -``` - -Three movements to keep straight: - -1. Edge state update — always happens, regardless of target fate. -2. Target node fate — conditional on `parent_node_id` and terminal - status. -3. Subtree cascade — only fires for terminal parents, strictly downward. - -## Invariants - -- Cancellation cascades strictly downward along parent->subtask links. - `cancel_orphans_on_*_fn` handles this. Enforced by: only these three - Inngest functions may issue `TaskCancelledEvent` with cause - `"parent_terminal"`. -- `TaskCancelledEvent` is the ONLY legitimate trigger for a running task's - cleanup. Direct DB CANCELLED writes without the event break the - cascade. Enforced by: runtime cleanup (sandbox teardown, worker - shutdown) is wired to the event handler, not to the DB status. -- A CANCELLED managed subtask is re-activatable if dependencies - re-satisfy; a CANCELLED static workflow node is NOT. Enforced by the - guard at `propagation.py:546-583`. -- Failure propagation is manager-aware: dynamic subtasks stay PENDING so - their manager can adapt; static nodes (no manager) — currently - auto-cancel, intended PENDING. -- Edge state transitions are monotonic within a run: - PENDING -> (SATISFIED | INVALIDATED), no reverse. Enforced at the seam - in `on_task_completed_or_failed`. - -## Extension points - -- **Add a new `CancelCause`:** extend the literal type in - `runtime/events/task_events.py`; ensure every emission site sets one. - The dashboard will render the new cause once it is added to the - renderer's switch. Do not reuse an existing cause for a new meaning. -- **Change propagation semantics:** single seam at - `propagation.py::on_task_completed_or_failed`; coordinate with - `is_workflow_complete_v2` and `test_propagation.py`. These three - co-evolve; changing one without the others leaves the runtime in an - inconsistent state. -- **Add a new terminal kind** (e.g. `TIMED_OUT` as distinct from FAILED): - extend `TaskExecutionStatus`, teach the propagation function the new - terminal's branch, and update finalization. -- **Hook a policy on failure** (retry, escalate, notify): plug into the - propagation seam's FAILED branch before the target-fate decision. - -## Anti-patterns - -- **Silently marking a node FAILED via direct DB write.** Must emit - `TaskFailedEvent` so the cascade fires. A direct write bypasses - `on_task_completed_or_failed`, leaves edges in PENDING, and strands - dependents. -- **Assuming "failed -> children cancel" today.** True for static - siblings, NOT for managed subtasks. Check `parent_node_id` before - reasoning about a failure's downstream effect. -- **Re-activating a CANCELLED static workflow node.** Explicitly forbidden - at `propagation.py:546-583`. The guard exists because static nodes have - no planner to own the re-activation decision. -- **Issuing `TaskCancelledEvent` without a `CancelCause`.** Breaks - dashboard rendering and loses the reason for the cancel. Every - emission site must set a cause. -- **Checking `is_workflow_complete_v2` without accounting for the intended - semantics flip.** Code that assumes "all nodes reach a terminal state" - will hang once static siblings stop auto-cancelling. -- **Coupling sandbox teardown to the DB status column instead of - `TaskCancelledEvent`.** Makes teardown invisible to the cascade and - easy to double-fire. - -## Follow-ups - -- `docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md` — - align static-sibling behavior with fractal-OS semantics; also fixes the - `is_workflow_complete_v2` hang. This is the single RFC that flips the - current-vs-intended diff above. -- Terminology note: the `propagate_task_failure_fn` module may still - encode the old auto-cancel path; audit and align when the RFC lands. - Leaving two code paths that disagree on static-sibling fate is the - single most likely source of future bugs. -- Finalization rewrite. Once static siblings can stay PENDING - indefinitely, `is_workflow_complete_v2` needs the "blocked by a failed - dependency chain" rule. Land it in the same PR as the semantics flip; - the two are inseparable. -- Dashboard UX. Distinguish "blocked by failed dependency" from - "cancelled". Today they render identically once we stop cancelling; - the new state needs its own badge. -- Test coverage. `test_propagation.py` currently encodes the CURRENT - behavior. When the RFC lands, flip those tests; audit for tests that - implicitly depend on auto-cancel as a shortcut to reach workflow - terminal. diff --git a/docs/architecture/cross_cutting/sandbox_lifecycle.md b/docs/architecture/cross_cutting/sandbox_lifecycle.md deleted file mode 100644 index 3826244a0..000000000 --- a/docs/architecture/cross_cutting/sandbox_lifecycle.md +++ /dev/null @@ -1,94 +0,0 @@ -# Cross-Cutting — Sandbox Lifecycle - -## 1. Purpose - -The sandbox is a shared, long-lived resource that spans Worker execution AND criteria evaluation. Getting its lifecycle correct is an Ergon invariant that crosses the providers, runtime, and evaluator layers. A worker writes into the sandbox's filesystem during generator turns; criteria read that filesystem after the worker terminates; the sandbox must stay alive across that boundary. This document is the authoritative reference for lifecycle questions — timeouts, teardown timing, reconnect semantics, cancellation, and the rejected alternatives. - -## 2. Core abstractions - -| Name | Kind | Freeze status | Owner | -| --- | --- | --- | --- | -| Per-task sandbox | lifecycle mode | Frozen; the only supported mode today. | Providers layer. | -| `BaseSandboxManager.create(task_id, event_sink)` | method | Signature frozen; `event_sink` becomes required after RFC 2026-04-17-sandbox-event-sink-activation. | Providers layer. | -| `BaseSandboxManager.close(sandbox_id)` | method | Frozen. Idempotent. Safe from any cancellation path. | Providers layer. | -| `BaseSandboxManager.reconnect(sandbox_id)` | method | **Pending API.** Criteria will use this to attach to the task's still-live sandbox. See follow-ups. | Providers layer. | -| Sandbox-timeout floor | invariant | Enforced by manager `create` only after RFC 2026-04-17-sandbox-lifetime-covers-criteria lands. | Providers layer. | - -The per-task sandbox is owned by the worker runtime for the duration of `Worker.execute()`, then ownership transfers to the evaluator harness for the duration of `check_evaluators`, then the harness calls `close`. There is never a moment when two components both believe they own the sandbox; handoff is explicit at `check_evaluators` entry. - -## 3. Control flow - -``` -task starts -> Manager.create() -> sandbox_id persisted on the execution row - | - v -worker yields GenerationTurns; uses sandbox.commands.run(...) for tools - | - v -worker terminates (COMPLETED | FAILED | CANCELLED) - | sandbox is NOT yet closed - v -check_evaluators fires; for each evaluator binding: - +-> fan out one criterion execution - | criterion calls (future) CriterionRuntime.get_sandbox() - | -> Manager.reconnect(sandbox_id) - | -> runs e.g. swebench-harness test scripts inside the same sandbox - | criterion writes score to RunTaskEvaluation - | - v -all criteria done -> Manager.close(sandbox_id) via finalization path - v -finalize_success -``` - -Data movement notes: - -- The `sandbox_id` is the only handle that crosses the worker-to-evaluator boundary. It is written to the execution row by the worker runtime at create time and read back by `check_evaluators` when fanning out criteria. -- Worker's on-disk state (files written to `/workspace/final_output/`, tool-call intermediates, language-specific build artifacts) is what criteria actually consume. That is the load-bearing reason the sandbox must outlive the worker. -- Criteria write scores to `RunTaskEvaluation` rows, not to the sandbox. Sandbox is read-only from the criterion's perspective (by convention; E2B does not enforce this). - -## 4. Invariants - -1. **Sandbox lives until all criteria for the task have completed.** Teardown runs after `check_evaluators` finishes, NOT at task completion, NOT during `finalize_success`. Confirm by reading the teardown call at `check_evaluators.py:82`. This was a point of confusion in earlier drafts of this doc — the correction is that teardown follows criteria, not the other way around. -2. **Sandbox timeout on creation MUST be at least `task_timeout + max_criterion_timeout`.** Criteria running against a timed-out sandbox is a data-loss bug: the criterion reconnects, the sandbox is dead, the score is lost. Pending enforcement in RFC 2026-04-17-sandbox-lifetime-covers-criteria. Today this is a convention — managers set a generous timeout by inspection, not by formula. -3. **Criteria MUST reconnect via the manager, never by constructing `AsyncSandbox` directly.** Direct construction loses template pinning, loses event emission, and creates a fresh container that cannot see the worker's on-disk state. Enforced end-to-end by the smoke tier on every PR: `DefaultCriterionRuntime.ensure_sandbox` prefers `manager.reconnect(sandbox_id)` over `manager.create(...)` when a task sandbox_id is available, and the smoke criterion `_verify_sandbox_setup` hooks run their env health checks through `context.runtime.run_command(...)` — never via `AsyncSandbox.connect`. -4. **`close(sandbox_id)` is idempotent and safe to call from any cancellation path.** Calling it twice is a no-op on the second call. Calling it from the cancellation cleanup hook is the only correct way to release a leaking sandbox. -5. **Per-task environment setup lives in `_install_dependencies`.** For benchmarks that require per-task environment setup (clone a specific commit, install version-pinned deps, apply a harness spec), that work runs inside `BaseSandboxManager._install_dependencies(sandbox, task_id)` — not inside the worker's `execute()`, not inside a separate `on_run_start` hook, and not inside the criterion. Managers that need per-task data (payload, instance-id metadata) read it from the data layer via `queries.task_executions.get_task_payload(task_id)`; `SandboxSetupRequest` carries only `task_id`, not the full payload. -6. **`_install_dependencies` is idempotent per `sandbox_key`.** `BaseSandboxManager.create()` early-returns when `sandbox_key` is already in `_sandboxes`, so repeat `ensure_sandbox()` / `create()` calls for the same key do NOT re-run setup scripts. This guarantee is load-bearing now that per-task setup (SWE-Bench clone + install, MiniF2F env bootstrap) lives in `_install_dependencies`; losing it would mean criterion-level `ensure_sandbox()` calls silently re-clone and re-install. Locked in by `tests/unit/sandbox/test_ensure_sandbox_idempotence.py`. - -## 5. Failure modes - -- **Sandbox killed mid-task (OOM, network blip, E2B platform incident).** The next `sandbox.commands.run` call from inside the worker raises. System-owner steering: the task transitions to FAILED with the raised error captured as the failure reason. Managers of dynamic subtasks then figure out re-coordination per fractal-OS semantics — a failed subtask surfaces to its parent, which decides whether to retry, substitute, or fail upward. Static workflow nodes have no manager to catch the failure; downstream siblings in a static DAG inherit the failure per the rules laid out in `cross_cutting/error_propagation.md`. -- **Sandbox times out after task but before criteria run.** The worker completed fine, `check_evaluators` fans out, the first criterion's reconnect hits a dead sandbox. Pending fix: bump timeout on creation to `task_timeout + max_criterion_timeout`. Until that RFC lands, managers set a generous literal timeout. -- **Cancellation.** `ergon run cancel` emits a `TaskCancelledEvent`. The `cleanup_cancelled_task_fn` cleanup hook marks the execution row cancelled and releases the execution's sandbox through the shared sandbox lifecycle helper. Repeated cleanup is idempotent: the second release sees an already-closed or missing sandbox and exits cleanly. -- **Criterion raises mid-evaluation.** The criterion's error bubbles to the evaluator harness, which records a failed `RunTaskEvaluation` and continues fanning out remaining criteria. The sandbox stays up until every criterion completes (success or failure); only then does teardown fire. A raising criterion does not short-circuit teardown. - -## 6. Lifecycle modes considered but NOT implemented - -- **Per-run** — share one sandbox across every task in a DAG. Rejected: cross-task isolation is a correctness property, not just a cleanliness one. A compromised tool call in task A could poison task B's filesystem. The per-task teardown gives every task a clean slate at start. -- **Per-experiment-group** — warm pool of prebuilt sandboxes shared across runs. Deferred: this would cut training-time cold-start cost meaningfully, but adds eviction policy, poisoning mitigation, and debugging complexity. No current benchmark justifies the investment. -- **Per-turn** — fresh sandbox per LLM call. Rejected: too slow (seconds per turn in E2B), and it breaks the load-bearing assumption that tool results from turn N are on disk for turn N+1. Most workers cd around, cat files, build artifacts, and cross-reference outputs between turns; a fresh sandbox per turn would require rebuilding that state from scratch every call. -- The current design is "per-task, optional reuse if a benchmark opts in." No benchmark opts in today — the mechanism for opt-in (a `lifecycle_mode` enum on the manager) does not yet exist. - -## 7. Extension points - -### 7.1 Opt into sandbox reuse (future) - -Not implemented. The future shape is a `lifecycle_mode: ClassVar[LifecycleMode]` enum on `BaseSandboxManager` subclasses with variants like `PER_TASK` (current behavior) and `PER_EXPERIMENT_GROUP`. Adding this requires resolving the cross-task isolation question; group-level sandboxes would need explicit reset hooks between tasks. No RFC drafted. - -### 7.2 Override teardown trigger - -A subclass that needs non-standard teardown timing (e.g., to inspect sandbox state post-mortem before close) can subclass the harness entry points in `check_evaluators` or `finalize_success`. This is rarely the right move — the teardown timing is an invariant that the rest of the system depends on. Coordinate with the workflow finalization invariants documented in `02_runtime_lifecycle.md` before changing anything here. - -## 8. Anti-patterns - -- **Closing the sandbox in `finalize_success` instead of after `check_evaluators`.** If `finalize_success` fires before criteria complete, every criterion sees a dead sandbox. Correct path: teardown runs inside the `check_evaluators` completion path (see `check_evaluators.py:82`). No current offenders — this is preserved by the existing harness structure. -- **Constructing a fresh sandbox inside a criterion.** Breaks provenance (the criterion is now scoring a clean environment instead of the worker's actual output state) and isolates the criterion from everything the worker put on disk. The correct path is `CriterionRuntime.get_sandbox()` once RFC 2026-04-17-criterion-runtime-di-container lands. Until then, criteria reconnect via whatever manager API is available and should never call `AsyncSandbox.create` directly. -- **Leaking sandboxes on cancellation.** Cancellation cleanup must go through `cleanup_cancelled_task_fn`; writing `CANCELLED` directly to graph or execution rows bypasses both descendant cancellation and sandbox release. -- **Assuming teardown happens on task COMPLETED.** It does not. The sandbox is alive for the entire evaluator fan-out. Any code that assumes a COMPLETED task implies a torn-down sandbox is wrong; use the sandbox-closed event (or `check_evaluators` completion) instead. - -## 9. Follow-ups - -- `docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md` — enforce the timeout invariant (`task_timeout + max_criterion_timeout`); add `reconnect(sandbox_id)` to `BaseSandboxManager` if the method is missing in the form criteria need. -- `docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md` — replace the stubbed `release-sandbox` step with a real `Manager.close(sandbox_id)` call. -- `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` — add `CriterionRuntime.get_sandbox()` so criteria attach via the manager, and `CriterionRuntime.read_resource(name)` so they read published outputs via a stable API instead of ad-hoc `RunResource` DB reads. diff --git a/docs/bugs/TEMPLATE.md b/docs/bugs/TEMPLATE.md deleted file mode 100644 index d92f9abd5..000000000 --- a/docs/bugs/TEMPLATE.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -status: open # open | fixed -opened: YYYY-MM-DD -fixed_pr: null # set to PR number when moved to fixed/ -priority: P1 # P0 = production broken; P1 = silent data loss or ux break; P2 = correctness; P3 = cleanup -invariant_violated: null # e.g. docs/architecture/03_providers.md#sandbox-event-sink -related_rfc: null # if a fix is being designed, link RFC here ---- - -# Bug: - -## Symptom - -What the user, operator, or system observes. Be concrete — log lines, wrong -values, missing events, etc. Not "X feels slow" — "X takes 45s when it should -take 3s, measured at commit <sha>." - -## Repro - -Exact steps. If there's a minimal test that reproduces it, point at that file -and line. If the bug is "runs forever / never happens in test", explain the -timing or state condition that triggers it. - -## Root cause - -If known. Link to the offending file and line (`path.py:123`). If not yet known, -state "unknown — investigation needed" and list what's been ruled out. - -## Scope - -Who's affected, how often, which workflows. "Every SWE-bench run" vs "only when -E2B template is stale" vs "observed once in staging." - -## Proposed fix - -One paragraph. If the fix is non-trivial, open an RFC and link it in -`related_rfc` above; this section then points to the RFC. - -## On fix - -When moving from `open/` to `fixed/`: - - Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. - - If this bug violated an architecture invariant, confirm the invariant is - restored (or the doc updated to reflect a revised invariant). diff --git a/docs/bugs/fixed/2026-04-17-sandbox-event-sink-unactivated.md b/docs/bugs/fixed/2026-04-17-sandbox-event-sink-unactivated.md deleted file mode 100644 index affd1b887..000000000 --- a/docs/bugs/fixed/2026-04-17-sandbox-event-sink-unactivated.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -status: fixed -opened: 2026-04-17 -fixed_pr: 11 -priority: P2 -invariant_violated: docs/architecture/03_providers.md#sandboxeventsink -related_rfc: docs/rfcs/accepted/2026-04-17-sandbox-event-sink-activation.md ---- - -# Bug: SandboxEventSink Protocol is unactivated across every benchmark - -## Symptom - -The `SandboxEventSink` Protocol at `ergon_core/core/providers/sandbox/event_sink.py:7-36` is -defined as the canonical path by which sandbox managers emit lifecycle events -(`sandbox_created`, `sandbox_command`, `sandbox_closed`). Two implementations exist — -`NoopSandboxEventSink` (the default) and `DashboardEmitterSandboxEventSink` (forwards to the -dashboard emitter). In practice, NO manager is ever constructed with a non-noop sink. Every -sandbox manager call site across every benchmark omits the `event_sink=` kwarg, so every -manager defaults to `NoopSandboxEventSink()`. - -Consequence: sandbox events do not reach the dashboard at all on the live-update path. There -is no second, inline path — a grep of `dashboard_emitter.sandbox_` across `ergon_core`, -`ergon_builtins`, `ergon_cli`, and `ergon_infra` returns zero production call sites (only -`event_sink.py`'s own forwarder, which is never constructed). The dashboard's sandbox view -updates only on page refresh, when the cold-start REST snapshot -(`build_run_snapshot()` at `ergon_core/core/api/runs.py:343`) rereads the mutable tables -that persist sandbox rows. While a run is live, sandbox lifecycle is invisible in the UI. - -## Repro - -``` -grep -rn "SandboxManager(" ergon_builtins/ | grep -v "def " -``` - -Output shows five instantiation sites, none pass `event_sink=`: - -- `ergon_builtins/workers/baselines/minif2f_react_worker.py:111` — `MiniF2FSandboxManager()` -- `ergon_builtins/workers/baselines/swebench_worker.py:123` — `SWEBenchSandboxManager()` -- `ergon_builtins/benchmarks/swebench_verified/criterion.py:72` — `SWEBenchSandboxManager()` -- `ergon_builtins/workers/research_rubrics/researcher_worker.py:74` — `ResearchRubricsSandboxManager()` -- `ergon_builtins/workers/research_rubrics/stub_worker.py:78` — `ResearchRubricsSandboxManager()` - -Confirm no alternate inline path exists: - -``` -grep -rn "dashboard_emitter.sandbox_" ergon_core/ ergon_builtins/ ergon_cli/ ergon_infra/ -``` - -Returns zero production call sites (only the `DashboardEmitterSandboxEventSink` forwarder -inside `event_sink.py`, which nothing constructs). No sandbox events reach the dashboard -during a live run; the view populates only via the REST snapshot on cold start. - -## Root cause - -`BaseSandboxManager.__init__` accepts `event_sink: SandboxEventSink | None = None`, -defaulting to the noop when None is passed. No call site passes a real sink. The Protocol -and its `DashboardEmitterSandboxEventSink` forwarder landed, but no follow-up PR ever wired -production managers to the forwarder. The `dashboard_emitter.sandbox_created/command/closed` -methods defined at `ergon_core/core/dashboard/emitter.py:246-321` exist only so the -forwarder has something to call; they have zero direct callers elsewhere. - -The result is silent architectural drift: the Protocol exists, has two implementations, is -referenced by documentation, and is exercised by zero production code on the live path. -Sandbox lifecycle still lands in the database (rows populate normally), so the REST -snapshot serves the dashboard correctly on page load — but the live Socket.io stream never -carries a sandbox event. A reader of `event_sink.py` would reasonably assume the sink -carries the data end-to-end; it does not. - -## Scope - -- Every benchmark that uses a sandbox is affected: smoke-test (default template), minif2f, - swebench-verified, researchrubrics. -- User-visible impact: sandbox lifecycle is not visible in the dashboard during a live run. - The view appears correct on initial page load (REST snapshot reads the tables) but stays - stale until refresh — sandbox created/command/closed transitions do not stream in. -- Data persistence itself is unaffected: `RunResource` rows, execution rows, and any - sandbox-id linkage continue to be written through their normal paths. The bug is purely - in the live-update lane. -- Tests that assert sandbox-event emission on the live path, if any, are effectively - asserting nothing today. The fix will require adding coverage against a recording sink. - -## Proposed fix - -Follow the revised `docs/rfcs/active/2026-04-17-sandbox-event-sink-activation.md`. The -approach shifted away from "make `event_sink=` required at every construction site" -after the singleton-per-subclass pattern at -`ergon_core/ergon_core/core/providers/sandbox/manager.py:69-72` — combined with the -`__init__` stomp at `manager.py:76-78` — was re-examined. Passing the sink at five call -sites is a behavioral no-op (there is only one `dashboard_emitter` in the process) and -creates a test-isolation hazard tracked separately at -`docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`. - -1. Add a `@classmethod set_event_sink(cls, sink)` on `BaseSandboxManager` that assigns - `cls._event_sink`. -2. In FastAPI app init (`ergon_core/ergon_core/core/api/app.py`, inside `lifespan`), - call `Manager.set_event_sink(DashboardEmitterSandboxEventSink(dashboard_emitter))` - once per manager subclass. -3. Remove the `event_sink=` kwarg from the production `__init__` signature (keep a - narrow test-only override path if needed); leave all 5 construction sites unchanged. -4. Add an integration test using a recording sink installed via `set_event_sink` in - fixture setup, asserting every lifecycle transition produces exactly one sink call - and the dashboard receives the corresponding `DashboardSandbox*Event`. - -## On fix - -- Set `status: fixed`, `fixed_pr: <PR#>`. -- Confirm `docs/architecture/03_providers.md` invariant "every sandbox event flows through - the sink" is stated without the "partially wired" softener, since wiring now exists. -- Move this file to `docs/bugs/fixed/`. diff --git a/docs/bugs/fixed/2026-04-18-ci-docker-caching.md b/docs/bugs/fixed/2026-04-18-ci-docker-caching.md deleted file mode 100644 index f6bec5923..000000000 --- a/docs/bugs/fixed/2026-04-18-ci-docker-caching.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -status: fixed -opened: 2026-04-18 -fixed_pr: TBD -priority: P3 -invariant_violated: null -related_rfc: null ---- - -## Fix summary (2026-04-23) - -- `docker-compose.ci.yml` `api` service now declares `cache_from: [type=gha,scope=ergon-api]` + `cache_to: [type=gha,scope=ergon-api,mode=max]` — BuildKit reads/writes the GitHub Actions cache. -- `.github/workflows/e2e-benchmarks.yml` adds `docker/setup-buildx-action@v3` before `docker compose up --build` in both jobs, and exports `DOCKER_BUILDKIT=1` + `COMPOSE_DOCKER_CLI_BUILD=1` so compose honours `cache_from`/`cache_to`. -- `postgres` kept at tag `15`; `inngest/inngest` left at `:latest` with a TODO. Digest-pinning of base images is a follow-up. -- Measurement: to be recorded on the landing PR (before/after wall-clock of `e2e-benchmarks.yml` on a no-op change). - -# Bug: CI Docker layers are not cached between e2e workflow runs - -## Symptom - -CI jobs that use `docker-compose.ci.yml` rebuild Docker image layers on -every run. The result is a long tail of slow e2e CI wall-clock times: the -Postgres + Inngest + app stack gets rebuilt from scratch each time the -`e2e-benchmarks.yml` workflow fires, even when no Dockerfile or -dependency has changed. The same cost will be paid on every PR once the -integration tier (see -`docs/rfcs/active/2026-04-18-testing-posture-reset.md`) lands, because it -uses the same compose stack. - -## Repro - -Inspect the two files: - -- `.github/workflows/e2e-benchmarks.yml` — no `actions/cache` step for - Docker Buildx layers, no GHA Docker cache import, no pre-pull of - pinned images. -- `docker-compose.ci.yml` — no `cache_from` or `cache_to` directives, - no BuildKit configuration, no pinned image digests used as a cache - base. - -Triggering `workflow_dispatch` on `e2e-benchmarks.yml` twice in a row -shows a full rebuild on both runs. - -## Root cause - -No Docker layer caching is configured in CI. Neither the workflow nor -the compose file declares a cache source or target, so BuildKit has -nothing to read from. The images are also not pinned to specific -digests, so image-layer caching via `docker pull` does not apply -consistently either. - -## Scope - -- Every run of `e2e-benchmarks.yml`. -- Every future PR after the testing-posture-reset RFC lands and the - integration tier starts using the same compose file on the hot path. -- Developers running the compose stack locally for the first time pay a - similar cost, though less visible than CI. -- Not a correctness bug — no data loss, no wrong results. Pure CI - latency and cost. - -## Proposed fix - -Three orthogonal improvements, do all three: - -1. Add `actions/cache` for Docker Buildx layers in - `e2e-benchmarks.yml`. Cache key by Dockerfile hash + compose-file - hash + lockfiles. -2. Add `cache_from` + `cache_to` directives in `docker-compose.ci.yml` - so BuildKit reads and writes a registry-backed cache (or a GHA - cache, `type=gha`). -3. Pin third-party base images (Postgres, Inngest) to specific digests - and `docker pull` them explicitly in a CI step before the compose - build, so the base layers are always cache hits. - -Alternative: switch to GitHub's native Docker build-push action which -bundles all three behaviors. Lower blast radius per step, fewer knobs to -tune. - -## On fix - -When moving from `open/` to `fixed/`: - - Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. - - Measure the improvement: before/after wall-clock on - `e2e-benchmarks.yml` for a no-op change. Record in the PR - description. - - If integration-tier Docker caching lands in the same PR, note that - here. diff --git a/docs/bugs/fixed/2026-04-18-dashboard-emitter-methods-not-wired.md b/docs/bugs/fixed/2026-04-18-dashboard-emitter-methods-not-wired.md deleted file mode 100644 index 5192d2e4c..000000000 --- a/docs/bugs/fixed/2026-04-18-dashboard-emitter-methods-not-wired.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -status: fixed -opened: 2026-04-18 -fixed_pr: 17 -priority: P1 -invariant_violated: docs/architecture/05_dashboard.md#invariants -related_rfc: docs/rfcs/accepted/2026-04-18-dashboard-event-wiring-enforcement.md ---- - -# Bug: 9 of 12 DashboardEmitter methods are defined but never invoked - -## Symptom - -The dashboard's live delta stream is effectively two events wide: graph -mutations and context events. Everything else the UI renders — task -status changes outside graph nodes, sandbox create/command/close, -resource publication, task evaluation updates, thread messages, task -cancellations, and workflow start/completion — does not update live. -Users see these fields only if they reload the page, which triggers a -cold-start REST snapshot via `build_run_snapshot`. - -The architecture invariant "every persistent backend state change has a -corresponding `dashboard/*` event" -(`docs/architecture/05_dashboard.md#invariants`) is violated silently. - -## Repro - -From the repo root: - -``` -rg -n '\.(task_status_changed|workflow_started|workflow_completed|resource_published|thread_message_created|task_evaluation_updated|task_cancelled)\(' \ - ergon_core ergon_builtins ergon_infra -``` - -Zero matches outside the emitter definition itself. The only dashboard -emitter call sites in the runtime are: - -- `ergon_core/ergon_core/core/dashboard/emitter.py:467` — `cohort_updated` -- `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py:81` — - `on_context_event` registered as listener on the context event repo -- `ergon_core/ergon_core/core/runtime/services/task_management_service.py:113` - — `graph_mutation` registered as listener on `WorkflowGraphRepository` - -For the sandbox methods, the only calls live inside -`ergon_core/ergon_core/core/providers/sandbox/event_sink.py:88,107,124`, -but nothing constructs a `DashboardEmitterSandboxEventSink` at runtime -(tracked separately in -`docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md`). - -## Root cause - -Each of the 9 dead methods was added to `DashboardEmitter` -(`ergon_core/ergon_core/core/dashboard/emitter.py:51`) ahead of its -corresponding call site, and the call sites were never added. The UI -happens to look healthy because the cold-start REST snapshot populates -every field; the absence of deltas is invisible on a static page load. - -Enforcement of the underlying invariant is review-only, and the review -did not catch the gap. - -## Scope - -All users of the live dashboard. Any workflow that runs longer than a -page load (i.e. every real workflow) is affected. Existing tests pass -because no test asserts that these deltas reach the frontend. - -## Proposed fix - -Per-method wiring pass: for each of the 9 dead methods, find the -state-mutation site that should trigger it (e.g. -`task_status_changed` at the point of the status write in -`task_management_service`), add the emit call, and add a smoke test -that asserts the Inngest event was produced. - -Enforcement is a separate concern and is covered by -`docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md`: -a pytest contract test that fails when any `DashboardEmitter` method -has zero call sites in `ergon_core/`, `ergon_builtins/`, or -`ergon_infra/`. Land the fix before the contract test so the test -passes at merge time. - -## Fix - -7 methods wired in this PR (sandbox 3 were already wired via PR #11): - -- `workflow_started` — `ergon_core/ergon_core/core/runtime/inngest/start_workflow.py` -- `workflow_completed` — `ergon_core/ergon_core/core/runtime/inngest/complete_workflow.py` -- `task_status_changed` — `ergon_core/ergon_core/core/runtime/services/task_execution_service.py` -- `task_cancelled` — `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` -- `task_evaluation_updated` — `ergon_core/ergon_core/core/runtime/inngest/evaluate_task_run.py` -- `resource_published` — `ergon_core/ergon_core/core/runtime/inngest/persist_outputs.py` -- `thread_message_created` — `ergon_core/ergon_core/core/runtime/services/communication_service.py` - -## On fix - -When moving from `open/` to `fixed/`: -- Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. -- Confirm the "every state change has a dashboard event" invariant in - `docs/architecture/05_dashboard.md#invariants` is restored and the - contract test from the related RFC is green. diff --git a/docs/bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md b/docs/bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md deleted file mode 100644 index 1ab532398..000000000 --- a/docs/bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -status: fixed -opened: 2026-04-18 -fixed_pr: 16 -priority: P2 -invariant_violated: docs/architecture/06_builtins.md#anti-patterns -related_rfc: docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md ---- - -# Bug: SWE-bench criterion spawns its own sandbox - -## Symptom - -The SWE-bench-verified criterion at -`ergon_builtins/benchmarks/swebench_verified/criterion.py:72` instantiates -`SWEBenchSandboxManager()` directly inside `_spawn_eval_sandbox`. Criterion -execution for SWE-bench therefore always spins up a fresh sandbox environment -rather than consuming the task's existing sandbox. This violates the stated -contract that criteria should run in the task's sandbox, not spawn their own. - -Observable effect: every SWE-bench eval pays the cost of an extra sandbox -create call. Harder to observe: if the first criterion-spawned sandbox hangs -or fails, the error surface is different from a task-sandbox failure, and any -future assumption that "a criterion sees the same sandbox state as the worker" -is wrong for this benchmark. - -## Repro - -Read `ergon_builtins/benchmarks/swebench_verified/criterion.py`. The function -`_spawn_eval_sandbox(run_id)` at lines 66-78 constructs a new -`SWEBenchSandboxManager()` (line 72), allocates a fresh `sandbox_key` -(line 73), calls `manager.create(...)` (line 74), and returns the new -sandbox. Every invocation of the SWE-bench criterion hits this path. - -## Root cause - -The criterion was written before the `CriterionRuntime` DI container pattern -existed as a way to hand a criterion access to the task's sandbox. The -criterion needs a sandbox to run the patch-verification tests; the simplest -available path at the time was to instantiate a sandbox manager directly and -bring up a new environment. - -The file and line are known: -`ergon_builtins/benchmarks/swebench_verified/criterion.py:72`. - -## Scope - -SWE-bench-verified only. No other criterion in `ergon_builtins/` spawns a -sandbox today; they either run purely on the task output, read blobs via -`read_resource`, or call LLM judges. But the pattern is contagious — if it -gets copied into the next criterion that needs sandbox access, the -"one sandbox per task, reused by criteria" invariant is gone. - -## Proposed fix - -Blocked on `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`. -That RFC adds `get_sandbox()` to the `CriterionRuntime` Protocol, returning -the task's sandbox (or `None` if it has been torn down). Once the RFC lands: - -1. Delete `_spawn_eval_sandbox` and the direct `SWEBenchSandboxManager()` - call. -2. Replace with `sandbox = await runtime.get_sandbox()` at the criterion - callsite. -3. Handle the `None` case: the current code path assumes a sandbox; with the - DI container, a missing sandbox is an explicit short-circuit with a - "sandbox unavailable" score. - -This work is scheduled as Task 0 of the criterion-runtime-di-container RFC's -migration plan, so the fix lands as part of the RFC rather than as a separate -commit. - -## On fix - -When moving from `open/` to `fixed/`: - - Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. - - Confirm the criterion no longer constructs `SWEBenchSandboxManager` - directly (grep: zero hits under `ergon_builtins/**/criterion*.py`). - - Confirm `docs/architecture/06_builtins.md#anti-patterns` still lists "a - Criterion spawning its own sandbox" as an anti-pattern and that the - swebench-verified callout is removed or updated to reference the fixed - state. diff --git a/docs/bugs/open/2026-04-17-dashboard-process-local-state.md b/docs/bugs/open/2026-04-17-dashboard-process-local-state.md deleted file mode 100644 index 1fa739723..000000000 --- a/docs/bugs/open/2026-04-17-dashboard-process-local-state.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -status: open -opened: 2026-04-17 -fixed_pr: null -priority: P3 -invariant_violated: docs/architecture/05_dashboard.md#known-limitation -related_rfc: null ---- - -# Bug: Dashboard state is Next.js-process-local; no HA deployment is possible - -## Symptom - -The Next.js dashboard holds all live run state in in-process singletons: - -- `global.__socketIO` — the Socket.io server instance — at - `ergon-dashboard/src/lib/socket/server.ts:36-47`. -- `store` — the `DashboardStore` singleton — at `ergon-dashboard/src/lib/state/store.ts`. - -Two consequences follow: - -1. A multi-replica dashboard deployment has DIVERGENT state across replicas. An Inngest - handler runs on one replica, mutates that replica's `DashboardStore`, and broadcasts - via that replica's Socket.io. Browser clients connected to a DIFFERENT replica will - not see the update until a `request:run` refetches from Postgres. -2. On a dashboard process restart, in-memory state is lost. Browsers recover via the - cold-start `request:run` handler, which rebuilds from `build_run_snapshot` over - Postgres. That works, but makes every restart a cold cache and a burst of DB load. - -## Repro - -Scale the dashboard to N > 1 replicas in any of: - -- Kubernetes with `replicas: N`. -- Docker Compose with `deploy.replicas: N`. -- Multiple `next dev` instances behind a load balancer (nginx, HAProxy, Caddy). - -Point browsers at the LB. Trigger an Inngest event (e.g., start a benchmark run). Observe: -each Inngest event is delivered to ONE replica; only that replica's currently-connected -Socket.io clients see the update in realtime. Clients on other replicas see nothing until -they reconnect or explicitly `request:run`, which then hits Postgres. - -## Root cause - -The Next.js Inngest handlers at `ergon-dashboard/src/inngest/functions/index.ts` write -directly into a process-local singleton. The Socket.io server is also process-local. -There is no shared-state layer — no Redis, no pub/sub, no session-affine routing — -coordinating state across replicas. - -This is not a bug in the sense of "feature broken." It is a design constraint. The -dashboard was built for single-node development and has never needed HA. The problem is -that the constraint is currently UNDOCUMENTED at the architecture level, so a future -operator could attempt a multi-replica deployment, see no error, and only discover the -divergence under load. - -## Scope - -- Blocks HA deployment of the dashboard tier. -- Works correctly for single-node dev, staging, and single-replica production — which - describes every deployment today. -- Not visible to single-node users. No data loss; Postgres remains the source of truth - for finalized runs. The issue is strictly realtime UX under horizontal scale. -- Also mildly degrades restart behavior (cold cache on every reboot) but this is - tolerable. - -## Proposed fix - -Future RFC (none filed yet). Three candidate approaches: - -1. **Redis-backed store**. Move `DashboardStore` state to Redis; every replica reads and - writes the shared store. Socket.io broadcasts use Redis pub/sub - (`socket.io-redis-adapter`) for cross-replica event fan-out. This is the canonical - pattern for horizontally-scaled Socket.io deployments. -2. **Session affinity**. Load-balancer routing based on `runId` (sticky by header or - cookie), so all traffic for a given run lands on the same replica. Sharded by runId. - Simpler but brittle under replica churn. -3. **Postgres-backed store with short-poll fallback**. Drop the in-memory store entirely; - every read hits Postgres; Socket.io delivers invalidation pings only. Simpler but - loses realtime granularity for cross-replica events and adds DB load. - -Option 1 is the recommended direction. File as "proposed future RFC" in -`docs/architecture/05_dashboard.md#follow-ups`. - -Until then, this limitation is DOCUMENTED in -`docs/architecture/05_dashboard.md#known-limitation`. Any HA deploy attempt without -addressing this is explicitly out of scope. - -## On fix - -- Set `status: fixed`, `fixed_pr: <PR#>`. -- Remove the "known limitation" section from `docs/architecture/05_dashboard.md` or - rewrite it to reflect the new design (Redis adapter, session affinity, whichever - lands). -- Move this file to `docs/bugs/fixed/`. diff --git a/docs/bugs/open/2026-04-18-blob-store-no-gc.md b/docs/bugs/open/2026-04-18-blob-store-no-gc.md deleted file mode 100644 index 6439783ad..000000000 --- a/docs/bugs/open/2026-04-18-blob-store-no-gc.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -status: open -opened: 2026-04-18 -fixed_pr: null -priority: P4 -invariant_violated: null -related_rfc: null ---- - -# Bug: blob store has no garbage collection - -## Symptom - -`SandboxResourcePublisher` writes blobs under `<ERGON_BLOB_ROOT>/<hash[:2]>/<hash>` -and never deletes them. See the write path at -`ergon_core/ergon_core/core/providers/sandbox/resource_publisher.py:166-175` -(`_write_blob`), invoked from `sync()` at line 93 and from `publish_value()` at -line 142. There is no corresponding delete path anywhere in the module, and no -retention policy, no size cap, and no sweeper. The blob store grows unboundedly -across runs. - -## Repro - -Conceptual — every `RunResource` publish writes bytes to disk via -`_write_blob`. No delete path exists. After N runs of any benchmark that -publishes artifacts (e.g. minif2f proofs, swebench patches), disk usage under -`ERGON_BLOB_ROOT` grows as `O(N * avg_artifact_size)` forever. Content-hash -dedup helps for byte-identical artifacts but does not bound growth for -workloads whose outputs vary across runs. - -## Root cause - -The publisher was designed as write-once content-addressable storage; see the -class docstring at `resource_publisher.py:27-33` ("Never updates. Content-hash -dedup makes repeated calls safe."). Deletion policy was out of scope for the -first cut, and no orphan-detection pass has ever been added. - -## Scope - -Research workloads with bounded disk budgets hit this eventually. Acceptable -today because research runs are manual and disk is cheap on the dev machine. -It grows into a real problem when (a) runs are scheduled on a shared host, -(b) CI accumulates artifacts across thousands of test runs, or (c) long-running -training loops publish heavy checkpoints. - -## Proposed fix - -Deferred. Options on the table: - - 1. Reference counting via `RunResource` rows — delete the blob when the last - referencing row is deleted. - 2. Age-based sweeper — delete blobs whose newest `RunResource` reference is - older than N days. - 3. Size-based LRU — when `<ERGON_BLOB_ROOT>` exceeds a threshold, evict the - least-recently-accessed entries. - 4. Never. Research-grade tool; the user manages disk out of band. - -No decision today. Filed as a P4 tracking item so future disk pressure surfaces -this faster. - -## On fix - - - Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. - - Move the file from `docs/bugs/open/` to `docs/bugs/fixed/`. - - Update `docs/architecture/03_providers.md` to document the chosen GC policy. diff --git a/docs/bugs/open/2026-04-18-dashboard-reconnect-stale-ui.md b/docs/bugs/open/2026-04-18-dashboard-reconnect-stale-ui.md deleted file mode 100644 index a84df4455..000000000 --- a/docs/bugs/open/2026-04-18-dashboard-reconnect-stale-ui.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -status: open -opened: 2026-04-18 -fixed_pr: null -priority: P2 -invariant_violated: null -related_rfc: null ---- - -# Bug: Dashboard UI freezes after Socket.io reconnect - -## Symptom - -When the browser's Socket.io connection drops mid-run (network blip, -laptop sleep/wake, WiFi handoff), the UI stops receiving updates and -stays frozen at the pre-disconnect state indefinitely. A full page -reload restores the view. The Socket.io client itself reconnects -automatically, but no state re-sync happens. - -## Repro - -1. Open a run workspace page at `/runs/<id>` while a workflow is - producing graph mutations or context events. -2. In browser devtools, Application → WebSockets (or Network → WS), - close the active Socket.io connection. -3. Observe the client auto-reconnect (new WS entry appears). -4. Continue the workflow. New graph mutations and context events fire - on the backend but never reach the UI. -5. Reload the page — all the missed state appears, confirming the - backend did its job and only the client-side resync is broken. - -## Root cause - -Three cooperating gaps: - -- `ergon-dashboard/src/providers/SocketProvider.tsx` exposes - `subscribe` at line 117 but the reconnect handler does not re-emit - `subscribe(runId)` for any active subscriptions. On reconnect the - socket is in a fresh session with no room memberships, so - `run:<id>` broadcasts go nowhere for that client. -- `ergon-dashboard/src/hooks/useRunState.ts` fires - `socket.emit("request:run", runId)` at line 497 only when `runId` - changes, not on a new `isConnected` transition. A reconnect does - not re-hydrate the snapshot. -- `ergon-dashboard/src/lib/socket/server.ts` has no replay logic on - room rejoin; `subscribe` at line 114 is a plain `socket.join(room)`. -- The reducer has no idempotency keys, so even if we did replay, the - current reducer would risk double-apply on events bracketing the - disconnect. - -## Scope - -Any user whose connection drops mid-run. In local dev this is rare; in -any real deployment (WiFi networks, mobile, laptops that sleep) this is -routine and makes the live dashboard effectively unreliable for long -runs. - -## Proposed fix - -Three-part, all in one PR: - -1. `SocketProvider.tsx`: track active subscriptions in a ref; on - `socket.io-client`'s `reconnect` event, re-emit `subscribe(runId)` - for each one. -2. `useRunState.ts`: add an effect keyed on `isConnected` that, when - transitioning false → true with a live `runId`, re-emits - `request:run`. -3. Add idempotency keys so snapshot + delta converge cleanly: - `(run_id, task_id, turn_id, event_index)` for context events and - `(run_id, sequence)` for graph mutations. The reducer drops events - whose key is already applied. - -Server-side replay buffering is out of scope — the -`request:run` re-fetch covers the same ground more simply. - -## On fix - -When moving from `open/` to `fixed/`: -- Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. -- Remove the reconnect anti-pattern bullet from - `docs/architecture/05_dashboard.md` section 6 or mark it resolved. diff --git a/docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md b/docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md deleted file mode 100644 index 096c04dcc..000000000 --- a/docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -status: open -opened: 2026-04-18 -fixed_pr: null -priority: P3 -invariant_violated: docs/architecture/03_providers.md#sandboxeventsink -related_rfc: docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md ---- - -# Bug: `BaseSandboxManager` singleton + `__init__` stomp races across construction sites - -## Symptom - -`BaseSandboxManager.__init__` unconditionally overwrites `self._event_sink` whenever a -non-None `event_sink` is passed -(`ergon_core/ergon_core/core/providers/sandbox/manager.py:76-78`): - -```python -def __init__(self, event_sink: SandboxEventSink | None = None): - if event_sink is not None: - self._event_sink = event_sink -``` - -Combined with the singleton-per-subclass pattern at `manager.py:69-72`: - -```python -def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance -``` - -— every re-construction of the same subclass returns the SAME instance and re-runs -`__init__`, so any second construction with a different `event_sink=` silently replaces -the first. Class-level state dicts (`_sandboxes`, `_run_ids`, `_display_task_ids`, -`_creation_locks` at `manager.py:61-66`) are also shared, so all tasks funneled through -any construction site operate on the same cross-context registries. - -No production site hits this today: all 5 construction sites -(`ergon_builtins/workers/baselines/minif2f_react_worker.py:111`, -`ergon_builtins/workers/baselines/swebench_worker.py:123`, -`ergon_builtins/benchmarks/swebench_verified/criterion.py:72`, -`ergon_builtins/workers/research_rubrics/researcher_worker.py:74`, -`ergon_builtins/workers/research_rubrics/stub_worker.py:78`) construct without args, and -`dashboard_emitter` is a process-wide singleton -(`ergon_core/ergon_core/core/dashboard/emitter.py:451`) — so swapping one -`DashboardEmitterSandboxEventSink` for another is an identity substitution in effect. -The latent race is real but currently masked by the uniform argument-less construction. - -## Repro - -Conceptual — no production repro exists; this documents the hazard: - -```python -sink_a = RecordingSandboxEventSink() -sink_b = DashboardEmitterSandboxEventSink(dashboard_emitter) - -m1 = MiniF2FSandboxManager(event_sink=sink_a) -m2 = MiniF2FSandboxManager(event_sink=sink_b) # silently overwrites m1's sink - -assert m1._event_sink is sink_a # FAILS — it is sink_b -assert m1 is m2 # holds — singleton -``` - -Any test that constructs a manager with a recording sink while another code path holds -the same subclass with the production sink will see its `RecordingSandboxEventSink` -stomped (or will stomp the production sink, depending on order). - -## Root cause - -Two interacting design choices: (a) `__new__` returns a cached `cls._instance` so the -same object backs every construction; (b) `__init__` still runs on every construction -and is last-write-wins for any attribute it assigns. The `event_sink` kwarg is the only -current trigger, but the same shape would bite any future `__init__` parameter that sets -instance state. - -## Scope - -- **No production impact today.** All 5 construction sites omit `event_sink=`, so the - stomp branch at `manager.py:77-78` never runs. -- **Latent risk** for: concurrent tests that swap sinks per-test without a full reset; - multi-sink scenarios (e.g. adding a logging-only sink alongside the dashboard sink); - any future refactor that starts threading `event_sink=` through constructors. -- Will become a live hazard the moment someone adopts the pre-revision proposal to pass - `event_sink=` at construction sites — which is why - `docs/rfcs/active/2026-04-17-sandbox-event-sink-activation.md` was rewritten to use a - class-level setter instead. - -## Proposed fix - -Resolved by either of: - -1. **Narrowly:** `docs/rfcs/active/2026-04-17-sandbox-event-sink-activation.md` — the - class-level `set_event_sink` setter removes the need to pass `event_sink` at - construction, making the stomp branch at `manager.py:77-78` vestigial. Delete the - `event_sink=` kwarg from production `__init__` as part of that RFC's implementation. -2. **Broadly:** `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md` — removes - the singleton entirely in favor of instance-owned managers, eliminating the class - hierarchy of shared state dicts along with the stomp. This is the correct long-term - fix; the narrow fix above is sufficient to close this bug. - -## On fix - -- Set `status: fixed`, `fixed_pr: <PR#>`. -- If the broader RFC lands, confirm `manager.py:61-72` no longer holds class-level - mutable state and update `docs/architecture/03_providers.md` to drop the - singleton-pattern note. -- Move this file to `docs/bugs/fixed/`. diff --git a/docs/bugs/open/2026-04-22-edge-added-mutation-unhandled.md b/docs/bugs/open/2026-04-22-edge-added-mutation-unhandled.md deleted file mode 100644 index 8f1456a55..000000000 --- a/docs/bugs/open/2026-04-22-edge-added-mutation-unhandled.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -status: open -opened: 2026-04-22 -fixed_pr: null -priority: P2 -invariant_violated: null -related_rfc: docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md ---- - -# Bug: `edge.added` mutation falls through to `unhandledMutations` - -## Symptom - -The contract test -[`graphMutations.test.ts:113`](ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts:113) -iterates `MutationTypeSchema.options` and asserts each mutation type has a -handler case that prevents it from being appended to `state.unhandledMutations`. -The assertion fails for `mutation_type: "edge.added"`: - -``` -mutation_type 'edge.added' is handled (does not fall through to unhandledMutations) - AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: - true !== false -``` - -Every other `MutationType` variant (`node.added`, `node.removed`, -`node.status_changed`, `node.field_changed`, `edge.removed`, -`edge.status_changed`, `annotation.set`, `annotation.deleted`) passes this -contract test. - -## Repro - -```bash -pnpm -C ergon-dashboard run test -- --test-name-pattern="edge.added" -``` - -Or run the full contract suite via `pnpm run check:fe` — the failing case is -in `src/features/graph/contracts/graphMutations.test.ts` at line 113. - -The synthetic mutation the test sends is built at -[`graphMutations.test.ts:74`](ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts:74): - -```ts -"edge.added": { - mutation_type: "edge.added", - source_node_id: nodeId, // 00000000-...-0001 (== rootTaskId) - target_node_id: "00000000-...-0002", - status: "pending", -}, -``` - -Applied against `emptyState()` (no tasks, empty `edges` map), the reducer -routes this into `unhandledMutations` instead of the `edge.added` arm. - -## Root cause - -Unknown — investigation needed. The reducer appears to have an `edge.added` -case arm already (grep shows a block around line 74 of the reducer source), -so the issue is not a missing arm. Likely candidates to check: - - - The arm runs but returns early (e.g. requires both source and target - nodes to exist in `state.tasks`, which they don't in the synthetic - `emptyState`) and then the outer switch's default pushes onto - `unhandledMutations` anyway. - - The arm's guard rejects the mutation because `target_node_id` points at - an unknown node and the reducer treats "unknown target" as unhandled - rather than as a buffered/pending edge. - - The `new_value` shape in the test doesn't match what the reducer's - runtime schema narrowing expects for the `edge.added` branch, so the - switch falls through. - -Confirm by reading `applyGraphMutation` in -`ergon-dashboard/src/features/graph/state/graphMutationReducer.ts` and tracing -what happens for an `edge.added` whose endpoints aren't in `state.tasks`. - -## Scope - - - Blocks `pnpm run check:fe` (CI gate for the dashboard). - - Violates the contract established by - [`2026-04-18-dashboard-event-wiring-enforcement.md`](docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md): - every `MutationType` must have a matching reducer arm. If `edge.added` - truly isn't handled for edges whose endpoints arrive later, runs where - edge events precede node events will silently drop edges. - - Affects any dashboard consumer of the live event stream, not just tests - — if the reducer really is dropping `edge.added` onto `unhandledMutations` - in production, edges won't render until (or unless) a retry/backfill path - replays the mutation. - -## Proposed fix - -Two sub-cases depending on root cause: - - 1. **Reducer bug**: the arm exists but exits without returning the mutated - state; fix is to ensure every `edge.added` returns through the arm (not - the default). In-place fix in `graphMutationReducer.ts`. - 2. **Semantic gap**: the reducer legitimately rejects `edge.added` when - endpoints are missing. Fix is either (a) relax the guard and buffer the - edge, applying it on the node-added that satisfies it, or (b) update the - contract test's `emptyState` to pre-seed the two node endpoints before - dispatching `edge.added`, and document the ordering requirement in the - reducer. - -Pick after root-cause investigation. If the answer is "buffer edges", this -may deserve promotion to an RFC because it changes the reducer's invariant. - -## On fix - -When moving to `fixed/`: - - Set `status: fixed` and `fixed_pr: <PR#>`. - - Confirm the full `MutationTypeSchema.options` loop passes in - `graphMutations.test.ts`. - - If the fix changed the reducer's contract (e.g. added buffering), - update `docs/architecture/` to describe the new invariant and link it - from `invariant_violated` above. diff --git a/docs/bugs/open/2026-04-23-inngest-function-failures.md b/docs/bugs/open/2026-04-23-inngest-function-failures.md deleted file mode 100644 index 593131a36..000000000 --- a/docs/bugs/open/2026-04-23-inngest-function-failures.md +++ /dev/null @@ -1,458 +0,0 @@ ---- -status: open -opened: 2026-04-23 -fixed_pr: null -priority: P1 -invariant_violated: null -related_rfc: null ---- - -# Bug: Four inngest function failures surfaced while bringing up local smoke - -Four independent bugs, all blocking or degrading the canonical-smoke end-to-end -loop, surfaced together while running `SMOKE_COHORT_SIZE=1 minif2f` against a -fresh local stack on 2026-04-23. Each also affects production flows (not -just smoke), which is why they're bundled into one report. - -Discovery query (now documented in `CLAUDE.md § Debugging → Inngest function -failures`): - -```bash -FROM_TIME=$(date -u -v-15M +%Y-%m-%dT%H:%M:%SZ) -curl -s -X POST http://localhost:8289/v0/gql -H 'Content-Type: application/json' \ - -d "{\"query\":\"query { runs(first: 40, orderBy: [{field: QUEUED_AT, direction: DESC}], filter: { from: \\\"${FROM_TIME}\\\", status: [FAILED] }) { edges { node { function { slug } output } } } }\"}" -``` - -Counts from one 15-minute window during a single cohort-of-1 submission: - -| Count | Function | Layer | -|------:|----------|-------| -| 18 | `ergon-dashboard-handle-graph-mutation` | Dashboard Zod | -| 11 | `ergon-core-cleanup-cancelled-task` | Backend DB | -| 8 | `ergon-core-task-execute` | Backend | -| 3 | `ergon-dashboard-dashboard-task-evaluation-updated` | Dashboard Zod | - -The subsections below capture symptom, repro, root cause (where known), -scope, and proposed fix for each. Any of them can be split into its own -`docs/bugs/open/*.md` when picked up — the grouping is for "discovered -together," not "must be fixed together." - -C and D are both instances of the same broken pipeline; § E captures the -systemic fix that subsumes their individual proposed fixes. - ---- - -## A. `ergon-core-task-execute` — stuck subtasks with null `task_id` - -### Symptom - -Every smoke subtask (`d_root`, `l_1`, `s_a`, `s_b`, etc.) transitions to -`RUNNING` in `run_task_executions` and never progresses. `error_json` stays -`null`, `completed_at` stays `null`, the parent run sits in `EXECUTING` -indefinitely. `wait_for_terminal` in the pytest driver eventually trips its -270 s timeout. Inngest reports `inngest/function.failed` for the -corresponding `task-execute` invocation ~1 s after `task/ready` fires. - -### Repro - -```bash -bash scripts/smoke_local_up.sh -# export env vars from the script's stanza -SMOKE_COHORT_SIZE=1 scripts/smoke_local_run.sh minif2f -# Watch: -docker compose exec postgres psql -U ergon -d ergon -c \ - "SELECT task_slug, status FROM run_graph_nodes WHERE run_id=<id> ORDER BY level;" -``` - -Subtasks wedge at `RUNNING` within a few seconds of dispatch. - -### Root cause - -Inngest error payload (captured via the GraphQL query above): - -``` -1 validation error for PreparedTaskExecution -task_id - UUID input should be a string, bytes or UUID object - [type=uuid_type, input_value=None, input_type=NoneType] -``` - -Two stacked bugs in `ergon_core/ergon_core/core/runtime/inngest/execute_task.py`: - -1. **`TaskExecutionService.prepare(...)` returns `task_id=None`** for subtask - graph nodes. `PreparedTaskExecution` is a pydantic model with a non-null - `task_id: UUID`, so the step output fails Pydantic validation inside - Inngest. Root cause of *why* `prepare` produces `None` is not yet - identified — the `task-execute` payload itself carries a valid - `payload.task_id`; something in `prepare` either drops it or looks up a - null-valued column. Investigation needed. - -2. **The `except` handler can't run** because `prepared` is referenced before - assignment. `execute_task.py:251-262`: - - ```python - prepared = await ctx.step.run("prepare-execution", _prepare, ...) # line 75 - # ... main body ... - except Exception as exc: # line 251 - error_msg = str(exc) - logger.exception("task-execute failed task_id=%s: %s", ...) - await svc.finalize_failure( - FailTaskExecutionCommand( - execution_id=prepared.execution_id, # NameError - ... - ) - ) - ``` - - When `step.run("prepare-execution", ...)` raises, `prepared` is never - bound. The except block then hits `prepared.execution_id` and raises - `NameError`, which is what Inngest finally records. The real error — - Pydantic's complaint — is buried in an earlier step-run attempt; our own - handler erases it before it can persist `error_json` or emit - `TaskFailedEvent`. - -### Scope - -Every workflow where `prepare` returns `task_id=None`. Observed on every -smoke subtask. Likely affects any production benchmark whose -subtask-graph-node row has the right shape to trip the same lookup path — -needs investigation to know for sure. The NameError bug, separately, -silently hides *every* pre-prepare failure across all task-execute runs, -not just this one case. - -### Proposed fix - -Two changes, one cheap and one that needs investigation: - -1. **Defensive except block (cheap, do immediately).** Pull the identifiers - the failure handler needs out of `payload` (which *is* guaranteed bound - at line 59) before entering the try block, or guard with - `if 'prepared' in locals()`. Preferred form: hoist `execution_id: UUID | - None = None` and `node_id` above the try, assign inside, branch on - presence when finalizing. This alone won't fix the stuck-subtask bug, - but it ensures the *real* failure is surfaced via `error_json` / - `TaskFailedEvent` on every future regression. - -2. **Fix the null `task_id` from prepare (investigation needed).** Read - `TaskExecutionService.prepare` + `PrepareTaskExecutionCommand` handling; - find where `task_id` is sourced for subtask graph nodes. Likely a - graph-node column lookup that's returning null when the node was dynamically - added by `add_subtask` rather than statically defined. Write a unit test - that drives `prepare` against a dynamically-spawned node row and asserts - `result.task_id is not None`. - ---- - -## B. `ergon-core-cleanup-cancelled-task` — enum rejects `CANCELLED` - -### Symptom - -Every cleanup-cancelled-task invocation fails with: - -``` -(psycopg2.errors.InvalidTextRepresentation) invalid input value -for enum taskexecutionstatus: "CANCELLED" -LINE 1: UPDATE run_task_executions SET status='CANCELLED' WHERE ... -``` - -Task rows that should be marked cancelled remain in whatever state they held -when the parent workflow cancelled. Dashboard widgets relying on cancelled -status misreport the cohort. - -### Repro - -Any workflow that triggers `TASK_CANCEL` or `RUN_CANCEL`. During the smoke -session, we observed 11 failures in 15 minutes — triggered by Inngest's -`cancel=[...RUN_CANCEL, *TASK_CANCEL]` wiring on `execute_task_fn` firing -whenever a run fails (bug A above). - -### Root cause - -The Postgres enum `taskexecutionstatus` was created without `CANCELLED`. -The application-side `TaskExecutionStatus` enum (likely in -`ergon_core/core/persistence/shared/enums.py` — confirm) includes -`CANCELLED`, but the migration that CREATE TYPE'd the DB enum omitted it (or -added values later were never ALTER TYPE ADD VALUE'd into the live DB). - -### Scope - -Every cancellation. Stale state for operators who cancel a run from the -dashboard or CLI. Downstream visualizations show tasks frozen in -non-terminal state when they should be cancelled. - -### Proposed fix - -Alembic migration: `ALTER TYPE taskexecutionstatus ADD VALUE IF NOT EXISTS -'CANCELLED';`. Must run outside a transaction (Postgres restriction on enum -mutation) — use `op.execute` with `autocommit_block` or put it in a -one-shot migration with the `transactional = False` pattern Alembic -supports. Add a unit test that exercises each enum value end-to-end against -real Postgres (not sqlite, not mock). - ---- - -## C. `ergon-dashboard-handle-graph-mutation` — missing `task_key` / `assigned_worker_key` - -### Symptom - -Dashboard returns HTTP 500 for every `dashboard/graph.mutation` event: - -``` -[ - { "expected": "string", "code": "invalid_type", - "path": ["task_key"], - "message": "Invalid input: expected string, received undefined" }, - { "expected": "string", "code": "invalid_type", - "path": ["assigned_worker_key"], - "message": "Invalid input: expected string, received undefined" } -] -``` - -Dashboard graph UI doesn't reflect dynamic add/remove/reassign events. -Every subtask mutation (smoke spawns 9 per run) is silently dropped by the -dashboard — the tree stays frozen at its initial state. - -### Repro - -Run any workflow that spawns subtasks. 18 failures from one minif2f run. - -### Root cause - -Same class as the `task_tree={}` bug the sub-agent just fixed in -`start_workflow.py` — the backend emits a `dashboard/graph.mutation` payload -that doesn't match the dashboard's Zod schema. Investigation needs to find: - -- the emitter (likely `ergon_core/core/dashboard/emitter.py::graph_mutation` - or similar) -- the dashboard Zod schema - (`ergon-dashboard/src/lib/contracts/events.ts` — search for graph.mutation - / `parseDashboardGraphMutation*`). - -The emitter is passing fields under different names — probably `task_slug` -instead of `task_key` and `assigned_worker_slug` instead of -`assigned_worker_key`. The emitter-side contract file -(`ergon_core/core/dashboard/event_contracts.py`) uses `slug`; dashboard -expects `key`. One side needs to change (or the contract file should -declare both aliases, matching the `name` / `name_field` alias trick already -used for `TaskTreeNode`). - -### Scope - -Every dynamic task graph mutation on every run. Dashboard freezes at the -initial tree. Smoke runs' 9-leaf DAG never renders its spawned children. - -### Proposed fix - -**Superseded by § E — systemic fix.** The scoped fix would be: rename -dashboard-side `task_key` → `task_slug` and `assigned_worker_key` → -`assigned_worker_slug` in two files (`graphMutations.ts:21,26,30` + -`graphMutationReducer.ts:204,211,279`) plus tests. ~5 line edits. But -that patches one symptom; § E fixes the whole drift class at once. - ---- - -## D. `ergon-dashboard-dashboard-task-evaluation-updated` — missing criterion fields - -### Symptom - -Dashboard returns HTTP 500 for every `dashboard/task.evaluation_updated` -event: - -``` -[ - { path: ["evaluation","criterionResults",0,"id"], - message: "Invalid input: expected string, received undefined" }, - { path: ["evaluation","criterionResults",0,"stageNum"], - message: "expected number, received undefined" }, - { path: ["evaluation","criterionResults",0,"stageName"], ... }, - { path: ["evaluation","criterionResults",0,"criterionNum"], ... }, - { path: ["evaluation","criterionResults",0,"criterionDescription"], ... } -] -``` - -Evaluation results never render on the dashboard. For smoke, this means -criterion scores are invisible even though they're correctly persisted in -`run_task_evaluations`. - -### Repro - -Any run that produces an evaluation. 3 failures from one minif2f run. - -### Root cause - -Same class as B. `DashboardTaskEvaluationUpdatedEvent` in -`ergon_core/core/dashboard/event_contracts.py` embeds an `evaluation: dict` -(opaque); whatever builds that dict doesn't populate -`criterionResults[].{id, stageNum, stageName, criterionNum, -criterionDescription}`. Dashboard Zod expects those fields on every -`criterionResults` row. - -The backend sender is in -`ergon_core/core/runtime/inngest/evaluate_task_run.py` (`dashboard_emitter. -task_evaluation_updated` at line 211 per `grep -rn`). It builds the -`evaluation` dict from… the `RunTaskEvaluationDto`, presumably. Check -whether the DTO has these fields and is just passing them through, or -whether they exist on the eval record but aren't serialized into the -criterion-results list. - -### Scope - -Every evaluation on every run. Silent for operators — dashboard just -doesn't update the score panel. - -### Proposed fix - -**Superseded by § E — systemic fix.** The scoped fix would be: replace -the `evaluation: dict[str, Any]` placeholder on -`DashboardTaskEvaluationUpdatedEvent` with a proper pydantic DTO that -carries `criterionResults[].{id, stageNum, stageName, criterionNum, -criterionDescription}`, thread those fields through the emitter in -`evaluate_task_run.py:211`. But doing so in isolation just moves the -drift problem — the hand-written dashboard Zod is the real liability. - ---- - -## E. Systemic: pydantic → Zod codegen is broken; two schema sources have drifted - -### Symptom - -Bugs C and D above are both **"dashboard hand-wrote a Zod schema that -disagrees with what the backend pydantic actually emits."** Neither is -a semantic problem; both are drift between two parallel sources of -truth. The `task_tree={}` bug that the sub-agent just fixed in -`start_workflow.py` was the *third* instance of the same class. This -keeps happening because the type pipeline that was *supposed* to prevent -it is broken on three dimensions. - -### Repro - -Grep: `find . -name "*.py" -exec grep -l 'dict\[str, Any\]' {} \;` on -dashboard event contracts returns several opaque fields. Try -`pnpm run generate:contracts` in `ergon-dashboard/` — step 1 -(`generate:contracts:schemas`) fails because -`scripts/export_contract_schemas.py` does not exist in the repo. - -### Root cause - -The pipeline that *should* give end-to-end type safety: - -``` -Python pydantic model (source of truth) - → scripts/export_contract_schemas.py → events/schemas/*.schema.json - → pnpm exec json-schema-to-zod → events/<Name>.ts (generated Zod) - → dashboard imports from "@/generated/events" → parser used at runtime -``` - -Three independent failures stack: - -1. **The exporter script is missing.** `package.json:12` references - `scripts/export_contract_schemas.py`; that file is not in the repo. - Either never committed or deleted. Tracked `.schema.json` files - under `ergon-dashboard/src/generated/events/schemas/` are stale - snapshots — no one can regenerate them. - -2. **Pydantic escape hatches — `dict[str, Any]` on multiple events.** - `DashboardTaskEvaluationUpdatedEvent.evaluation: dict[str, Any]`, - `DashboardWorkflowStartedEvent.task_tree: dict[str, Any]`, and others - export as `{type: object, additionalProperties: true}` → Zod - generates `z.any()` → validates nothing. Even if the exporter is - restored, these types contribute zero safety until they're tightened - to proper nested pydantic DTOs. - -3. **Parallel hand-written Zod in `lib/contracts/events.ts`.** The - dashboard keeps a *second* strict parser per event type — - `parseDashboardTaskEvaluationUpdatedData` at `events.ts:310`, - `parseDashboardGraphMutationData` at `events.ts:421`, etc. These - were written by hand with field expectations the backend never - actually committed to. Inngest handlers use the strict parser; it's - the drift surface. The generated schemas are ignored at runtime. - -### Scope - -Every `dashboard/*` event. Any time the backend adds or renames a -field, one of: (a) the generated schema silently accepts it via -`z.any()`, (b) the hand-written parser silently rejects it, (c) both. -The failure mode is always "dashboard 500s, backend is fine, no shared -test exercises the boundary." Recurring symptom, no structural fix. - -### Proposed fix - -Four-step restore-and-tighten: - -1. **Restore `scripts/export_contract_schemas.py`.** Walk every - `InngestEventContract` subclass under - `ergon_core.core.dashboard.event_contracts`, call - `model.model_json_schema()`, write to the path named in - `ergon-dashboard/src/generated/events/schemas/manifest.json`. - Probably ~40 lines. If git history has a prior version, resurrect - it; otherwise write a fresh one. First run will likely show diffs - against currently-tracked schemas, proving the drift. - -2. **Tighten pydantic — no `dict[str, Any]` on dashboard events.** - Audit every `Dashboard*Event`; replace opaque-dict fields with real - nested DTOs. Candidates found: - - - `DashboardTaskEvaluationUpdatedEvent.evaluation` → - proper `DashboardTaskEvaluationDto` (criteria results etc.) - - `DashboardWorkflowStartedEvent.task_tree` → - proper `TaskTreeNode` recursive model (one already exists in the - same file but is currently unused here) - - any other `dict[str, Any]` / `Any` leaks surfaced while auditing - -3. **Regenerate, delete hand-written parsers.** Run - `pnpm run generate:contracts`. Delete every hand-authored - `parseDashboard*Data` in `lib/contracts/events.ts` and the - parallel hand-written Zod in - `features/graph/contracts/graphMutations.ts`. Route every dashboard - consumer through `@/generated/events/*` so there is one source of - truth. - -4. **CI check.** Add a job to `ci-fast.yml` that runs - `pnpm run generate:contracts` and fails the build if it produces a - diff against tracked files. Now any backend type change that - forgets to regenerate fails in PR, not in prod. - -After this lands, bugs C and D are solved by construction and every -future dashboard contract change is covered. Ongoing cost: two commands -(`pnpm run generate:contracts && git add src/generated`) whenever a -`Dashboard*Event` type changes. - -### Risk - -The strict hand-written parsers may have caught *other* drifts that -haven't surfaced yet — deleting them and regenerating could unmask -those. Mitigation: do step 2 (tighten) thoroughly; the generated Zod -is only as good as the pydantic source. Run the full e2e smoke suite -before landing step 3. - ---- - -## Ordering / priority - -1. **A1 (defensive except block)** — trivial, do first. Makes every future - `task-execute` failure actually write `error_json`, which lets the next - investigator see what's really going wrong without doing the GraphQL - dance. -2. **A2 (node_id / task_id unification)** — highest-value root cause; - unblocks smoke entirely. Plan in `README.md § Open refactors`. -3. **B (CANCELLED enum)** — migration-only; unblocks any future cancel flow. -4. **E (contract-gen pipeline)** — fixes C and D by construction, plus every - future dashboard-contract drift of the same class. Four sub-steps - (restore exporter → tighten pydantic → regenerate + delete hand-written - → CI check); step 1 is fastest and surfaces the blast radius. Smoke's - pytest assertions pass without this (they hit DB directly) but the - Playwright dashboard-state assertions will fail until E lands. - -C and D are not listed separately — § E subsumes them. - -## Not in scope of this report - -- The `task_tree={}` bug in `start_workflow.py` — already patched in a - separate change (same class as C and D). -- The uvicorn logger swallowing issue — mitigated by adding - `logging.basicConfig` to `app.py`; no separate bug file since the - Inngest-GraphQL path is now the preferred debug surface anyway (see - `CLAUDE.md`). -- The smoke criterion firing before children complete — by design; the - smoke parent worker should `await` its subtree before returning. Tracked - as a smoke-fixture fix, not a core bug. diff --git a/docs/bugs/open/2026-05-14-pr-3-prepare-run-node-detached-instance.md b/docs/bugs/open/2026-05-14-pr-3-prepare-run-node-detached-instance.md deleted file mode 100644 index 7099bd524..000000000 --- a/docs/bugs/open/2026-05-14-pr-3-prepare-run-node-detached-instance.md +++ /dev/null @@ -1,170 +0,0 @@ ---- -status: open -opened: 2026-05-14 -fixed_pr: null -priority: P0 -invariant_violated: null -related_rfc: docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/ ---- - -# Bug: PR 3 `_prepare_run_node` accesses ExperimentDefinition after session close - -## Symptom - -Every benchmark smoke (`minif2f`, `swebench-verified`, `researchrubrics`) -on `feature/v2-pr-*` branches starting at PR 3 hangs in -`status: "running"` with `context_event_count: 0` and `error: null` -until the harness times out at 270s. The run never produces a single -context chunk, never transitions out of running, and never surfaces -the underlying exception. - -Confirmed regression boundary from CI history: - -- **PR 52** (PR 1 run-tier snapshot, sha `79d043c`): smokes ✅ -- **PR 52** (PR 2 typed run-node boundary, sha `03409b3`): smokes ✅ -- **PR 53** (PR 3 worker_execute typed run nodes, sha `8651fd3`): smokes ❌ -- **PR 54** (PR 4 sync fanout): smokes ❌ -- **PR 55** (PR 4.5 ANN gaps): smokes ❌ -- **PR 56** (PR 5 object-bound API): smokes ❌ (first surfaced as - `ConfigurationError: task.worker is None` — masked the underlying - hang. PR 5's `_legacy_worker_bridge.py` fix restored compat with - `TaskSpec`-returning benchmarks; smokes now hit the same hang the - earlier PRs hit.) - -## Repro - -`gh run view --log --job 75958883560` (PR 56 minif2f) shows the -sequence: - -```text -mutations: [ - {sequence: 0, mutation_type: "node.added", target_task_slug: "mathd_algebra_478"}, - {sequence: 1, mutation_type: "annotation.set", ...}, - {sequence: 2, mutation_type: "node.status_changed", ...}, # pending → ready - {sequence: 3, mutation_type: "node.status_changed", ...}, # ready → running -] -executions: [{status: "running", error: null, task_slug: "mathd_algebra_478"}] -graph_nodes: [{status: "running", level: 0, parent_node_id: null, ...}] -context_event_count: 0 -``` - -The api container log dump (`docker compose logs api --tail 200`) is -silent past Alembic startup — per CLAUDE.md the uvicorn reloader eats -worker-side stdout in CI. The Inngest GraphQL endpoint isn't reachable -from CI, so the function-level error (if any) is lost. - -## Root cause - -**Confirmed empirically** via local stack + Inngest GraphQL probe at -`http://localhost:8289/v0/gql`. The hypothesis about sandbox-key -divergence was wrong; the actual error is a SQLAlchemy -`DetachedInstanceError` in `ergon-core-task-execute`: - -``` -Instance <ExperimentDefinition at 0x...> is not bound to a Session; -attribute refresh operation cannot proceed -(https://sqlalche.me/e/20/bhk3) -``` - -Worker-execute never fires — the orchestrator (`execute_task`) -crashes during `prepare`, before sandbox_setup. Inngest swallows the -SQLAlchemy error and silently retries with exponential backoff. The -test sees `status: "running"` (task was set RUNNING before the crash) -and `context_event_count: 0` (no worker ever ran) until the 270s -timeout. - -**Source:** `ergon_core/.../application/tasks/execution.py:88-169` -(`TaskExecutionService._prepare_run_node`, introduced in PR 3). - -The function takes this shape: - -```python -with get_session() as session: - view = await self._graph_repo.node(...) - definition = session.get(ExperimentDefinition, command.definition_id) - ... - session.add(execution); session.flush() - await self._graph_repo.update_node_status(...) - session.commit() # ← expires all ORM instances - # (expire_on_commit=True default) -# ← session closes here - -return PreparedTaskExecution( - ..., - benchmark_type=definition.benchmark_type, # ← DetachedInstanceError - execution_id=execution.id, # ← DetachedInstanceError -) -``` - -`session.commit()` expires every ORM instance loaded in the session -(SQLAlchemy default `expire_on_commit=True`). The next attribute -access triggers a refresh from the DB — but the `with` block has -already closed the session, so the refresh fails. - -Pre-PR-3, the equivalent code paths (`_prepare_legacy_graph_native` -and `_prepare_legacy_definition`) read the same fields *inside* the -session block before commit, so the bug didn't exist. - -## Why CI logs hid this - -CLAUDE.md warns that uvicorn's reloader eats handler stdout in the api -container, so `docker compose logs api` was empty. The smoke test -only sees the harness-level `TimeoutError` because Inngest absorbs the -SQLAlchemy error from `task-execute` into its retry queue. The -authoritative source — Inngest's GraphQL endpoint — is reachable only -from the host network, not from CI runners. - -## Scope - -- All three canonical smokes (`minif2f`, `swebench-verified`, - `researchrubrics`) fail on every push to `feature/v2-pr-3+` branches. -- Unit / state / integration tests are unaffected — the regression is - observable only against a live sandbox lifecycle. -- Production unaffected — none of the v2 PRs have shipped to `main` - yet (latest `main` is sha `3aaa844` 2026-05-13). - -## Proposed fix - -Capture the ORM-derived scalars (`definition.benchmark_type` and -`execution.id`) into local variables *inside* the session block before -`session.commit()` expires the instances. Two-line fix. - -```python -# inside the `with get_session() as session:` block, before commit -benchmark_type = definition.benchmark_type -execution_id = execution.id -... -session.commit() - -# outside the block — use the captured locals -return PreparedTaskExecution( - ..., - benchmark_type=benchmark_type, - execution_id=execution_id, -) -``` - -Alternative considered: pass `expire_on_commit=False` to the session -factory. Rejected — that's a global behaviour change that would mask -similar bugs elsewhere. The narrow fix is to read the values you need -before the lifetime boundary, which is also clearer to future readers. - -**Follow-ups worth queuing (not blocking this fix):** - -1. The architecture guard `test_repository_layer_conventions.py` - should add a check that no `PreparedTaskExecution`-style return - value is constructed outside the session block from ORM instances - loaded inside. Hand-rolled grep would catch - `definition.<attr>` after `session.commit()`. -2. Inngest's retry on SQLAlchemy errors hides bugs. Worth a separate - bug to either classify SQLAlchemy errors as `NonRetriableError` at - the orchestrator boundary, or to flush the function-error payload - into the api container logs so `docker compose logs api` would - have surfaced this without GraphQL. - -## On fix - -When moving from `open/` to `fixed/`: - - Set `status: fixed` and `fixed_pr: <PR#>` in frontmatter. - - Update the v2 implementation-plan doc set (PR 3 plan) to call - out the sandbox-key alignment. diff --git a/docs/dashboard/snapshot-parity-fix.md b/docs/dashboard/snapshot-parity-fix.md deleted file mode 100644 index 4b0d01755..000000000 --- a/docs/dashboard/snapshot-parity-fix.md +++ /dev/null @@ -1,359 +0,0 @@ -# Fix Plan: Dashboard Snapshot Parity + Generation Logging - -**Status:** Draft -**Related docs:** -- `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md` — generation turn persistence (generation logging fix lives here) -- `docs/event-wal/STATE_UNIFICATION_PLAN.md` — graph WAL as single source of truth (upstream dependency) - -**Goal:** After a page reload, `GET /runs/{run_id}` returns exactly the same run state that the live WebSocket stream would have built — including dynamic tasks, generation turns, and the timeline. Postgres is the single source of truth; the in-memory socket store is a read-through cache. - ---- - -## 1. Problem Statement - -The dashboard has two paths to run state: - -| Path | Source | Used when | -|------|--------|-----------| -| REST snapshot | `GET /runs/{run_id}` → `build_run_snapshot` → PG | Page load / reload | -| Live stream | WebSocket `graph:mutation`, `task:status`, `generation:turn` events | While the run is active | - -These two paths currently diverge in four ways. After reload, the user sees an incomplete or empty dashboard. - ---- - -## 2. The Four Gaps - -### Gap 1 — Dynamic tasks invisible on reload - -**Root cause:** `build_run_snapshot` reads only `ExperimentDefinitionTask` (the static workflow definition). Dynamic nodes — spawned at runtime and stored in `RunGraphNode` with `definition_task_id IS NULL` — are never queried. - -**Live stream behaviour:** Every `graph:mutation node.added` event (including dynamic nodes) is applied via `applyGraphMutation`, keyed by `RunGraphNode.id`. - -**Impact:** After reload, the task DAG is missing any nodes added dynamically. The graph renders the skeleton from the definition only. - -### Gap 2 — Canonical task key is inconsistent - -**Root cause:** The REST snapshot currently keys the task map by `ExperimentDefinitionTask.id`. The live stream keys every task (static and dynamic) by `RunGraphNode.id` — because `graph:mutation` events use `target_id = RunGraphNode.id`. These are different UUIDs. - -**Impact:** Even for static tasks, `executionsByTask`, `resourcesByTask`, and `evaluationsByTask` built from the REST snapshot may not match task IDs the frontend uses after applying live mutations. The per-task data panels are unreliable. - -**Decision:** Use `RunGraphNode.id` as the canonical task key everywhere — in the snapshot task map, in execution/resource/evaluation keying, and in all helper functions. This aligns with the live stream and with the graph layer's identity model. - -### Gap 3 — `generation_turns_by_task` is never populated - -**Root cause:** `RunSnapshotDto.generation_turns_by_task` has `default_factory=dict` and `build_run_snapshot` never sets it. `RunGenerationTurn` rows are not loaded in the snapshot builder at all. - -**Impact:** The generation turns panel is structurally impossible to populate on reload — always blank regardless of what's in PG. - -### Gap 4 — `GET /runs/{run_id}/mutations` endpoint does not exist - -**Root cause:** `RunGraphMutation` rows are written to PG by `WorkflowGraphRepository` on every mutation, but there is no FastAPI route that reads them. The frontend Timeline feature calls `/api/runs/{runId}/mutations` which proxies to this missing endpoint. - -**Impact:** The Timeline tab always 502s. The mutation scrubber cannot function. - ---- - -## 3. The Fix — Two Phases - -These are separated because Phase 1 is entirely backend API changes (no breaking changes, no new infrastructure) and Phase 2 is a larger architectural change to the worker interface already planned in `02_INCREMENTAL_PERSISTENCE.md`. - ---- - -## Phase 1: Snapshot Parity (immediate) - -All changes are in `ergon_core/ergon_core/core/api/runs.py` except where noted. - -### 1.1 Replace `_build_task_tree` with a node+edge-based builder - -**Drop `ExperimentDefinitionTask` from the task map entirely.** - -`initialize_from_definition()` in `WorkflowGraphRepository` already copies every definition task into `RunGraphNode` when a run starts. So for any initialized run, all tasks — static and dynamic — already exist as `RunGraphNode` rows with `task_key`, `description`, `status`, and `assigned_worker_key`. There is no information in `ExperimentDefinitionTask` that isn't already on the node. - -Replace `_build_task_tree` with a new function that takes only nodes and edges: - -```python -def _build_task_map( - nodes: list[RunGraphNode], - edges: list[RunGraphEdge], - worker_by_binding: dict[str, ExperimentDefinitionWorker], - task_timestamps: dict[UUID, tuple[datetime | None, datetime | None]], -) -> tuple[dict[str, RunTaskDto], str, int, int, int, int, int]: - """Build the flat task map from graph nodes and edges. - - Returns (task_map, root_node_id, total, total_leaf, completed, failed, running). - """ - if not nodes: - return {}, "", 0, 0, 0, 0, 0 - - # Build initial task map — all nodes start as leaves at level 0 - task_map: dict[str, RunTaskDto] = {} - for node in nodes: - nid = str(node.id) - worker = worker_by_binding.get(node.assigned_worker_key or "") - started_at, completed_at = task_timestamps.get(node.id, (None, None)) - task_map[nid] = RunTaskDto( - id=nid, - name=node.task_key, - description=node.description, - status=node.status, - parent_id=None, - child_ids=[], - depends_on_ids=[], - is_leaf=True, - level=0, - assigned_worker_id=str(worker.id) if worker else None, - assigned_worker_name=node.assigned_worker_key, - started_at=started_at, - completed_at=completed_at, - ) - - # Wire parent/child and dependency edges (same logic as applyEdgeAdded) - for edge in edges: - src = str(edge.source_node_id) - tgt = str(edge.target_node_id) - source_task = task_map.get(src) - target_task = task_map.get(tgt) - if source_task is None or target_task is None: - continue - - if target_task.parent_id is None: - # parent → child edge - task_map[tgt] = target_task.model_copy( - update={"parent_id": src, "level": source_task.level + 1} - ) - task_map[src] = source_task.model_copy( - update={"child_ids": [*source_task.child_ids, tgt], "is_leaf": False} - ) - elif target_task.parent_id != src: - # dependency edge - task_map[tgt] = target_task.model_copy( - update={"depends_on_ids": [*target_task.depends_on_ids, src]} - ) - - # Find root (no parent), compute counts from leaves - root_id = next((t.id for t in task_map.values() if t.parent_id is None), "") - total = len(task_map) - leaves = [t for t in task_map.values() if t.is_leaf] - total_leaf = len(leaves) - completed = sum(1 for t in leaves if t.status == "completed") - failed = sum(1 for t in leaves if t.status == "failed") - running = sum(1 for t in leaves if t.status == "running") - - return task_map, root_id, total, total_leaf, completed, failed, running -``` - -`build_run_snapshot` now loads nodes and edges instead of definition tasks and dependencies: - -```python -nodes_stmt = select(RunGraphNode).where(RunGraphNode.run_id == run_id) -nodes = list(session.exec(nodes_stmt).all()) - -edges_stmt = select(RunGraphEdge).where(RunGraphEdge.run_id == run_id) -edges = list(session.exec(edges_stmt).all()) -``` - -No `ExperimentDefinitionTask`, no `ExperimentDefinitionTaskDependency`, no definition-to-node ID mapping step. - -`_current_task_statuses` is also eliminated — status is already on each `RunGraphNode` row, read directly in `_build_task_map`. - -### 1.2 Fix execution, resource, and evaluation keying - -**Always use `ex.node_id`:** - -```python -execution_task_map: dict[UUID, UUID] = { - ex.id: ex.node_id for ex in executions if ex.node_id is not None -} -``` - -The `if ex.node_id is not None` guard exists because `RunTaskExecution.node_id` is a nullable column. It is nullable for three reasons: -1. The legacy `TelemetryRepository.create_task_execution()` path never sets `node_id` — only `definition_task_id`. -2. `GraphNodeLookup.node_id()` can return `None` if called before the graph is initialized. -3. Historical rows in the database predate the `node_id` column. - -The new execution service paths (`_prepare_node`, `_prepare_definition`) always set `node_id`, so executions from current code will always have it. Executions where `node_id IS NULL` are legacy rows; they will not appear in `execution_task_map` and their turns will be silently dropped from `generation_turns_by_task`. This is acceptable for now. - -**Schema tightening (not in Phase 1):** Once `STATE_UNIFICATION_PLAN.md` retires the `RunTaskStateEvent` propagation path, `TelemetryRepository.create_task_execution()` can be updated to require `node_id`. The DB column stays nullable for historical rows, but the application-layer type becomes `node_id: UUID` (non-optional). This is tracked alongside the type-tightening work in `02_INCREMENTAL_PERSISTENCE.md` §6.3. - -`_task_keyed_executions` and `_task_keyed_resources` key by `str(ex.node_id)` — matching the task map keys from step 1.1. - -`_task_keyed_evaluations` currently keys by `str(ev.definition_task_id)`. Since evaluations link to definition tasks (not nodes), a secondary lookup is needed to remap: - -```python -# Built from nodes loaded in step 1.1 -defn_to_node: dict[UUID, UUID] = { - n.definition_task_id: n.id - for n in nodes - if n.definition_task_id is not None -} -``` - -Then in `_task_keyed_evaluations`: key by `str(defn_to_node.get(ev.definition_task_id))` instead of `str(ev.definition_task_id)`. - -### 1.4 Populate `generation_turns_by_task` - -Load `RunGenerationTurn` rows for the run and group by the task's `node_id` via the execution lookup: - -```python -# In build_run_snapshot, after loading executions: - -gen_turns_stmt = ( - select(RunGenerationTurn) - .where(RunGenerationTurn.run_id == run_id) - .order_by(RunGenerationTurn.task_execution_id, RunGenerationTurn.turn_index) -) -gen_turns = list(session.exec(gen_turns_stmt).all()) - -gen_turns_by_task: dict[str, list[RunGenerationTurnDto]] = defaultdict(list) -for turn in gen_turns: - task_key = execution_task_map.get(turn.task_execution_id) - if task_key is None: - continue - gen_turns_by_task[str(task_key)].append( - RunGenerationTurnDto( - id=str(turn.id), - task_execution_id=str(turn.task_execution_id), - worker_binding_key=turn.worker_binding_key, - turn_index=turn.turn_index, - prompt_text=turn.prompt_text, - raw_response=turn.raw_response, - response_text=turn.response_text, - tool_calls=turn.tool_calls_json, - tool_results=turn.tool_results_json, - policy_version=turn.policy_version, - has_logprobs=turn.token_ids_json is not None, - created_at=turn.created_at.isoformat() if turn.created_at else None, - ) - ) -``` - -Then pass `generation_turns_by_task=dict(gen_turns_by_task)` to `RunSnapshotDto(...)`. - -### 1.5 Add `GET /{run_id}/mutations` endpoint - -```python -@router.get("/{run_id}/mutations", response_model=list[RunGraphMutationDto]) -def get_mutations(run_id: UUID) -> list[RunGraphMutationDto]: - """Return the append-only mutation log for a run, ordered by sequence. - - Used by the Timeline scrubber to replay DAG state at any point in time. - """ - with get_session() as session: - run = session.get(RunRecord, run_id) - if run is None: - raise HTTPException(status_code=404, detail=f"Run {run_id} not found") - - stmt = ( - select(RunGraphMutation) - .where(RunGraphMutation.run_id == run_id) - .order_by(RunGraphMutation.sequence) - ) - mutations = list(session.exec(stmt).all()) - - return [ - RunGraphMutationDto( - id=str(m.id), - run_id=str(m.run_id), - sequence=m.sequence, - mutation_type=m.mutation_type, - target_type=m.target_type, - target_id=str(m.target_id), - actor=m.actor, - old_value=m.old_value, - new_value=m.new_value, - reason=m.reason, - created_at=m.created_at.isoformat(), - ) - for m in mutations - ] -``` - -`RunGraphMutationDto` needs adding to `schemas.py`. Its field names and types must match the frontend's `GraphMutationDtoSchema` in `graphMutations.ts` exactly. - -### Phase 1 file map - -``` -ergon_core/ergon_core/core/api/runs.py - - _build_task_tree(): deleted - - _current_task_statuses(): deleted (status now read directly from RunGraphNode in _build_task_map) - + _build_task_map(nodes, edges, worker_by_binding, task_timestamps): new, replaces both - ~ _task_keyed_executions(): always use ex.node_id - ~ _task_keyed_resources(): execution_task_map keyed by node_id - ~ _task_keyed_evaluations(): remap via defn_to_node lookup - ~ build_run_snapshot(): load RunGraphNode + RunGraphEdge instead of ExperimentDefinitionTask + deps - ~ build_run_snapshot(): build defn_to_node map for evaluation remapping - ~ build_run_snapshot(): execution_task_map keyed by node_id - ~ build_run_snapshot(): load RunGenerationTurn, populate generation_turns_by_task - + GET /{run_id}/mutations endpoint - -ergon_core/ergon_core/core/api/schemas.py - + RunGraphMutationDto (matches GraphMutationDtoSchema in the frontend) -``` - ---- - -## Phase 2: Generation Logging (incremental turn persistence) - -**Status: Already implemented.** `02_INCREMENTAL_PERSISTENCE.md` has been fully shipped. This section is kept for context on why Phase 1 alone gives partial parity and what the implemented system provides. - -### Why Phase 1 alone gives only partial parity - -Phase 1 populates `generation_turns_by_task` from whatever `RunGenerationTurn` rows exist in PG at snapshot time. If turns were still batch-written post-completion, reload during a live run would return `generation_turns_by_task: {}` because no rows would exist yet. - -### What the implemented system delivers - -`Worker.execute()` is an async generator. The runtime calls `GenerationTurnRepository.persist_single()` for each yielded `GenerationTurn` and notifies `DashboardEmitter.on_turn_persisted()` immediately after each commit. This means: - -- Each turn is in PG the moment it completes — before the next turn starts -- `generation:turn` socket events are emitted per-turn (not batched post-completion) -- Reload at any point during execution shows all turns completed so far -- Worker crash loses at most one in-flight turn; prior turns are marked `execution_outcome="failure"` via `mark_execution_outcome()` - -With Phase 2 already in place, Phase 1's population of `generation_turns_by_task` from PG achieves full live parity immediately — there is no gap between PG state and the socket stream for generation turns. - ---- - -## 4. Sequencing - -``` -Phase 2 (generation logging) — DONE (02_INCREMENTAL_PERSISTENCE.md shipped) - ✓ Worker.execute() async generator - ✓ Per-turn persist_single() in worker_execute_fn - ✓ Per-turn generation:turn dashboard event via repository listener - ✓ execution_outcome tracking + crash recovery - -Phase 1 (snapshot parity) — implement now - ├── 1.1 Replace _build_task_tree with _build_task_map(nodes, edges) - ├── 1.2 Fix execution/resource/evaluation keying to use node_id - ├── 1.3 Dynamic nodes included automatically (no special handling) - ├── 1.4 Populate generation_turns_by_task - └── 1.5 Add mutations endpoint -``` - -Phase 2 is already done. Phase 1 can ship immediately and independently — no new infrastructure required. - ---- - -## 5. Invariant to Verify - -After both phases, this test should pass: - -``` -1. Start a run -2. Fetch GET /runs/{run_id} mid-execution (REST snapshot) -3. Fetch sync:run from socket (in-memory store) -4. Assert: tasks maps are equal (same keys, same statuses) -5. Assert: generationTurnsByTask is equal (same turns) -6. Assert: GET /runs/{run_id}/mutations returns non-empty list -7. Reload the page and assert the UI matches the live view -``` - ---- - -## 6. What This Doesn't Change - -- **`sync:run`** — still serves the in-memory store. Its role is "fast first paint for runs the server has seen since startup." REST snapshot is the authoritative fallback. -- **Socket event stream** — no changes. `graph:mutation`, `task:status`, `generation:turn` etc. are unchanged. -- **Frontend** — no changes required. `deserializeRunState` already reads `generationTurnsByTask`; `deserializeTask` already accepts the `RunTask` shape. The mutations proxy route already exists. Once the backend is correct, the frontend works. -- **`STATE_UNIFICATION_PLAN.md`** — Phase 1 works regardless of whether that migration has happened. `RunGraphNode.status` is already being written (it's how `_current_task_statuses` works today). That plan improves the write path, not the read path we're fixing here. diff --git a/docs/dead-code-audit-2026-04-25.md b/docs/dead-code-audit-2026-04-25.md deleted file mode 100644 index 8551b54aa..000000000 --- a/docs/dead-code-audit-2026-04-25.md +++ /dev/null @@ -1,184 +0,0 @@ -# Dead Code Audit - 2026-04-25 - -This document records a static dead-code audit of `ergon_core/ergon_core/core` -plus nearby call sites in `ergon_builtins`, `ergon_cli`, and tests. It is meant -to be an executable cleanup plan, not just an inventory. - -## Method - -- Searched repo-wide call sites with `rg`. -- Used focused read-throughs of the runtime propagation, persistence, generation, - evaluation, and RL paths. -- Installed and ran `vulture`: - - `python -m vulture ergon_core/ergon_core/core --min-confidence 80` - - `python -m vulture ergon_core/ergon_core tests ergon_builtins ergon_cli --min-confidence 60` -- Treated ORM models, Pydantic models, FastAPI route functions, registry-loaded - components, and public package exports conservatively. - -## Decision Legend - -| Decision | Meaning | -| --- | --- | -| Delete | Remove once a focused test/lint pass confirms no hidden caller. | -| Port/use | Old code contains useful behavior that should be moved into the active path before deletion. | -| Keep | Useful internal API, currently imported, or deliberately kept as an extension point. | - -## Executive Summary - -The largest risk is `core/runtime/execution/propagation.py`, which mixes active -graph-native propagation with stale legacy propagation helpers. The active path -is now service-driven: - -1. `task/completed` event calls `TaskPropagationService.propagate()`. -2. `TaskPropagationService.propagate()` marks the source node completed. -3. It calls `on_task_completed_or_failed()` to satisfy edges and activate ready nodes. -4. It calls `is_workflow_complete_v2()` / `is_workflow_failed_v2()` for terminal detection. - -Failures follow the same active helper through `TaskPropagationService.propagate_failure()`. -The old helpers still in the file either walk `ExperimentDefinitionTaskDependency` -directly or implement stale semantics. They should be removed before they are -accidentally reused. - -## Estimated Size Impact - -If every row currently marked `Delete` is removed, the cleanup removes about: - -- 6 entire modules. -- 38 audit-listed delete items. -- 13 top-level classes. -- 17 top-level functions. -- 45 class methods. -- 2 unused type aliases. -- 1,079 physical Python lines, or 941 nonblank Python lines. - -Against `ergon_core/ergon_core/core`, that is roughly 5.7% of physical Python -lines and 5.9% of nonblank Python lines (`18,919` physical lines, `15,883` -nonblank lines total). This estimate counts the deleted symbols/modules -themselves, not the small import/export/test cleanup that will come with them. - -Propagation alone accounts for 8 top-level functions and about 221 physical -lines. That is the most important qualitative reduction because it removes stale -alternative control flow, not just unused helpers. - -## Propagation Decision Table - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Runtime propagation | `core/runtime/execution/propagation.py` | `on_task_completed` | No repo-wide caller. Static dependency walker over `ExperimentDefinitionTaskDependency`. Replaced by `TaskPropagationService.propagate()` plus `on_task_completed_or_failed()`. Vulture flags it. | Delete | Legacy completion path. It writes source completion and dependency satisfaction in one helper, while the active service now separates node finalization from graph propagation. | Medium | Run propagation integration tests after removal. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `on_task_completed_by_node` | No repo-wide caller. Docstring says deprecated. Vulture flags it. | Delete | Superseded by `on_task_completed_or_failed()`. It also has stale behavior: it considers dependencies satisfied when source nodes are any terminal status, not only `COMPLETED`. | High if reused | Add/keep test coverage for fan-in failure remaining `BLOCKED`, then remove. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_completed` | Only used by dead `on_task_completed`. | Delete | No active caller. Completion is now handled in `TaskPropagationService.propagate()` with `WorkflowGraphRepository.update_node_status()`. | Low | Search after deleting `on_task_completed`. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_ready` | Used by `get_initial_ready_tasks` and dead `on_task_completed`. | Keep for now | Still used by workflow initialization for root tasks. Could be made private or moved near initialization later. | Low | Keep until `WorkflowInitializationService` is refactored. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `get_current_task_status` | Only used by dead `on_task_completed`. | Delete | Legacy helper needed only for static dependency walker. | Low | Search after deleting `on_task_completed`. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `is_workflow_complete` | No repo-wide caller. Vulture flags it. | Delete | Replaced by `is_workflow_complete_v2()`, which understands terminal statuses and failed/cancelled behavior. | Medium | Run terminal-state propagation tests. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `is_workflow_failed` | No repo-wide caller. Vulture flags it. | Delete | Replaced by `is_workflow_failed_v2()`, which handles `BLOCKED` as settled and waits for no pending/running work. | Medium | Run failure/blocking propagation tests. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_running_by_node` | No repo-wide caller. Vulture flags it. Graph-native prepare writes node status directly through `WorkflowGraphRepository`. | Delete | Unused wrapper around active repository method. | Low | Search after removal. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_completed_by_node` | No repo-wide caller. Vulture flags it. | Delete | Source completion is handled directly in `TaskPropagationService.propagate()`. | Low | Search after removal. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_running` | Imported by `TaskExecutionService._prepare_definition()`. | Keep | Active static definition execution path still uses it. | Low | Do not remove in first cleanup. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_failed` | Imported by `TaskExecutionService.finalize_failure()` for static tasks. | Keep | Active failure finalization path still uses it. | Low | Do not remove in first cleanup. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `mark_task_failed_by_node` | Imported by `TaskExecutionService.finalize_failure()` for dynamic graph nodes. | Keep | Active dynamic failure finalization path still uses it. | Low | Do not remove in first cleanup. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `get_initial_ready_tasks` | Imported by `WorkflowInitializationService.initialize()`. | Keep | Active workflow initialization path uses it to mark root tasks ready. | Medium | Consider moving into initialization service later, but not dead. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `on_task_completed_or_failed` | Imported by `TaskPropagationService`. Covered by propagation integration tests. | Keep | Active v2 propagation implementation. | High if changed | Treat as production-critical. | -| Runtime propagation | `core/runtime/execution/propagation.py` | `is_workflow_complete_v2`, `is_workflow_failed_v2` | Imported by `TaskPropagationService`. | Keep | Active terminal-state checks. | High if changed | Keep integration coverage around `BLOCKED`, `FAILED`, `CANCELLED`, and empty graph behavior. | - -## Runtime Events and Tracing - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Evaluation events | `core/runtime/events/evaluation_events.py` | `TaskEvaluationEvent` | Only re-exported from `events/__init__.py`; no in-repo instantiation. Current evaluation dispatch uses `EvaluateTaskRunRequest` and raw `task/evaluate` trigger names. | Delete | Runtime does not use it, and external compatibility is not a concern for this repo. | Medium | Remove export and run tests/import checks. | -| Evaluation events | `core/runtime/events/evaluation_events.py` | `CriterionEvaluationEvent` | Only re-exported from `events/__init__.py`; no active criterion event function found. | Delete | Orphaned planned feature/event contract, and external compatibility is not a concern for this repo. | Medium | Remove export and run tests/import checks. | -| Tracing | `core/runtime/tracing.py` | `action_context` | No repo-wide caller. | Delete | Unused context helper. Current code uses task/evaluation-specific context helpers and `emit_span()`. | Low | Search for `action_context` after removal. | -| Tracing | `core/runtime/tracing.py` | `TraceSink.add_event`, `TraceSink.child_context` | No active call sites; implemented on protocol/noop/otel sinks. | Keep | Reasonable extension points for tracing API, even if currently unused. | Low | Keep unless simplifying tracing API intentionally. | -| Delegation errors | `core/runtime/errors/delegation_errors.py` | `TaskNotPendingError` | Exported from `errors/__init__.py`, but no raises in repo. Comment says kept for backwards compatibility. | Delete | Backwards compatibility is not needed; no active runtime behavior depends on it. | Low | Remove export and run import checks. | -| Graph errors | `core/runtime/errors/graph_errors.py` | `AnnotationNotFoundError` | Re-exported but not raised in repo. | Delete | No active graph path raises it, and external compatibility is not a concern. | Low | Remove export and run import checks. | - -## Persistence - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Definitions persistence | `core/persistence/definitions/repositories.py` | `DefinitionRepository` | No repo-wide import or instantiation. Reads happen through `queries`, `graph_repository`, services, or direct sessions. | Delete | Superseded read repository. Docstring points writes elsewhere. | Medium | Ensure no package-level import consumers. | -| Saved specs persistence | `core/persistence/saved_specs/repositories.py` | `SavedSpecsRepository`, `saved_specs_repository` | No repo-wide import or call site. | Delete | Repository wrapper appears never wired. | Medium | Keep ORM models; only remove repository if imports stay clean. | -| Saved specs persistence | `core/persistence/saved_specs/models.py` | Saved spec ORM models | Models are schema objects and may be loaded by Alembic metadata. | Keep | ORM models are not dead just because repository is unused. | High | Do not remove without migration/schema audit. | -| Telemetry persistence | `core/persistence/telemetry/repositories.py` | `TelemetryRepository.get_run`, `get_task_executions`, `get_resources`, `create_run`, `create_task_execution`, `complete_task_execution`, `create_resource` | `TelemetryRepository` is used by `EvaluationPersistenceService`, but only evaluation methods are called in repo. Run/task/resource lifecycles are handled through services or direct ORM queries. | Delete | Stale facade methods. The active evaluation methods should stay, but these seven methods are not wired. | Medium | Delete the seven methods only; keep `create_task_evaluation`, `get_task_evaluations`, and `refresh_run_evaluation_summary`. | -| Telemetry persistence | `core/persistence/telemetry/repositories.py` | `TelemetryRepository.create_task_evaluation`, `refresh_run_evaluation_summary`, `get_task_evaluations` | Used by evaluation persistence flow. | Keep | Active evaluation persistence. | Medium | Do not remove. | -| Telemetry persistence | `core/persistence/telemetry/repositories.py` | `GenerationTurnRepository.persist_single` | Current worker execution persists `RunContextEvent`, not `RunGenerationTurn`. Earlier investigation treated this as a missing write, but the frontend/backend action-log direction is context events. | Delete | Do not revive the old generation-turn write path. Finish migrating readers/UI to `RunContextEvent`, then delete this stale writer. | High | Add tests proving yielded turns render from context events in both live updates and persisted snapshots. | -| Telemetry persistence | `core/persistence/telemetry/repositories.py` | `GenerationTurnRepository.add_listener`, `persist_turns`, `get_for_execution`, `get_for_run`, `mark_execution_outcome` | No generation-turn writes are active. `get_for_execution` is only used by the stale base `Worker.get_output()` path; `ReActWorker` already reads context events. Context event listeners are handled by `ContextEventRepository.add_listener`. | Delete | Once base `Worker.get_output()` and API readers use context events, the generation-turn repository surface is obsolete. | Medium | Migrate base output/readers first, then delete the repository class. | -| Query namespace | `core/persistence/queries.py` | `queries.evaluations` / `EvaluationsQueries` | No repo-wide `queries.evaluations` usage. Evaluation reads use direct `select(RunTaskEvaluation)` in run read paths or evaluation-specific services. | Delete | Unused namespace branch. With no external consumers, keeping a complete-but-unused query facade is unnecessary. | Medium | Remove `EvaluationsQueries` and `Queries.evaluations`; run import/type checks. | -| Query namespace | `core/persistence/queries.py` | `ResourcesQueries.list_latest_for_execution` | No repo-wide caller found. Other code uses `list_by_execution`, `list_by_run`, `latest_by_path`, `append`, or direct queries. | Delete | Unused convenience method. | Low | Remove method; run tests that touch resources. | -| Shared typing | `core/persistence/shared/types.py` | `BenchmarkSlug`, `ExecutionId` | No repo-wide imports beyond definition. Sibling aliases like `TaskSlug`, `AssignedWorkerSlug`, `NodeId`, `RunId`, `DefinitionId`, and `EdgeId` are used. | Delete | They are not part of the live type surface. | Low | Remove only these aliases. | - -## Generation, Judges, and Evaluation Helpers - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| LLM judge provider | `core/providers/judges/llm_judge.py` | Entire module, `LLMJudgeResponse`, `call_llm_judge` | No imports from `core.providers.judges`. Active judge behavior lives on `DefaultCriterionRuntime.call_llm_judge()` and includes runtime options like model, max tokens, and temperature. | Delete | Stale duplicate. If a shared helper is wanted later, extract it from the active runtime method rather than keeping this older subset. | Medium | Delete module and run evaluator tests. | -| VLLM generation | `core/providers/generation/vllm_model.py` | Entire module, `resolve_model_target`, `ResolvedModel`, `_discover_vllm_model_name`, `VLLMDiscoveryError` | Production imports use `core/providers/generation/model_resolution.py` and builtins vLLM backend. Only tests reference this stale module. | Delete | Duplicate model-resolution path. Keeping two implementations risks divergent behavior. | Medium | Remove/replace stale tests that target this module. | -| PydanticAI format parsing | `core/providers/generation/pydantic_ai_format.py` | `extract_text`, `extract_tool_calls` | No repo-wide callers. `extract_logprobs` is used by builtins `react_worker`. Active text/tool extraction uses typed `ModelResponse` / `GenerationTurn.response_parts`, not serialized dict parsing. | Delete | Unused dict parsers. Keep `extract_logprobs` and the module because logprob extraction is active. | Medium | Remove these two functions only; run generation-turn tests. | -| PydanticAI format parsing | `core/providers/generation/pydantic_ai_format.py` | `extract_logprobs` | Imported by `ergon_builtins/workers/baselines/react_worker.py`. | Keep | Active behavior. | Low | None. | -| Evaluation schema re-exports | `core/runtime/evaluation/evaluation_schemas.py` | `LLMJudgeResponse` | Listed in `__all__`, no imports from this file found. There is another `LLMJudgeResponse` in stale judge provider. | Delete | Redundant schema export with no in-repo caller. | Low | Remove and run evaluator/import tests. | -| Evaluation schema re-exports | `core/runtime/evaluation/evaluation_schemas.py` | `CommandResult`, `SandboxResult` re-exports | Imported into this module and re-exported, but no callers import these names from this module. | Delete | Barrel/re-export noise with no in-repo caller. | Low | Remove and run evaluator/import tests. | - -## RL and Rollout - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| RL polling | `core/rl/polling.py` | `PollTimeoutError`, `poll_until_all_complete` | No repo-wide caller. Docstring says used by TRL adapter, but adapter polls through `RolloutService`. | Delete | Stale helper. | Low | Search after deletion. | -| TRL adapter | `core/rl/trl_adapter.py` | `make_ergon_rollout_func` | No repo-wide caller. Module is explicitly marked deprecated in favor of HTTP adapter. Vulture flags unused `trainer` parameter. | Delete | Sunset path. `ergon_infra/adapters/trl_http.py` is the current replacement. | Medium | Confirm no external in-process TRL users. | -| RL package exports | `core/rl/__init__.py` | `VLLM_LOGPROB_SETTINGS` | Alias to `LOGPROB_SETTINGS`; no repo-wide caller. | Delete | Compatibility alias only, and external compatibility is not needed. | Low | Remove and run import checks. | - -## Miscellaneous Core - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Core utils | `core/utils.py` | `get_mime_type` | No repo-wide caller. | Delete | Small unused helper. | Low | Search after deletion. | -| OpenRouter budget | `tests/real_llm/openrouter_budget.py` | `OpenRouterBudget` | Referenced from real-LLM fixtures/benchmarks rather than active production modules. | Keep test-local | Useful for real-LLM test budget gating. Not part of core runtime. | Low | None. | -| Dashboard emitter | `core/dashboard/emitter.py` | `_RunContextEvent` import | Vulture flags unused import. | Delete | Straight unused import cleanup. | Low | Run lint/type check. | -| RL extraction | `core/rl/extraction.py` | `add_special_tokens` parameter on `Tokenizer.encode()` protocol | Vulture flags it, but it is part of a `Protocol` signature matching common tokenizer APIs. Callers intentionally use bare `tokenizer.encode(...)`. | Keep | Static-analysis false positive. The parameter documents compatibility with tokenizer implementations such as Hugging Face tokenizers. | Low | If vulture noise matters, suppress/allowlist instead of deleting the protocol parameter. | - -## Frontend / Dashboard Context-Event Migration - -| Area | File | Symbol / module | Current evidence | Decision | Why | Risk | Follow-up test/check | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Run state hydration | `ergon-dashboard/src/lib/runState.ts` | `deserializeRunState()` / `deserializeGenerationTurns()` | Snapshot hydration reads `generationTurnsByTask` but initializes `contextEventsByTask` as an empty `Map`. | Port/use | Snapshot state should hydrate `contextEventsByTask` from the backend and stop treating `generationTurnsByTask` as the action source. | High | Add a refresh/snapshot test proving tool calls render from persisted context events. | -| Run state live updates | `ergon-dashboard/src/hooks/useRunState.ts` | socket listener set | The hook subscribes to `generation:turn` but not `context:event`, even though the socket server broadcasts context events. | Port/use | Live action updates should flow through `context:event`. | High | Add a live delta test that posts/sends a context event and sees it in the task workspace/event stream. | -| Task action UI | `ergon-dashboard/src/components/workspace/TaskWorkspace.tsx` | `GenerationTracePanel` use | The workspace renders `GenerationTracePanel` from `runState.generationTurns`; `ContextEventLog` exists but is not mounted here. | Port/use | The task workspace should render agent actions from `ContextEventLog` / context-derived events. | High | Replace/demote the generations panel and assert tool calls/tool results appear for the selected task. | -| Unified event stream | `ergon-dashboard/src/lib/runEvents.ts` | generation/context event conversion | The stream includes both generation turns and context events, but context events are currently generic summaries and may never arrive in state. | Port/use | Use context events for action-level timeline entries; remove generation-turn timeline entries once the old source is gone. | Medium | Snapshot and live-stream tests should cover `tool_call`, `tool_result`, `assistant_text`, and `thinking`. | -| Dashboard event bridge | `ergon-dashboard/src/inngest/functions/index.ts` | `dashboard/generation.turn_completed` handler | Generation-turn dashboard handler remains, but Python no longer appears to emit generation-turn events; context-event handler is the active bridge. | Delete | Stale frontend event listener around the old generation-turn source. | Medium | Delete after context-event UI path is wired; verify no `generation:turn` tests depend on it. | -| Dashboard REST/API contracts | `ergon_core/core/api/schemas.py`, `ergon_core/core/runtime/services/run_read_service.py`, `ergon_core/core/api/runs.py`, `ergon-dashboard/src/generated/rest/contracts.ts` | `RunGenerationTurnDto`, `generation_turns_by_task`, `/runs/{run_id}/generations` | Backend snapshots still expose generation turns, and FE generated contracts still include `generationTurnsByTask`. | Delete | Once context-event hydration is exposed and consumed, the generation-turn API surface is stale. | High | Add/verify `contextEventsByTask` in contracts, then remove generation-turn endpoint/DTO fields. | -| Telemetry schema | `core/persistence/telemetry/models.py` and migrations | `RunGenerationTurn` table/model | The table/model exist for the old turn summary. Current canonical replay/action log is `RunContextEvent`. | Delete | Remove after all readers and tests migrate to context events. | High | Requires migration cleanup or a new migration strategy; do after application code no longer references it. | - -## Recommended Cleanup Order - -1. Propagation cleanup first: - - Delete only the stale helpers listed as `Delete`. - - Keep active helpers used by `TaskExecutionService`, `WorkflowInitializationService`, and `TaskPropagationService`. - - Run propagation integration tests. - -2. Low-risk stale modules: - - `core/utils.get_mime_type` - - `core/rl/polling.py` - - dashboard unused import - -3. Duplicate implementation cleanup: - - Decide whether to delete or centralize `core/providers/judges/llm_judge.py`. - - Delete stale `core/providers/generation/vllm_model.py` and update/remove tests that target it. - -4. Persistence cleanup: - - Remove unused repository wrappers only after confirming no external package imports. - - Do not remove ORM models as part of repository cleanup. - -5. Public-ish exports: - - Delete event/schema/error aliases directly; this repo has no external consumers to protect. - -6. Context-event dashboard migration: - - Wire snapshots and live socket updates to `contextEventsByTask`. - - Render task actions from context events. - - Remove generation-turn dashboard listeners, DTOs, API endpoint, repository, and table/model after readers are gone. - -## Propagation-Specific Conclusion - -The current active propagation logic is not missing from the old helpers; it is -more complete than the old helpers. The old helpers are stale because they either -operate on static definition dependencies or use pre-`BLOCKED` terminal semantics. -The highest-value cleanup is to remove the stale helpers from `propagation.py` -so future work cannot accidentally call them. - diff --git a/docs/event-wal/01_AUDIT.md b/docs/event-wal/01_AUDIT.md deleted file mode 100644 index a2f37f16a..000000000 --- a/docs/event-wal/01_AUDIT.md +++ /dev/null @@ -1,419 +0,0 @@ -# Event WAL Audit: Lossless State Reconstruction from Postgres - -**Goal:** Given a run where 6 of 10 DAG tasks completed, worker 7 fails 10 tool calls in after 46 seconds — can we rebuild the full DAG and its execution state from PG alone? - -**Answer today: no.** The DAG structure and task-level state survive, but the 10 turns of work inside the failed task are lost. Below is the full audit. - ---- - -## 1. Two Parallel State Systems - -Ergon currently has two independent DAG state representations that don't share writes. - -### System A: Propagation Layer (legacy — table dropped) - -**Files:** `propagation.py` - -This was what the Inngest runtime used. The `RunTaskStateEvent` table and its -`run_task_state_events` schema have been dropped. State is now tracked -exclusively via the Graph Layer (System B). - -**What it captured (historical):** -- Every task status transition (PENDING → RUNNING → COMPLETED/FAILED) -- Old status, new status, execution ID, error metadata -- `created_at` timestamps on each event - -**Superseded by:** `RunGraphMutation` / `RunGraphNode` (System B). - -### System B: Graph Layer - -**Files:** `run_graph_*` models, `WorkflowGraphRepository` - -The more sophisticated system — append-only mutation WAL with sequence numbers, point-in-time annotation queries (`get_annotation_at`), structural invariant enforcement (acyclicity via Kahn's). The `RunGraphAnnotation` docstring explicitly says it exists for *"counterfactual replay and credit assignment in the training pipeline."* - -**Tables:** -- `run_graph_nodes` — mutable task nodes per run, free-form status string -- `run_graph_edges` — mutable dependency edges per run -- `run_graph_annotations` — append-only namespaced metadata WAL (current = latest sequence, point-in-time = sequence ≤ N) -- `run_graph_mutations` — append-only audit log of every change (node.added, node.status_changed, edge.added, annotation.set, etc.) with old_value/new_value diffs and actor attribution - -**What it captures:** -- Every structural change to the DAG -- Full old/new value diffs with actor and reason -- Point-in-time reconstruction at any mutation sequence number - -**The problem (resolved):** Previously, the Inngest functions wrote to System A -(`RunTaskStateEvent`) but not to System B (`RunGraphMutation`). The -`RunTaskStateEvent` table has since been dropped. System B is now the -sole state source. - ---- - -## 2. What Survives the Failure Scenario - -### Tasks 1–6 (completed): Fully captured ✓ - -| Table | What's there | -|-------|-------------| -| `run_task_executions` | started_at, completed_at, final_assistant_message, status=COMPLETED | -| `run_generation_turns` | All turns: raw_request, raw_response, tool_calls, tool_results, logprobs | -| `run_actions` | Each action with started_at, completed_at, action_type, input/output | -| `run_resources` | Sandbox outputs downloaded and registered | -| `run_task_evaluations` | Scores, feedback, pass/fail per evaluator | - -### Task 7 (failed mid-execution): Partially captured ⚠️ - -| Table | What's there | -|-------|-------------| -| `run_task_executions` | status=RUNNING, started_at set, completed_at=NULL | - -Then, after the `except` block fires: - -| Table | What's written on failure | -|-------|--------------------------| -| `run_task_executions` | status=FAILED, completed_at=now, error_json={"message": ...} | - -### Task 7's 10 turns of work: **LOST** ✗ - -This is the critical gap. In `worker_execute.py`: - -```python -result = await worker.execute(task, context=worker_context) # ← crash here -_persist_generation_turns(payload, result) # ← never reached -``` - -`_persist_generation_turns()` is called **after** `worker.execute()` returns. If the worker crashes at turn 10: -- The `result` object never exists -- `_persist_generation_turns` never runs -- All 10 `RunGenerationTurn` rows that should exist **don't** -- The raw LLM exchanges, tool calls, tool results, logprobs — all gone - -The "lossless per-turn records" are only lossless if the worker completes successfully. - -### Task 7's sandbox state: **LOST** ✗ - -`persist_outputs_fn` only runs after worker success (it's `step.invoke`'d sequentially after `worker_execute_fn` returns). Crash = sandbox outputs never downloaded, `RunResource` rows never created. - -### Tasks 8–10 (never started): Correctly absent ✓ - -If they depended on task 7: they were never marked ready. If independent of 7: they may have started/completed on their own timeline. The propagation layer handles this correctly via `RunGraphNode` status. - ---- - -## 3. Additional Gaps - -### 3a. Per-Turn Timestamps - -`RunGenerationTurn.created_at` is set when the batch is persisted — all turns from a single task execution get roughly the same timestamp. For continuous-time trajectory claims and replay fidelity, each turn needs: -- `started_at`: when the LLM call was dispatched -- `completed_at`: when the response arrived - -These already exist on `RunAction` but not on `RunGenerationTurn`. Easy to add, but only meaningful once turns are persisted incrementally (see §4.1). - -### 3b. Batch State Is Memory-Only - -`RolloutService._batches` is a Python dict in the API process. If the API container restarts mid-training, all `batch_id → run_ids` mappings are lost. The trainer gets 404 and the training step fails. The code comments acknowledge this explicitly: - -> *"Acceptable because the API container doesn't restart mid-training; if it does the trainer gets a 404 and the training step fails explicitly."* - -### 3c. Thread ↔ Execution Linkage - -`ThreadMessage` has `run_id` but no `task_execution_id`. To reconstruct "what messages did agent A send during task 7's execution," you must infer from timestamps. No direct FK path. - -### 3d. Generation Turn `raw_request` Is Empty - -In `GenerationTurnRepository.persist_turns()`: - -```python -raw_request={}, # ← always empty -raw_response=turn.raw_response, -``` - -The request side of the LLM exchange is never persisted. For lossless reconstruction of what the agent saw before responding, this is a gap — you have the response but not the prompt/context that produced it. - -### 3e. Prompt Fidelity in Trajectory Extraction - -`RolloutService._extract_trajectories()` hardcodes the prompt: - -```python -prompt_text = tokenizer.apply_chat_template( - [{"role": "user", "content": "Complete the benchmark task."}], - ... -) -``` - -This is not the actual prompt the agent saw. For on-policy RL training and for the paper's "lossless trajectory" claim, the prompt should come from the persisted `raw_request` (once that's populated — see §3d). - ---- - -## 4. Recommended Fixes (Priority Order) - -### 4.1 Incremental Turn Persistence [Critical] - -**Problem:** Generation turns are batch-written post-completion. Crash loses all turns. - -**Fix:** Persist each turn to PG as it happens during worker execution. Options: - -- **A) Callback/sink pattern:** Pass a `TurnSink` to the worker that writes each `RunGenerationTurn` as it completes. The worker calls `sink.persist(turn)` after each LLM response + tool result cycle. -- **B) WorkerContext method:** Expose `context.persist_turn(turn)` that workers call incrementally. The context holds the session factory and execution IDs. -- **C) Per-turn Inngest steps:** Wrap each turn in its own `ctx.step.run()` for Inngest-level durability. Biggest architectural change but gives per-turn crash recovery semantics. - -**Recommendation:** Option B. It's the smallest change to the worker interface, it doesn't require restructuring the Inngest function, and it gives us incremental writes. Option C is worth considering for the full paper but is a larger refactor. - -**Side effect:** Once turns are written incrementally, `created_at` on each turn naturally reflects actual wall time, closing gap §3a for free. - -### 4.2 Unify State Systems [Resolved] - -**Problem (resolved):** Propagation layer and graph layer were parallel systems -with no shared writes. - -**Resolution:** `RunTaskStateEvent` and its `run_task_state_events` table have -been dropped (migration `307fcca3a621_drop_run_task_state_events`). The graph -mutation log (`RunGraphMutation` / `RunGraphNode`) is now the single source of -truth for task state. - -### 4.3 Populate `raw_request` [Medium] - -**Problem:** `RunGenerationTurn.raw_request` is always `{}`. - -**Fix:** In `GenerationTurnRepository.persist_turns()`, change: - -```python -raw_request=turn.raw_request if hasattr(turn, 'raw_request') else {}, -``` - -Or better: ensure the `GenerationTurn` API type carries `raw_request` and workers populate it. This gives full round-trip fidelity on the LLM exchange. - -### 4.4 Batch State to PG [Medium] - -**Problem:** `RolloutService._batches` is memory-only. - -**Fix:** Add a `rollout_batches` table: - -``` -rollout_batches: - id: UUID (PK) - definition_id: UUID (FK → experiment_definitions) - status: str (PENDING / RUNNING / COMPLETE) - created_at: datetime - -rollout_batch_runs: - batch_id: UUID (FK → rollout_batches) - run_id: UUID (FK → runs) -``` - -On API restart, `poll()` can reconstruct batch state from PG instead of returning 404. - -### 4.5 Thread ↔ Execution FK [Low] - -**Problem:** Can't directly query messages sent during a specific task execution. - -**Fix:** Add optional `task_execution_id: UUID | None` FK to `ThreadMessage`. Workers set it when sending messages during task execution; None for out-of-band messages. - -### 4.6 Per-Turn Timestamps [Low — Free with §4.1] - -**Problem:** All turns share the same `created_at`. - -**Fix:** Add `started_at` / `completed_at` to `RunGenerationTurn`. With incremental persistence (§4.1), `created_at` already reflects real time, but explicit start/end gives duration per turn which matters for the continuous-time trajectory paper claims. - ---- - -## 5. Coverage Tracker - -| § | Fix | Priority | Addressed by | -|---|---|---|---| -| 4.1 | Incremental turn persistence | Critical | `02_INCREMENTAL_PERSISTENCE.md` — async generator workers, per-yield PG writes | -| 4.2 | Unify state systems | Resolved | `RunTaskStateEvent` dropped; graph WAL is single source | -| 4.3 | Populate `raw_request` | Medium | `02_INCREMENTAL_PERSISTENCE.md` §6.2 + §11 (prompt fidelity fix) | -| 4.4 | Batch state to PG | Medium | Inline spec below — standalone small PR, ~1 day | -| 4.5 | Thread ↔ Execution FK | Low | Inline spec below — single column addition, ~half day | -| 4.6 | Per-turn timestamps | Low | `02_INCREMENTAL_PERSISTENCE.md` — `created_at` on each yield is wall-clock time | -| — | Workflow resumption | — | `03_WORKFLOW_RESUMPTION.md` — DAG-aware restart + worker resume from buffer | -| — | Prompt fidelity | — | `02_INCREMENTAL_PERSISTENCE.md` §11 — use persisted raw_request in extraction | - -**§4.4 and §4.5** are small, self-contained fixes. Inline specs below — no separate plan doc. - -### Unified implementation order across all docs - -``` -Phase 1: 02_ INCREMENTAL PERSISTENCE (4-5 days) - ├── Async generator Worker.execute() - ├── WorkerOutput replaces WorkerResult (breaking) - ├── get_output(context) reads from PG via repository - ├── from_buffer() for worker resumption - ├── Migrate all workers (ReAct, Stub, SmokeTest, TrainingStub) - ├── repo.persist_single() with dashboard listener - ├── PG schema: execution_outcome column - ├── Type tightening (enums, Literals, NewTypes, nullable fixes) - └── Tests - -Phase 2: 01_ INLINE FIXES (can overlap with Phase 1) (1.5 days) - ├── §4.4 Batch state to PG - ├── §4.5 Thread ↔ Execution FK - └── Tests - -Phase 3: STATE_UNIFICATION (complete) - ├── RunTaskStateEvent table dropped (migration 307fcca3a621) - └── Graph WAL is sole state source - -Phase 4: 03_ WORKFLOW RESUMPTION (5-7 days) - ├── Level 1: DAG-aware restart (resume_workflow_fn, RunResumeEvent) - ├── Level 2: Worker resumption from buffer (from_buffer in execute_task_fn) - ├── Level 3: Sandbox pause on failure (E2B pause/connect) - ├── CLI + observability - └── Tests - -Phase 5: PROMPT FIDELITY FIX (0.5 day) - ├── _extract_trajectories() uses persisted raw_request - └── Test -``` - ---- - -### §4.4 Batch State to PG — Inline Spec - -**When:** After 02_INCREMENTAL_PERSISTENCE Phase 1. Before 03_WORKFLOW_RESUMPTION -(resumption needs `poll()` to handle non-terminal states, which is easier if batch -state is already durable). - -**Estimated effort:** ~1 day. - -**Migration — add two tables:** - -```python -class RolloutBatch(SQLModel, table=True): - __tablename__ = "rollout_batches" - - id: UUID = Field(default_factory=uuid4, primary_key=True) - definition_id: UUID = Field(foreign_key="experiment_definitions.id", index=True) - status: BatchStatus = Field(default=BatchStatus.PENDING, index=True) - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - -class RolloutBatchRun(SQLModel, table=True): - __tablename__ = "rollout_batch_runs" - - id: UUID = Field(default_factory=uuid4, primary_key=True) - batch_id: UUID = Field(foreign_key="rollout_batches.id", index=True) - run_id: UUID = Field(foreign_key="runs.id", index=True) -``` - -**Files to modify:** - -``` -ergon_core/ergon_core/core/persistence/telemetry/models.py - + add RolloutBatch and RolloutBatchRun models - -ergon_core/ergon_core/core/rl/rollout_service.py - - remove self._batches: dict[UUID, _BatchState] - - remove _BatchState class - + submit(): write RolloutBatch + RolloutBatchRun rows instead of dict insert - + poll(): query RolloutBatch/RolloutBatchRun instead of dict lookup - + cancel(): query from PG instead of dict pop - -ergon_core/migrations/versions/XXXX_rollout_batch_state.py - + create rollout_batches and rollout_batch_runs tables -``` - -**Tests:** - -``` -tests/state/test_rollout_batch_state.py - - submit() creates batch + batch_run rows in PG - - poll() returns correct status from PG - - cancel() marks runs cancelled via PG query - - API restart: poll() still finds the batch (no 404) -``` - ---- - -### §4.5 Thread ↔ Execution FK — Inline Spec - -**When:** Same PR as §4.4 or immediately after. No dependencies. - -**Estimated effort:** ~half day. - -**Migration — add one column:** - -```python -# ThreadMessage — add optional FK -task_execution_id: UUID | None = Field( - default=None, - foreign_key="run_task_executions.id", - index=True, -) -``` - -**Files to modify:** - -``` -ergon_core/ergon_core/core/persistence/telemetry/models.py - + ThreadMessage: add task_execution_id: UUID | None - -ergon_core/ergon_core/core/runtime/services/communication_service.py - + save_message(): accept and persist task_execution_id - + get_thread_messages(): optionally filter by task_execution_id - -ergon_core/ergon_core/core/runtime/services/communication_schemas.py - + CreateMessageRequest: add task_execution_id: UUID | None = None - + MessageResponse: add task_execution_id: UUID | None = None - -ergon_core/migrations/versions/XXXX_thread_execution_fk.py - + add task_execution_id column to thread_messages -``` - -**Tests:** - -``` -tests/state/test_thread_execution_link.py - - save message with task_execution_id → persisted correctly - - query messages by task_execution_id → returns only matching messages - - save message without task_execution_id → None, backward compatible -``` - ---- - -## 6. Reconstruction Test - -Once §4.1 and §4.2 are implemented, the following should be possible: - -```python -def reconstruct_run_state(session: Session, run_id: UUID, at_time: datetime) -> RunSnapshot: - """Rebuild the complete DAG execution state at a given wall-clock time.""" - - # 1. Graph structure — from mutation log - mutations = get_mutations_before(session, run_id, at_time) - graph = replay_mutations(mutations) # nodes, edges, statuses - - # 2. Per-task execution state — from graph layer - task_states = get_graph_nodes_before(session, run_id, at_time) - # Each task: which status, which execution, when - - # 3. Per-turn agent behavior — from generation turns - turns = get_turns_before(session, run_id, at_time) - # Each turn: what the agent saw, what it said, what tools returned, when - - # 4. Inter-agent communication — from thread messages - messages = get_messages_before(session, run_id, at_time) - - # 5. Evaluations (for completed tasks) — from task evaluations - evals = get_evaluations_before(session, run_id, at_time) - - return RunSnapshot(graph, task_states, turns, messages, evals) -``` - -For the failure scenario: at t=46s, this returns tasks 1–6 completed with full turns, task 7 running with 10 turns of partial progress, tasks 8–10 in whatever state the DAG dictates. No data loss. - ---- - -## 7. Implications for the Workshop Paper - -The paper claims Ergon is a "durable event-driven substrate" for agentic RL. The current implementation doesn't fully support that claim for the most interesting case (mid-execution failure). Specifically: - -- **"Lossless trajectory collection"** — only true if the worker completes. §4.1 makes it true regardless. -- **"Fault-tolerant execution"** — Inngest provides task-level retry/recovery, but sub-task state (the 10 turns) is lost. §4.1 preserves it. -- **"Continuous-time event log"** — the graph mutation WAL has sequence numbers and timestamps, but generation turns (the actual agent behavior) don't have per-turn wall-clock timing. §4.1 + §4.6 close this. -- **"Reconstruct POSG trajectories from the event log"** — requires both the DAG state and the per-agent turns to be in PG and queryable at any point in time. §4.1 + §4.2 make this possible. - -The strongest version of the paper can say: *"Every agent action, tool result, LLM exchange, DAG state transition, and inter-agent message is durably persisted to an append-only event log with wall-clock timestamps. The complete execution state of a multi-agent workflow can be reconstructed at any point in time from this log alone, including after arbitrary worker failures."* - -That sentence requires §4.1 and §4.2 to be true. §4.3–§4.6 strengthen it but aren't strictly necessary for the core claim. diff --git a/docs/event-wal/02_INCREMENTAL_PERSISTENCE.md b/docs/event-wal/02_INCREMENTAL_PERSISTENCE.md deleted file mode 100644 index 29ba94753..000000000 --- a/docs/event-wal/02_INCREMENTAL_PERSISTENCE.md +++ /dev/null @@ -1,930 +0,0 @@ -# Tech Plan: Incremental Turn Persistence via Async Generator Workers - -**Status:** Draft -**Depends on:** 01_AUDIT.md §4.1 -**Goal:** No data loss on worker crash. Every completed turn is in PG -before the next one starts. No Redis. No streaming infrastructure. - ---- - -## 1. Architecture - -Workers yield `GenerationTurn` objects. The runtime persists each one -to PG as it arrives. That's the whole design. - -``` -Worker.execute() — async generator - │ - │ yield GenerationTurn ←── turn 0 (written to PG immediately) - │ yield GenerationTurn ←── turn 1 (written to PG immediately) - │ yield GenerationTurn ←── turn 2 (written to PG immediately) - │ ... ↑ crash here = turns 0-2 in PG, turn 3 lost - │ yield GenerationTurn ←── turn N - │ return WorkerOutput ←── final output text + metadata - │ - ▼ -worker_execute_fn (runtime) - │ - │ async for turn in worker.execute(...): - │ write RunGenerationTurn row to PG - │ emit dashboard event - │ turn_count += 1 - │ - │ output = worker.get_output() (or from generator return value) - │ - ▼ -PG: RunGenerationTurn rows (one per yield, wall-clock timestamps) -``` - -**No Redis.** No TurnSink. No StreamEvent. No AgentEventPayload. -No flush_to_pg(). No materialize_turns(). No stream keys. No TTLs. -No orphan reconciliation. - -**One type flows through the system:** `GenerationTurn`. The worker -produces it. The runtime persists it. The extraction pipeline reads it. -No conversion, no post-hoc assembly, no vendor-specific flush logic. - -**Workers choose their granularity:** -- Yield per-turn during execution → incremental persistence, live dashboard, stronger crash recovery -- Yield all turns at the end → functionally identical to today, zero migration effort - ---- - -## 2. Worker Interface Changes - -### 2.1 execute() becomes an async generator - -```python -# ergon_core/api/worker.py - -from collections.abc import AsyncGenerator - -class Worker(ABC): - type_slug: ClassVar[str] - - @abstractmethod - async def execute( - self, - task: BenchmarkTask, - *, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - """Run the worker's task behavior, yielding turns as they complete. - - Each yielded GenerationTurn is persisted to PG immediately by the - runtime. Workers that can detect turn boundaries mid-execution - yield incrementally (one turn per ReAct loop iteration). Workers - that can't yield all turns at the end in one batch. - - The final output text and metadata are returned via get_output(), - called by the runtime after the generator exhausts. - """ - ... - - def __init__( - self, - *, - name: str, - model: str | None = None, - metadata: Mapping[str, Any] | None = None, - ) -> None: - self.name = name - self.model = model - self.metadata: dict[str, Any] = dict(metadata or {}) - self._turn_repo = GenerationTurnRepository() - - def get_output(self, context: WorkerContext) -> WorkerOutput: - """Build output from persisted turns. Override for custom output. - - Called by the runtime after the async generator is fully consumed. - Default reads turns from PG via self._turn_repo and returns the - last turn's response text. Workers that need structured output, - summaries, or custom logic override this — self._turn_repo is - available to all subclasses. - - The turns are already in PG (persisted by the runtime on each yield), - so this is a read — not a computation. - """ - with get_session() as session: - turns = self._turn_repo.get_for_execution(session, context.execution_id) - last_turn = turns[-1] if turns else None - return WorkerOutput( - output=last_turn.response_text if last_turn else "", - success=True, - ) - - @classmethod - def from_buffer( - cls, - turns: list[GenerationTurn], - task: BenchmarkTask, - **kwargs, - ) -> "Self | None": - """Construct a worker pre-seeded with recovered turn history. - - Returns a new worker instance whose execute() will continue - from where the previous execution left off, or None if this - worker type doesn't support resumption. - """ - return None -``` - -### 2.2 WorkerOutput replaces WorkerResult - -`WorkerResult` is deleted. Turns are yielded by the generator. The non-turn -fields move to `WorkerOutput`: - -```python -# ergon_core/api/results.py - -class WorkerOutput(BaseModel): - """Non-turn output from a worker execution. - - Turns are yielded by the async generator. This carries everything else. - """ - model_config = {"frozen": True} - - output: str - success: bool = True - artifacts: dict[str, Any] = Field(default_factory=dict) - metadata: dict[str, Any] = Field(default_factory=dict) -``` - -**Breaking change.** `WorkerResult` is removed. All workers must be migrated -to the async generator pattern in the same commit series. No backward compat -shim — this is a clean cut. - ---- - -## 3. ReActWorker Implementation - -### 3.1 Eager yielding (incremental persistence) - -```python -# ergon_builtins/workers/baselines/react_worker.py - -class ReActWorker(Worker): - type_slug = "react-v1" - - def __init__(self, *, name, model=None, tools=None, - system_prompt=None, max_iterations=10): - super().__init__(name=name, model=model) - self.tools = tools or [] - self.system_prompt = system_prompt - self.max_iterations = max_iterations - - async def execute(self, task, *, context): - resolved = resolve_model_target(self.model) - agent = Agent( - model=resolved.model, - instructions=self.system_prompt, - tools=self.tools, - output_type=_AgentOutput, - ) - - task_prompt = _format_task(task) - node_count = 0 - prev_message_count = 0 - - async with agent.iter(task_prompt, model_settings=model_settings) as run: - async for _node in run: - node_count += 1 - - current_messages = run.result.new_messages() if run.result else [] - if len(current_messages) > prev_message_count: - new_turns = _build_turns(current_messages[prev_message_count:]) - for turn in new_turns: - yield turn # ← persisted to PG by the runtime - prev_message_count = len(current_messages) - - if node_count >= self.max_iterations: - break - - def get_output(self, context): - """Extract structured _AgentOutput from the last turn's raw_response.""" - with get_session() as session: - turns = self._turn_repo.get_for_execution(session, context.execution_id) - if not turns: - return WorkerOutput(output="", success=False) - last_turn = turns[-1] - raw = last_turn.raw_response - # Extract the structured output PydanticAI embedded in the response - output_text = _extract_agent_output(raw) - return WorkerOutput( - output=output_text, - success=True, - metadata={"turn_count": len(turns), "model": self.model}, - ) - - @classmethod - def from_buffer(cls, turns, task, **kwargs): - worker = cls(**kwargs) - worker._seed_messages = [] - for turn in turns: - if turn.raw_request: - worker._seed_messages.append(_reconstruct_request(turn.raw_request)) - worker._seed_messages.append(_reconstruct_response(turn.raw_response)) - return worker -``` - -No `self._output`. No stashing state during execution. `get_output()` reads -from PG via the repository — the turns are already there because the runtime -persisted each yield. The override just adds PydanticAI-specific extraction -of the structured `_AgentOutput`. - -### 3.2 What a lazy worker looks like - -A minimal worker that doesn't care about incremental persistence: - -```python -class SimpleWorker(Worker): - type_slug = "simple-v1" - - async def execute(self, task, *, context): - response = await call_llm(task.description) - yield GenerationTurn(raw_response=response, tool_results=[]) - - # get_output() — uses the base implementation (last turn's response_text) - # No override needed. -``` - -No framework knowledge needed. No hooks. No sinks. No stashing state. -Just yield a `GenerationTurn` when you have one. The base `get_output()` -reads the last turn from PG and returns its response text. - ---- - -## 4. Runtime Integration (worker_execute_fn) - -```python -# ergon_core/core/runtime/inngest/worker_execute.py - -async def worker_execute_fn(ctx: inngest.Context) -> WorkerExecuteResult: - payload = WorkerExecuteRequest.model_validate(ctx.event.data) - - worker_cls = WORKERS.get(payload.worker_type) - worker = worker_cls(name=payload.worker_binding_key, model=payload.model_target) - - task = BenchmarkTask( - task_key=payload.task_key, - instance_key=str(payload.execution_id), - description=payload.task_description, - ) - context = WorkerContext( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - sandbox_id=payload.sandbox_id, - ) - - turn_count = 0 - try: - async for turn in worker.execute(task, context=context): - _persist_turn( - run_id=payload.run_id, - execution_id=payload.execution_id, - worker_binding_key=payload.worker_binding_key, - turn=turn, - turn_index=turn_count, - execution_outcome="success", - ) - turn_count += 1 - - output = worker.get_output(context) - - except Exception as exc: - # Turns already in PG (each yield was persisted). - # Mark any already-written turns as from a failed execution. - _mark_turns_as_failed(payload.execution_id) - raise - - return WorkerExecuteResult(success=output.success, final_assistant_message=output.output) -``` - -**On crash:** The `except` block fires. Turns 0 through N-1 (everything -yielded before the crash) are already in PG with wall-clock timestamps. -Only the in-flight turn (the one that would have been yielded next) is lost. -`_mark_turns_as_failed()` updates the `execution_outcome` column on the -already-persisted turns so downstream consumers know this was a partial execution. - -**No finally block.** No flush. No cleanup. Each yield is its own PG write. -There's nothing to flush because there's no buffer. - ---- - -## 5. _persist_turn() — One PG Write Per Turn - -```python -def _persist_turn( - *, - run_id: UUID, - execution_id: UUID, - worker_binding_key: str, - turn: GenerationTurn, - turn_index: int, - execution_outcome: ExecutionOutcome = "success", -) -> None: - with get_session() as session: - session.add(RunGenerationTurn( - id=new_id(), - run_id=run_id, - task_execution_id=execution_id, - worker_binding_key=worker_binding_key, - turn_index=turn_index, - raw_request=turn.raw_request or {}, - raw_response=turn.raw_response, - response_text=extract_text(turn.raw_response), - tool_calls_json=extract_tool_calls(turn.raw_response), - tool_results_json=turn.tool_results or None, - logprobs_json=( - [lp.model_dump() for lp in turn.logprobs] - if turn.logprobs else None - ), - policy_version=turn.policy_version, - execution_outcome=execution_outcome, - created_at=utcnow(), # wall-clock time of this turn - )) - session.commit() -``` - -One write, one commit, per turn. ~5-10ms overhead per turn. For a 10-turn -execution, 50-100ms total. Negligible compared to LLM inference latency. - -`created_at` is set at persist time, which IS the wall-clock time the turn -completed (because we persist immediately after yield). No separate -`started_at` / `completed_at` needed — `created_at` on consecutive turns -gives you the inter-turn duration. - ---- - -## 6. PG Schema Changes - -### 6.1 RunGenerationTurn additions - -```python -class RunGenerationTurn(SQLModel, table=True): - # ... existing fields ... - - # NEW: execution outcome at time of persist - execution_outcome: ExecutionOutcome | None = Field( - default=None, index=True, - ) - # "success" = persisted during successful execution - # "failure" = execution failed (set retroactively by _mark_turns_as_failed) - # None = legacy rows from before this feature -``` - -Note: `started_at` / `completed_at` removed from the plan. `created_at` -(already exists on the model) now carries real wall-clock time because -each turn is persisted immediately. Inter-turn duration = `created_at[N+1] - created_at[N]`. - -### 6.2 raw_request population - -Fix in `_persist_turn()` (above): `raw_request=turn.raw_request or {}`. -The `GenerationTurn.raw_request` field already exists and the `ReActWorker` -already populates it. The bug was in `GenerationTurnRepository.persist_turns()` -which hardcoded `raw_request={}`. With `_persist_turn()` replacing that -code path, the fix is automatic. - -### 6.3 Type tightening (same migration / PR) - -While touching the models, tighten `str` fields to use existing enums and `Literal` types. -No behavioral change — pure type narrowing that catches bugs at write time. - -**Use existing enums instead of `str`:** - -```python -# RunRecord -status: str → status: RunStatus - -# RunTaskExecution -status: str → status: TaskExecutionStatus - -# ExperimentCohort -status: str → status: ExperimentCohortStatus - -# TrainingSession (needs new enum) -status: str → status: TrainingStatus # "running" | "completed" | "failed" -``` - -**Introduce `Literal` unions for closed value sets:** - -```python -# RunGraphMutation -MutationType = Literal[ - "node.added", "node.removed", "node.status_changed", "node.field_changed", - "edge.added", "edge.removed", "edge.status_changed", - "annotation.set", "annotation.deleted", -] -mutation_type: str → mutation_type: MutationType - -# RunGraphMutation + RunGraphAnnotation -GraphTargetType = Literal["node", "edge"] -target_type: str → target_type: GraphTargetType - -# RunGenerationTurn (new field from §6.1) -ExecutionOutcome = Literal["success", "failure"] -execution_outcome: str | None → execution_outcome: ExecutionOutcome | None - -# WorkflowCompleteResult / WorkflowFailedResult -status: str = "completed" → status: Literal["completed"] = "completed" -status: str = "failed" → status: Literal["failed"] = "failed" - -# RunResource -kind: str → kind: Literal["output"] -``` - -**Introduce `NewType` aliases for stringly-typed identifiers:** - -```python -# ergon_core/core/persistence/shared/types.py - -from typing import NewType - -WorkerBindingKey = NewType("WorkerBindingKey", str) -BenchmarkSlug = NewType("BenchmarkSlug", str) -``` - -**Fix nullable fields that shouldn't be:** - -```python -# DashboardTaskEvaluationUpdatedEvent -task_id: UUID | None = None → task_id: UUID - -# DashboardSandboxCommandEvent — add missing run_id -run_id: UUID -``` - -**Replace empty-string defaults with explicit optionality:** - -```python -WorkerExecuteRequest.task_description: str = "" → str | None = None -EvaluateTaskRunRequest.evaluator_binding_key: str = "" → str | None = None -EvaluateTaskRunRequest.agent_reasoning: str = "" → str | None = None -EvaluateTaskRunRequest.sandbox_id: str = "" → str | None = None -RunCleanupResult.status: str = "" → str | None = None -``` - -**Tighten bare `list` types on RunGenerationTurn JSON columns:** - -```python -tool_calls_json: list | None → list[dict[str, object]] | None -tool_results_json: list | None → list[dict[str, object]] | None -token_ids_json: list | None → list[int] | None -logprobs_json: list | None → list[dict[str, object]] | None -``` - -**Type untyped function parameters:** - -```python -# GenerationTurnRepository.persist_turns() -turns: list → turns: list[GenerationTurn] -``` - -**Note:** `RunGraphNode.status` and `RunGraphEdge.status` stay as `str` — -the graph layer is intentionally domain-agnostic. - -### 6.4 Migration - -One Alembic migration: -- Add `execution_outcome` column to `run_generation_turns` -- Column type changes are Python-side only (no DDL) - ---- - -## 7. What We Removed (and Why) - -| Removed | Why | -|---|---| -| Redis | No streaming buffer needed. Each turn is written to PG on yield. | -| TurnSink | No buffer to manage. The runtime writes directly to PG. | -| StreamEvent / AgentEventPayload | No stream to parse. Workers yield typed `GenerationTurn` objects. | -| flush_to_pg() | No flush needed. Each yield is its own PG write. | -| materialize_turns() | No post-hoc assembly. Workers produce `GenerationTurn` during execution. | -| RunStreamEvent table | No stream events to persist. Turn-level granularity is sufficient. | -| Orphan key reconciliation | No Redis keys to orphan. | -| TTLs, MAXLEN, stream cleanup | No streams. | -| get_redis(), redis_client.py | No Redis. | -| docker-compose redis service | No Redis. | - -**What we kept:** -- `GenerationTurn` — the one type that flows through the system -- `RunGenerationTurn` — the PG table the RL extraction pipeline reads -- `from_buffer()` — worker-specific resumption from recovered turns -- `WorkerContext` — unchanged (no turn_sink field needed) -- The RL extraction pipeline (`extraction.py`) — unchanged, reads `RunGenerationTurn` as before - ---- - -## 8. Failure Matrix - -| Failure mode | Turns persisted? | Recovery | -|---|---|---| -| Worker exception (Python crash) | All yielded turns in PG | Turns marked `execution_outcome="failure"`. Resume via 03_ plan. | -| Worker OOM / SIGKILL | All yielded turns in PG | Same — each yield committed individually. Process death doesn't roll back. | -| PG down | No turns persisted | Task fails on first yield. No silent degradation. | - -The critical improvement over today: **worker OOM/SIGKILL no longer loses -everything.** Each yielded turn is an independent PG commit. Process death -can't roll them back. The only lost work is the in-flight turn (one LLM call). - ---- - -## 9. Dashboard Events — Repository-Level Notification - -### 9.1 Current flow (slow, indirect) - -``` -worker_execute_fn → _emit_generation_turn_events() → Inngest event - → Next.js Inngest function → store.update() → Socket.io broadcast -``` - -Each turn notification bounces through Inngest (HTTP round-trip to Inngest -server, then Inngest calls the Next.js function endpoint). Adds 100-500ms -latency per turn. Dashboard updates are batched post-execution, not live. - -### 9.2 New flow (direct, per-turn) - -The repository notifies listeners when a turn is persisted. The dashboard -emitter is a listener. No Inngest hop. - -``` -_persist_turn() → GenerationTurnRepository.persist_single() - ├── session.add(RunGenerationTurn) - ├── session.commit() - └── notify_listeners(turn) ← fires after commit - │ - └── DashboardEmitter.generation_turn_completed() - │ - └── Inngest event → Next.js → Socket.io broadcast -``` - -The Inngest hop from emitter to dashboard still exists (the Python API -and the Next.js dashboard are separate processes — Socket.io lives in -Next.js). But the trigger is now per-turn and immediate, not batched. - -### 9.3 Repository change - -```python -# ergon_core/core/persistence/telemetry/repositories.py - -class GenerationTurnRepository: - - def __init__(self) -> None: - self._listeners: list[Callable[[RunGenerationTurn], Awaitable[None]]] = [] - - def add_listener(self, listener: Callable[[RunGenerationTurn], Awaitable[None]]) -> None: - self._listeners.append(listener) - - async def persist_single( - self, - session: Session, - *, - run_id: UUID, - execution_id: UUID, - worker_binding_key: str, - turn: GenerationTurn, - turn_index: int, - execution_outcome: ExecutionOutcome = "success", - ) -> RunGenerationTurn: - row = RunGenerationTurn( - id=new_id(), - run_id=run_id, - task_execution_id=execution_id, - worker_binding_key=worker_binding_key, - turn_index=turn_index, - raw_request=turn.raw_request or {}, - raw_response=turn.raw_response, - response_text=extract_text(turn.raw_response), - tool_calls_json=extract_tool_calls(turn.raw_response), - tool_results_json=turn.tool_results or None, - logprobs_json=( - [lp.model_dump() for lp in turn.logprobs] - if turn.logprobs else None - ), - policy_version=turn.policy_version, - execution_outcome=execution_outcome, - created_at=utcnow(), - ) - session.add(row) - session.commit() - - for listener in self._listeners: - try: - await listener(row) - except Exception: - logger.warning("Turn listener failed", exc_info=True) - - return row -``` - -### 9.4 Wiring - -At startup (or in `worker_execute_fn`), register the dashboard emitter -as a listener: - -```python -repo = GenerationTurnRepository() -repo.add_listener(dashboard_emitter.on_turn_persisted) -``` - -The emitter method: - -```python -# ergon_core/core/dashboard/emitter.py - -class DashboardEmitter: - async def on_turn_persisted(self, row: RunGenerationTurn) -> None: - """Called by the repository after a turn is committed to PG.""" - if not self._enabled: - return - evt = DashboardGenerationTurnEvent( - run_id=row.run_id, - task_execution_id=row.task_execution_id, - worker_binding_key=row.worker_binding_key, - worker_name=row.worker_binding_key, - turn_index=row.turn_index, - response_text=row.response_text, - tool_calls=row.tool_calls_json, - policy_version=row.policy_version, - ) - await inngest_client.send( - inngest.Event(name=evt.name, data=evt.model_dump(mode="json")) - ) -``` - -### 9.5 What this replaces - -- **Delete** `_emit_generation_turn_events()` in `worker_execute.py` — no longer called -- **Delete** `_emit_generation_turn_events_from_pg()` — never needed -- The `DashboardGenerationTurnEvent` contract and the Next.js Inngest - function that handles it are unchanged — same event shape, just emitted - per-turn instead of batched - -### 9.6 What `worker_execute_fn` looks like now - -```python -async for turn in worker.execute(task, context=context): - await repo.persist_single( - session_factory(), - run_id=payload.run_id, - execution_id=payload.execution_id, - worker_binding_key=payload.worker_binding_key, - turn=turn, - turn_index=turn_count, - ) - turn_count += 1 -``` - -One call per turn. The repository handles PG write + dashboard notification. -`worker_execute_fn` doesn't know or care about the dashboard. The emitter -is a listener registered elsewhere. - -### 9.7 Future: direct Socket.io (no Inngest hop) - -The Inngest hop between the Python emitter and the Next.js Socket.io -server still adds latency. The long-term path is: Python emits directly -to a shared pub/sub (Redis pub/sub, or a WebSocket connection to the -dashboard server) instead of going through Inngest. This eliminates the -last async hop and gives sub-100ms dashboard updates. - -This is out of scope for this plan but the listener pattern on the -repository makes it trivial to add later — just register a different -listener that publishes to Redis pub/sub instead of Inngest. - ---- - -## 10. Implementation Order - -### Single phase — breaking change (4-5 days) - -All workers migrate in one commit series. No backward compat period. - -1. Delete `WorkerResult`, add `WorkerOutput` in `results.py` -2. Change `Worker.execute()` return type to `AsyncGenerator[GenerationTurn, None]` -3. Add `get_output()` method to `Worker` ABC -4. Add `from_buffer()` classmethod to `Worker` ABC -5. Migrate all workers: `ReActWorker`, `StubWorker`, `SmokeTestWorker`, `TrainingStubWorker` -6. Implement `_persist_turn()` — one PG write per yielded turn -7. Rewrite `worker_execute_fn`: consume generator, persist per-yield, handle crash -8. Remove `_persist_generation_turns()` (old batch path) -9. Move dashboard events into the turn loop -10. Add `execution_outcome` column to `RunGenerationTurn` + migration -11. Apply type tightening from §6.3 -12. Tests - ---- - -## 11. Prompt Fidelity Fix - -Depends on raw_request being populated (§6.2 fixes this). Separate small PR -after the main incremental persistence work lands. - -In `RolloutService._extract_trajectories()`, replace the hardcoded prompt: - -```python -# BEFORE -prompt_text = tokenizer.apply_chat_template( - [{"role": "user", "content": "Complete the benchmark task."}], - tokenize=False, add_generation_prompt=True, -) - -# AFTER -first_turn = turns_by_run.get(run_id, [None])[0] -if first_turn and first_turn.raw_request: - prompt_text = _extract_prompt_from_raw_request(first_turn.raw_request, tokenizer) -else: - prompt_text = tokenizer.apply_chat_template( - [{"role": "user", "content": "Complete the benchmark task."}], - tokenize=False, add_generation_prompt=True, - ) -``` - ---- - -## 12. PydanticAI Type Mapping Reference - -The existing `react_worker.py` (lines 142-179) already implements the -conversion. Reference, don't reinvent. - -``` -PydanticAI types → GenerationTurn fields -───────────────────────────────────────────────────────── -ModelResponse → raw_response (via dataclasses.asdict + _make_json_safe) -ModelRequest (preceding) → raw_request (via dataclasses.asdict + _make_json_safe) -ToolReturnPart (from next → tool_results (list of {tool_call_id, tool_name, result}) - ModelRequest) -provider_details.logprobs → logprobs (list of TokenLogprob) -``` - -PydanticAI uses dataclasses for message types. The ReActWorker serializes -them with `dataclasses.asdict()` + `_make_json_safe()`. Other agent SDKs -serialize their own types differently — this mapping is PydanticAI-specific -and lives in `ergon_builtins`, not in core. - -The inverse (`from_buffer()` reconstruction): - -```python -# In ReActWorker — PydanticAI-specific, not in core -def _reconstruct_response(raw: dict) -> ModelResponse: - return ModelResponse(**raw) - -def _reconstruct_request(raw: dict) -> ModelRequest: - return ModelRequest(**raw) -``` - -Round-trip through `dataclasses.asdict()` / `**raw` is lossless for -PydanticAI's message types. - ---- - -## 13. Transaction Boundaries - -**Rule:** Repository methods **flush but don't commit.** The caller **owns -the commit.** This matches the existing `WorkflowGraphRepository` pattern. - -Exception: `GenerationTurnRepository.persist_single()` commits per-turn -because each turn must survive independently (the whole point of -incremental persistence). If the process dies between turns, the committed -ones are in PG. - -| Caller | Commits? | Why | -|---|---|---| -| `GenerationTurnRepository.persist_single()` | Yes — per-turn | Each turn must survive process death independently | -| `propagation.on_task_completed()` | Yes — existing | Already commits at end | -| `graph_repo.update_node_status()` | No — flushes | Caller commits | -| `_create_execution_rows()` in resume | No — flushes | `resume_workflow_fn` commits | -| `TaskExecutionService.prepare()` | Yes — existing | Already commits | - ---- - -## 14. File Map - -### ADD - -``` -ergon_core/ergon_core/core/persistence/shared/types.py - # NewType aliases: WorkerBindingKey, BenchmarkSlug - -ergon_core/migrations/versions/XXXX_incremental_persistence.py - # adds execution_outcome to run_generation_turns - -tests/state/test_incremental_persistence.py - # generator worker yields N turns → N RunGenerationTurn rows in PG - # worker crash at turn 5 → turns 0-4 in PG, marked as failure - # lazy worker yields all at end → same result, just batched - # from_buffer() → pre-seeded worker produces turns continuing from buffer - -tests/state/test_type_invariants.py - # RunRecord.status only accepts RunStatus values - # RunTaskExecution.status only accepts TaskExecutionStatus values - # RunGraphMutation.mutation_type only accepts known Literal values -``` - -### MODIFY - -``` -# ── Worker interface ────────────────────────────────────── - -ergon_core/ergon_core/api/worker.py - ~ execute() return type: WorkerResult → AsyncGenerator[GenerationTurn, None] - + __init__: add self._turn_repo = GenerationTurnRepository() - + get_output(context): base reads from PG via self._turn_repo - + from_buffer() classmethod (default returns None) - -ergon_core/ergon_core/api/results.py - - delete WorkerResult - + add WorkerOutput model (output, success, artifacts, metadata — no turns) - -ergon_core/ergon_core/api/generation.py - (no changes — GenerationTurn is already correct) - -ergon_core/ergon_core/api/worker_context.py - (no changes — no turn_sink field needed) - -# ── ReAct worker ────────────────────────────────────────── - -ergon_builtins/ergon_builtins/workers/baselines/react_worker.py - ~ execute(): return type → async generator, yield turns during loop - ~ get_output(context): override to extract PydanticAI structured output - + from_buffer(): reconstruct PydanticAI message history - + _seed_messages field for resumption - -ergon_builtins/ergon_builtins/workers/baselines/stub_worker.py - ~ migrate to async generator pattern - -ergon_builtins/ergon_builtins/workers/baselines/smoke_test_worker.py - ~ migrate to async generator pattern - -ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py - ~ migrate to async generator pattern - -# ── Runtime / Inngest ───────────────────────────────────── - -ergon_core/ergon_core/core/runtime/inngest/worker_execute.py - ~ rewrite: consume async generator, repo.persist_single() per yield - + _mark_turns_as_failed(): update execution_outcome on crash - - remove _persist_generation_turns() (old batch path) - - remove _emit_generation_turn_events() (replaced by repo listener) - -ergon_core/ergon_core/core/persistence/telemetry/repositories.py - + GenerationTurnRepository.persist_single(): single-turn PG write - + GenerationTurnRepository._listeners: list of post-commit callbacks - + GenerationTurnRepository.add_listener(): register a callback - ~ persist_turns(): keep for backward compat (extraction pipeline) or deprecate - -ergon_core/ergon_core/core/dashboard/emitter.py - + DashboardEmitter.on_turn_persisted(): listener method, emits per-turn event - -# ── PG models ───────────────────────────────────────────── - -ergon_core/ergon_core/core/persistence/telemetry/models.py - + RunGenerationTurn.execution_outcome: ExecutionOutcome | None - ~ RunRecord.status: str → RunStatus - ~ RunTaskExecution.status: str → TaskExecutionStatus - ~ ExperimentCohort.status: str → ExperimentCohortStatus - + TrainingSession: needs TrainingStatus enum - ~ RunGenerationTurn: tighten bare list types on JSON columns - -ergon_core/ergon_core/core/persistence/graph/models.py - ~ RunGraphMutation.mutation_type: str → MutationType - ~ RunGraphMutation.target_type / RunGraphAnnotation.target_type: str → Literal - -ergon_core/ergon_core/core/persistence/shared/enums.py - + TrainingStatus enum - -# ── DTOs / event contracts ──────────────────────────────── - -ergon_core/ergon_core/core/runtime/services/inngest_function_results.py - ~ WorkflowCompleteResult.status → Literal["completed"] - ~ WorkflowFailedResult.status → Literal["failed"] - ~ RunCleanupResult.status: str = "" → str | None = None - -ergon_core/ergon_core/core/runtime/services/child_function_payloads.py - ~ empty-string defaults → str | None = None - -ergon_core/ergon_core/core/runtime/services/graph_dto.py - ~ mutation_type / target_type tightened - -ergon_core/ergon_core/core/dashboard/event_contracts.py - ~ DashboardTaskEvaluationUpdatedEvent.task_id: not nullable - + DashboardSandboxCommandEvent: add run_id - -# ── Prompt fidelity (separate follow-up PR) ─────────────── - -ergon_core/ergon_core/core/rl/rollout_service.py - ~ _extract_trajectories(): use persisted raw_request instead of hardcoded prompt - -# ── Infrastructure ──────────────────────────────────────── - -(no docker-compose changes — no Redis) -(no new dependencies — no redis[hiredis]) -``` - -### DELETE - -``` -ergon_core/ergon_core/api/results.py → WorkerResult class (replaced by WorkerOutput) -worker_execute.py → _persist_generation_turns() function -worker_execute.py → _emit_generation_turn_events() function -``` diff --git a/docs/event-wal/03_WORKFLOW_RESUMPTION.md b/docs/event-wal/03_WORKFLOW_RESUMPTION.md deleted file mode 100644 index 0af536403..000000000 --- a/docs/event-wal/03_WORKFLOW_RESUMPTION.md +++ /dev/null @@ -1,668 +0,0 @@ -# Workflow Resumption from Durable State - -**Status:** Draft -**Depends on:** 01_AUDIT.md, 02_INCREMENTAL_PERSISTENCE.md -**Goal:** Given a failed run where PG has lossless task state + turn buffers (thanks to the Redis → PG flush), resume the workflow from the point of failure rather than restarting from scratch. - ---- - -## 1. The Problem - -Today, a failed task means a failed workflow. The `task/failed` event propagates -up to `workflow/failed`, which marks the run as FAILED and triggers cleanup. -The only recovery path is to start a new run from scratch. - -With the incremental persistence work (02), PG now has: -- DAG structure and every task's state (which completed, which failed, which never started) -- Lossless turn buffer for the failed task (everything up to the crash) -- Worker type and config for every task (so we can reconstruct workers) - -This is everything needed to resume — not retry from scratch, but continue -from the exact point of failure. The question is: what's the mechanism? - ---- - -## 2. What "Resume" Means - -There are three levels, from least to most ambitious: - -### Level 1: Re-dispatch failed tasks (DAG-aware restart) - -The simplest. Look at the run's task states in PG, find every task that's -FAILED or was never started (blocked by a failed dependency), and re-emit -`task/ready` events for them. Completed tasks are not re-run. - -This is **not** worker-level resumption — the failed task restarts from scratch -with a fresh sandbox and fresh worker. But the DAG doesn't restart — tasks 1–6 -keep their results, only task 7+ re-execute. - -**What's needed:** -- A new event: `run/resume` -- A new Inngest function: `resume_workflow_fn` that reads task states from PG - and re-emits `task/ready` for failed/blocked tasks -- Update `RunRecord.status` from FAILED back to EXECUTING -- Re-create `RunTaskExecution` rows for the tasks being retried - -**Complexity:** Low. This is mostly wiring — the propagation layer already knows -how to dispatch ready tasks, we just need to re-enter the DAG mid-flight. - -### Level 2: Resume failed workers from turn buffer - -The interesting one. Instead of restarting task 7 from scratch, reconstruct the -worker's state from the persisted turn buffer and continue from where it left off. - -The worker crashed 10 turns in. PG has those 10 turns (from the Redis flush). -`Worker.from_buffer()` reconstructs a pre-seeded worker instance. The worker's -`execute()` picks up from turn 10 instead of turn 0. - -**What's needed (in addition to Level 1):** -- `resume_workflow_fn` detects that the failed task has a turn buffer in PG -- Loads the turns, calls `worker_cls.from_buffer(turns, task, **kwargs)` -- If `from_buffer()` returns a worker (not None), uses that for re-execution -- If `from_buffer()` returns None, falls back to Level 1 (fresh restart) -- The sandbox situation: the original sandbox is dead. Options: - - (a) Create a fresh sandbox and accept the worker resumes without filesystem state - - (b) If sandbox outputs were persisted incrementally (future work), restore them - - (c) Accept that sandbox-dependent workers can't resume, only restart - -**Complexity:** Medium. The worker interface is already there (`from_buffer()`). -The gap is the orchestration logic that detects a resumable failure and routes -to the seeded worker instead of a fresh one. - -### Level 3: Checkpoint-based mid-turn resumption - -The most ambitious. Not just "continue from the last completed turn" but -"continue from the exact token/tool-call where the crash happened." This -would require reconstructing the LLM context window mid-generation, which -most inference backends don't support. - -**What's needed:** Inference-level checkpoint restore. Not practical today. - -**Recommendation:** Not worth pursuing. Level 2 already recovers all completed -turns; the cost of replaying the in-progress turn (one LLM call) is negligible. - ---- - -## 3. Recommended Design: Levels 1 + 2 - -### 3.1 New event - -```python -class RunResumeEvent(InngestEventContract): - """Emitted to resume a failed run from its last good state.""" - - name: ClassVar[str] = "run/resume" - - run_id: UUID - definition_id: UUID - resume_mode: Literal["restart_failed", "resume_from_buffer"] -``` - -- `restart_failed`: Level 1 — re-dispatch failed tasks fresh -- `resume_from_buffer`: Level 2 — attempt worker resumption from turn buffer, - fall back to restart if `from_buffer()` returns None - -### 3.2 Two responsibilities, cleanly separated - -**The orchestrator (`resume_workflow_fn`)** handles: -- Validating the run is resumable -- Classifying tasks (completed/failed/blocked/pending) -- Deciding which failed tasks to resume vs restart -- Picking which execution to resume from (latest failed attempt) -- Re-entering the DAG — emitting `task/ready` events for tasks that need re-execution -- Updating run status - -**The worker execution (`execute_task_fn`)** handles: -- Detecting a `resume_from_execution_id` on the incoming event -- Loading turns from the previous execution's buffer -- Calling `worker_cls.from_buffer()` to get a seeded worker -- Falling back to a fresh worker if `from_buffer()` returns None -- Running the rest of execution identically (sandbox, persist, propagate) - -The orchestrator doesn't touch workers. The worker execution doesn't touch the DAG. - -### 3.3 Orchestrator: `resume_workflow_fn` - -```python -@inngest_client.create_function( - fn_id="workflow-resume", - trigger=inngest.TriggerEvent(event="run/resume"), - retries=1, - output_type=WorkflowResumeResult, -) -async def resume_workflow_fn(ctx: inngest.Context) -> WorkflowResumeResult: - payload = RunResumeEvent.model_validate(ctx.event.data) - - # 1. Validate resumable - run, definition = _load_resumable_run(payload.run_id) - - # 2. Classify tasks from PG state - task_states = _classify_tasks(run, definition) - # COMPLETED → skip - # FAILED → re-dispatch (with resume_from_execution_id if mode is resume_from_buffer) - # BLOCKED → will unblock via normal DAG propagation once dependencies complete - # PENDING → never reached, will dispatch via normal propagation - - # 3. Build dispatch list - dispatch = [] - for task in task_states.failed: - resume_exec_id = None - if payload.resume_mode == "resume_from_buffer": - resume_exec_id = task.latest_failed_execution_id - dispatch.append((task, resume_exec_id)) - - # 4. Update run status, create new RunTaskExecution rows - _mark_run_resuming(run) - _create_execution_rows(dispatch) - - # 5. Emit task/ready events — with resume_from_execution_id where applicable - events = [ - inngest.Event( - name=TaskReadyEvent.name, - data=TaskReadyEvent( - run_id=payload.run_id, - definition_id=payload.definition_id, - task_id=task.id, - resume_from_execution_id=resume_exec_id, - ).model_dump(mode="json"), - ) - for task, resume_exec_id in dispatch - ] - await inngest_client.send(events) - - return WorkflowResumeResult(...) -``` - -The orchestrator emits standard `task/ready` events. The only difference from -a normal dispatch is the optional `resume_from_execution_id` field. No new -event type needed. - -### 3.4 Worker execution: resume branch in `execute_task_fn` - -```python -async def execute_task_fn(ctx: inngest.Context) -> TaskExecuteResult: - payload = TaskReadyEvent.model_validate(ctx.event.data) - - # Resume-aware worker construction — small branch, same execution path after - if payload.resume_from_execution_id is not None: - worker = _build_resumed_worker(payload) - # If from_buffer() returned None (worker doesn't support resumption), - # _build_resumed_worker falls back to a fresh worker — Level 1 behaviour. - else: - worker = _build_fresh_worker(payload) - - # ... rest is identical: sandbox setup, execute, flush Redis → PG, propagate -``` - -```python -def _build_resumed_worker(payload: TaskReadyEvent) -> Worker: - """Load turn buffer from previous execution, attempt from_buffer().""" - worker_cls = WORKERS.get(payload.worker_type) - - turns = _load_turns_from_execution(payload.resume_from_execution_id) - if turns: - task = _load_benchmark_task(payload) - resumed = worker_cls.from_buffer( - turns, task, - name=payload.worker_binding_key, - model=payload.model_target, - ) - if resumed is not None: - return resumed - - # Fallback: fresh worker (Level 1) - return worker_cls(name=payload.worker_binding_key, model=payload.model_target) -``` - -**Why this split matters:** -- The operator chooses "resume" vs "restart" at the run level (via the API/CLI) -- The orchestrator chooses *which execution to resume from* (DAG knowledge) -- The worker execution handles *how* to resume (framework knowledge via `from_buffer()`) -- Each layer only knows what it needs to know - -### 3.5 `TaskReadyEvent` change - -```python -class TaskReadyEvent(InngestEventContract): - name: ClassVar[str] = "task/ready" - - run_id: UUID - definition_id: UUID - task_id: UUID - resume_from_execution_id: UUID | None = None # NEW — None for fresh, set for resume -``` - -No new event type. The standard `task/ready` event gains one optional field. -Normal dispatches leave it as None. Resume dispatches set it to the failed -execution's ID. `execute_task_fn` branches on its presence. - -### 3.6 State transitions - -``` -RunRecord.status: - - PENDING → EXECUTING → COMPLETED (happy path) - PENDING → EXECUTING → FAILED (current failure path) - FAILED → RESUMING → EXECUTING → COMPLETED (resume path) - FAILED → RESUMING → EXECUTING → FAILED (resume also fails) -``` - -New status: `RESUMING` — transient state while `resume_workflow_fn` is -classifying tasks and dispatching. Moves to `EXECUTING` once tasks are dispatched. - -```python -class RunStatus(StrEnum): - PENDING = "pending" - EXECUTING = "executing" - RESUMING = "resuming" # NEW - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" -``` - -### 3.7 RunTaskExecution handling - -When resuming a task, we create a **new** `RunTaskExecution` row with an -incremented `attempt_number` (field already exists on the model — no -migration needed for this). The previous execution's row stays as-is -(status=FAILED, with its turn buffer intact in `RunGenerationTurn`). - -This preserves the full history: attempt 1 failed with 10 turns, attempt 2 -resumed from those 10 turns and completed with 15 total. - -The `RunTaskExecution` for the resumed attempt gets a reference back: - -```python -class RunTaskExecution(SQLModel, table=True): - # ... existing fields ... - - # NEW: links to the execution we resumed from (None for fresh executions) - resumed_from_execution_id: UUID | None = Field( - default=None, - foreign_key="run_task_executions.id", - ) -``` - ---- - -## 4. API Surface - -### 4.1 HTTP endpoint - -``` -POST /runs/{run_id}/resume -Body: { "mode": "restart_failed" | "resume_from_buffer" } -``` - -Validates the run is in FAILED state, emits `RunResumeEvent`, returns 202 Accepted. - -### 4.2 CLI command - -``` -ergon run resume <run_id> [--mode restart_failed|resume_from_buffer] -``` - -Default mode: `resume_from_buffer` (try Level 2, fall back to Level 1). - ---- - -## 5. What This Means for the Paper - -The paper's durability claim becomes concrete and demonstrable: - -> *"When a worker fails mid-execution, the system preserves all completed -> work (DAG state, turn buffers, tool outputs) and can resume the workflow -> from the point of failure. On a 10-task DAG where task 7 fails after -> 10 tool calls, resumption re-executes only task 7 (from turn 10, not -> turn 0) and propagates to tasks 8–10. Tasks 1–6 are not re-run."* - -The experiment to demonstrate this: -1. Run a multi-task workflow -2. Inject a failure mid-execution (kill the worker after N turns) -3. Show that PG has lossless state (turns 1–N preserved) -4. Resume the run -5. Show that the resumed worker continues from turn N -6. Show total wall-clock time is dominated by the remaining work, not replay - -This is the "fault injection → recovery → measure overhead" experiment from -the earlier paper discussion, and it's now mechanically possible. - ---- - -## 6. Implementation Order - -### Phase 1: Level 1 — DAG-aware restart (2-3 days) - -1. Add `RESUMING` to `RunStatus` enum -2. Add `RunResumeEvent` to event contracts -3. Add `resume_from_execution_id` to `RunTaskExecution` -4. Implement `resume_workflow_fn` (classify tasks, re-dispatch failed ones) -5. Add optional `resume_from_execution_id` field to `TaskReadyEvent` -6. Add `POST /runs/{run_id}/resume` endpoint -7. Tests: fail task 7 → resume → tasks 1-6 skipped, task 7 restarted fresh, 8-10 propagate - -### Phase 2: Level 2 — Worker resumption from buffer (2-3 days) - -1. Add `_build_resumed_worker()` to `execute_task_fn` (loads turns, calls `from_buffer()`) -2. Implement `ReActWorker.from_buffer()` with `_seed_messages` (already in 02_ plan) -3. Update `resume_workflow_fn` to detect resumable tasks and set `resume_from_execution_id` -4. Tests: fail task 7 at turn 10 → resume → worker continues from turn 10 → completes - -### Phase 3: Sandbox pause on failure + resume (1-2 days) - -1. In `worker_execute_fn` finally block: if `not success`, call `sandbox.pause()` instead of `sandbox.kill()` -2. Set `on_timeout="pause"` when creating sandboxes (covers SIGKILL case) -3. On resume: `Sandbox.connect(sandbox_id)` before `from_buffer()`; if fails, fall back to Level 1 -4. On successful resume completion: `sandbox.kill()` to clean up -5. Tests: worker exception → sandbox paused → resume → full state; sandbox dead → fallback to Level 1 - -### Phase 4: CLI + observability (1 day) - -1. Add `ergon run resume` CLI command -2. Dashboard shows resume status (which tasks were skipped, resumed, restarted) -3. Tracing spans for the resume lifecycle - ---- - -## 7. Interaction with Other Plans - -- **02_INCREMENTAL_PERSISTENCE.md**: Required. Without lossless turn buffers in PG, - Level 2 resumption has nothing to resume from. -- **01_AUDIT.md §4.2 (Graph WAL unification)**: Nice to have. If the graph mutation - log tracks resume events, you get a complete audit trail of "run failed at sequence N, - resumed at sequence M, tasks X/Y/Z re-dispatched." -- **Batch state durability (RolloutService)**: The trainer's `poll()` needs to handle - the case where a run transitions FAILED → RESUMING → EXECUTING → COMPLETED. Currently - `poll()` treats FAILED as terminal. This needs a small change: if a run is in RESUMING - or if `resume_from_execution_id` is set on any execution, `poll()` should wait rather - than returning a failure. - ---- - -## 8. File Map - -### ADD - -``` -ergon_core/ergon_core/core/runtime/inngest/resume_workflow.py - - resume_workflow_fn: classify tasks, emit task/ready events with resume_from_execution_id - - _classify_tasks(): read PG, return completed/failed/blocked/pending - - _create_execution_rows(): new RunTaskExecution rows for re-dispatched tasks - -ergon_core/ergon_core/core/runtime/events/infrastructure_events.py - + RunResumeEvent - -ergon_core/ergon_core/core/runtime/services/inngest_function_results.py - + WorkflowResumeResult - -tests/state/test_workflow_resumption.py - - Level 1: fail task 7 → resume restart → 1-6 skipped, 7 restarted fresh, 8-10 propagate - - Level 2: fail task 7 at turn 10 → resume from buffer → worker pre-seeded with 10 turns - - from_buffer() returns None → falls back to Level 1 (fresh restart) - - resume non-failed run → error - -tests/state/test_sandbox_resume.py - - worker crash → sandbox paused → Sandbox.connect() restores full state - - sandbox crash → connect fails → falls back to Level 1 (fresh sandbox) - - turn_complete event has sandbox_id in payload - - resume from latest turn → correct sandbox_id resolved -``` - -### MODIFY - -``` -ergon_core/ergon_core/core/persistence/shared/enums.py - + RunStatus.RESUMING - -ergon_core/ergon_core/core/persistence/telemetry/models.py - + RunTaskExecution.resumed_from_execution_id: UUID | None - -ergon_core/ergon_core/core/runtime/events/task_events.py - + TaskReadyEvent: add resume_from_execution_id: UUID | None = None - -ergon_core/ergon_core/core/runtime/inngest/execute_task.py - + resume branch: if payload.resume_from_execution_id, _build_resumed_worker() - + add _build_resumed_worker(): load turns from PG, call from_buffer(), fallback to fresh - + add _replay_sandbox_state(): load tool events, filter sandbox tools, replay in order - -ergon_core/ergon_core/core/providers/sandbox/manager.py - + add pause_sandbox() / resume_sandbox() methods wrapping E2B pause/connect - -ergon_core/ergon_core/core/runtime/inngest_registry.py - + register resume_workflow_fn in ALL_FUNCTIONS - -ergon_core/ergon_core/core/api/app.py (or runs.py) - + POST /runs/{run_id}/resume endpoint - -ergon_core/ergon_core/core/rl/rollout_service.py - + poll() handles RESUMING status (don't treat as terminal) - -migrations/versions/XXXX_workflow_resumption.py - + resumed_from_execution_id column on run_task_executions - + (RunStatus.RESUMING is Python-side only, no DDL) -``` - ---- - -## 9. Sandbox State on Resume - -### 9.1 The problem - -Level 2 resumption gives the worker its conversation history (via `from_buffer()`) -but the sandbox may be dead. For benchmarks where the agent writes files, runs -commands, or modifies the environment (gdpeval, code tasks), the conversation -references artifacts that may no longer exist. - -There are two distinct failure modes with different recovery properties: - -- **Worker crash (our process dies, sandbox still alive on E2B):** The sandbox - is intact. We just need to reconnect to it. -- **Sandbox crash (E2B container dies):** The sandbox state is gone. The - conversation history is preserved but the filesystem is not. - -### 9.2 Solution: E2B pause on failure only - -E2B natively supports `sandbox.pause()` and `Sandbox.connect(sandbox_id)`. -Pause preserves **both filesystem and memory state** — all files, running -processes, loaded variables, environment. Resume takes ~1 second. Pause takes -~4 seconds per GiB of RAM. - -**We only pause on failure, never during normal execution.** Paused sandboxes -aren't cleaned up by normal timeout culling — if we paused on every turn, -we'd leak paused sandboxes. The pause happens in the `finally` block of -`worker_execute_fn`, only when the execution failed. - -``` -Normal execution: - sandbox created → worker runs → sandbox killed on completion - (no pausing, no overhead) - -On worker crash (Python exception — finally block runs): - 1. finally block detects failure - 2. sandbox.pause() → ~2-4 seconds, full state saved - 3. flush Redis → PG → turns persisted with sandbox_id - 4. sandbox stays paused (not killed) - -On resume: - 1. Load failed execution → get sandbox_id from RunTaskExecution - 2. Sandbox.connect(sandbox_id) → ~1 second, full state restored - 3. Worker.from_buffer(turns) → conversation history restored - 4. execute() continues with matching sandbox + history - 5. On completion (success or failure): sandbox.kill() - -On SIGKILL (finally doesn't run): - Sandbox survives on E2B with on_timeout="pause" → auto-pauses - after timeout. Resume path tries Sandbox.connect() — works if - sandbox hasn't been manually killed. -``` - -**Zero overhead on the happy path.** The sandbox is never paused during -normal execution. The 2-4 second pause cost is paid only on failure, which -is when you're already in a degraded state and 4 seconds is negligible. - -The `sandbox_id` is already stored in PG when the sandbox is created -(via `sandbox_setup_fn`). No new storage needed — the resume path looks -up the failed execution's sandbox ID from the existing data. - -### 9.3 Sandbox crashes: fall back to Level 1 - -If the sandbox itself is dead (E2B infra failure, container OOM, etc.), -there's nothing to pause or resume. The sandbox state is gone. - -**Recovery:** Fall back to Level 1 (DAG-aware restart). The failed task -re-executes from scratch with a fresh sandbox and fresh worker. The -conversation buffer is preserved in PG for observability/debugging but -can't be used for resumption because the sandbox context it references -no longer exists. - -The resume path handles this gracefully: - -```python -def _build_resumed_execution(payload): - """Try Level 2 (sandbox + conversation resume), fall back to Level 1.""" - sandbox_id = _get_sandbox_id_from_execution(payload.resume_from_execution_id) - - # Try to reconnect to the paused sandbox - sandbox = None - if sandbox_id: - try: - sandbox = Sandbox.connect(sandbox_id) - except (SandboxNotFoundError, SandboxTimeoutError): - pass # Sandbox dead → will fall through to Level 1 - - if sandbox is not None: - # Sandbox alive → try Level 2 (conversation + sandbox resume) - turns = _load_turns(payload.resume_from_execution_id) - worker = worker_cls.from_buffer(turns, task, **kwargs) - if worker is not None: - return worker, sandbox - # from_buffer() returned None → can't resume conversation, - # but sandbox is alive. Kill it, start fresh. - sandbox.kill() - - # Level 1: fresh sandbox, fresh worker - return worker_cls(name=..., model=...), _create_fresh_sandbox() -``` - -### 9.4 Options we considered and rejected - -**Option A: Replay tool calls from the stream** - -Re-execute sandbox-affecting tool calls (file_write, shell_exec, etc.) in order -against a fresh sandbox to reconstruct filesystem state. - -*Why we rejected it:* -- Tool results are not the same as tool side effects. `shell_exec("make build")` - returns `"Compiled successfully"` but the side effect is a binary file on disk. - The stream has the return value, not the side effect. -- To truly reconstruct state, you'd need to replay ALL tools in order — including - expensive ones (API calls, long compilations, GPU operations). A 40-tool execution - with $0.50/call tools costs $20 to replay. -- Non-determinism: network-dependent commands, timestamps, random seeds. Replayed - state may diverge from what the agent's conversation history references. -- Classification problem: workers would need to declare which tools are - "sandbox-affecting" vs "external." This is error-prone and framework-specific. - -**Option B: Restore files from stream events** - -Instead of replaying tools, extract file contents from `file_write` tool inputs -in the stream and write them directly to a fresh sandbox. - -*Why we rejected it:* -- Only covers `file_write`. Doesn't cover files created by shell commands - (`make build`, `python setup.py`, `pip install`), which is most of the - interesting sandbox state. -- Shell command side effects are not in the stream at all — the stream has - the command input and the stdout/stderr result, not the filesystem diff. -- Binary files may be truncated or absent from the stream payload. - -**Option C: Periodic sandbox filesystem snapshots to object storage** - -Tar the sandbox filesystem periodically, upload to S3/GCS, restore on resume. - -*Why we rejected it:* -- Slow: tar + upload of a sandbox filesystem could be 10-30 seconds depending - on size. Much slower than E2B's native pause (~4 seconds for full memory + fs). -- Incomplete: captures filesystem but not memory state (running processes, - loaded variables, open connections). -- Infrastructure overhead: need object storage, upload/download plumbing, - snapshot scheduling. -- E2B's native pause/resume does this better, faster, and more completely. - -**Option D: E2B snapshots (one-to-many)** - -E2B offers `createSnapshot()` which captures sandbox state and allows spawning -multiple new sandboxes from it. More flexible than pause/resume. - -*Why we deferred it (not rejected):* -- Snapshots are slightly more complex than pause/resume (snapshot → new sandbox - vs pause → reconnect to same sandbox). -- For single-worker-resume, pause/resume is simpler and sufficient. -- Snapshots become useful if we ever need to fork a sandbox (e.g., branching - proof search, parallel exploration). Worth revisiting for the MAS use case - but not needed for the initial resumption implementation. - -### 9.5 Durability tiers (what the paper can claim) - -| Failure mode | Conversation | Sandbox | Recovery | Cost | -|---|---|---|---|---| -| Worker exception (finally runs) | Lossless (Redis → PG) | Lossless (pause in finally) | Level 2: resume conversation + sandbox | ~1s reconnect | -| Worker SIGKILL (finally doesn't run) | Lossless (Redis survives process) | Alive (on_timeout=pause) | Level 2: resume conversation + sandbox | ~1s reconnect | -| Sandbox crash (container dies) | Lossless (Redis → PG) | Lost | Level 1: restart task fresh | Full re-execution of failed task | -| Worker + sandbox both crash | Lossless (Redis → PG) | Lost | Level 1: restart task fresh | Full re-execution of failed task | -| Redis crash | Lost (current execution only) | Alive but orphaned | No recovery for in-flight data | Re-execute from last PG state | - -The paper can say: *"Worker failures recover losslessly — conversation state -from the event log, sandbox state via E2B pause/resume. Infrastructure failures -(sandbox crash, Redis crash) preserve all completed work at the DAG level and -restart only affected tasks."* - -### 9.6 Sequencing - -Sandbox pause on failure is **Phase 3 of 03_WORKFLOW_RESUMPTION**. - -- Phase 1 (Level 1 DAG restart) and Phase 2 (Level 2 conversation resumption) - work without sandbox state — valuable for research/tool-use benchmarks. -- Phase 3 adds `sandbox.pause()` in the failure path only. Zero overhead on - happy path. For sandbox-heavy benchmarks (gdpeval, code tasks), this closes - the gap between "conversation resumed but sandbox empty" and "full state - recovery." - -**Implementation:** - -1. In `worker_execute_fn` finally block: `sandbox.pause()` instead of - `sandbox.kill()` when `not success` -2. Set `on_timeout="pause"` on sandbox creation (covers SIGKILL) -3. On resume: `Sandbox.connect(sandbox_id)` → if alive, Level 2; if dead, Level 1 -4. On successful resume completion: `sandbox.kill()` to clean up - ---- - -## 10. Open Questions - -1. **Multiple failures.** If the resumed task also fails, can you resume again? - Yes — create another `RunTaskExecution` with `attempt_number=3` and - `resumed_from_execution_id` pointing to attempt 2. The turn buffer from - attempt 2 is the new resume point. No limit on attempts (but a configurable - max would be sensible). - -2. **Partial DAG resume.** What if tasks 3 and 7 both failed? The resume - dispatches both. Task 3's dependents (4, 5, 6) may need re-execution even - though they completed in the original run, because task 3's output changed. - This is a DAG invalidation problem. Simplest approach: only re-dispatch - FAILED tasks and their transitive dependents (tasks that transitively depend - on a FAILED task get re-dispatched regardless of their original status). - Tasks with no path to a FAILED task keep their original results. - -3. **Paused sandbox cleanup.** Paused sandboxes persist indefinitely on E2B. - If a failed run is never resumed, the paused sandbox leaks. Need a cleanup - mechanism: either a TTL-based sweep ("kill paused sandboxes older than X - hours if the run is still FAILED") or explicit cleanup when the operator - decides not to resume (the `run/cleanup` event kills the paused sandbox - instead of a running one). - -4. **E2B pause cost at scale.** Need to confirm storage cost for paused sandboxes - during large-scale RL training. If hundreds of rollouts fail concurrently - (e.g., model checkpoint is bad), that's hundreds of paused sandboxes. - Probably fine — E2B docs say indefinite retention — but worth verifying. diff --git a/docs/experiments/rq1-cli-specialism/changelog.md b/docs/experiments/rq1-cli-specialism/changelog.md deleted file mode 100644 index 2f56b398d..000000000 --- a/docs/experiments/rq1-cli-specialism/changelog.md +++ /dev/null @@ -1,295 +0,0 @@ -# RQ1 CLI Specialism Overnight Changelog - -## Goal - -Use the PR #39 workflow-CLI ResearchRubrics agent to produce rollout-card artifacts that support RQ1: returns remain a useful guardrail, but rollout cards preserve richer delegation and role-specialism behaviour that scalar returns discard. - -## 2026-04-26 23:30 UTC+1 - Preflight - -- Worktree: `/Users/charliemasters/Desktop/synced_vm_002/ergon/.worktrees/feature/finish-agent-workflow-cli` -- Branch: `feature/finish-agent-workflow-cli` -- PR: https://github.com/DeepFlow-research/ergon/pull/39 -- Commit at start: `ae7a0a8 Finish agent workflow CLI task editing` -- PR checks: all current checks passing by `gh pr checks 39`: - - `Integration tests (Python)`: pass - - `Lint + type-check (Frontend)`: pass - - `Lint + type-check (Python)`: pass - - `Unit tests (Python)`: pass - - `smoke [minif2f]`: pass - - `smoke [researchrubrics]`: pass - - `smoke [swebench-verified]`: pass -- Local `.env`: not present in the PR worktree. Real-LLM commands source `/Users/charliemasters/Desktop/synced_vm_002/ergon/.env` without copying it. -- Required keys after sourcing main `.env`: `OPENROUTER_API_KEY`, `EXA_API_KEY`, and `E2B_API_KEY` are set. -- Local services: - - `docker compose ps` in the worktree showed no compose-owned services. - - `http://127.0.0.1:3001/` responded. - - `http://127.0.0.1:9000/` responded with HTTP 404, which still indicates a process is listening; harness fixture treats connection success as stack-up. - -## Run Log - -Runs append below. Each entry should include command, env knobs, rollout artifact path, run ID, terminal status, score notes, graph/subtask notes, and prompt/config changes. - -## 2026-04-26 23:36 UTC+1 - Preflight Smoke Blocker - -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 uv run pytest tests/real_llm/benchmarks/test_smoke_stub.py -v -s --assume-stack-up` -- Result: - - Failed during test collection before any benchmark/model spend. -- Root cause: - - `telemetry.models` imports `ergon_core.api.json_types`, which executes `ergon_core.api.__init__`. - - `ergon_core.api.__init__` eagerly imported `RunResourceView` from `api.run_resource`. - - `api.run_resource` imports `RunResourceKind` from `telemetry.models` while `telemetry.models` is partially initialized. -- Fix: - - Added `tests/unit/runtime/test_import_boundaries.py` as a regression. - - Changed `ergon_core/ergon_core/api/__init__.py` to lazily expose `RunResourceKind` and `RunResourceView` via `__getattr__`. -- Verification: - - `uv run pytest tests/unit/runtime/test_import_boundaries.py -q` -> `1 passed` - - `uv run ruff format ergon_core/ergon_core/api/__init__.py tests/unit/runtime/test_import_boundaries.py && uv run ruff check ergon_core/ergon_core/api/__init__.py tests/unit/runtime/test_import_boundaries.py` -> `All checks passed` -- Commit: - - `e23c276 Fix run resource API import boundary` - -## 2026-04-26 23:45 UTC+1 - Stack Rebuild - -- Rebuilt the shared `ergon` compose project from the PR #39 worktree: - - `COMPOSE_PROJECT_NAME=ergon docker compose up -d --build --wait` -- Reason: - - The running stack was built before PR #39, so the API/Inngest runtime might not know `researchrubrics-workflow-cli-react`. -- Result: - - `ergon-api-1`, `ergon-dashboard-1`, `ergon-inngest-dev-1`, and `ergon-postgres-1` are running. - - API root returns HTTP 404 but the process is reachable; the real-LLM fixture only requires connection success. - -## 2026-04-26 23:47 UTC+1 - Baseline Workflow-CLI Batch 1 - -- Intent: - - Run 5 ResearchRubrics samples with the current PR #39 workflow-CLI prompt. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Failed after creating run `2626bae9-b058-4b1b-9803-8e6186468023`. -- Failure: - - Harness endpoint `GET /api/test/read/run/2626bae9-b058-4b1b-9803-8e6186468023/state` returned HTTP 500. - - Local DB/API inspection showed `psycopg2.errors.UndefinedColumn: column run_resources.copied_from_resource_id does not exist`. -- Root cause: - - The long-lived local Postgres DB was stamped at Alembic head `0a1b2c3d4e5f`, but was missing the already-existing migration `a2b3c4d5e6f7_add_copied_from_resource_id.py` effect. This is local schema drift, not a missing migration in the branch. -- Local repair: - - Applied idempotent local DDL: - - `ALTER TABLE run_resources ADD COLUMN IF NOT EXISTS copied_from_resource_id UUID NULL` - - `CREATE INDEX IF NOT EXISTS ix_run_resources_copied_from_resource_id ON run_resources (copied_from_resource_id)` - - Add FK constraint `fk_run_resources_copied_from_resource_id_run_resources` if absent. - - Verification: information schema now reports one `copied_from_resource_id` column. -- Post-repair canary: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 uv run pytest tests/real_llm/benchmarks/test_smoke_stub.py -v -s --assume-stack-up` - - Result: `1 passed` in 27.15s. - -## 2026-04-26 23:45 UTC+1 - Baseline Workflow-CLI Batch 1b - -- Intent: - - Retry 5 ResearchRubrics samples after local schema repair. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Passed, but not useful for headline RQ1 evidence. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T224530Z-3caf7e5c-e09f-47a8-8afb-58fd2693b761/` - - Run ID: `3caf7e5c-e09f-47a8-8afb-58fd2693b761` - - Wall clock: 235.6s - - Budget: $0.477609 -- Findings: - - The hardcoded `researchrubrics` benchmark loaded only 2 private/default smoke rows: `smoke-001`, `smoke-002`. - - Graph had 2 root nodes, 0 edges, 0 child subtasks, 2 resources, 1 evaluation. - - Worker did call `workflow inspect task-tree` once per task, but did not spawn/coordinate specialist subtasks. - - Evaluator returned score 0.0 because the API container did not have `OPENAI_API_KEY`. -- Fixes after analysis: - - `ResearchRubricsBenchmark._payload_from_row` now accepts vanilla dataset rows with `prompt` when `ablated_prompt` is absent. - - `tests/real_llm/benchmarks/test_researchrubrics.py` now honors `ERGON_REAL_LLM_BENCHMARK`, defaulting to `researchrubrics`. - - `docker-compose.yml` now passes `OPENAI_API_KEY`, `EXA_API_KEY`, and `HF_API_KEY` to the API container alongside the existing E2B/OpenRouter keys. - - Focused tests: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -q` -> `10 passed`. - - Vanilla load check: `ResearchRubricsVanillaBenchmark(limit=5)` -> 5 rows loaded. - - Stack rebuilt with exported env; API container verified all provider keys present. - -## 2026-04-27 00:00 UTC+1 - Vanilla 5-Sample Workflow-CLI Batch 1 - -- Intent: - - Run the actual 5-row ScaleAI ResearchRubrics benchmark after enabling vanilla rows and backend evaluator env. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Run reached terminal `failed`, but pytest timed out waiting for resources/evaluations because all tasks failed before report persistence. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T230154Z-ab57a0df-2a6d-4174-95f5-87185f717707/` - - Run ID: `ab57a0df-2a6d-4174-95f5-87185f717707` - - Row counts: 5 graph nodes, 0 graph edges, 25 mutations, 121 context events, 10 sandbox events, 0 resources, 0 evaluations. -- Findings: - - This was the intended 5 real-row ScaleAI benchmark: five sample IDs were created. - - Behavior was rich but not successful: 116 tool calls total, including 111 `exa_search` and 5 `workflow inspect task-tree`. - - No task called `write_report_draft` or `final_result`; all failed with generic `Worker execution failed`. - - Failure mode appears to be search-budget exhaustion / max-iteration behavior on large vanilla rubrics, not missing provider keys. - - No child subtasks: the workflow tool was available but graph editing was not manager-capable, and the prompt only suggested inspection/resource-copying. -- Core/harness fixes: - - `_wait_for_post_terminal_artifacts` now returns for terminal `failed`/`cancelled` runs with no running executions, so failed-before-output rollouts still dump artifacts. - - `_require_keys` now includes `openai_api_key`. - - Broke a context-event import cycle by storing context `turn_logprobs` as open JSON payloads instead of importing `TokenLogprob` from `ergon_core.api.generation`. - - Added import-boundary coverage for context models. - - Tests: `uv run pytest tests/unit/runtime/test_import_boundaries.py tests/unit/state/test_research_rubrics_benchmark.py -q` -> `12 passed`. - -## 2026-04-27 00:04 UTC+1 - Prompt Hillclimb Variant 1 - -- Prompt/tool changes: - - Workflow-CLI ReAct worker now passes `manager_capable=True` to `make_workflow_cli_tool`. - - Prompt asks level-0 tasks to create exactly three specialist child tasks before research: - - source scout - - rubric compliance checker - - synthesis reviewer - - Prompt tells non-root tasks not to create recursive children. - - Prompt caps own work to at most 6 `exa_search` calls before writing `final_output/report.md`. -- Verification: - - `uv run pytest tests/unit/state/test_research_rubrics_workers.py tests/unit/state/test_workflow_cli_tool.py -q` -> `10 passed`. - - API restarted; provider keys still present in container. -- Next run command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Diagnostic run: - - Run ID: `4d3721d0-aacb-4f04-bea9-9217c0549f9e` - - Stopped pytest manually after confirming it was polluted by the async workflow bridge bug. - - Positive signal: at least one root task attempted the desired `workflow manage add-task` specialist pattern before searching: - - source scout - - rubric compliance checker - - synthesis reviewer - - Bug found: agent-side workflow manage commands called the sync CLI bridge, which used `asyncio.run()` inside an already-running event loop. API log showed `RuntimeWarning: coroutine '_handle_manage' was never awaited`. -- Core fix: - - Added `execute_workflow_command_async(...)` in `ergon_cli.commands.workflow`. - - `execute_workflow_command(...)` now remains a sync wrapper for CLI callers. - - `make_workflow_cli_tool(...)` now awaits the async executor. - - Tests: `uv run pytest tests/unit/cli/test_workflow_cli.py tests/unit/state/test_workflow_cli_tool.py -q` -> `10 passed`. - -## 2026-04-27 00:13 UTC+1 - Prompt Hillclimb Variant 1b - -- Intent: - - Re-run Variant 1 with the fixed async workflow bridge. -- Status: - - Cancelled after diagnostic success and provider failures. -- Diagnostic result: - - Run ID: `9a83787a-dac2-45a1-9d3f-823f65984716` - - Early poll showed 20 graph nodes: 5 roots + 15 level-1 specialist children. - - Each root created source scout, rubric compliance, and synthesis reviewer children. - - This is the desired RQ1 graph-specialism signal. - - However, several roots failed on provider/schema errors (`finish_reason=None`) before reports/evaluations landed; remaining children were pending/blocked. - - Cancelled run via `uv run ergon run cancel 9a83787a-dac2-45a1-9d3f-823f65984716`. - -## 2026-04-27 00:22 UTC+1 - Prompt Hillclimb Variant 1c - -- Intent: - - Keep the specialist-subtask prompt, but switch from OpenRouter Sonnet to direct OpenAI to avoid the `finish_reason=None` OpenRouter/PydanticAI failure. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_MODEL=openai:gpt-4o-mini ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Cancelled after partial artifact dump. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T232803Z-3b258073-ab38-4a22-ac18-766c27d8aa1e/` - - Run ID: `3b258073-ab38-4a22-ac18-766c27d8aa1e` - - Row counts: 11 graph nodes, 37 mutations, 90 context events, 3 resources, 2 evaluations. -- Findings: - - Direct OpenAI avoided the OpenRouter `finish_reason=None` issue. - - Two root tasks completed and produced evaluations: - - score `0.11382113821138211`, passed `true` - - score `0.014084507042253521`, passed `false` - - Three roots failed before final output; two failed roots created specialist children, which were blocked by parent failure. - - This is a partial "rich behavior vs return" data point: returns are low/partial, but rollout-card structure exposes role-specialist decomposition not captured by scalar return. - -## 2026-04-27 00:29 UTC+1 - Prompt Hillclimb Variant 1d - -- Intent: - - Same specialist prompt, direct OpenAI, stronger model (`openai:gpt-4o`) to improve returns while preserving graph-specialism signal. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_MODEL=openai:gpt-4o ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Cancelled after partial artifact dump because dynamic child tasks remained pending. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T233740Z-356b7189-229b-4ef4-849c-f3c87964feb4/` - - Run ID: `356b7189-229b-4ef4-849c-f3c87964feb4` - - Row counts: 20 graph nodes, 43 mutations, 62 context events, 5 resources, 4 evaluations. -- Findings: - - Best evidence so far: 5 roots, 15 specialist children, 4/5 root reports completed, 4 evaluations landed. - - Scores: - - `0.1267605633802817`, passed `true` - - `0.11382113821138211`, passed `true` - - `0.07142857142857142`, passed `false` - - `0.0`, passed `false` - - Dynamic children remained `pending` rather than being scheduled after creation. -- Core fix: - - `WorkflowService.add_task` now emits `task/ready` for the created dynamic node after commit. - - Added an injectable task-ready dispatcher and unit test coverage. - - Tests: `uv run pytest tests/unit/runtime/test_workflow_service.py tests/unit/cli/test_workflow_cli.py tests/unit/state/test_workflow_cli_tool.py -q` -> `22 passed`. - -## 2026-04-27 00:38 UTC+1 - Prompt Hillclimb Variant 1e - -- Intent: - - Same GPT-4o specialist prompt, now with dynamic child scheduling fixed. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_MODEL=openai:gpt-4o ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Pytest artifact-dump wrapper passed, but run terminal status is `failed`. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T233920Z-0700b668-a640-49f2-80f9-a5c87bc160a9/` - - Run ID: `0700b668-a640-49f2-80f9-a5c87bc160a9` - - Row counts: 20 graph nodes, 70 mutations, 257 context events, 5 resources, 5 evaluations, 20 executions. - - Run summary: `final_score=0.7134802212615627`, `normalized_score=0.14269604425231255`, `evaluators_count=5`. -- Findings: - - Scheduling fix worked: Inngest logs show dynamic `task/ready` events for child `node_id`s and `task-execute` initialized for those children. - - Graph-specialism signal preserved: 5 roots and 15 specialist children. - - Returns improved versus prior failed/partial variants: all 5 root tasks completed and all 5 root evaluations landed; 1/5 evaluations passed. - - Remaining failure mode: most specialist children started and then failed generically (`Worker execution failed`), causing the overall run to fail even though root reports/evaluations landed. - - Root cause for child recursion: the prompt told agents to inspect `task-tree`; child agents can see other level-0 roots in that output and at least one child incorrectly called `manage add-task`. -- Prompt fix for next run: - - Delegation decision now uses only `workflow("inspect task-workspace --format json")` and `task_workspace.task.level`. - - Prompt explicitly says to ignore level-0 tasks shown elsewhere in task-tree. - - Non-root specialist children are told not to call `workflow("manage add-task`), to use at most 2 workflow inspections and 3 `exa_search` calls, and to write `final_output/report.md`. -- Verification: - - Red test first: `uv run pytest tests/unit/state/test_research_rubrics_workers.py::TestResearcherWorker::test_workflow_cli_prompt_uses_current_task_level_for_delegation -q` failed on the missing `task-workspace --format json` instruction. - - Green tests: `uv run pytest tests/unit/state/test_research_rubrics_workers.py tests/unit/state/test_workflow_cli_tool.py -q` -> `11 passed`. - -## 2026-04-27 00:55 UTC+1 - Prompt Hillclimb Variant 1f - -- Intent: - - Same GPT-4o specialist prompt, but with delegation keyed to the current task workspace rather than global task-tree rows. -- Command: - - `ERGON_REAL_LLM=1 ERGON_REAL_LLM_BUDGET_USD=50 ERGON_REAL_LLM_MODEL=openai:gpt-4o ERGON_REAL_LLM_BENCHMARK=researchrubrics-vanilla ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react ERGON_REAL_LLM_LIMIT=5 uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s --assume-stack-up` -- Status: - - Pytest artifact-dump wrapper passed, but run terminal status is `failed`. -- Rollout: - - Directory: `tests/real_llm/.rollouts/20260426T234424Z-7fc055f5-03c3-4cab-8117-04e844696482/` - - Run ID: `7fc055f5-03c3-4cab-8117-04e844696482` - - Row counts: 20 graph nodes, 70 mutations, 235 context events, 5 resources, 5 evaluations, 20 executions. - - Run summary: `final_score=0.7597894539417135`, `normalized_score=0.1519578907883427`, `evaluators_count=5`. -- Findings: - - Best overnight evidence so far. - - Graph-specialism signal: 5 roots created exactly 15 specialist children; `manage add-task` appears exactly 15 times and no recursive child creation was observed. - - Return guardrail: all 5 root tasks completed, all 5 root evaluations landed, and the aggregate normalized score improved slightly over 1e (`0.1519578907883427` vs `0.14269604425231255`). - - Specialist execution improved but remains noisy: 5/15 children completed, 10/15 failed with generic `Worker execution failed`, so the run-level status is still `failed`. - - This supports the RQ1 story: the scalar terminal status is poor, but the rollout card exposes a stable specialist-delegation pattern, role-specific child descriptions, root report completion, and recoverable child-worker behavior. -- Backend harness endpoint check: - - `GET http://127.0.0.1:9000/api/test/read/run/7fc055f5-03c3-4cab-8117-04e844696482/state` returned HTTP 200 with `status=failed`, `graph_nodes=20`, `mutations=70`, `evaluations=5`, `executions=20`, `resource_count=5`, `context_event_count=235`. - - The same path on dashboard port `3001` returned 404; the harness route is backend API, not dashboard. - -## Morning Handoff Notes - -- Best variant: Prompt Hillclimb Variant 1f. -- Best artifact path: `tests/real_llm/.rollouts/20260426T234424Z-7fc055f5-03c3-4cab-8117-04e844696482/` -- Candidate RQ1 headline evidence: - - Returns/status alone: run is `failed`, but all 5 root tasks completed and all 5 evaluations landed. - - Rollout-card structure: 5 root tasks, exactly 15 specialist child tasks, 70 graph mutations, 235 context events. - - Specialism behavior: root tasks consistently decomposed into source-scout, rubric-checker/compliance, and synthesis-reviewer roles. - - Cross-community analysis hook: this single rollout card supports post-hoc role-diversity / worker-specialism measurements that are invisible in terminal status or scalar return. -- Main residual issue: - - Dynamic specialist children now schedule and some complete, but child failures still propagate run failure. Next core-code direction would be either (a) make advisory child tasks non-fatal for parent benchmark return, or (b) harden child-worker prompting/tooling so specialist children reliably write `final_output/report.md`. - -## 2026-04-27 10:05 UTC+1 - Model Resolution Refactor - -- Intent: - - Make Ergon cloud model targets (`openai:*`, `anthropic:*`, `google:*`) route through OpenRouter instead of direct provider APIs or PydanticAI fallback inference. -- Changes: - - Upgraded `pydantic-ai` from `0.7.2` to `0.8.1`, the latest resolvable version in this environment. - - Centralized dispatch in `ergon_core/core/providers/generation/model_resolution.py`. - - Added `openrouter.py` for OpenRouter-hosted cloud targets and `openai_compatible.py` for `vllm:` plus `openai-compatible:` endpoint targets. - - Removed the builtins model-backend registration path and the old `cloud_passthrough.py` / `vllm_backend.py` modules. -- Note: - - The installed PydanticAI version exposes `OpenRouterProvider` but not `OpenRouterModel`; the implementation uses `OpenAIChatModel(..., provider=OpenRouterProvider(...))`, which gives the desired OpenRouter routing semantics. diff --git a/docs/features/context-event-log.md b/docs/features/context-event-log.md deleted file mode 100644 index ca4103d1a..000000000 --- a/docs/features/context-event-log.md +++ /dev/null @@ -1,1396 +0,0 @@ -# Feature RFC: Context Event Log - -**Status:** RFC -**Replaces:** `run_generation_turns` table + `RunGenerationTurn` model -**Affects:** RL extraction, worker resumption, dashboard streaming, REST snapshot - ---- - -## 1. Problem - -`RunGenerationTurn` was designed to store one row per model call. It has three fundamental gaps that make lossless RL training and context reconstruction impossible: - -**Gap 1 — No full context capture.** `prompt_text` stores a single formatted string on the first turn only. For a multi-turn agentic trajectory, we have no record of what messages were actually sent to the model at turns 1+. You cannot reconstruct what the model saw without replaying the entire episode. - -**Gap 2 — KV-cache alignment is not guaranteed.** Because prompt context is reconstructed from loose text at extraction time (applying the chat template in `rollout_service.py`), the `prompt_ids` fed to TRL may differ from what the model saw during generation. This breaks KL divergence estimation and importance sampling. - -**Gap 3 — Events are conflated into opaque blobs.** Tool calls, tool results, thinking tokens, and assistant text are all packed into `raw_response` (a framework-native dict) plus a few extracted JSON columns. They cannot be streamed individually, do not have independent token IDs or logprobs, and cannot be queried by type. The frontend, RL pipeline, and context reconstruction code all re-parse the same blob differently and inconsistently. - -The result: the current schema works for simple single-turn benchmarks with a text output but breaks down for multi-turn agentic runs, extended thinking, lossless RL, or replaying exactly what an agent saw. - ---- - -## 2. Terminology - -**Turn** — one task execution, bounded by a `task_execution_id`. Maps 1:1 to a worker invocation today. In future, a turn may be paused at a tool-approval boundary and resumed; the event log handles this naturally since it is append-only. - -**Generation** — one output unit from the model within a turn: a text response, a tool call, or a thinking block. A generation has logprobs when the backend (vLLM) supports them. - -**Context event** — any discrete entry in the conversation sequence. The full ordered sequence of context events for a `task_execution_id` is the exact reconstruction of what was live in the model's context window at any point during that turn. - -**The invariant:** read all context events for a `task_execution_id` ordered by `sequence` → reconstruct the exact message list the model saw. No inference, no re-tokenisation, no parsing of framework blobs required. - ---- - -## 3. Target Schema: `run_context_events` - -### 3.1 Table - -```sql -CREATE TABLE run_context_events ( - id UUID PRIMARY KEY, - run_id UUID NOT NULL REFERENCES runs(id), - task_execution_id UUID NOT NULL REFERENCES run_task_executions(id), - worker_binding_key TEXT NOT NULL, - - -- Ordering - sequence INTEGER NOT NULL, -- monotonically increasing within task_execution_id - created_at TIMESTAMPTZ NOT NULL, -- wall clock when persisted - - -- Timing (model-generated events only; NULL for environment events) - started_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ, - - -- Discriminated payload - event_type TEXT NOT NULL, -- see §3.2 - payload JSONB NOT NULL, -- typed by event_type; see §3.3 - - -- RL metadata (on model-generated events only) - policy_version TEXT -); - -CREATE UNIQUE INDEX ON run_context_events (task_execution_id, sequence); -CREATE INDEX ON run_context_events (run_id); -CREATE INDEX ON run_context_events (task_execution_id); -CREATE INDEX ON run_context_events (event_type); -``` - -The `UNIQUE` constraint on `(task_execution_id, sequence)` enforces append-only ordering and makes the log idempotent under retry. - -### 3.2 Event types - -| `event_type` | Produced by | Has `turn_token_ids` / `turn_logprobs` | -|---|---|---| -| `system_prompt` | Worker / runtime at turn start | No | -| `user_message` | Worker or agent-to-agent delegation | No | -| `assistant_text` | Model | On first model-output event of the turn (vLLM only; `None` until provider support added) | -| `tool_call` | Model | On first model-output event of the turn (vLLM only; `None` until provider support added) | -| `tool_result` | Environment | No | -| `thinking` | Model (extended thinking only) | On first model-output event of the turn (vLLM only; `None` until provider support added) | - -### 3.3 Payload shapes (Python discriminated union) - -Each payload embeds `event_type` as a `Literal` field so Pydantic can discriminate on read — the same pattern as `GraphMutationValue` in `graph_dto.py`. - -```python -# ergon_core/ergon_core/core/persistence/context/event_payloads.py - -from typing import Annotated, Any, Literal -from pydantic import BaseModel, Field -from ergon_core.api.generation import TokenLogprob - - -# Exported type alias — use everywhere event_type is stored as a string field -ContextEventType = Literal[ - "system_prompt", - "user_message", - "assistant_text", - "tool_call", - "tool_result", - "thinking", -] - - -class SystemPromptPayload(BaseModel): - event_type: Literal["system_prompt"] = "system_prompt" - text: str - - -class UserMessagePayload(BaseModel): - event_type: Literal["user_message"] = "user_message" - text: str - from_worker_key: str | None = None # set for agent-to-agent messages - - -class AssistantTextPayload(BaseModel): - event_type: Literal["assistant_text"] = "assistant_text" - text: str - turn_id: str # links events from the same generation call - turn_token_ids: list[int] | None = None # set on FIRST model-output event of the turn only - turn_logprobs: list[TokenLogprob] | None = None # set on FIRST model-output event of the turn only - - -class ToolCallPayload(BaseModel): - event_type: Literal["tool_call"] = "tool_call" - tool_call_id: str - tool_name: str - args: dict[str, Any] - turn_id: str # links events from the same generation call - turn_token_ids: list[int] | None = None # None if another event in this turn holds them - turn_logprobs: list[TokenLogprob] | None = None # None if another event in this turn holds them - - -class ToolResultPayload(BaseModel): - event_type: Literal["tool_result"] = "tool_result" - tool_call_id: str # links back to the ToolCallPayload with the same id - tool_name: str - result: Any # tool returns are intentionally open — any JSON-serialisable value - is_error: bool = False - - -class ThinkingPayload(BaseModel): - event_type: Literal["thinking"] = "thinking" - text: str - turn_id: str # links events from the same generation call - turn_token_ids: list[int] | None = None # set on FIRST model-output event of the turn only - turn_logprobs: list[TokenLogprob] | None = None # set on FIRST model-output event of the turn only - - -ContextEventPayload = Annotated[ - SystemPromptPayload - | UserMessagePayload - | AssistantTextPayload - | ToolCallPayload - | ToolResultPayload - | ThinkingPayload, - Field(discriminator="event_type"), -] -``` - -### 3.4 ORM model - -```python -# ergon_core/ergon_core/core/persistence/context/models.py - -from pydantic import TypeAdapter -from sqlalchemy.dialects.postgresql import JSONB -from ergon_core.core.persistence.context.event_payloads import ( - ContextEventPayload, - ContextEventType, -) - - -class RunContextEvent(SQLModel, table=True): - __tablename__ = "run_context_events" - - id: UUID = Field(default_factory=new_id, primary_key=True) - run_id: UUID = Field(foreign_key="runs.id", index=True) - task_execution_id: UUID = Field( - foreign_key="run_task_executions.id", index=True - ) - worker_binding_key: str = Field(index=True) - sequence: int - event_type: ContextEventType = Field(index=True) - payload: dict[str, Any] = Field(sa_column=Column(JSONB)) - # Note: SQLAlchemy maps JSONB to TEXT in SQLite (used in tests) — no fixture changes needed. - started_at: datetime | None = Field(default=None, sa_type=TZDateTime) - completed_at: datetime | None = Field(default=None, sa_type=TZDateTime) - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - policy_version: str | None = None - - def parsed_payload(self) -> ContextEventPayload: - return TypeAdapter(ContextEventPayload).validate_python(self.payload) -``` - ---- - -## 4. Changes to the Write Path - -### 4.1 Enriching `GenerationTurn` - -`GenerationTurn` (the public worker API in `ergon_core/ergon_core/api/generation.py`) must be updated to carry the full typed input message list and drop the old `prompt_text` field. - -Define two discriminated unions alongside `GenerationTurn` — one for request parts (input to the model) and one for response parts (output from the model). Part kind names mirror PydanticAI's field names so construction from framework objects is direct, with no intermediate dict serialisation. - -```python -# ergon_core/ergon_core/api/generation.py - -from typing import Annotated, Literal -from pydantic import BaseModel, Field - - -# --- Request parts (ModelRequest input) --- - -class SystemPromptPart(BaseModel): - part_kind: Literal["system-prompt"] = "system-prompt" - content: str - - -class UserPromptPart(BaseModel): - part_kind: Literal["user-prompt"] = "user-prompt" - content: str - - -class ToolReturnPart(BaseModel): - part_kind: Literal["tool-return"] = "tool-return" - tool_call_id: str - tool_name: str - content: str - - -ModelRequestPart = Annotated[ - SystemPromptPart | UserPromptPart | ToolReturnPart, - Field(discriminator="part_kind"), -] - - -# --- Response parts (ModelResponse output) --- - -class TextPart(BaseModel): - part_kind: Literal["text"] = "text" - content: str - - -class ToolCallPart(BaseModel): - part_kind: Literal["tool-call"] = "tool-call" - tool_name: str - tool_call_id: str - args: dict[str, Any] - - -class ThinkingPart(BaseModel): - part_kind: Literal["thinking"] = "thinking" - content: str - - -ModelResponsePart = Annotated[ - TextPart | ToolCallPart | ThinkingPart, - Field(discriminator="part_kind"), -] - - -class GenerationTurn(BaseModel): - # Set by the framework adapter (_build_turns in react_worker.py). - # Workers do not set these directly. - messages_in: list[ModelRequestPart] = Field(default_factory=list) - response_parts: list[ModelResponsePart] = Field(default_factory=list) - tool_results: list[ToolReturnPart] = Field(default_factory=list) - - # turn_token_ids and turn_logprobs are both turn-level: flat lists covering all - # model-generated tokens in this generation call in generation order (text, thinking, - # tool args — whichever vLLM exposes). Both stored on the FIRST model-output context - # event only; group by turn_id to find them. - # turn_token_ids: None until the vLLM provider is updated to extract token IDs from - # provider_details (PydanticAI currently exposes logprobs but drops token IDs). - turn_token_ids: list[int] | None = None - turn_logprobs: list[TokenLogprob] | None = None - policy_version: str | None = None - started_at: datetime | None = None - completed_at: datetime | None = None -``` - -`prompt_text` and `raw_response` are removed entirely. The framework adapter (`_build_turns` in `react_worker.py`) is responsible for populating all three list fields; workers never set them directly. - -### 4.2 Changes to `_build_turns` in `react_worker.py` - -Currently `_build_turns` only extracts the `ModelResponse` into a `GenerationTurn` and pairs tool results from the subsequent `ModelRequest`. It must also carry the full `ModelRequest` parts for each turn so the runtime can emit `system_prompt`, `user_message`, and `tool_result` context events. - -> **PydanticAI message utilities:** -> - The caller passes `result.all_messages()` (or `result.new_messages()`) from the `RunResult` into `_build_turns` — do not accumulate messages manually. -> - PydanticAI also provides a `history_processors` hook for modifying history before each model call (filtering, truncation, summarisation). This is a different concern — it runs in the hot path of agent execution, not as a post-processing step. Do not conflate the two. -> - `ModelMessagesTypeAdapter` (`pydantic_ai.messages`) handles full round-trip serialisation of `list[ModelMessage]` if you ever need to serialise the raw list for debugging. Do not use `dataclasses.asdict` on PydanticAI message objects. -> - The turn-grouping and part-extraction logic below is still hand-written: PydanticAI has no concept of "turns" as defined here. - -```python -def _build_turns(messages: list[ModelMessage]) -> list[GenerationTurn]: - """Build GenerationTurn objects from PydanticAI message history. - - Each turn pairs one ModelResponse (the model's output) with the - ModelRequest that preceded it (the context sent in) and the - subsequent ModelRequest that carried tool results back. - """ - turns: list[GenerationTurn] = [] - pending_response: ModelResponse | None = None - pending_request_in: ModelRequest | None = None - - for message in messages: - if isinstance(message, ModelRequest): - if pending_response is not None: - # Close the pending turn: this request carries its tool results - turns.append(_to_turn( - pending_request_in, - pending_response, - tool_result_request=message, - )) - pending_response = None - pending_request_in = None - # This request is the context for the NEXT model call - pending_request_in = message - elif isinstance(message, ModelResponse): - pending_response = message - - if pending_response is not None: - turns.append(_to_turn(pending_request_in, pending_response, tool_result_request=None)) - - return turns - - -def _to_turn( - request_in: ModelRequest | None, - response: ModelResponse, - tool_result_request: ModelRequest | None, -) -> GenerationTurn: - return GenerationTurn( - messages_in=_extract_request_parts(request_in) if request_in else [], - response_parts=_extract_response_parts(response), - tool_results=_extract_tool_results(tool_result_request) if tool_result_request else [], - logprobs=extract_logprobs(response), - ) - - -def _extract_request_parts(request: ModelRequest) -> list[ModelRequestPart]: - parts: list[ModelRequestPart] = [] - for part in request.parts: - if isinstance(part, PydanticSystemPromptPart): - parts.append(SystemPromptPart(content=part.content)) - elif isinstance(part, PydanticUserPromptPart) and isinstance(part.content, str): - parts.append(UserPromptPart(content=part.content)) - elif isinstance(part, PydanticToolReturnPart): - parts.append(ToolReturnPart( - tool_call_id=part.tool_call_id, - tool_name=part.tool_name, - content=part.content, - )) - # Other part kinds (e.g. image content) are not yet supported — skip. - return parts - - -def _extract_response_parts(response: ModelResponse) -> list[ModelResponsePart]: - parts: list[ModelResponsePart] = [] - for part in response.parts: - if isinstance(part, PydanticTextPart): - parts.append(TextPart(content=part.content)) - elif isinstance(part, PydanticToolCallPart): - parts.append(ToolCallPart( - tool_name=part.tool_name, - tool_call_id=part.tool_call_id, - args=part.args_as_dict(), - )) - elif isinstance(part, PydanticThinkingPart): - parts.append(ThinkingPart(content=part.content)) - # Other part kinds are not yet supported — skip. - return parts -``` - -### 4.3 New `ContextEventRepository` - -```python -# ergon_core/ergon_core/core/persistence/context/repository.py - -class ContextEventRepository: - """Append-only write path for run_context_events.""" - - def __init__(self) -> None: - self._listeners: list[Callable[[RunContextEvent], Awaitable[None]]] = [] - self._sequence_counters: dict[UUID, int] = {} # task_execution_id → next seq - - def add_listener(self, listener: Callable[[RunContextEvent], Awaitable[None]]) -> None: - self._listeners.append(listener) - - def _make_event( - self, - run_id: UUID, - execution_id: UUID, - worker_binding_key: str, - sequence: int, - payload: ContextEventPayload, - *, - started_at: datetime | None = None, - completed_at: datetime | None = None, - policy_version: str | None = None, - ) -> RunContextEvent: - return RunContextEvent( - run_id=run_id, - task_execution_id=execution_id, - worker_binding_key=worker_binding_key, - sequence=sequence, - event_type=payload.event_type, - payload=payload.model_dump(mode="json"), - started_at=started_at, - completed_at=completed_at, - policy_version=policy_version, - ) - - async def persist_turn( - self, - session: Session, - *, - run_id: UUID, - execution_id: UUID, - worker_binding_key: str, - turn: GenerationTurn, - ) -> list[RunContextEvent]: - """Decompose one GenerationTurn into ordered context events and persist them. - - Emits events in this order: - 1. system_prompt (from messages_in SystemPromptPart, first turn only) - 2. user_message (from messages_in UserPromptPart, first turn only) - 3. thinking (from response_parts ThinkingPart, if present) - 4. assistant_text / tool_call (from response_parts TextPart / ToolCallPart) - 5. tool_result (from tool_results, one per ToolReturnPart) - - Returns the list of persisted events (in sequence order). - """ - events: list[RunContextEvent] = [] - seq = self._next_sequence(execution_id) - - # Each generation call gets a unique turn_id so model-output events can be - # grouped. turn_token_ids and turn_logprobs are both stored on the FIRST - # model-output event only; subsequent events in the same turn have None. - turn_id = str(uuid4()) - turn_token_ids_remaining = turn.turn_token_ids - turn_logprobs_remaining = turn.turn_logprobs - - # 1–2. Context events from messages_in (first turn only — system + user) - for part in turn.messages_in: - if isinstance(part, SystemPromptPart): - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - SystemPromptPayload(text=part.content), - )) - seq += 1 - elif isinstance(part, UserPromptPart): - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - UserMessagePayload(text=part.content), - )) - seq += 1 - # ToolReturnPart entries are handled as tool_result events below - - # 3–4. Events from response_parts (model-generated) - for part in turn.response_parts: - if isinstance(part, ThinkingPart): - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - ThinkingPayload( - text=part.content, - turn_id=turn_id, - turn_token_ids=turn_token_ids_remaining, - turn_logprobs=turn_logprobs_remaining, - ), - started_at=turn.started_at, - completed_at=turn.completed_at, - policy_version=turn.policy_version, - )) - turn_token_ids_remaining = None # assigned to first event; clear for rest - turn_logprobs_remaining = None - seq += 1 - elif isinstance(part, TextPart): - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - AssistantTextPayload( - text=part.content, - turn_id=turn_id, - turn_token_ids=turn_token_ids_remaining, - turn_logprobs=turn_logprobs_remaining, - ), - started_at=turn.started_at, - completed_at=turn.completed_at, - policy_version=turn.policy_version, - )) - turn_token_ids_remaining = None - turn_logprobs_remaining = None - seq += 1 - elif isinstance(part, ToolCallPart): - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - ToolCallPayload( - tool_call_id=part.tool_call_id, - tool_name=part.tool_name, - args=part.args, - turn_id=turn_id, - turn_token_ids=turn_token_ids_remaining, - turn_logprobs=turn_logprobs_remaining, - ), - started_at=turn.started_at, - completed_at=turn.completed_at, - policy_version=turn.policy_version, - )) - turn_token_ids_remaining = None - turn_logprobs_remaining = None - seq += 1 - - # 5. Tool results - for tr in turn.tool_results: - events.append(self._make_event( - run_id, execution_id, worker_binding_key, seq, - ToolResultPayload( - tool_call_id=tr.tool_call_id, - tool_name=tr.tool_name, - result=tr.content, - ), - )) - seq += 1 - - self._sequence_counters[execution_id] = seq - - for event in events: - session.add(event) - session.commit() - - for event in events: - for listener in self._listeners: - try: - await listener(event) - except Exception: - logger.warning("Context event listener failed", exc_info=True) - - return events - - def get_for_execution( - self, session: Session, execution_id: UUID - ) -> list[RunContextEvent]: - stmt = ( - select(RunContextEvent) - .where(RunContextEvent.task_execution_id == execution_id) - .order_by(RunContextEvent.sequence) - ) - return list(session.exec(stmt).all()) - - def get_for_run( - self, session: Session, run_id: UUID - ) -> list[RunContextEvent]: - stmt = ( - select(RunContextEvent) - .where(RunContextEvent.run_id == run_id) - .order_by(RunContextEvent.task_execution_id, RunContextEvent.sequence) - ) - return list(session.exec(stmt).all()) -``` - -### 4.4 Changes to `worker_execute.py` - -Replace the call to `repo.persist_single()` with `context_event_repo.persist_turn()`: - -```python -# Before (current) -repo = GenerationTurnRepository() -repo.add_listener(dashboard_emitter.on_turn_persisted) -... -await repo.persist_single(session, run_id=..., execution_id=..., turn=turn, ...) - -# After -context_event_repo = ContextEventRepository() -context_event_repo.add_listener(dashboard_emitter.on_context_event) -... -# Build turns from ALL messages — not an incremental slice. -result: RunResult = await agent.run(...) -turns = _build_turns(result.all_messages()) -for turn in turns: - # Register the execution→node mapping before emitting events so the - # DashboardEmitter can resolve task_node_id in on_context_event. - dashboard_emitter.register_execution(execution_id=execution_id, task_node_id=node_id) - await context_event_repo.persist_turn( - session, run_id=run_id, execution_id=execution_id, - worker_binding_key=worker_binding_key, turn=turn, - ) -``` - -### 4.5 Listener Chain (Python → Inngest → Socket → Client) - -The full event flow for every `RunContextEvent` row committed to Postgres: - -``` -persist_turn() commits row - → calls on_context_event(event: RunContextEvent) - → sends Inngest event "dashboard/context.event" - → Inngest function runs in dashboard server - → store.addContextEvent(runId, taskNodeId, event) - → broadcastContextEvent(runId, taskNodeId, event) - → socket.io "context:event" to run room - → client updates local state -``` - -#### `DashboardContextEventEvent` contract - -```python -# ergon_core/ergon_core/core/dashboard/event_contracts.py - -class DashboardContextEventEvent(InngestEventContract): - name: ClassVar[str] = "dashboard/context.event" - - id: UUID # RunContextEvent.id — used as dedup key on the frontend - run_id: UUID - task_execution_id: UUID - task_node_id: UUID # resolved from _execution_task_map at emit time - worker_binding_key: str - sequence: int - event_type: ContextEventType # imported from event_payloads - payload: ContextEventPayload # full typed discriminated union; serialised via model_dump(mode="json") - created_at: datetime - started_at: datetime | None - completed_at: datetime | None -``` - -`task_node_id` is resolved by the emitter from the in-memory `execution_task_map` held by the run context. The frontend should not need to do this lookup — it receives the node ID it needs to index into the task graph directly. - -#### `DashboardEmitter.on_context_event` - -```python -# ergon_core/ergon_core/core/dashboard/emitter.py - -async def on_context_event(self, event: RunContextEvent) -> None: - """Called by ContextEventRepository after each event is committed. - - Resolves task_node_id from the execution map and emits a typed - Inngest event. Non-blocking: errors are caught and logged. - """ - if not self._enabled: - return - try: - task_node_id = self._execution_task_map.get(event.task_execution_id) - if task_node_id is None: - logger.warning( - "on_context_event: no task_node_id for execution %s", event.task_execution_id - ) - return - evt = DashboardContextEventEvent( - run_id=event.run_id, - task_execution_id=event.task_execution_id, - task_node_id=task_node_id, - worker_binding_key=event.worker_binding_key, - sequence=event.sequence, - event_type=event.event_type, - payload=event.payload, - created_at=event.created_at, - started_at=event.started_at, - completed_at=event.completed_at, - ) - await inngest_client.send( - inngest.Event(name=evt.name, data=evt.model_dump(mode="json")) - ) - except Exception: - logger.warning("Failed to emit dashboard/context.event", exc_info=True) -``` - -`_execution_task_map: dict[UUID, UUID]` must be added to `DashboardEmitter`. It is populated via a new `register_execution` method called from `worker_execute.py` before `persist_turn` (see §4.4): - -```python -def register_execution(self, execution_id: UUID, task_node_id: UUID) -> None: - """Register execution_id → task graph node_id mapping. - Called from worker_execute.py before persist_turn so that on_context_event - can resolve task_node_id without a DB lookup.""" - self._execution_task_map[execution_id] = task_node_id -``` - -#### Frontend Inngest function - -`parseDashboardContextEventData` is a Zod parse helper that validates raw Inngest event data against the `DashboardContextEventEvent` Zod schema (defined in `ergon-dashboard/src/lib/contracts/events.ts`). It throws on validation failure. The `id` field on `ContextEventState` maps to `event.data.id` from the Inngest payload — it is used as the deduplication key in `addContextEvent` and the socket handler. - -```typescript -// ergon-dashboard/src/inngest/functions/onContextEvent.ts - -export const onContextEvent = inngest.createFunction( - { id: "dashboard-context-event" }, - { event: "dashboard/context.event" }, - async ({ event }) => { - const payload = parseDashboardContextEventData(event.data); - - const contextEvent: ContextEventState = { - id: payload.id, - taskExecutionId: payload.task_execution_id, - taskNodeId: payload.task_node_id, - workerBindingKey: payload.worker_binding_key, - sequence: payload.sequence, - eventType: payload.event_type as ContextEventType, - payload: payload.payload as ContextEventPayload, - createdAt: payload.created_at, - startedAt: payload.started_at ?? null, - completedAt: payload.completed_at ?? null, - }; - - store.addContextEvent(payload.run_id, payload.task_node_id, contextEvent); - broadcastContextEvent(payload.run_id, payload.task_node_id, contextEvent); - - return { success: true }; - }, -); -``` - -#### Socket event - -```typescript -// ergon-dashboard/src/lib/socket/types.ts - -export interface ServerToClientEvents { - // existing events ... - "context:event": (data: { - runId: string; - taskNodeId: string; - event: ContextEventState; - }) => void; -} -``` - ---- - -## 5. Read Paths - -### 5.1 RL Extraction - -`extraction.py` currently groups `RunGenerationTurn` rows by `worker_binding_key` and builds flat token sequences from `response_text` / `logprobs_json`. The `env_mask` is inferred heuristically from the presence of tool calls. With the new schema this becomes exact and explicit. - -#### New function signature - -The function signature is unchanged — callers pass pre-fetched rows; `extraction.py` does not query the DB directly. - -```python -# ergon_core/ergon_core/core/rl/extraction.py - -def extract_agent_trajectories( - context_events: list[RunContextEvent], - eval_scores: dict[str, float], # task_execution_id (str) → score - tokenizer: Tokenizer, - *, - reward_strategy: RewardStrategy | None = None, -) -> list[AgentTrajectory]: - """Build TRL-compatible trajectories from context events. - - One AgentTrajectory is returned per unique worker_binding_key. - Events must be pre-ordered by (task_execution_id, sequence) — the - caller is responsible for this ordering (matches get_for_run output). - """ -``` - -`prompt_text` is removed from the signature — the prompt is now reconstructed from `system_prompt` + `user_message` events directly, so callers no longer need to pass it separately. - -#### Extraction logic - -```python -def extract_agent_trajectories(...) -> list[AgentTrajectory]: - # Group events by worker_binding_key, preserving sequence order within each group - by_worker: dict[str, list[RunContextEvent]] = defaultdict(list) - for event in context_events: - by_worker[event.worker_binding_key].append(event) - - trajectories: list[AgentTrajectory] = [] - strategy = reward_strategy or IndependentTaskReward() - - for worker_key, events in by_worker.items(): - prompt_text = _build_prompt_text(events) - prompt_ids = tokenizer.encode(prompt_text) if prompt_text else [] - - completion_ids: list[int] = [] - logprobs: list[float] = [] - env_mask: list[int] = [] - execution_ids: set[str] = set() - - for event in events: - parsed = event.parsed_payload() - event_type = event.event_type - - if event_type in ("system_prompt", "user_message"): - # Prompt context — not part of the completion sequence - execution_ids.add(str(event.task_execution_id)) - continue - - if event_type in ("assistant_text", "tool_call", "thinking"): - # Model-generated tokens — env_mask = 1 - token_ids = _get_token_ids(parsed, tokenizer) - token_logprobs = _get_logprobs(parsed, len(token_ids)) - completion_ids.extend(token_ids) - logprobs.extend(token_logprobs) - env_mask.extend([1] * len(token_ids)) - - elif event_type == "tool_result": - # Environment tokens — env_mask = 0 - assert isinstance(parsed, ToolResultPayload) - result_tokens = tokenizer.encode(str(parsed.result)) - completion_ids.extend(result_tokens) - logprobs.extend([0.0] * len(result_tokens)) - env_mask.extend([0] * len(result_tokens)) - - execution_ids.add(str(event.task_execution_id)) - - reward = strategy.compute(worker_key, execution_ids, eval_scores) - - trajectories.append(AgentTrajectory( - agent_id=worker_key, - prompt_ids=prompt_ids, - completion_ids=completion_ids, - logprobs=logprobs, - env_mask=env_mask, - reward=reward, - turns=_count_turns(events), - )) - - return trajectories - - -def _build_prompt_text(events: list[RunContextEvent]) -> str: - """Concatenate system_prompt + user_message events for the first execution turn.""" - parts: list[str] = [] - for event in events: - if event.event_type == "system_prompt": - parsed = event.parsed_payload() - assert isinstance(parsed, SystemPromptPayload) - parts.append(parsed.text) - elif event.event_type == "user_message": - parsed = event.parsed_payload() - assert isinstance(parsed, UserMessagePayload) - parts.append(parsed.text) - elif event.event_type in ("assistant_text", "tool_call", "thinking", "tool_result"): - break # Stop at first model output — prompt is everything before it - return "\n\n".join(parts) - - -def _get_token_ids(parsed: ContextEventPayload, tokenizer: Tokenizer) -> list[int]: - """Extract token IDs from a model-generated event payload. - - Uses turn_token_ids if present (vLLM path — turn-level flat list on the first - event of each turn). Falls back to tokenising the text content (non-vLLM path). - The fallback produces correct token counts but IDs won't match the model's - original forward pass, which is acceptable for reward-only RL but not for - importance sampling. See also the multi-event alignment note on _get_logprobs. - """ - if isinstance(parsed, AssistantTextPayload): - return parsed.turn_token_ids if parsed.turn_token_ids is not None else tokenizer.encode(parsed.text) - if isinstance(parsed, ToolCallPayload): - args_text = json.dumps(parsed.args) - return parsed.turn_token_ids if parsed.turn_token_ids is not None else tokenizer.encode(args_text) - if isinstance(parsed, ThinkingPayload): - return parsed.turn_token_ids if parsed.turn_token_ids is not None else tokenizer.encode(parsed.text) - raise ValueError(f"_get_token_ids called on non-model event: {type(parsed)}") - - -def _get_logprobs(parsed: ContextEventPayload, n_tokens: int) -> list[float]: - """Extract per-token logprob scalars from turn_logprobs, padding with 0.0 if unavailable. - - NOTE: This slicing is only correct for single-event turns (text-only or tool-call-only). - For multi-event turns (text + tool_call), the flat turn_logprobs list covers ALL tokens in - generation order; slicing per-event misaligns the logprobs. The correct path for multi-event - turns is to group by turn_id, collect all token_ids across the group, then align turn_logprobs - once against the concatenated list. This is a Phase 1 implementation concern; the schema is - correct. Until multi-event alignment is implemented, single-event turns are exact and - multi-event turns fall back to partial logprob coverage or 0.0 padding. - """ - lps: list[TokenLogprob] | None = getattr(parsed, "turn_logprobs", None) - if lps is None: - return [0.0] * n_tokens - scalars = [lp.logprob for lp in lps] - # Align length to n_tokens - if len(scalars) < n_tokens: - scalars.extend([0.0] * (n_tokens - len(scalars))) - return scalars[:n_tokens] - - -def _count_turns(events: list[RunContextEvent]) -> int: - """Count distinct generation turns (unique turn_ids across model-output events).""" - seen: set[str] = set() - for event in events: - if event.event_type in ("assistant_text", "tool_call", "thinking"): - parsed = event.parsed_payload() - turn_id: str | None = getattr(parsed, "turn_id", None) - if turn_id: - seen.add(turn_id) - return len(seen) -``` - -#### Key improvements over the old path - -| | Old (`RunGenerationTurn`) | New (`RunContextEvent`) | -|---|---|---| -| `env_mask` | Inferred heuristically from tool call presence | Exact: derived from `event_type` | -| `token_ids` | Always re-tokenised from text (lossy) | Native from vLLM when available; text fallback otherwise | -| `logprobs` | From `logprobs_json` blob, manually aligned | Per-event typed field, aligned by `_get_logprobs` | -| Prompt | Passed in by caller as `prompt_text` | Reconstructed from `system_prompt` + `user_message` events | -| Thinking tokens | Dropped entirely | First-class `thinking` events, included in completion | - -### 5.2 Context Reconstruction / `from_buffer()` - -`ReActWorker.from_buffer()` currently does: -```python -worker._seed_messages.append(ModelResponse(**turn.raw_response)) -``` - -With the new schema, this becomes an assembly step that reconstructs PydanticAI `ModelRequest`/`ModelResponse` pairs from the event log: - -```python -@classmethod -def from_buffer(cls, execution_id: UUID, session: Session, ...) -> Self | None: - repo = ContextEventRepository() - events = repo.get_for_execution(session, execution_id) - seed_messages = assemble_pydantic_ai_messages(events) - worker = cls(...) - worker._seed_messages = seed_messages - return worker -``` - -`assemble_pydantic_ai_messages(events)` reconstructs the alternating `ModelRequest`/`ModelResponse` sequence from the structured event fields only: - -- `system_prompt` events → `SystemPromptPart` in the first `ModelRequest` -- `user_message` events → `UserPromptPart` in the first `ModelRequest` -- `thinking` events → `ThinkingPart` in the current `ModelResponse` -- `assistant_text` events → `TextPart` in the current `ModelResponse` -- `tool_call` events → `ToolCallPart` in the current `ModelResponse` -- `tool_result` events → `ToolReturnPart` in the next `ModelRequest` - -Grouping rule: consecutive model-generated events (`thinking`, `assistant_text`, `tool_call`) with no intervening `tool_result` belong to the same `ModelResponse`. A `tool_result` event closes the current `ModelResponse` and opens a new `ModelRequest`. This means thinking tokens are always included in the reconstructed history — they are first-class `ThinkingPart` entries, not dropped. - -This function lives in `ergon_core/ergon_core/core/persistence/context/assembly.py`. - -```python -# ergon_core/ergon_core/core/persistence/context/assembly.py - -from pydantic_ai.messages import ( - ModelRequest, - ModelResponse, - SystemPromptPart as PydanticSystemPromptPart, - UserPromptPart as PydanticUserPromptPart, - TextPart as PydanticTextPart, - ToolCallPart as PydanticToolCallPart, - ToolReturnPart as PydanticToolReturnPart, - ThinkingPart as PydanticThinkingPart, - ArgsDict, - ModelMessage, -) -from ergon_core.core.persistence.context.models import RunContextEvent -from ergon_core.core.persistence.context.event_payloads import ( - SystemPromptPayload, - UserMessagePayload, - AssistantTextPayload, - ToolCallPayload, - ToolResultPayload, - ThinkingPayload, -) - - -def assemble_pydantic_ai_messages(events: list[RunContextEvent]) -> list[ModelMessage]: - """Reconstruct the PydanticAI ModelRequest/ModelResponse sequence from stored events. - - Events must be pre-sorted by sequence (ascending) — matches get_for_execution output. - - Grouping rules: - - system_prompt + user_message events → SystemPromptPart / UserPromptPart in the first ModelRequest - - thinking / assistant_text / tool_call events → parts of the current ModelResponse - - tool_result event → closes the current ModelResponse, opens a new ModelRequest with ToolReturnPart - - trailing response (turn with no tool results) → flushed at end - """ - messages: list[ModelMessage] = [] - current_request_parts: list = [] - current_response_parts: list = [] - - for event in events: - parsed = event.parsed_payload() - - if event.event_type == "system_prompt": - assert isinstance(parsed, SystemPromptPayload) - current_request_parts.append(PydanticSystemPromptPart(content=parsed.text)) - - elif event.event_type == "user_message": - assert isinstance(parsed, UserMessagePayload) - current_request_parts.append(PydanticUserPromptPart(content=parsed.text)) - - elif event.event_type in ("thinking", "assistant_text", "tool_call"): - # First model-generated event of a turn: flush the pending request - if current_request_parts and not current_response_parts: - messages.append(ModelRequest(parts=current_request_parts)) - current_request_parts = [] - - if event.event_type == "thinking": - assert isinstance(parsed, ThinkingPayload) - current_response_parts.append(PydanticThinkingPart(content=parsed.text)) - elif event.event_type == "assistant_text": - assert isinstance(parsed, AssistantTextPayload) - current_response_parts.append(PydanticTextPart(content=parsed.text)) - elif event.event_type == "tool_call": - assert isinstance(parsed, ToolCallPayload) - current_response_parts.append(PydanticToolCallPart( - tool_name=parsed.tool_name, - tool_call_id=parsed.tool_call_id, - args=ArgsDict(args_dict=parsed.args), - )) - - elif event.event_type == "tool_result": - assert isinstance(parsed, ToolResultPayload) - # Close the current ModelResponse - if current_response_parts: - messages.append(ModelResponse(parts=current_response_parts)) - current_response_parts = [] - # Open a new ModelRequest with the tool return - # str(parsed.result): worker JSON-serialises complex results before storage, - # so the string representation is adequate for re-injection. - current_request_parts.append(PydanticToolReturnPart( - tool_call_id=parsed.tool_call_id, - tool_name=parsed.tool_name, - content=str(parsed.result), - )) - - # Flush any trailing response (final turn produced no tool results) - if current_response_parts: - messages.append(ModelResponse(parts=current_response_parts)) - - return messages -``` - -`ArgsDict` is `pydantic_ai.messages.ArgsDict`. Confirm exact import path during Phase 1 implementation — PydanticAI's internal message module paths can shift between minor versions. - -### 5.3 Dashboard Streaming - -The emitter and Inngest function are fully specced in §4.5. This section covers the frontend state model, live reducer, and UI components. - -#### TypeScript types - -```typescript -// ergon-dashboard/src/lib/contracts/contextEvents.ts - -export type ContextEventType = - | "system_prompt" - | "user_message" - | "assistant_text" - | "tool_call" - | "tool_result" - | "thinking"; - -// Discriminated union — mirrors Python ContextEventPayload -export type ContextEventPayload = - | { event_type: "system_prompt"; text: string } - | { event_type: "user_message"; text: string; from_worker_key: string | null } - | { event_type: "assistant_text"; text: string; turn_id: string; turn_logprobs: TokenLogprob[] | null } - | { event_type: "tool_call"; tool_call_id: string; tool_name: string; args: Record<string, unknown>; turn_id: string; turn_logprobs: TokenLogprob[] | null } - | { event_type: "tool_result"; tool_call_id: string; tool_name: string; result: unknown; is_error: boolean } - | { event_type: "thinking"; text: string; turn_id: string; turn_logprobs: TokenLogprob[] | null }; - -export interface TokenLogprob { - token: string; - logprob: number; -} - -export interface ContextEventState { - id: string; - taskExecutionId: string; - taskNodeId: string; - workerBindingKey: string; - sequence: number; - eventType: ContextEventType; - payload: ContextEventPayload; - createdAt: string; - startedAt: string | null; - completedAt: string | null; -} -``` - -Zod schema in `ergon-dashboard/src/lib/contracts/events.ts` must match the `DashboardContextEventEvent` Python contract field-for-field. - -#### Store changes - -`WorkflowRunState` replaces `generationTurns: GenerationTurnState[]` with a per-task map: - -```typescript -// ergon-dashboard/src/lib/state/store.ts - -export interface WorkflowRunState { - // ... existing fields unchanged ... - - // Replaces: generationTurns: GenerationTurnState[] - contextEventsByTask: Map<string, ContextEventState[]>; - // keyed by task_node_id (string UUID); events are in sequence order within each key -} -``` - -New store method: - -```typescript -addContextEvent(runId: string, taskNodeId: string, event: ContextEventState): void { - const run = this.runs.get(runId); - if (!run) return; - const existing = run.contextEventsByTask.get(taskNodeId) ?? []; - // Insert in sequence order — live events typically arrive in order, - // but guard against redelivery by deduplicating on id - if (existing.some(e => e.id === event.id)) return; - run.contextEventsByTask.set(taskNodeId, [...existing, event].sort((a, b) => a.sequence - b.sequence)); -} -``` - -#### Client socket handler - -```typescript -socket.on("context:event", ({ runId, taskNodeId, event }) => { - setRunState(prev => { - const existing = prev.contextEventsByTask.get(taskNodeId) ?? []; - if (existing.some(e => e.id === event.id)) return prev; // deduplicate - const updated = new Map(prev.contextEventsByTask); - updated.set(taskNodeId, [...existing, event].sort((a, b) => a.sequence - b.sequence)); - return { ...prev, contextEventsByTask: updated }; - }); -}); -``` - -#### Snapshot hydration - -On initial load, `build_run_snapshot` returns `context_events_by_task` (see §5.4). The client hydrates `contextEventsByTask` from this field directly — the structure is identical. - -#### UI components - -``` -ergon-dashboard/src/features/graph/components/ - ContextEventLog.tsx — container: takes ContextEventState[], renders in order - ContextEventEntry.tsx — dispatcher: switches on eventType, renders child - events/ - SystemPromptEvent.tsx — grey collapsed header; expand to show full text - UserMessageEvent.tsx — indigo chat bubble; shows from_worker_key label if set - ThinkingEvent.tsx — purple italicised block; collapsed by default (can be long) - AssistantTextEvent.tsx — white assistant bubble; plain text render - ToolCallEvent.tsx — amber block; tool name as header, args as collapsible JSON - ToolResultEvent.tsx — green block (red if is_error); result as collapsible JSON -``` - -**`ContextEventLog`** — receives all `ContextEventState[]` for a task node. Renders events in `sequence` order. Each event is separated by a thin timeline connector. - -**`ContextEventEntry`** — pure dispatcher: -```tsx -function ContextEventEntry({ event }: { event: ContextEventState }) { - switch (event.eventType) { - case "system_prompt": return <SystemPromptEvent payload={event.payload} />; - case "user_message": return <UserMessageEvent payload={event.payload} />; - case "thinking": return <ThinkingEvent payload={event.payload} timestamps={event} />; - case "assistant_text": return <AssistantTextEvent payload={event.payload} timestamps={event} />; - case "tool_call": return <ToolCallEvent payload={event.payload} timestamps={event} />; - case "tool_result": return <ToolResultEvent payload={event.payload} />; - } -} -``` - -**Event-type component contracts:** - -| Component | Key props | Default collapsed? | Notes | -|---|---|---|---| -| `SystemPromptEvent` | `text` | Yes | Full text in expand | -| `UserMessageEvent` | `text`, `from_worker_key` | No | Show delegation source label when set | -| `ThinkingEvent` | `text`, `startedAt`, `completedAt` | Yes | Duration badge; italic text | -| `AssistantTextEvent` | `text`, `startedAt`, `completedAt` | No | Duration badge | -| `ToolCallEvent` | `tool_name`, `args`, `tool_call_id`, timing | Yes (args) | Header shows tool name; args as syntax-highlighted JSON | -| `ToolResultEvent` | `tool_name`, `result`, `is_error` | Yes (result) | Red border if `is_error`; result as JSON | - -`ToolCallEvent` and `ToolResultEvent` should be visually linked by `tool_call_id` — when expanded, the tool result shows a back-reference to the call that produced it (and vice versa). Implementation can use a subtle connector line or matching colour stripe. - -### 5.4 REST Snapshot - -`GET /runs/{run_id}` (`build_run_snapshot`) currently loads `RunGenerationTurn` rows and groups by `execution_task_map` into `generation_turns_by_task`. This is replaced by loading `RunContextEvent` rows grouped by execution: - -```python -context_events_stmt = ( - select(RunContextEvent) - .where(RunContextEvent.run_id == run_id) - .order_by(RunContextEvent.task_execution_id, RunContextEvent.sequence) -) -context_events = list(session.exec(context_events_stmt).all()) - -context_events_by_task: dict[str, list["RunContextEventDto"]] = defaultdict(list) -for event in context_events: - task_node_id = execution_task_map.get(event.task_execution_id) - if task_node_id is None: - continue - context_events_by_task[str(task_node_id)].append( - RunContextEventDto( - id=str(event.id), - task_execution_id=str(event.task_execution_id), - sequence=event.sequence, - event_type=event.event_type, - payload=event.payload, - created_at=event.created_at.isoformat(), - ) - ) -``` - -The `RunSnapshotDto` field `generation_turns_by_task` is renamed to `context_events_by_task` and typed accordingly. - -```python -# ergon_core/ergon_core/core/api/schemas.py - -class RunContextEventDto(BaseModel): - id: str - task_execution_id: str - sequence: int - event_type: str # ContextEventType — kept as plain str for forward compat - payload: dict[str, Any] - created_at: str # ISO 8601 - started_at: str | None = None - completed_at: str | None = None -``` - ---- - -## 6. File Map - -### New files -``` -ergon_core/ergon_core/core/persistence/context/ - __init__.py - models.py — RunContextEvent ORM model - event_payloads.py — ContextEventPayload discriminated union (all 6 types) - repository.py — ContextEventRepository (persist_turn, get_for_execution, get_for_run) - assembly.py — assemble_pydantic_ai_messages() for from_buffer() reconstruction -ergon_core/migrations/versions/<hash>_add_run_context_events.py -``` - -### Changed files -``` -ergon_core/ergon_core/api/generation.py - + ModelRequestPart union (SystemPromptPart | UserPromptPart | ToolReturnPart) - + ModelResponsePart union (TextPart | ToolCallPart | ThinkingPart) - ~ GenerationTurn: messages_in, response_parts, tool_results (typed); drop raw_response, prompt_text - -ergon_builtins/ergon_builtins/workers/baselines/react_worker.py - ~ _build_turns(): populate messages_in from ModelRequest parts - ~ _to_turn(): accepts request_in + response + tool_result_request - ~ from_buffer(): use assemble_pydantic_ai_messages() instead of raw_response - -ergon_core/ergon_core/core/runtime/inngest/worker_execute.py - ~ replace GenerationTurnRepository + persist_single with ContextEventRepository + persist_turn - -ergon_core/ergon_core/core/dashboard/emitter.py - + on_context_event() listener - + DashboardContextEventEvent contract - -ergon_core/ergon_core/core/dashboard/event_contracts.py - + DashboardContextEventEvent - -ergon_core/ergon_core/core/api/runs.py - ~ build_run_snapshot(): load RunContextEvent instead of RunGenerationTurn - ~ RunSnapshotDto: generation_turns_by_task → context_events_by_task - -ergon_core/ergon_core/core/api/schemas.py - + RunContextEventDto - ~ RunSnapshotDto: generation_turns_by_task → context_events_by_task - -ergon_core/ergon_core/core/rl/extraction.py - ~ extract_agent_trajectories(): read RunContextEvent instead of RunGenerationTurn - -ergon-dashboard/src/features/graph/components/ - + ContextEventLog.tsx - + ContextEventEntry.tsx - + events/SystemPromptEvent.tsx - + events/UserMessageEvent.tsx - + events/ThinkingEvent.tsx - + events/AssistantTextEvent.tsx - + events/ToolCallEvent.tsx - + events/ToolResultEvent.tsx - -ergon-dashboard/src/lib/contracts/ - + contextEvents.ts — ContextEventState, ContextEventPayload, ContextEventType TS types - ~ events.ts — + Zod schema for DashboardContextEventEvent - -ergon-dashboard/src/lib/state/store.ts - ~ WorkflowRunState: generationTurns → contextEventsByTask: Map<string, ContextEventState[]> - + addContextEvent() store method - -ergon-dashboard/src/lib/socket/types.ts - + "context:event" to ServerToClientEvents - -ergon-dashboard/src/inngest/functions/ - + onContextEvent.ts — Inngest handler: parses event, calls store + broadcast -``` - -### Deleted as part of this work - -These are removed in Phase 3 once all readers have been migrated. They are not deleted in Phase 1 or 2. - -``` -ergon_core/ergon_core/core/persistence/telemetry/models.py - DELETE class RunGenerationTurn (all fields + parsed_tool_calls / parsed_tool_results / - parsed_token_ids / parsed_logprobs / _parse_optional_list / - _validate_json_columns methods) - KEEP ExecutionOutcome type alias (still used by RunTaskExecution; no longer used by RunContextEvent) - KEEP all other model classes - -ergon_core/ergon_core/core/persistence/telemetry/repositories.py - DELETE class GenerationTurnRepository (entire class: __init__, add_listener, - persist_single, persist_turns, - get_for_execution, get_for_run, - mark_execution_outcome) - KEEP TelemetryRepository (remove only the GenerationTurnRepository call site inside it) - -ergon_core/ergon_core/core/dashboard/emitter.py - DELETE on_turn_persisted() method - DELETE import of RunGenerationTurn - KEEP everything else - -ergon_core/ergon_core/core/dashboard/event_contracts.py - DELETE DashboardGenerationTurnEvent class - KEEP all other contracts - -ergon_core/ergon_core/core/api/runs.py - DELETE GET /runs/{run_id}/generations endpoint - DELETE generation_turns_by_task field from RunSnapshotDto - KEEP context_events_by_task (added in Phase 2) -``` - -The `run_generation_turns` table is **never dropped** — historical data lives there. The ORM model class is removed but the table is left in place. - ---- - -## 7. Migration Plan - -The migration is additive. The old table is never dropped until every reader has been migrated. No data is backfilled — historical runs keep `RunGenerationTurn` rows; new runs write `RunContextEvent` rows only. - -### Phase 0 — Schema (no behaviour change) - -- Add Alembic migration: `run_context_events` table. -- Add ORM model, payload types, repository — no callers yet. -- Deploy. Verify table exists. - -### Phase 1 — Dual write - -- Update `_build_turns()` to populate `messages_in` on `GenerationTurn`. -- Update `worker_execute.py` to call **both** `GenerationTurnRepository.persist_single()` and `ContextEventRepository.persist_turn()` for every turn. -- Update `DashboardEmitter` to emit `DashboardContextEventEvent` alongside the existing `DashboardGenerationTurnEvent`. -- Deploy. Verify both tables are populated for new runs. Existing reads are unchanged. - -### Phase 2 — Migrate reads - -In any order (each is independently deployable): - -- **RL extraction** (`extraction.py`): read from `RunContextEvent`. Gate on `len(context_events) > 0`; fall back to `RunGenerationTurn` if no events exist (historical run). -- **REST snapshot** (`runs.py`): load `RunContextEvent` into `context_events_by_task`. Keep `generation_turns_by_task` populated from `RunGenerationTurn` until the frontend is updated. -- **Worker resumption** (`react_worker.py` `from_buffer()`): use `assemble_pydantic_ai_messages()`. If no context events exist for the execution (historical run), `from_buffer()` returns `None` (same as today's "execution not found" path) — do not fall back to `raw_response` blob reconstruction. -- **Frontend**: update generation panel to consume `context_events_by_task` from the snapshot and `dashboard/context.event` from the socket. Remove `generation_turns_by_task` consumption. -- **`GET /runs/{run_id}/generations` endpoint**: mark deprecated in OpenAPI docs. - -Deploy and verify each migration independently before proceeding. - -### Phase 3 — Decommission - -Once all readers have been migrated and no new `RunGenerationTurn` rows are being written: - -- Remove the dual-write call to `GenerationTurnRepository.persist_single()` from `worker_execute.py`. -- Remove the `DashboardGenerationTurnEvent` emit from `DashboardEmitter`. -- Remove `GET /runs/{run_id}/generations` endpoint. -- Remove `generation_turns_by_task` from `RunSnapshotDto`. -- Archive `RunGenerationTurn` model and `GenerationTurnRepository` (keep the table; do not drop it — historical data lives there). - -### Migration go/no-go criteria - -| Gate | Check | -|---|---| -| Phase 0 → 1 | `run_context_events` table exists in prod, migration clean | -| Phase 1 → 2 | Both tables populate correctly for 10+ live runs, event counts match turn counts | -| Phase 2 → 3 | All readers verified, frontend uses context_events_by_task, RL extraction reads from new table | -| Phase 3 | Zero active callers of `GenerationTurnRepository.persist_single()` | - ---- - -## 8. Known Gaps Deferred to Later Work - -**Token IDs from vLLM.** `turn_token_ids` (on the first model-output event of each turn, alongside `turn_logprobs`) will remain `None` until the vLLM provider is updated to extract token IDs from `provider_details`. PydanticAI currently exposes logprobs via `provider_details["logprobs"]` but does not surface the corresponding token ID integers. The schema and `GenerationTurn` DTO are ready; populating the field requires a provider-side improvement and does not block the schema migration. - -**Logprobs for tool call argument tokens.** `turn_logprobs` stores the flat `choice.logprobs.content` list from vLLM, and is held on the first model-output event of each turn (grouped by `turn_id`). Whether vLLM includes tool call argument tokens in `logprobs.content` is empirically unknown — vLLM generates everything in one forward pass then splits, so it *may* include them. A spike (`scripts/spike_logprob_splitting.py`) has been written; run it with `VLLM_BASE_URL` set against a live vLLM instance to get a definitive answer. Until the spike runs, `turn_logprobs` is stored as `None`; populate once the live probe confirms coverage. - -**RL logprob alignment for multi-event turns.** The current `_get_logprobs` helper slices `turn_logprobs` to `n_tokens` per event, which is correct only for single-event turns. For turns that produce multiple model-output events (e.g., ThinkingPart + TextPart + ToolCallPart), the correct approach is to group events by `turn_id`, accumulate all `token_ids` across the group, then align `turn_logprobs` once against the concatenated token sequence. This avoids misaligning text-token logprobs onto tool-call-token positions. Implement `turn_id`-grouped alignment in Phase 1; it is required for importance-sampling–based RL. Reward-only RL is unaffected (rewards depend on outcomes, not individual token logprobs). - -**Extended thinking extraction.** `ThinkingPayload` is defined. Extraction from PydanticAI's `ModelResponse` (`part_kind == "thinking"`) requires verifying the actual field name in the PydanticAI dataclass for the Claude backend. This should be confirmed during Phase 1 implementation. - -**Agent-to-agent messages.** `UserMessagePayload.from_worker_key` is defined for delegation messages (where one agent sends a message to another). Populating this requires the task management toolkit to emit a `user_message` context event when a delegation result is returned. Not required for Phase 1; add in a follow-on when agent-to-agent communication logging is needed. - -**System prompt deduplication.** In a multi-turn execution the system prompt is constant but currently emitted once (on the first turn's `messages_in`). The assembly function must handle this correctly — reconstruct it as a single leading `SystemPromptPart` regardless of how many turns exist. This is a `assembly.py` implementation concern, not a schema concern. - ---- - -## 9. What This Doesn't Change - -- Worker `execute()` signature — still `async def execute(...) -> AsyncGenerator[GenerationTurn, None]` -- `WorkerContext` — unchanged -- Graph WAL (`run_graph_mutations`) — separate concern -- Task execution records (`RunTaskExecution`) — unchanged -- Evaluation records (`RunTaskEvaluation`) — unchanged -- All existing `graph:mutation`, `task:status` socket events — unchanged diff --git a/docs/integration-spec/0-task-status-design.md b/docs/integration-spec/0-task-status-design.md deleted file mode 100644 index d5c01b28c..000000000 --- a/docs/integration-spec/0-task-status-design.md +++ /dev/null @@ -1,717 +0,0 @@ -# Task Status Design - -> **Status:** DRAFT — for review before schema changes, integration test updates, or core logic changes. -> -> This document is the canonical reference for task status vocabulary, terminology, and propagation rules. It supersedes status semantics described elsewhere in the integration spec. The intended workflow: agree this doc → update schemas → rewrite integration tests against these semantics → fix core logic to pass. - ---- - -## 1. Terminology - -Two orthogonal relationships connect tasks. Propagation rules differ completely between them, so the terms must be used precisely. - -### Containment (vertical axis) - -Created by `plan_subtasks()` / `add_subtask()`. Stored via `parent_node_id` on `RunGraphNode`. Represents "this task owns the execution context for its children." - -| Term | Definition | -|------|-----------| -| **Parent** | The task that spawned this task. One level up via `parent_node_id`. | -| **Children** | Tasks spawned by this task. One level down via `parent_node_id`. Collectively a sub-DAG — children can have dependency edges between themselves. | -| **Ancestors** | All parents, grandparents, etc. up the containment chain. | -| **Descendants** | All children, grandchildren, etc. — the full containment subtree. | - -### Dependency (horizontal axis) - -Created by the experiment definition or by `plan_subtasks()` dependency specs. Stored as `RunGraphEdge` rows. Represents "this task's outputs are inputs to its successors." - -| Term | Definition | -|------|-----------| -| **Predecessors** | Tasks this task depends on. Incoming `RunGraphEdge`. Must COMPLETE before this task can start. "To the left." | -| **Successors** | Tasks that depend on this task. Outgoing `RunGraphEdge`. Waiting for this task's outputs. "To the right." | - -**Hard constraint:** Cross-containment dependency edges are forbidden. A `RunGraphEdge` must not connect two nodes with different `parent_node_id` values. All dependency edges within a sub-DAG are between siblings (same `parent_node_id`). This keeps the two axes independent. - ---- - -## 2. Status Definitions - -| Status | Terminal? | Meaning | -|--------|-----------|---------| -| `PENDING` | No | Created. Not all predecessors have COMPLETED. Normal waiting state. | -| `READY` | No | All predecessors COMPLETED. Eligible for dispatch via `task/ready`. | -| `BLOCKED` | No | At least one predecessor is FAILED, or parent context is unavailable. Cannot proceed without operator action. | -| `RUNNING` | No | Worker executing in sandbox. | -| `COMPLETED` | Yes | Worker returned success. Outputs available in `RunResource`. | -| `FAILED` | Yes | Worker raised an error during execution. Outputs unavailable. | -| `CANCELLED` | Yes | Explicitly stopped by operator, or orphaned by parent cancellation. | - -### Design principles - -**BLOCKED is not PENDING.** PENDING means "waiting for normal upstream progress." BLOCKED means "progress has stopped — a dependency failed and operator action is required." Without this distinction, a stuck run with BLOCKED tasks looks identical to an active run with PENDING tasks. - -**FAILED is reserved for tasks whose worker actually errored.** A task that never ran because its predecessor failed is BLOCKED, not FAILED. FAILED means "the execution attempt ran and something went wrong inside it." - -**CANCELLED means orphaned or explicitly stopped.** A child whose parent was cancelled has no execution context to return to — it is CANCELLED. A task explicitly stopped by the operator is CANCELLED. A task that cannot proceed because a predecessor failed is BLOCKED, not CANCELLED. - -**Successors are never automatically cancelled when a predecessor fails.** They become BLOCKED. The operator decides: restart the failed predecessor, manually cancel the successors, or cancel the run. The system holds state, not makes decisions. - ---- - -## 3. Conditional FSM - -### Transition table - -| From | To | Trigger | Who can trigger | -|------|----|---------|----------------| -| `PENDING` | `READY` | All predecessors COMPLETED | Propagation (automatic) | -| `PENDING` or `READY` | `BLOCKED` | Any predecessor reaches FAILED | Propagation (automatic) | -| `BLOCKED` | `PENDING` | The FAILED predecessor is restarted (→ PENDING) | Propagation (automatic) | -| `READY` | `RUNNING` | Worker picks up `task/ready` | Inngest dispatch | -| `RUNNING` | `COMPLETED` | Worker returns success | Worker | -| `RUNNING` | `FAILED` | Worker raises error | Worker | -| `FAILED` | `PENDING` | Operator calls `restart_task()` | Operator | -| `COMPLETED` | `PENDING` | Operator calls `restart_task()` | Operator | -| Any non-terminal | `CANCELLED` | Operator calls `cancel_task()`, or parent → CANCELLED | Operator / cascade | - -### Propagation rules per transition - ---- - -#### PENDING → READY - -**Trigger:** All incoming `RunGraphEdge` rows are `EDGE_SATISFIED` (all predecessors COMPLETED). - -**Propagation:** -- Emit `task/ready`. Nothing else changes. - ---- - -#### PENDING or READY → BLOCKED - -**Trigger:** Any predecessor reaches FAILED. - -**Propagation:** -- **Successors →** cascade BLOCKED rightward: any successor that is PENDING or READY → BLOCKED. RUNNING successors are not interrupted (they have live executions; let them finish or fail). -- **Descendants →** none (task has not started; no children exist yet). -- **Parent →** no direct effect. Parent is waiting for this task to be terminal; BLOCKED is non-terminal, so parent is now stuck. - ---- - -#### BLOCKED → PENDING - -**Trigger:** The predecessor that was FAILED is restarted (transitions to PENDING). - -**Propagation:** -- Incoming edge from restarted predecessor resets to EDGE_PENDING. -- This task returns to PENDING (not READY — predecessor must complete again before this becomes READY). -- **Successors →** cascade PENDING rightward: successors that were BLOCKED because of this task → PENDING (they can re-evaluate once the chain resolves). - ---- - -#### READY → RUNNING - -**Trigger:** Worker picks up `task/ready` event and begins execution. - -**Propagation:** None. - ---- - -#### RUNNING → COMPLETED - -**Trigger:** Worker returns success; `persist_outputs` writes to `RunResource`. - -**Propagation:** -- **Outgoing edges →** flip to `EDGE_SATISFIED`. -- **Successors (horizontal) →** for each successor, re-evaluate all incoming edges: - - All `EDGE_SATISFIED` → successor → READY - - Any predecessor FAILED → successor stays BLOCKED - - Any predecessor still PENDING/RUNNING → successor stays PENDING -- **Descendants (vertical) →** this task does NOT finalise as COMPLETED until all descendants are terminal. The propagation loop waits. If descendants are BLOCKED, this task is stuck. -- **Parent →** when this task becomes terminal, parent re-evaluates its completion condition. No transition is triggered yet; the parent waits for all its children. - ---- - -#### RUNNING → FAILED - -**Trigger:** Worker raises an unrecoverable error. - -**Propagation:** -- **Successors (horizontal) →** each successor that is PENDING or READY → BLOCKED. RUNNING successors are not interrupted. -- **Descendants (vertical) →** each descendant that is PENDING or READY → BLOCKED. RUNNING descendants are not interrupted (they continue executing; they will succeed or fail on their own). -- **Parent →** no direct effect. Parent is waiting for all children to be terminal. BLOCKED descendants are non-terminal, so parent is now stuck. -- **Run-level →** `RunRecord.status` stays `RUNNING`. The run is not automatically failed. It is stuck. - -**What does NOT happen:** -- Successors are not CANCELLED. -- Descendants are not FAILED (they didn't run and error — their predecessor did). -- `RunRecord` does not become FAILED. - ---- - -#### FAILED → PENDING (restart) - -**Trigger:** Operator calls `restart_task()`. - -**Propagation:** -- **Own outgoing edges →** reset to `EDGE_PENDING`. -- **Successors (horizontal) →** successors that are BLOCKED because of this task → PENDING. Their incoming edge from this task resets to `EDGE_PENDING`. They re-evaluate when this task eventually completes. -- **Descendants (vertical) →** full subtree reset: **all descendants → PENDING** regardless of prior status, including previously COMPLETED ones. This is intentional: the new execution gets a fresh container; prior outputs are bound to the old container's context. Roots within the subtask group are dispatched via `task/ready`. Non-roots wait for propagation within the subtask group. -- **Parent →** no direct effect. - -**Design note on plan_subtasks:** When the restarted task re-executes, it will call `plan_subtasks()` again. The prior children (now reset to PENDING) must be cancelled first, and the new execution creates a fresh set of children. Resetting to PENDING and then calling `plan_subtasks()` again would create duplicate child generations. The correct restart sequence is: cancel prior descendants → reset own edges → dispatch `task/ready` → new execution creates fresh children via `plan_subtasks()`. - ---- - -#### Any non-terminal → CANCELLED - -**Trigger:** Operator calls `cancel_task()` (cause: `manager_decision`) or parent reaches CANCELLED (cause: `parent_cancelled`). - -**Propagation:** -- **Descendants (vertical) →** all descendants → CANCELLED (orphaned; their execution context is gone). -- **Successors (horizontal) →** stay BLOCKED. The cancelled task still did not produce outputs; successors cannot proceed. They are waiting, not orphaned. The operator must separately cancel them or find another path forward. -- **Parent →** no direct effect. CANCELLED is terminal; parent re-evaluates its completion condition. - ---- - -### Upward propagation (child → parent) - -When a child reaches a terminal status, the parent re-evaluates whether it can finalise. - -| Child terminal status | Parent effect | -|----------------------|--------------| -| COMPLETED | Re-evaluate: if all children terminal → parent finalises (see below) | -| FAILED | Children that are BLOCKED are non-terminal → parent stuck | -| CANCELLED | Terminal. If all children terminal → parent evaluates | -| BLOCKED | Non-terminal → parent stuck | - -**Parent finalisation:** When all descendants are terminal, the parent evaluates: - -| Descendant terminal states | Parent result | -|---------------------------|--------------| -| All in `{COMPLETED, CANCELLED}` | Parent → COMPLETED (propagate normally) | -| Any FAILED | Parent → FAILED (its sub-execution did not fully succeed) | - ---- - -## 4. RunRecord Lifecycle - -`RunRecord` tracks the run-level view. It does **not** get a BLOCKED status. - -| Status | Meaning | -|--------|---------| -| `CREATED` | Run record created; workflow not yet started | -| `RUNNING` | Workflow executing — includes stuck/blocked state | -| `COMPLETED` | All tasks terminal in `{COMPLETED, CANCELLED}`; scores aggregated | -| `FAILED` | Operator explicitly failed the run, or a run-level timeout fired | -| `CANCELLED` | Operator cancelled the run; all non-terminal tasks cascade to CANCELLED | - -**Stuck detection:** A run with BLOCKED tasks shows `RUNNING`. Monitoring detects stuck runs by: `RunRecord.status == RUNNING` with no `RunGraphMutation` rows newer than a configurable threshold (e.g. 30 minutes). This is preferable to a `BLOCKED` RunRecord status because the condition is transient — once the operator restarts the failed task, the run is live again. - ---- - -## 5. Schema Changes Required - -### A. Add `BLOCKED` to node status and task execution status - -```sql --- Postgres enum extension (non-transactional; run outside a transaction block) -ALTER TYPE taskexecutionstatus ADD VALUE IF NOT EXISTS 'blocked'; -``` - -- Add `BLOCKED = "blocked"` to `TaskExecutionStatus` in `ergon_core/core/persistence/shared/enums.py` -- Update `TERMINAL_STATUSES` in `status_conventions.py` — BLOCKED is **non-terminal** -- Alembic migration required - -### B. Add `triggered_by_mutation_id` to `RunGraphMutation` (assumption L) - -```sql -ALTER TABLE rungraphmutation -ADD COLUMN triggered_by_mutation_id UUID REFERENCES rungraphmutation(id); -``` - -- Add field to `RunGraphMutation` model -- All cascade-produced mutations must populate this field -- Alembic migration required - -### C. Move `sandbox_id` from `RunRecord.summary_json` to `RunTaskExecution` (assumption A) - -```sql -ALTER TABLE runtaskexecution ADD COLUMN sandbox_id VARCHAR; -``` - -- Remove sandbox_id from `RunRecord.summary_json` -- `run/cleanup` collects all `RunTaskExecution.sandbox_id` values for the run and terminates each -- Alembic migration required - -### D. Remove `StubSandboxManager` and `is_stub_sandbox_id` from production code (assumption B) - -No schema change — code cleanup only (four files, see violated assumption B). - -### E. Add `batch_operation_id` to `RunGraphMutation` (bulk operations) - -```sql -ALTER TABLE rungraphmutation -ADD COLUMN batch_operation_id UUID; - -CREATE INDEX ON rungraphmutation (batch_operation_id) -WHERE batch_operation_id IS NOT NULL; -``` - -- Add field to `RunGraphMutation` model; `NULL` for non-batch mutations -- All mutations (direct + cascade) from a single bulk operator request share the same `batch_operation_id` -- Alembic migration required - ---- - -## 6. Impact on Integration Spec - -The following tests in `2-control-flow-spec.md` need rewriting against these semantics: - -| Test | Current (wrong) | Correct | -|------|----------------|---------| -| Test 3 — failure cascade | Successors → CANCELLED with cause `dep_invalidated` | Successors → BLOCKED; nothing is cancelled | -| Test 6 — manager_decision cancel | Successors → BLOCKED (this part was correct) | Confirmed correct; add assertion `RunRecord.status == RUNNING` not FAILED | -| Test 7 — parent failure cascade | Children → FAILED | Children → BLOCKED; FAILED is only for tasks whose worker errored | -| Violated assumption J | "children should be FAILED not CANCELLED" | Correct target is BLOCKED, not FAILED | - ---- - -## 8. Bulk Status Operations - -Operators need to atomically move multiple tasks to different statuses in a single request — e.g., restart three tasks, cancel two others, and manually unblock one. This section specifies the semantics required for that to be safe. - -### Request shape - -``` -PATCH /runs/{run_id}/tasks/status -{ - "changes": [ - { "task_id": "<uuid>", "target_status": "PENDING", "cause": "operator_restart" }, - { "task_id": "<uuid>", "target_status": "CANCELLED", "cause": "manager_decision" }, - { "task_id": "<uuid>", "target_status": "PENDING", "cause": "operator_unblock" } - ] -} -``` - -Returns: the full set of `RunGraphMutation` rows created (direct + cascades). - ---- - -### Execution protocol - -#### Step 1 — Pre-flight validation (before any write) - -For every `(task_id, target_status)` entry: - -1. **FSM validity**: the transition `current_status → target_status` must appear in the transition table (section 3), including the two new operator transitions added below. Reject the entire batch on any violation. -2. **Duplicate detection**: if the same `task_id` appears more than once in the batch, reject the entire batch. -3. **Task existence**: all `task_id` values must exist in `RunGraphNode` for this run. Reject on unknown ID. - -No writes occur during this phase. A batch that fails validation leaves the database unchanged. - -#### Step 2 — Apply in topological order - -Sort the `changes` list by the dependency graph topology (predecessors before successors). Within the same topological rank: **restarts before cancellations**, so that cascade-BLOCKED unlocking resolves before cancellations potentially re-BLOCK downstream tasks. - -Apply each transition as a normal `RunGraphMutation` write (status column update + WAL row). All writes share the same `batch_operation_id` (see schema change E below). - -#### Step 3 — Single propagation pass - -After all specified transitions are written, run one propagation pass over the full affected subgraph (all successors and descendants of every changed task). Do not run propagation after each individual change — this avoids transient intermediate states being observed or emitted as events. - -#### Step 4 — Cascade WAL attribution - -Every cascade mutation produced by the propagation pass in Step 3 gets: -- `triggered_by_mutation_id` = the `RunGraphMutation.id` of the specific batch item that caused it -- `batch_operation_id` = same UUID as the rest of the batch - -This makes "what did this bulk edit actually touch?" a single-key query on `batch_operation_id`. - ---- - -### New operator FSM transitions - -Two transitions not in the section 3 table are required for bulk edits to be useful: - -| From | To | Cause | Notes | -|------|----|-------|-------| -| `BLOCKED` | `PENDING` | `operator_unblock` | Operator asserts "ignore the failed predecessor; re-evaluate this task." Incoming edge from the failed predecessor is reset to `EDGE_PENDING`. The failed predecessor is still FAILED — the operator is bypassing it, not fixing it. | -| `FAILED` | `CANCELLED` | `manager_decision` | Operator gives up on a task without restarting it. Propagates: all descendants → CANCELLED; successors stay BLOCKED (they still lack this task's outputs). | - -`operator_unblock` requires explicit intent — it must not fire automatically. It is only reachable via a bulk or single-task operator call, never from propagation logic. - ---- - -### Atomicity guarantee - -The entire batch — Steps 2, 3, and 4 — executes inside a single database transaction. Either all mutations commit or none do. Inngest events (`task/ready`, etc.) are sent only after the transaction commits. - ---- - -### Schema change E — `batch_operation_id` on `RunGraphMutation` - -```sql -ALTER TABLE rungraphmutation -ADD COLUMN batch_operation_id UUID; -``` - -- Populated for all mutations (direct + cascade) that originate from a bulk operator request. -- `NULL` for single-task operator actions and for propagation-only cascades not part of a batch. -- No FK constraint — it is a correlation tag, not a parent row reference. -- Add index: `CREATE INDEX ON rungraphmutation (batch_operation_id) WHERE batch_operation_id IS NOT NULL;` - -`batch_operation_id` is orthogonal to `triggered_by_mutation_id` (assumption L): the former groups by operator request, the latter traces causal lineage within that request. - ---- - -### Impact on integration spec - -Add to `2-control-flow-spec.md`: - -| Test | Spec | -|------|------| -| Test B-1 — batch restart | 3-task linear chain: A→B→C all BLOCKED after A fails. Bulk restart A + manually unblock B. Assert: A → PENDING; B → PENDING (via `operator_unblock`); C → PENDING (via cascade); all share `batch_operation_id`; `triggered_by_mutation_id` on C points to B's mutation. | -| Test B-2 — batch conflict rejection | Batch with same task appearing twice with different targets. Assert: 422, no mutations written. | -| Test B-3 — batch FSM violation rejection | Batch with `RUNNING → PENDING` (not in FSM). Assert: 422, no mutations written. | -| Test B-4 — batch FAILED → CANCELLED | Task with descendant subtasks, all BLOCKED. Operator bulk-cancels the FAILED task. Assert: FAILED task → CANCELLED; descendants → CANCELLED; successors remain BLOCKED. | - ---- - -## 9. Concurrency and Race Prevention - -The current system has no explicit locking model. `only_if_not_terminal` is a weak guard — it prevents writing over a terminal status, but it does not prevent two concurrent non-terminal writes from interleaving. Under load, this produces silent lost updates and stuck runs. - ---- - -### 9.1 Races that exist today - -| Race | Scenario | Observed symptom | -|------|----------|-----------------| -| **Fan-in double dispatch** | Two predecessors of task C complete ~simultaneously. Both propagation handlers read each other's status before the write commits → both conclude "all predecessors COMPLETED" → both emit `task/ready` for C | C executes twice; second execution corrupts or duplicates outputs | -| **`task/ready` duplicate delivery** | Inngest provides at-least-once delivery. `task/ready` re-delivered after transient failure → second worker picks up the same task | Same as above | -| **Restart vs completion** | Worker commits COMPLETED for task A while operator restart is in-flight (FAILED→PENDING). COMPLETED propagation runs after the restart commits, clobbers PENDING | Task stuck in inconsistent state; successors may not be correctly unblocked | -| **Bulk batch vs concurrent propagation** | Bulk batch is modifying task B; simultaneously a worker completes B's predecessor → propagation also writes to B | Lost update — one write silently overwrites the other; final state depends on timing | -| **Operator cancel vs propagation** | Propagation is computing B→READY. Concurrently, operator cancels B. Both writes race | B ends up READY when it should be CANCELLED; a worker picks it up | -| **BLOCKED cascade interleave** | Task A fails → successors of A cascade to BLOCKED. Concurrently, task D (another predecessor of B) completes → propagation tries to set B→READY | B oscillates or lands in wrong state | - ---- - -### 9.2 Required locking model - -#### Rule 1 — Row-level lock before every status write - -Any code path that writes `RunGraphNode.status` must first acquire a `SELECT ... FOR UPDATE` lock on the row. No exceptions. This serializes concurrent writes to the same node. - -```sql --- Pattern: lock then check-and-write -SELECT status FROM run_graph_node WHERE id = $1 FOR UPDATE; --- (validate FSM transition is legal) -UPDATE run_graph_node SET status = $2 WHERE id = $1; -``` - -#### Rule 2 — Propagation locks the full affected subgraph before reading - -When a propagation pass begins (triggered by any terminal transition), it must: - -1. Compute the set of nodes it may write to (all successors + descendants of the triggering node). -2. Acquire `SELECT ... FOR UPDATE` on all of them **in `id` order** before reading any status. Ordering prevents deadlocks between two concurrent propagation passes that overlap on different subsets. -3. Read → compute → write within the same transaction. - -Locking in `id` order is a hard requirement. Two propagation passes that lock in arbitrary order will deadlock. - -#### Rule 3 — Bulk batch locks everything before pre-flight validation - -Pre-flight validation (Section 8, Step 1) must not read any `RunGraphNode` status before first locking all directly-specified nodes **plus their full containment subtrees**. This prevents a concurrent propagation pass from modifying a node between the validation read and the write. - -Lock acquisition order: same `id ASC` rule. - -#### Rule 4 — `task/ready` handler: CAS on READY→RUNNING - -The `task/ready` Inngest handler must, as its **first database operation**, execute: - -```sql -UPDATE run_graph_node -SET status = 'RUNNING' -WHERE id = $node_id AND status = 'READY' -RETURNING id; -``` - -If 0 rows returned: exit immediately without creating a `RunTaskExecution`. The task was already picked up by a concurrent worker or was cancelled between dispatch and pickup. This is the only safe guard against duplicate delivery; the `only_if_not_terminal` check alone is insufficient because RUNNING is non-terminal. - ---- - -### 9.3 Intra-batch conflict types - -The pre-flight validation in Section 8 covers direct duplicates. Under concurrency, two additional conflict classes matter: - -#### Class 1 — Cascade supersession (reject) - -A batch that cancels task A and also restarts a descendant of A is incoherent: the cancel cascade will immediately CANCEL the descendant, making the restart a no-op at best and a state corruption at worst. - -**Pre-flight check:** Compute the full cancellation cascade for every `→ CANCELLED` entry in the batch. If any node in that expansion also appears as a `→ PENDING` (restart or unblock) entry, reject the entire batch. - -``` -batch: cancel A, restart C (where C is a descendant of A) -cascade(cancel A) = {A, B, C, D, ...} -C ∈ cascade(cancel A) AND C ∈ restart targets → REJECT -``` - -#### Class 2 — Dependency incoherence (warn, allow, cancel wins) - -A batch restarts task A and also explicitly cancels A's only non-failed successor B. The restart will propagate to unblock B, but the explicit cancel overrides it. This is not incoherent — the operator is saying "restart A but don't proceed to B." Cancel wins because it is an explicit operator instruction. - -**Behavior:** Allow. Apply in topological order (restart A first, then cancel B). The explicit cancel in the batch takes precedence over any cascade that restart A would have triggered for B. Log a warning: "explicit cancel of B overrides cascade from restart of A." - -#### Class 3 — Redundant cascade (allow, idempotent) - -A batch restarts A (which would propagate B to PENDING via cascade) and also explicitly unblocks B. Both want B→PENDING. This is idempotent. - -**Behavior:** Allow. Pre-flight removes the explicit B entry from the write list (it will be produced by the cascade anyway) and records it as a no-op in the batch response. - ---- - -### 9.4 What the spec guarantees - -| Guarantee | Mechanism | -|-----------|-----------| -| Single status write is atomic | One transaction; row lock held throughout | -| Bulk batch is atomic | One transaction; all locks acquired before first write | -| Two concurrent writes to the same node cannot interleave | Row lock (`SELECT FOR UPDATE`) | -| `task/ready` is idempotent under duplicate delivery | READY→RUNNING CAS guard (Rule 4) | -| Propagation reads consistent state | Full subgraph locked before any read (Rule 2) | -| No deadlock between concurrent propagation passes | Locks acquired in `id ASC` order (Rules 2 & 3) | - -**What the spec does NOT guarantee:** - -- Global ordering of events across unrelated nodes in the same run. Two unrelated tasks completing simultaneously will have their propagation passes interleave at the run level — each node's writes are serialized, but the relative order of writes to different nodes is not. This is fine. -- Starvation freedom. A propagation pass over a large subtask group holds many row locks. Short single-task operations on nodes within that subtask group will wait. This is an acceptable tradeoff. - ---- - -### 9.5 Impact on integration spec - -Add to `2-control-flow-spec.md`: - -| Test | Spec | -|------|------| -| Test R-1 — fan-in double dispatch | Two-predecessor fan-in. Simulate both predecessors completing in the same millisecond (hold first commit with advisory lock until second is ready, release both). Assert: exactly one `task/ready` emitted for the fan-in task; exactly one `RunTaskExecution` row created. | -| Test R-2 — duplicate `task/ready` delivery | Inject duplicate `task/ready` event for an already-RUNNING task. Assert: handler exits without creating a second `RunTaskExecution`; node stays RUNNING. | -| Test R-3 — cascade supersession rejection | Batch: cancel parent A + restart child C. Assert: 422, no mutations written, error body identifies C as the superseded node. | -| Test R-4 — concurrent operator cancel vs propagation | Task B is PENDING; its predecessor completes (propagation starts, B→READY in-flight). Concurrently cancel B. Assert: B ends up CANCELLED (not READY); no `task/ready` emitted for B. | - ---- - -## 10. Agent-Facing Graph Tools - -The operator-facing interfaces (Sections 8–9) allow humans to manage stuck runs. This section specifies two toolkit tools that expose equivalent power to agents during execution — letting agents self-diagnose and self-recover without operator involvement. - -The design principle: **composability without raw access.** Agents get the full expressiveness of the FSM through validated interfaces. They do not get raw database writes that bypass the consistency invariants Sections 8–9 exist to enforce. - ---- - -### 10.1 Motivation - -The current toolkit exposes single-task atomic operations (`plan_subtasks`, `cancel_task`, `restart_task`). An agent that encounters a partially-stuck subtask group must either give up, call tools in sequence hoping no race occurs between calls, or wait for an operator. None of these are good. - -The analogy: LLMs get very far with bash not because bash bypasses OS invariants, but because it exposes the full expressive power of the OS through one composable interface. The equivalent here is an agent that can: -1. **Read** the full current graph state to self-diagnose -2. **Write** any valid combination of FSM transitions atomically - -These two tools together give agents a near-arbitrary action space over the run graph while keeping the consistency guarantees intact. - ---- - -### 10.2 `query_run_graph` — read tool - -**Signature:** -```python -query_run_graph( - run_id: UUID, - *, - scope: Literal["run", "subtree"] = "subtree", - include_wal: bool = False, -) -> RunGraphSnapshot -``` - -**Scope parameter:** -- `"subtree"` (default): returns only the calling task's containment subtree — its own descendants via `parent_node_id`. This is safe by default: an agent cannot observe sibling tasks' internal state or other agents' work. -- `"run"`: returns the full run graph. Allowed but recorded in the WAL with `actor="agent:query_run"`. Useful for agents that need to coordinate across the full run (e.g. a root orchestrator). - -**Return shape:** -```python -class RunGraphSnapshot: - nodes: list[NodeSnapshot] # id, task_slug, status, parent_node_id, level - edges: list[EdgeSnapshot] # id, source_node_id, target_node_id, status - mutations: list[MutationEntry] # only if include_wal=True; full WAL for scope - snapshot_at: datetime # timestamp of the consistent read - -class NodeSnapshot: - node_id: UUID - task_id: UUID | None - task_slug: str - status: NodeStatus # includes BLOCKED - parent_node_id: UUID | None - level: int - blocked_by: list[UUID] | None # node_ids of FAILED predecessors, if BLOCKED -``` - -`blocked_by` is the most important field: it tells the agent *which* predecessor caused the BLOCK, so it can decide whether to restart that predecessor, unblock itself, or escalate. - -**Consistency guarantee:** the snapshot is taken inside a single transaction with `REPEATABLE READ` isolation. The agent sees a consistent graph state, not a mix of pre- and post-propagation values. - -**No WAL by default.** Including the WAL (`include_wal=True`) is expensive on large runs; agents should only request it when diagnosing causality (e.g. "why did this task fail three times?"). - ---- - -### 10.3 `bulk_update_tasks` — write tool - -**Signature:** -```python -bulk_update_tasks( - changes: list[TaskStatusChange], - *, - scope_check: bool = True, -) -> BulkUpdateResult -``` - -```python -class TaskStatusChange: - task_id: UUID - target_status: NodeStatus - cause: AgentCause - -AgentCause = Literal[ - "agent_restart", # agent restarting a failed subtask - "agent_cancel", # agent explicitly cancelling a subtask it owns - "agent_unblock", # agent unblocking a BLOCKED subtask (operator_unblock equivalent) -] -``` - -**Relationship to Section 8:** This is the Section 8 bulk endpoint exposed as a toolkit function. It uses the same four-step protocol: pre-flight validation → topological application → single propagation pass → WAL attribution. All mutations share a `batch_operation_id`; cascade mutations carry `triggered_by_mutation_id`. - -The only differences from the operator endpoint: -1. **Scope restriction** (see below) -2. **Cause vocabulary**: `agent_*` causes are used instead of `manager_decision` / `operator_restart`, keeping operator and agent mutations distinguishable in the WAL -3. **`batch_operation_id` carries the calling task's `execution_id`**: every bulk mutation emitted by an agent is traceable back to the specific execution that issued it - -**Scope restriction — what agents can and cannot modify:** - -By default (`scope_check=True`), an agent may only modify tasks within its own containment subtree — nodes where `parent_node_id` is the calling task's node, or further descendants. Attempting to modify a sibling, a parent, or an unrelated task raises a `ScopeViolationError` and the entire batch is rejected. - -``` -Calling task: parent -├── child-A ← agent CAN modify (direct child) -│ └── grandchild-A1 ← agent CAN modify (descendant) -└── child-B ← agent CAN modify - -sibling (same level as parent) ← agent CANNOT modify -parent itself ← agent CANNOT modify its own status -``` - -Root orchestrator agents (tasks with `parent_node_id IS NULL`) may set `scope_check=False` to modify the full run graph. This opt-out is recorded in the WAL. - -**Why not allow agents to modify siblings?** Sibling tasks may be running concurrently under different agents. Cross-subtree writes create exactly the races Section 9 was designed to prevent, now driven by two concurrent agents instead of two concurrent propagation passes. The scope restriction is the agent-level equivalent of the locking model's `id ASC` rule: it prevents deadlock by making the locking hierarchy explicit. - -**FSM validation still runs.** `bulk_update_tasks` does not bypass the FSM. An agent that attempts `RUNNING → PENDING` gets a 422 just like an operator would. The agent's broader action space comes from being able to combine multiple valid transitions atomically, not from bypassing validity constraints. - -**Example — agent self-heals a stuck subtask group:** - -```python -# Agent observes its subtask group via query_run_graph: -snapshot = query_run_graph(run_id, scope="subtree") - -# Three tasks are BLOCKED because child-A failed. -# Agent decides: restart child-A; the others will unblock via propagation. -failed = [n for n in snapshot.nodes if n.status == "failed"] -blocked = [n for n in snapshot.nodes if n.status == "blocked"] - -result = bulk_update_tasks([ - TaskStatusChange(task_id=failed[0].task_id, target_status="pending", cause="agent_restart"), -]) -# Propagation automatically unblocks the BLOCKED tasks after child-A completes. -# Agent doesn't need to enumerate them — the system handles cascade. -``` - -**Example — agent implements a novel orchestration pattern:** - -```python -# Agent decides to abandon branch A and proceed with branch B. -# Cancels all of A's subtasks, unblocks B's subtasks. -result = bulk_update_tasks([ - TaskStatusChange(task_id=a1.task_id, target_status="cancelled", cause="agent_cancel"), - TaskStatusChange(task_id=a2.task_id, target_status="cancelled", cause="agent_cancel"), - TaskStatusChange(task_id=b1.task_id, target_status="pending", cause="agent_unblock"), -]) -``` - -This is the "bash-level flexibility" the operator interface cannot easily express: the agent is making a dynamic orchestration decision in a single atomic operation that the system designers did not anticipate. - ---- - -### 10.4 What agents cannot do - -These are explicit non-goals, not oversights: - -| Forbidden | Why | -|-----------|-----| -| Write raw SQL / bypass FSM | Invalidates locking model, corrupts WAL | -| Modify tasks outside own subtree (default) | Creates agent-vs-propagation races | -| Set `RunRecord.status` directly | Run-level lifecycle is operator/system-only | -| Access other runs' data | Isolation boundary | -| Emit Inngest events directly | Event dispatch is the system's responsibility post-mutation | -| Write mutations without `batch_operation_id` | Every agent write must be attributable | - ---- - -### 10.5 Shipping order - -**Ship `query_run_graph` first.** Read-only, no consistency risk, immediately useful for agent observability. Agents can self-diagnose and log what they see before any write tools exist. - -**Ship `bulk_update_tasks` only after Section 9 locking model is implemented.** The locking guarantees (Rule 1–4 from Section 9.2) must be in place before agents issue bulk writes. Without row-level locking, two concurrent agents modifying overlapping subtrees create exactly the races `bulk_update_tasks` is designed to avoid. - -**Do not ship `scope_check=False` as a default.** The opt-out exists for root orchestrators. It should require an explicit flag in the agent's worker configuration, not just a runtime parameter, so the expanded scope is auditable at definition time. - ---- - -### 10.6 Schema change F — `calling_execution_id` on `RunGraphMutation` - -To make agent-issued mutations distinguishable from operator and system mutations, add: - -```sql -ALTER TABLE run_graph_mutations -ADD COLUMN calling_execution_id UUID REFERENCES run_task_executions(id); -``` - -- Populated on every mutation produced by `query_run_graph` (`include_wal=True` reads) and `bulk_update_tasks` calls -- `NULL` for system propagation and operator mutations -- Combined with `batch_operation_id`: "find all mutations this specific agent execution issued across all its bulk calls" is `WHERE calling_execution_id = $1` - ---- - -### 10.7 Impact on integration spec - -Add to `2-control-flow-spec.md`: - -| Test | Spec | -|------|------| -| Test AG-1 — agent self-heal | Parent spawns subtask group; one subtask fails → others BLOCKED. Agent calls `query_run_graph` → observes BLOCKED nodes with `blocked_by`. Agent calls `bulk_update_tasks` to restart the failed task. Assert: BLOCKED tasks → PENDING via cascade; run eventually COMPLETED; all mutations carry `calling_execution_id` of the parent task's execution. | -| Test AG-2 — agent scope violation | Agent attempts `bulk_update_tasks` targeting a sibling task (outside own subtree). Assert: `ScopeViolationError`, zero mutations written. | -| Test AG-3 — agent bulk cancel branch | Agent cancels multiple subtasks in one call. Assert: all target tasks → CANCELLED; WAL entries share `batch_operation_id` and `calling_execution_id`; run's other tasks unaffected. | -| Test AG-4 — snapshot consistency | Agent calls `query_run_graph` while a propagation pass is in-flight. Assert: snapshot reflects a consistent pre- or post-propagation state, not a mix; `snapshot_at` timestamp is accurate. | - ---- - -## 7. Open Questions - -**Q1: Can CANCELLED tasks be restarted?** -Currently restart requires FAILED or COMPLETED. Should CANCELLED also be restartable? Logically yes — operator cancelled by mistake. Needs a decision before `restart_task` logic is rewritten. - -**Q2: Stuck detection threshold.** -What is the right "no WAL activity" threshold before a run is flagged as stuck? Depends on expected task duration. Should be configurable per experiment definition, not hardcoded. - -**Q3: BLOCKED propagation through the containment tree upward.** -If a child is BLOCKED, the parent is stuck (it cannot finalise). Should the parent itself transition to BLOCKED to make this visible, or is "RUNNING with stuck children" an acceptable observable state? Current design says parent stays RUNNING; this may make dashboards harder to interpret. - -**Q4: What happens to BLOCKED successors when the run is cancelled?** -If the operator cancels the run (`RunRecord → CANCELLED`), all non-terminal tasks should cascade to CANCELLED — including BLOCKED ones. This is clean. Needs explicit handling in the run cancellation handler. diff --git a/docs/integration-spec/1-audit.md b/docs/integration-spec/1-audit.md deleted file mode 100644 index 35b8a5120..000000000 --- a/docs/integration-spec/1-audit.md +++ /dev/null @@ -1,105 +0,0 @@ -# Integration Tier Audit - -## Directory Structure - -``` -tests/integration/ -├── conftest.py -├── smokes/ -│ └── test_smoke_harness.py -├── swebench_verified/ -│ ├── test_benchmark.py -│ ├── test_criterion.py -│ ├── test_rubric.py -│ ├── test_sandbox_manager.py -│ ├── test_smoke_e2e.py -│ ├── test_task_schemas.py -│ └── test_toolkit.py -└── minif2f/ - ├── test_sandbox_manager.py - └── test_verification_integration.py -``` - ---- - -## What Is Actually Covered - -| File | What it tests | Live infra required | -|------|--------------|-------------------| -| `smokes/test_smoke_harness.py` | seed → read → reset HTTP round-trip against a real server + Postgres | Server + Postgres | -| `swebench_verified/test_criterion.py` | `SWEBenchTestCriterion.evaluate()` with a mock `CriterionRuntime` | None | -| `swebench_verified/test_benchmark.py` | `build_instances()` with mocked `_load_rows` | None | -| `swebench_verified/test_task_schemas.py` | `SWEBenchInstance`, `SWEBenchTaskPayload` parsing, `_parse_test_list` | None | -| `swebench_verified/test_toolkit.py` | bash and str_replace_editor tools with a mock sandbox | None | -| `swebench_verified/test_rubric.py` | Rubric has one criterion named "test-resolution" with weight 1.0 | None | -| `swebench_verified/test_sandbox_manager.py` | Template resolution + `AsyncSandbox.create` call shape (mocked) | None | -| `swebench_verified/test_smoke_e2e.py` | Dockerfile and `e2b.toml.template` exist on disk | None | -| `minif2f/test_sandbox_manager.py` | Template resolution + sandbox create/verify lifecycle (mocked) — **3 tests uncollected** | None | -| `minif2f/test_verification_integration.py` | `ProofVerificationCriterion` — live test skipped without E2B key; static 'sorry' rejection test | Optional E2B | - -**There are no Postgres persistence round-trip tests for `RunRecord`, `RunTaskExecution`, `RunResource`, or `RunGraphNode`. There are no Inngest event schema or propagation tests. The single test that exercises real infrastructure is `test_smoke_harness.py::test_seed_then_read_then_reset_roundtrip`, and it only asserts HTTP response codes — not Postgres state.** - ---- - -## Existing Issues - -### Critical - -#### 1. Three tests have never been collected — `minif2f/test_sandbox_manager.py` - -Three functions are named `testresolve_*` instead of `test_*`. pytest never collects them. They have passed in CI since they were written only because they are invisible to the runner. - -```python -# Current — never collected: -def testresolve_template_falls_back_to_name_when_registry_missing(...): ... -def testresolve_template_prefers_registry_template_id(...): ... -def testresolve_template_falls_back_on_malformed_registry(...): ... - -# Should be: -def test_resolve_template_falls_back_to_name_when_registry_missing(...): ... -def test_resolve_template_prefers_registry_template_id(...): ... -def test_resolve_template_falls_back_on_malformed_registry(...): ... -``` - -**Fix:** Rename the three functions. Zero-risk, one commit. - ---- - -### High - -#### 2. The Inngest preflight gates tests that do not use Inngest - -`conftest.py` probes Inngest TCP connectivity at session start and calls `pytest.exit()` if it is unreachable. This applies to every file in `tests/integration/` regardless of whether the test needs Inngest. - -Eight of the ten test files require no live infrastructure at all — they are fully mocked unit tests sitting behind the integration preflight for no reason. A ten-line rubric test that checks a list length cannot run in any environment without a running Inngest server. - -**Fix:** Move the fully-mocked files to `tests/unit/` where they belong, or introduce a sub-conftest scoped only to the files that actually need Inngest. - -#### 3. Eight of ten files are misclassified unit tests - -Every file below needs no live infrastructure and is currently blocked behind the integration preflight: - -- `swebench_verified/test_benchmark.py` — mocks `_load_rows` -- `swebench_verified/test_criterion.py` — mocks `CriterionRuntime` -- `swebench_verified/test_rubric.py` — instantiates a local object -- `swebench_verified/test_sandbox_manager.py` — mocks `AsyncSandbox` -- `swebench_verified/test_smoke_e2e.py` — filesystem stat only -- `swebench_verified/test_task_schemas.py` — pure data parsing -- `swebench_verified/test_toolkit.py` — mocks `AsyncSandbox` -- `minif2f/test_sandbox_manager.py` — mocks `AsyncSandbox` - -These should live in `tests/unit/` and run under `pnpm run test:be:fast`. - ---- - -### Medium - -#### 4. `_reset_sandbox_singleton` fixture is duplicated - -The fixture that resets `BaseSandboxManager` class-level state is defined independently in both `minif2f/test_sandbox_manager.py` and `minif2f/test_verification_integration.py`. It is identical in both files. - -**Fix:** Extract to `tests/integration/minif2f/conftest.py`. - -#### 5. The one real integration test covers a narrow happy path - -`test_smoke_harness.py` has a single test: seed → read → reset, ending in a 404. It asserts HTTP status codes only. It does not assert Postgres state, does not test non-happy-path statuses, and does not verify that reset actually cascades to child rows. diff --git a/docs/integration-spec/2-control-flow-spec.md b/docs/integration-spec/2-control-flow-spec.md deleted file mode 100644 index 70328ccf2..000000000 --- a/docs/integration-spec/2-control-flow-spec.md +++ /dev/null @@ -1,717 +0,0 @@ -# Control Flow Test Specification - -## What the System Actually Is (and What Integration Tests Should Assert) - -The integration tier is testing the wrong things because it is not oriented around the actual system model. Understanding that model is a prerequisite for knowing what to test. - -### The system model - -Ergon is an Inngest-driven, append-only workflow engine. Every meaningful invariant lives in Postgres. The unit of truth is **the database state after events settle**, not HTTP response codes. - -| Table | Purpose | -|-------|---------| -| `RunRecord` | One row per benchmark run; owns the run-level lifecycle status | -| `RunGraphNode` / `RunGraphEdge` / `RunGraphMutation` | DAG execution state; WAL is the ground truth | -| `RunTaskExecution` | One row per execution attempt; records start/end, output, error | -| `RunResource` / `ThreadMessage` | Task outputs and agent-to-agent communication | - -### The state machine - -| Status | Terminal? | Meaning | -|--------|-----------|---------| -| `PENDING` | No | Created; not all predecessors COMPLETED. Normal waiting. | -| `READY` | No | All predecessors COMPLETED. Eligible for `task/ready` dispatch. | -| `BLOCKED` | No | At least one predecessor FAILED. Operator action required. | -| `RUNNING` | No | Worker executing in sandbox. | -| `COMPLETED` | Yes | Worker returned success. Outputs in `RunResource`. | -| `FAILED` | Yes | Worker raised an error during execution. | -| `CANCELLED` | Yes | Explicitly stopped by operator, or orphaned by parent cancellation. | - -**BLOCKED is not PENDING.** PENDING means "waiting for normal upstream progress." BLOCKED means "a predecessor FAILED and operator action is required." A stuck run with BLOCKED tasks must be distinguishable from an active run with PENDING tasks. - -**FAILED is reserved for worker errors.** A task that never ran because its predecessor FAILED is BLOCKED, not FAILED. FAILED means "the execution attempt ran and something went wrong inside it." - -**CANCELLED means intentionally stopped or orphaned.** A task whose parent was cancelled has no execution context — it is CANCELLED. A task that cannot proceed because a predecessor failed is BLOCKED, not CANCELLED. - -**`only_if_not_terminal` guard:** Prevents writing to a node already in `{COMPLETED, FAILED, CANCELLED}`. BLOCKED, PENDING, READY, and RUNNING are non-terminal and can be overwritten by valid transitions. - -**RunRecord statuses:** `CREATED → RUNNING → {COMPLETED | FAILED | CANCELLED}` - -`RunRecord` has no BLOCKED state. A run with BLOCKED tasks shows `RUNNING` — it is stuck, not failed. `RunRecord.status == FAILED` requires an explicit operator action or run-level timeout, not automatic propagation. `RunRecord.status == COMPLETED` requires all nodes in `{COMPLETED, CANCELLED}`. - -**Edge statuses:** `EDGE_PENDING → EDGE_SATISFIED` - -An edge flips to `EDGE_SATISFIED` when its source node reaches `COMPLETED`. On restart, edges flip back to `EDGE_PENDING`. - -### The Inngest event flow - -``` -benchmark/run-request - → [benchmark_run_start_fn] - persist ExperimentDefinition - INSERT RunRecord (status=CREATED) - emit workflow/started - -workflow/started - → [start_workflow_fn] - create RunGraphNode for each definition task - create RunGraphEdge for each dependency - mark root nodes READY - emit task/ready (fanout to all roots) - -task/ready - → [execute_task_fn] - CAS: UPDATE run_graph_node SET status='RUNNING' - WHERE id=? AND status='READY' RETURNING id; - → 0 rows returned: exit immediately (duplicate delivery or concurrent pickup) - INSERT RunTaskExecution (status=PENDING) - [sandbox_setup_fn] - [worker_execute_fn] - optionally: plan_subtasks() → inserts child RunGraphNodes, emits task/ready for roots - optionally: save_message() → inserts Thread + ThreadMessage - [persist_outputs_fn] → INSERT RunResource rows - UPDATE RunTaskExecution (status=COMPLETED) - UPDATE RunGraphNode → COMPLETED - INSERT RunGraphMutation - emit task/completed - - [on any failure] - UPDATE RunTaskExecution (status=FAILED, error_json=...) - UPDATE RunGraphNode → FAILED - INSERT RunGraphMutation - emit task/failed - -task/completed → [propagate_execution] - flip source edge → EDGE_SATISFIED - for each successor: - all incoming edges EDGE_SATISFIED → mark READY, emit task/ready - any predecessor FAILED → stays BLOCKED - otherwise → stays PENDING - if all nodes terminal → emit workflow/completed - -task/failed → [propagate_execution] - SELECT FOR UPDATE all affected successor + descendant rows (id ASC) - for each PENDING/READY successor (horizontal): → BLOCKED - INSERT RunGraphMutation (cause=dep_failed) - for each PENDING/READY descendant (vertical): → BLOCKED - INSERT RunGraphMutation (cause=parent_failed) - RUNNING successors/descendants: NOT interrupted (live executions continue) - (no task/blocked event — BLOCKED is a synchronous DB write, not an event dispatch) - RunRecord stays RUNNING — the run is stuck, not failed - -task/cancelled → [cancel_orphans_fn] - SELECT FOR UPDATE all descendant rows (id ASC) - cascade CANCELLED to all non-terminal children (cause=parent_terminal) - UPDATE RunTaskExecution → CANCELLED - UPDATE RunGraphNode → CANCELLED - INSERT RunGraphMutation - -workflow/completed - → [complete_workflow_fn] - aggregate scores from RunTaskExecution outputs - UPDATE RunRecord (status=COMPLETED, completed_at=now, summary_json={scores}) - emit run/cleanup - -workflow/failed - → [fail_workflow_fn] - UPDATE RunRecord (status=FAILED, error_message=...) - emit run/cleanup -``` - -**Key difference from prior spec:** `task/failed` no longer emits `task/cancelled` for downstream nodes. It writes BLOCKED directly to `RunGraphNode` rows. There is no `task/blocked` event — BLOCKED transitions are synchronous DB updates in the propagation pass. - -### Subtask spawning (dynamic DAG mutation) - -A worker can call `plan_subtasks()` during execution. This inserts child `RunGraphNode` rows under the currently-executing node and immediately emits `task/ready` for the new roots. The containment tree is recorded via `parent_node_id` (self-referential FK) and `level` (precomputed depth: `parent.level + 1`). - -**Key invariants:** -- Every node with `parent_node_id != NULL` has `level == parent.level + 1` -- Cycle detection runs at `plan_subtasks` time; any cycle rejected before any DB write -- The parent task does not finalise until all its children are terminal -- Cross-containment `RunGraphEdge` rows are forbidden: both endpoints must share the same `parent_node_id` - -### Cancellation causes - -| Cause | Trigger | -|-------|---------| -| `manager_decision` | Operator explicitly called `cancel_task()` | -| `parent_terminal` | Parent node reached CANCELLED | -| `dep_invalidated` | Predecessor restarted; this successor's prior outputs are stale (restart only — not failure) | -| `run_cancelled` | Run-level cancellation broadcast | - -**`dep_invalidated` does not apply to failure.** When a predecessor FAILS, successors become BLOCKED — no cancellation event is emitted. `dep_invalidated` fires only when a predecessor is *restarted* and a COMPLETED successor needs to be invalidated. - -### The restart flow - -`restart_task(node_id)` (valid on FAILED or COMPLETED nodes): - -1. Acquire `SELECT FOR UPDATE` on all affected rows (id ASC — see Section 9.2 of status design doc) -2. Reset node: `FAILED/COMPLETED → PENDING` -3. Reset own outgoing edges: `EDGE_SATISFIED → EDGE_PENDING` -4. BLOCKED successors → `PENDING` (predecessor being retried; re-evaluate when it completes) -5. COMPLETED successors → `CANCELLED` with cause `dep_invalidated` (prior outputs are stale) -6. Containment descendants: ALL → `CANCELLED` regardless of prior status; new execution creates fresh children via `plan_subtasks()` (resetting to PENDING instead would create duplicate child generations) -7. Emit `task/ready` for the node - -Nodes with `parent_node_id != NULL` cannot be restarted independently — only via their parent's restart. - ---- - -## Integration Tests - -Each test: stub workers (no E2B, no real LLM), submit real event to local Inngest dev server, poll until `RunRecord.status` is terminal **or** all expected BLOCKED/stuck nodes are confirmed, then assert exact Postgres state. - -Polling timeout: 30 s for standard tests, 120 s for `pytest.mark.slow`. - ---- - -### Test 1: Single-task happy path - -One task, worker returns `WorkerOutput(success=True)`. - -Assert: -- `RunRecord.status == COMPLETED`, `completed_at` set -- `RunGraphNode.status == COMPLETED` -- `RunTaskExecution.status == COMPLETED`, `started_at ≤ completed_at` -- At least one `RunResource` row with correct `run_id` and `task_execution_id` -- WAL contains entries for `PENDING → RUNNING → COMPLETED` - ---- - -### Test 2: Linear chain — propagation - -Three tasks A → B → C. All succeed. - -Assert: -- Completed in topological order (`completed_at`: A before B before C) -- All `RunGraphEdge` rows `EDGE_SATISFIED` -- WAL: `READY` transition for B only after A's `COMPLETED` entry; ditto C after B -- `RunRecord.status == COMPLETED` - ---- - -### Test 3: Failure cascade — successor becomes BLOCKED - -Three tasks A → B → C. A succeeds. B fails. C never ran. - -Assert: -- A: `COMPLETED` -- B: `FAILED`, `RunTaskExecution.error_json` non-null -- C: `BLOCKED` — **not** CANCELLED, **not** PENDING -- No `RunTaskExecution` row for C (it never started) -- WAL entry for C: cause `dep_failed`, timestamped after B's `FAILED` mutation -- `RunRecord.status == RUNNING` — the run is stuck, **not** FAILED -- No `RunResource` rows owned by C -- No `task/ready` emitted for C - -**Critical:** `RunRecord.status` must be `RUNNING`. A run is only FAILED when an operator explicitly fails it or a timeout fires. Automatic failure propagation does not change `RunRecord`. - ---- - -### Test 4: Diamond DAG — propagation convergence - -Four tasks: `root → left`, `root → right`, `left → sink`, `right → sink`. All succeed. - -Assert: -- `left` and `right` both reach `RUNNING` before `sink` becomes `READY` -- `sink` transitions to `READY` only after both `left` AND `right` are `COMPLETED` (both edges `EDGE_SATISFIED`) -- `sink` transitions to `READY` exactly once — the `only_if_not_terminal` / CAS guard is exercised: sink receives two "dep satisfied" signals but dispatches once -- All four `COMPLETED`, `RunRecord.status == COMPLETED` - ---- - -### Test 5: Subtask spawning — dynamic DAG - -Parent worker calls `plan_subtasks()` with: - -``` -root_child (no deps within subtask group) - ↓ dependency edge -leaf_child (depends on root_child; same parent_node_id) -``` - -Both are direct children of parent — same `parent_node_id`, same `level`. The `→` is a sibling dependency edge, not a nesting level. - -Assert: -- Two `RunGraphNode` rows with `parent_node_id == parent_node.id` -- Both have `level == parent.level + 1` -- `root_child` was the only node to transition to `READY` immediately after subtask insertion (`leaf_child` was `PENDING`) -- `leaf_child` transitions to `READY` only after `root_child` COMPLETED (WAL timestamps) -- Parent reaches `COMPLETED` only after both children are terminal -- No `RunGraphEdge` from parent to either child — containment is via `parent_node_id`, not edges - ---- - -### Test 6: Cancellation — manager_decision - -Two sibling tasks: `target` (with subtree) and `sibling` (independent). - -``` -target (cancelled while RUNNING) -├── target-child-A -│ ├── target-grandchild-A1 -│ └── target-grandchild-A2 -└── target-child-B - -sibling (must NOT be affected) -└── sibling-child -``` - -Call `cancel_task(target.node_id)` while target is RUNNING and subtree is live. - -Assert — cancellation target: -- `target.status == CANCELLED`, cause `manager_decision` -- `target.RunTaskExecution.status == CANCELLED` - -Assert — full subtree cascade: -- `target-child-A`, `target-child-B`: `CANCELLED`, cause `parent_terminal` -- `target-grandchild-A1`, `target-grandchild-A2`: `CANCELLED`, cause `parent_terminal` -- WAL: grandchild entries timestamped after their parent's `CANCELLED` mutation (level-by-level, not all-at-once) -- No `RunTaskExecution` with `status IN (RUNNING, COMPLETED)` for any descendant - -Assert — sibling isolation: -- `sibling.status` unchanged -- `sibling-child.status` unchanged -- WAL for sibling contains no `CANCELLED` entries - -Assert — run-level status: -- `RunRecord.status == RUNNING` (cancelling one task does not fail the run while other tasks remain) - -Assert — idempotency: -- Second `cancel_task(target.node_id)` returns an error -- WAL mutation count for `target` does not increase - ---- - -### Test 7: Parent-failure cascade — descendants become BLOCKED - -When a parent's worker fails (`RUNNING → FAILED`), non-terminal containment descendants become `BLOCKED`. BLOCKED means "this execution context collapsed; operator action required." FAILED is only for tasks whose own worker errored. CANCELLED is only for tasks explicitly stopped or orphaned. - -``` -parent (raises controlled exception) -├── child-A (PENDING when parent fails) -│ ├── grandchild-A1 (PENDING) -│ └── grandchild-A2 (RUNNING) -└── child-B (RUNNING when parent fails) -└── child-C (COMPLETED before parent fails — must survive) -``` - -Assert: -- Parent: `FAILED`, `RunTaskExecution.error_json` non-null -- `child-A`: `BLOCKED`, cause `parent_failed` — **not** CANCELLED, **not** FAILED -- `grandchild-A1`: `BLOCKED`, cause `parent_failed` -- `child-B`: **not interrupted** — continues to its own terminal state (RUNNING execution is live) -- `grandchild-A2`: **not interrupted** — continues to its own terminal state -- `child-C`: remains `COMPLETED` — already terminal, not overwritten -- WAL: grandchild BLOCKED entries timestamped after their parent's BLOCKED mutation -- `RunRecord.status == RUNNING` — stuck, not FAILED - -**What must NOT happen:** -- No CANCELLED mutations for any descendant (cancel = intentional stop) -- No FAILED mutations for child-A or grandchild-A1 (they did not run and error) -- RunRecord must not automatically become FAILED - -**Depth parametrisation:** Run at N=1, N=3, N=10 sublayer depths. Every descendant at every level must be BLOCKED. Catches iterative vs recursive cascade bugs where only the first level is processed. - ---- - -### Test 8: Restart — BLOCKED successors unblock; descendants cancel and regenerate - -Prior state: - -``` -parent (FAILED after first execution) -├── child-root (COMPLETED in prior run) -│ └── child-leaf (COMPLETED in prior run) -└── child-standalone (FAILED in prior run) - -blocked-successor (DAG successor of parent; currently BLOCKED) -completed-successor (DAG successor of parent; COMPLETED in prior run) -``` - -**Step 1 — refine:** `refine_task(parent.node_id, new_description)` updates description; status unchanged (FAILED). - -**Step 2 — restart:** `restart_task(parent.node_id)` - -Assert — parent reset: -- `parent.status == PENDING`, then `RUNNING` once `task/ready` fires -- WAL: `FAILED → PENDING` with cause `operator_restart` -- New `RunTaskExecution` with incremented `attempt_number` - -Assert — containment descendants cancelled (not reset to PENDING): -- `child-root`, `child-leaf`, `child-standalone`: all `CANCELLED` -- Not reset to PENDING — new execution creates fresh children via `plan_subtasks()` -- Prior child rows preserved in DB (append-only); status is CANCELLED - -Assert — BLOCKED successor unblocked: -- `blocked-successor`: `BLOCKED → PENDING` -- Incoming edge from parent resets to `EDGE_PENDING` -- WAL cause: `predecessor_restarted` - -Assert — COMPLETED successor invalidated: -- `completed-successor`: `COMPLETED → CANCELLED`, cause `dep_invalidated` -- Incoming edge from parent resets to `EDGE_PENDING` - -Assert — dispatch: -- `task/ready` fires for `parent` only — not for any containment descendants -- New execution creates fresh children; propagation drives non-root activations - -Assert — independent guard: -- `restart_task(child-root.node_id)` raises an error — containment nodes cannot be restarted independently - -**Step 3 — restarted execution completes:** -- New children reach `COMPLETED` via normal propagation -- `parent.status == COMPLETED` -- `blocked-successor` re-activates (edge → EDGE_SATISFIED → READY → COMPLETED) -- `RunRecord.status == COMPLETED` -- Exactly two `RunTaskExecution` rows for parent (attempt 1, attempt 2) - -**Depth parametrisation:** Run at N=1, N=3, N=10. Every containment descendant at every level must be CANCELLED on restart. `task/ready` must fire only for the new subtask roots — not every node simultaneously. - ---- - -### Test 9: Communication service — message routing - -Stub worker calls `save_message(from_agent_id=leaf-X, to_agent_id=parent, thread_topic=smoke-completion)`. - -Assert: -- `Thread` row exists scoped to `run_id` with correct `topic` -- `ThreadMessage` row with correct `from_agent_id`, `to_agent_id`, `run_id`, `task_execution_id` -- `sequence_num == 1` for first message -- Second message to same thread gets `sequence_num == 2` - ---- - -### Test 10: BLOCKED propagates rightward transitively - -Three tasks A → B → C. A fails. Neither B nor C has started. - -Assert: -- A: `FAILED` -- B: `BLOCKED` (direct successor) -- C: `BLOCKED` (transitive successor — blocked because B is blocked, which is blocked because A failed) -- WAL: B's BLOCKED timestamped after A's FAILED; C's BLOCKED timestamped after B's BLOCKED -- `RunRecord.status == RUNNING` - -**Distinct from Test 3:** Test 3 has B failing (B is the one whose worker errors). This test has A failing, cascading BLOCKED to both B and C via transitive propagation. The propagation must walk the full rightward chain, not just direct successors. - ---- - -### Test 11: BLOCKED → PENDING when predecessor restarted - -Precondition: state from Test 10 (A FAILED, B and C BLOCKED). - -Call `restart_task(A.node_id)`. - -Assert immediately after restart: -- A: `PENDING` -- B: `PENDING` (was BLOCKED because of A; unblocked by restart) -- C: `PENDING` (was BLOCKED because of B; cascade unblocked) -- All incoming edges reset to `EDGE_PENDING` -- WAL: PENDING transitions for B and C with cause `predecessor_restarted` -- `RunRecord.status == RUNNING` - -Assert after A, B, C complete normally: -- A → B → C all `COMPLETED` in order -- `RunRecord.status == COMPLETED` - ---- - -### Test 12: RUNNING successor not interrupted by predecessor failure - -Two tasks A → B. B is actively RUNNING when A fails. - -Assert immediately after A fails: -- A: `FAILED` -- B: **not** BLOCKED, **not** CANCELLED — remains `RUNNING` -- WAL: no BLOCKED mutation for B (RUNNING tasks are never interrupted by predecessor failure) -- `RunRecord.status == RUNNING` - -Assert after B reaches its own terminal state: -- If B completes: `B.status == COMPLETED`; `RunRecord.status == RUNNING` (stuck — A is FAILED and the run cannot complete while A is non-COMPLETED-or-CANCELLED) -- If B fails: `B.status == FAILED`; `RunRecord.status == RUNNING` - ---- - -### Test 13: Operator unblock — BLOCKED → PENDING without restarting the predecessor - -A: `FAILED`. B: `BLOCKED` (successor of A). Operator calls `unblock_task(B.node_id)` (cause `operator_unblock`). - -Assert: -- B: `PENDING` — not READY (A has not re-completed; B re-evaluates when predecessors complete) -- A: `FAILED` — unchanged -- B's incoming edge from A: `EDGE_PENDING` -- WAL entry for B: cause `operator_unblock` - -Assert this never fires automatically: -- No propagation path produces `BLOCKED → PENDING` without an explicit operator call -- B remains PENDING indefinitely until A is restarted and completes, or B is separately managed - ---- - -## Edge Cases and Boundary Conditions - -### EC-1: Fan-in convergence race under failure - -Diamond DAG: `root → left`, `root → right`, `left → sink`, `right → sink`. Left fails at the same moment right completes. Use a sleep barrier to make the race reproducible. - -Two propagation events race to `sink`: BLOCKED propagation (from left's failure) and READY evaluation (from right's completion). - -Assert: -- `sink.status == BLOCKED` — left is a failed predecessor; right completing alone is not sufficient -- `RunRecord.status == RUNNING` -- `sink` WAL has exactly one terminal mutation (BLOCKED); no READY or RUNNING entry -- If right's completion arrives first: right's edge satisfies, but sink re-evaluates, finds left FAILED → BLOCKED -- If left's failure arrives first: sink → BLOCKED; right's completion does not override it - -Mark `pytest.mark.slow`, run with `--count=5`. - -**Note:** Prior spec expected `sink == CANCELLED`. Under new semantics, a failed predecessor blocks successors; it does not cancel them. - ---- - -### EC-2: Duplicate `task/ready` delivery — CAS guard at READY→RUNNING - -The `task/ready` handler's first DB operation: - -```sql -UPDATE run_graph_node SET status='RUNNING' -WHERE id = $node_id AND status = 'READY' -RETURNING id; -``` - -0 rows returned → exit immediately, no RunTaskExecution created. - -Unit-tier test: seed `RunGraphNode` in `RUNNING` state, invoke `prepare-execution` logic directly. - -Assert: -- No second `RunTaskExecution` created -- No sandbox provisioned -- No duplicate `RUNNING` WAL entry -- Handler exits cleanly - ---- - -### EC-3: Cross-containment dependency edges are forbidden - -A `RunGraphEdge` where `source.parent_node_id != target.parent_node_id` must be rejected at graph construction time. - -Unit-tier test. - -Assert: -- `add_edge(source, target)` raises an error when `source.parent_node_id != target.parent_node_id` -- `add_edge` raises an error when one node has `parent_node_id` and the other does not -- Error raised before any DB write — no partial edge rows created - ---- - -### EC-4: Evaluation after restart — which attempt's score counts - -1. Task completes (attempt 1) → `RunTaskEvaluation` created with score X -2. `restart_task` called → task re-runs (attempt 2) → second `RunTaskEvaluation` with score Y - -Assert: -- Exactly two `RunTaskEvaluation` rows for same `definition_task_id` and `run_id` -- `RunRecord.summary_json` reflects score Y (most recent attempt), not X -- Attempt-1 `RunTaskEvaluation` preserved (append-only) but not used in summary -- Summary row identified by matching `execution_id` to the most recent `RunTaskExecution` for the node - -If not yet implemented: `pytest.mark.xfail(strict=True, reason="evaluation after restart: score selection across attempts not yet defined")`. - ---- - -### EC-5: Concurrency queue + node cancellation while queued - -`execute_task_fn` has `concurrency limit=15`. Saturate all 15 slots, fire a 16th `task/ready`, then cancel the 16th node via `cancel_task` while it is queued (before Inngest dispatches it). - -Assert: -- When Inngest dispatches the queued invocation, the READY→RUNNING CAS returns 0 rows (node is now CANCELLED) → handler exits -- No `RunTaskExecution` row created for the cancelled node -- No sandbox provisioned -- No `TaskCompletedEvent` or `TaskFailedEvent` emitted -- WAL: CANCELLED entry for node, no RUNNING entry - -Mark `pytest.mark.slow`. - ---- - -### EC-6: Multiple BLOCKED predecessors — task stays BLOCKED until all resolved - -Fan-in: A and B both feed into C. A fails, then B fails independently. - -Assert after both fail: -- C: `BLOCKED` (two failed predecessors) - -Call `restart_task(A.node_id)`. - -Assert: -- C: still `BLOCKED` — B is still FAILED; one predecessor restarting is insufficient -- A's outgoing edge resets to `EDGE_PENDING`; B's edge is still in blocked state - -Call `restart_task(B.node_id)`. - -Assert: -- C: `PENDING` — both predecessors now being retried; C re-evaluates -- Both incoming edges at `EDGE_PENDING` - -After A and B both complete: -- C: `READY` → `RUNNING` → `COMPLETED` - ---- - -### EC-7: RUNNING descendant survives parent FAILED - -Parent spawns a subtask that is actively RUNNING when parent's worker raises. - -Assert immediately after parent fails: -- Parent: `FAILED` -- Child: **not** interrupted — remains `RUNNING` -- WAL: no BLOCKED or CANCELLED mutation for child yet -- `RunRecord.status == RUNNING` - -After child reaches its own terminal state: -- If child `COMPLETED`: parent stays `FAILED` (already terminal); `RunRecord.status == RUNNING` (stuck) -- If child `FAILED`: parent stays `FAILED`; `RunRecord.status == RUNNING` - -The child's execution is independent of parent's failure — it runs to completion or failure on its own. - ---- - -## Bulk Operation Tests - -Full protocol: `0-task-status-design.md` § 8. These tests assert Postgres state after batch API requests. All mutations within a batch share the same `batch_operation_id`. - -### B-1: Batch restart + operator unblock - -Precondition: A→B→C linear chain; A failed; B and C BLOCKED. - -Bulk request: restart A + operator-unblock B. - -Assert: -- All mutations share one `batch_operation_id` -- A: `PENDING` (directly specified) -- B: `PENDING` (directly specified, cause `operator_unblock`) -- C: `PENDING` (cascade from B's unblocking) -- `triggered_by_mutation_id` on C's mutation points to B's mutation row -- All incoming edges `EDGE_PENDING` - -After A completes: B → C complete in order; `RunRecord.status == COMPLETED`. - -### B-2: Batch conflict — same task twice - -Batch: restart A AND cancel A. - -Assert: -- 422 response; zero mutations written; A status unchanged - -### B-3: Batch FSM violation - -Batch: `RUNNING → PENDING` for an actively running task (not in FSM). - -Assert: -- 422 response; zero mutations written - -### B-4: Batch FAILED → CANCELLED (give up without restarting) - -Task A: `FAILED`. Descendants B and C (containment): `BLOCKED`. - -Bulk request: cancel A (FAILED → CANCELLED). - -Assert: -- A: `CANCELLED` -- B, C: `CANCELLED` (cascade from A's cancellation — they are descendants) -- DAG successors of A: remain `BLOCKED` — they still lack A's outputs -- `RunRecord.status == RUNNING` (successors are BLOCKED; run is stuck) - -### B-5: Cascade supersession rejection - -Batch: cancel parent P + restart child C (where C is a containment descendant of P). - -Pre-flight must detect C is inside the cancel cascade of P. - -Assert: -- 422 response; zero mutations written -- Error body identifies C as the superseded target - ---- - -## Race and Concurrency Tests - -Full locking model: `0-task-status-design.md` § 9. These tests verify correctness guarantees under concurrent load. - -### R-1: Fan-in double dispatch — exactly-once task/ready - -Two-predecessor fan-in task C. Hold both predecessors A and B at commit boundary via advisory lock, release simultaneously. - -Assert: -- Exactly one `task/ready` event emitted for C -- Exactly one `RunTaskExecution` row for C -- C's WAL contains exactly one `READY` transition -- C eventually `COMPLETED` - -Mark `pytest.mark.slow`, run with `--count=10`. - -### R-2: Duplicate task/ready — CAS prevents double execution - -Integration-tier version of EC-2. Submit same `task/ready` event twice via Inngest API for an already-RUNNING task. - -Assert: -- Exactly one `RunTaskExecution` row -- Exactly one `RUNNING` WAL entry -- Second handler invocation exits without DB writes - -### R-3: Cascade supersession — atomic rejection under concurrent load - -Submit two concurrent API requests: bulk-cancel P + restart C (where C is a descendant of P). - -Assert: -- Exactly one request succeeds; the other returns 422 -- Final DB state is consistent (not partial — no C in PENDING while P is CANCELLED) - -### R-4: Concurrent cancel vs propagation — cancel always wins - -Task B is PENDING. A completes — propagation starts to move B to READY. Simultaneously cancel B via API. - -Assert: -- B's final status is `CANCELLED` regardless of propagation timing -- No `task/ready` emitted for B after it is CANCELLED -- No `RunTaskExecution` created for B - -Mark `pytest.mark.slow`. - ---- - -## Cross-Cutting Invariants - -Write these as shared assertion helpers, call from every test above. - -| Invariant | Assertion | -|-----------|-----------| -| WAL completeness | Every `RunGraphNode` has at least one `RunGraphMutation` entry | -| Execution coverage | Every node that reached `COMPLETED` or `FAILED` has a `RunTaskExecution` with non-null `completed_at` | -| No executions for un-run BLOCKED nodes | No `RunTaskExecution` with status `RUNNING` or `COMPLETED` for a node that is currently BLOCKED with no prior attempt | -| No orphaned executions | Every `RunTaskExecution.node_id` references an existing `RunGraphNode` in the same run | -| RunRecord terminal consistency | `COMPLETED` → all nodes in `{COMPLETED, CANCELLED}`; `FAILED` → explicit operator/timeout only; `RUNNING` → any non-terminal nodes present (including BLOCKED) | -| BLOCKED is non-terminal | No `RunGraphNode` in `{COMPLETED, FAILED, CANCELLED}` was transitioned there directly from `BLOCKED` without an intervening operator action in the WAL | -| FAILED is worker-only | Every FAILED node has a `RunTaskExecution` with non-null `error_json` | -| CANCELLED is intentional | Every CANCELLED node has a WAL cause in `{manager_decision, parent_terminal, dep_invalidated, run_cancelled}` | -| No RunRecord FAILED without operator | `RunRecord.status == FAILED` implies a WAL entry with cause `operator_decision` or `run_timeout` — automatic propagation never sets it | -| Edge–node consistency | Every `EDGE_SATISFIED` edge has a source node with `status == COMPLETED` | -| Level consistency | Every node with `parent_node_id != NULL` has `level == parent.level + 1` | -| Append-only WAL | Mutation count for any given node only ever increases; no rows deleted or updated | -| Timeline consistency | `started_at ≤ completed_at` on every `RunTaskExecution` | -| Batch attribution | All mutations from a bulk request share `batch_operation_id`; cascade mutations carry `triggered_by_mutation_id` pointing to the direct batch item that caused them | -| No BLOCKED RunRecord | `RunRecord.status` is never `BLOCKED` — stuck runs show `RUNNING` | - ---- - -## Summary - -The current `tests/integration/` has one real integration test checking HTTP response codes on a narrow happy path. This spec covers: - -- **13 primary control flow tests** (Tests 1–13): happy path, propagation, failure→BLOCKED, diamond convergence, subtask spawning, cancellation, parent-failure→BLOCKED, restart+unblocking, communication, transitive BLOCKED cascade, BLOCKED→PENDING on restart, RUNNING isolation, operator unblock -- **7 edge cases** (EC-1–7): fan-in race under failure, duplicate delivery CAS, cross-containment guard, evaluation across restarts, queued cancellation, multi-predecessor BLOCKED, RUNNING descendant survival -- **5 bulk operation tests** (B-1–5): batch atomicity, conflict rejection, FSM violation, give-up cancel, cascade supersession rejection -- **4 concurrency tests** (R-1–4): double dispatch, duplicate delivery, concurrent cancel under load, propagation race - -All tests assert Postgres state, not HTTP status codes. The WAL (`RunGraphMutation`) is the ground truth for causality and ordering. `RunRecord.status` is only FAILED via explicit operator action — never via automatic propagation. diff --git a/docs/integration-spec/3-structural-checks.md b/docs/integration-spec/3-structural-checks.md deleted file mode 100644 index 217ce7ac9..000000000 --- a/docs/integration-spec/3-structural-checks.md +++ /dev/null @@ -1,312 +0,0 @@ -# Structural Correctness Checks - -## Beyond the Nine Flows: Additional Bulletproofing Categories - -The nine control flows are about *runtime correctness* — does the system reach the right Postgres state after events settle? The categories below protect a different surface: *structural correctness* — does the codebase wire things up consistently in the first place? Some of these are best placed in the unit tier (no infra required), but they belong in the same TDD mandate. - ---- - -### 10. Event Pub→Sub Call Graph (static analysis — unit tier) - -**Problem:** A simple "every event name has a handler" check is one-directional. It catches orphaned event types (defined but nothing subscribes), but misses two equally dangerous failure modes: - -1. **Dead handlers** — a handler is registered in `ALL_FUNCTIONS` but nothing in the codebase ever emits the event it listens for. The handler is live but unreachable. -2. **Missing fan-out** — an event should trigger multiple handlers (Inngest supports N subscribers per event), but one is missing from `ALL_FUNCTIONS`. Publisher and one subscriber are fine; the second subscriber is silently absent. - -The fix is a bidirectional call graph: map every event to its expected publishers and expected subscribers, then assert both sides. - -**How to build the graph:** - -The subscriber side is trivially introspectable — `ALL_FUNCTIONS` is the source of truth: - -```python -from ergon_core.core.runtime.inngest_registry import ALL_FUNCTIONS - -def build_subscriber_map() -> dict[str, list[str]]: - """event_name → [handler_id, ...]""" - result: dict[str, list[str]] = {} - for fn in ALL_FUNCTIONS: - event = fn.trigger.event - result.setdefault(event, []).append(fn.id) - return result -``` - -The publisher side cannot be introspected automatically without AST analysis, so it is declared explicitly as a canonical fixture. This has a secondary benefit: the fixture **is the architecture document** for the event graph. Anyone reading it can trace the full flow. - -```python -# tests/unit/state/test_event_call_graph.py - -# Canonical pub→sub map. Each entry states: -# publishers: which Inngest function IDs emit this event (or "external" for CLI/API entrypoints) -# subscribers: which Inngest function IDs must handle it -EXPECTED_CALL_GRAPH: dict[str, dict[str, list[str]]] = { - "benchmark/run-request": { - "publishers": ["external:cli"], - "subscribers": ["benchmark-run-start"], - }, - "workflow/started": { - "publishers": ["benchmark-run-start"], - "subscribers": ["start-workflow"], - }, - "task/ready": { - # emitted by start-workflow (initial roots), by propagate-execution (after - # dep satisfied), and by execute-task (when plan_subtasks inserts new roots) - "publishers": ["start-workflow", "propagate-execution", "execute-task"], - "subscribers": ["execute-task"], - }, - "task/completed": { - "publishers": ["execute-task"], - "subscribers": ["propagate-execution"], - }, - "task/failed": { - "publishers": ["execute-task"], - "subscribers": ["propagate-execution"], - }, - "task/cancelled": { - "publishers": ["propagate-execution", "cancel-orphans"], - "subscribers": ["cancel-orphans"], - }, - "workflow/completed": { - "publishers": ["propagate-execution"], - "subscribers": ["complete-workflow"], - }, - "workflow/failed": { - "publishers": ["propagate-execution"], - "subscribers": ["fail-workflow"], - }, - "run/cancelled": { - "publishers": ["external:api"], - "subscribers": ["cancel-run"], - }, - "run/cleanup": { - "publishers": ["complete-workflow", "fail-workflow"], - "subscribers": ["cleanup-run"], - }, - # criterion/evaluate: publishers unknown, subscribers MISSING — this is the live bug - # this entry must be completed and a handler added before this test can pass - "criterion/evaluate": { - "publishers": [], # TODO: identify where this is emitted - "subscribers": [], # BUG: no handler registered in ALL_FUNCTIONS - }, -} -``` - -**Three assertions from one fixture:** - -```python -def test_every_declared_subscriber_is_registered(): - """All expected subscribers exist in ALL_FUNCTIONS.""" - registered = {fn.id for fn in ALL_FUNCTIONS} - for event, graph in EXPECTED_CALL_GRAPH.items(): - for expected_sub in graph["subscribers"]: - assert expected_sub in registered, ( - f"Event '{event}' expects handler '{expected_sub}' " - f"but it is not in ALL_FUNCTIONS" - ) - -def test_no_registered_handler_is_absent_from_call_graph(): - """Every handler in ALL_FUNCTIONS appears as a subscriber somewhere in the graph. - A handler that appears in no event's subscriber list is unreachable dead code.""" - declared_subs = {s for g in EXPECTED_CALL_GRAPH.values() for s in g["subscribers"]} - for fn in ALL_FUNCTIONS: - assert fn.id in declared_subs, ( - f"Handler '{fn.id}' is registered in ALL_FUNCTIONS but does not appear " - f"in EXPECTED_CALL_GRAPH — either add it or remove the handler" - ) - -def test_every_declared_event_type_has_a_model_class(): - """Every event slug in the call graph has a corresponding Pydantic event model. - Catches typos in the fixture and models that were deleted without updating the graph.""" - from ergon_core.core.runtime.events import all_event_models # hypothetical collector - known_slugs = {m.model_fields["name"].default for m in all_event_models()} - for event in EXPECTED_CALL_GRAPH: - assert event in known_slugs, ( - f"Event '{event}' in call graph has no corresponding event model class" - ) -``` - -**What this catches that the original one-liner missed:** - -| Failure mode | Simple slug check | Call graph | -|---|---|---| -| Event defined, no handler | ✅ | ✅ | -| Handler registered, nothing emits to it | ❌ | ✅ | -| Event should fan out to N handlers, only N-1 registered | ❌ | ✅ | -| Event slug in graph but no Pydantic model class | ❌ | ✅ | -| Handler slug typo in `ALL_FUNCTIONS` | ❌ | ✅ | - -**Concrete live example — `criterion/evaluate`:** The fixture above documents the bug explicitly. The test `test_every_declared_subscriber_is_registered` will fail on the `criterion/evaluate` entry the moment a subscriber is declared in the fixture but absent from `ALL_FUNCTIONS`. Until the fixture entry has a non-empty `subscribers` list AND a matching registered handler, the bug is recorded but the test doesn't crash the suite — which is the right behaviour during a fix. Add the handler, update the fixture, the test goes green. - ---- - -### 11. Inngest Function Catalog Integrity (static analysis — unit tier) - -**Problem:** `ALL_FUNCTIONS` could have duplicate slugs (two functions with the same name) or functions with no trigger at all. Inngest silently accepts duplicate registrations and last-write-wins, meaning one handler silently shadows another. - -**What to test:** - -```python -def test_no_duplicate_inngest_slugs(): - slugs = [fn.id for fn in inngest_registry.ALL_FUNCTIONS] - assert len(slugs) == len(set(slugs)), f"Duplicate slugs: {[s for s in slugs if slugs.count(s) > 1]}" - -def test_all_inngest_functions_have_event_trigger(): - for fn in inngest_registry.ALL_FUNCTIONS: - assert hasattr(fn.trigger, "event"), f"{fn.id} has no event trigger" -``` - ---- - -### 12. Registry Integrity (static analysis — unit tier) - -**Problem:** A slug can be registered in `BENCHMARKS`, `WORKERS`, `EVALUATORS`, or `CRITERIA` that points to a class that cannot be instantiated — wrong base class, missing required class vars, or import error at collection time. - -**What to test:** For each registry (`CORE_BENCHMARKS`, `WORKERS`, etc.), assert that every value is a class, that it is a subclass of the correct ABC, and that it can be constructed with minimal arguments (or at least that `__init__` doesn't immediately raise on a no-arg call where no args are required). The `onboarding_deps` benchmark contract in `test_benchmark_contract.py` is a partial model for this; extend the pattern to all registries. - -```python -@pytest.mark.parametrize("slug,cls", list(CORE_BENCHMARKS.items())) -def test_benchmark_registry_entries_are_valid_subclasses(slug, cls): - assert issubclass(cls, Benchmark), f"{slug} is not a Benchmark subclass" - assert hasattr(cls, "type_slug"), f"{slug} missing type_slug" -``` - ---- - -### 13. Event Payload Round-Trips (static analysis — unit tier) - -**Problem:** All Inngest events are Pydantic models. A broken `model_dump()` / `model_validate()` cycle (e.g. a UUID field that serialises to a non-string in one environment) means events cannot be deserialized by the Inngest dev server or the handler. - -**What to test:** For every concrete event class defined in `task_events.py`, `evaluation_events.py`, and `infrastructure_events.py`, construct a minimal valid instance, round-trip it through `model_dump()` + `model_validate()`, and assert equality. Use `mode="json"` on `model_dump` to catch UUID/datetime serialisation issues that only surface over the wire. - -```python -@pytest.mark.parametrize("event_cls,kwargs", [ - (TaskReadyEvent, {"run_id": uuid4(), "node_id": uuid4(), ...}), - ... -]) -def test_event_round_trips(event_cls, kwargs): - original = event_cls(**kwargs) - payload = original.model_dump(mode="json") - restored = event_cls.model_validate(payload) - assert original == restored -``` - ---- - -### 14. Toolkit Tool Name Uniqueness (static analysis — unit tier) - -**Problem:** The Inngest worker toolkit is assembled by combining tool lists from different providers. If two providers define a tool with the same `__name__`, the second silently shadows the first at the model-context level — the LLM sees duplicate names and the worker may call the wrong function. - -**What to test:** For each registered worker class, instantiate its toolkit (with a mock sandbox/runtime) and assert that all tool `__name__` values within that toolkit are unique. - -```python -def test_swebench_toolkit_tool_names_are_unique(): - tools = build_swebench_toolkit(sandbox=MockSandbox(), runtime=MockRuntime()) - names = [t.__name__ for t in tools] - assert len(names) == len(set(names)), f"Duplicate tool names: {set(n for n in names if names.count(n) > 1)}" -``` - ---- - -### 15. Worker / Benchmark / Evaluator ABC Compliance (static analysis — unit tier) - -**Problem:** Every concrete worker, benchmark, and evaluator must implement specific abstract methods. Python's ABC machinery only raises at *instantiation time*, not at import/collection time. A class that forgets to implement `execute()` passes all static checks until someone tries to run it. - -**What to test:** Attempt instantiation of every registered concrete class (with minimal stub arguments) and assert no `TypeError` is raised for missing abstract methods. This goes one step beyond the subclass check in category 12. - -```python -@pytest.mark.parametrize("slug,cls", list(WORKERS.items())) -def test_worker_is_fully_concrete(slug, cls): - # Should not raise TypeError: Can't instantiate abstract class - try: - instance = cls.__new__(cls) - except TypeError as e: - pytest.fail(f"Worker '{slug}' is not fully concrete: {e}") -``` - ---- - -### 16. DB Schema Reflection (Postgres only — integration tier, no Inngest) - -**Problem:** SQLModel generates table DDL from Python models. If a migration is applied in the wrong order or skipped, the live Postgres schema diverges from the models. This silently breaks writes to new columns or reads of renamed columns. - -**What to test:** Connect to the Postgres instance, reflect every table that `SQLModel.metadata` knows about, and assert that every column on every model exists in the reflected schema with the correct type and nullability. This test needs Postgres but not Inngest, so it should be gated separately from the event-flow tests. - -```python -def test_db_schema_matches_models(postgres_engine): - from sqlalchemy import inspect - inspector = inspect(postgres_engine) - for table_name, table in SQLModel.metadata.tables.items(): - reflected_cols = {c["name"] for c in inspector.get_columns(table_name)} - model_cols = {c.name for c in table.columns} - assert model_cols <= reflected_cols, ( - f"Table '{table_name}' missing columns: {model_cols - reflected_cols}" - ) -``` - ---- - -### 17. Sandbox Container Builds (Docker — slow tier) - -**Problem:** Each benchmark sandbox is built from a `Dockerfile` in the repo. A broken base image pin, a removed system package, or a pip install that fails silently produces a container that appears to build but crashes at runtime. This is only caught when someone actually tries to run a benchmark. - -**Two Dockerfiles to test:** -- `ergon_builtins/benchmarks/minif2f/Dockerfile` (Lean4 toolchain) -- `ergon_builtins/benchmarks/swebench_verified/Dockerfile` (Python dev environment) - -**What to test:** A `docker build` that must exit zero. These are slow (minutes each) and should be gated behind a `pytest.mark.docker_build` marker so they run in CI nightly but not on every PR push. A basic smoke: build succeeds, `docker run --rm <image> echo ok` exits zero. - -```python -@pytest.mark.docker_build -@pytest.mark.parametrize("dockerfile_path,context_dir", [ - ("ergon_builtins/benchmarks/minif2f/Dockerfile", "ergon_builtins/benchmarks/minif2f/"), - ("ergon_builtins/benchmarks/swebench_verified/Dockerfile", "ergon_builtins/benchmarks/swebench_verified/"), -]) -def test_sandbox_container_builds(dockerfile_path, context_dir, tmp_path): - result = subprocess.run( - ["docker", "build", "-f", dockerfile_path, "-t", f"ergon-test-{tmp_path.name}", context_dir], - capture_output=True, text=True, timeout=600, - ) - assert result.returncode == 0, f"Docker build failed:\n{result.stderr}" -``` - ---- - -### 18. Concurrency Race: `only_if_not_terminal` Under Concurrent Cancellation (full-stack integration) - -**Problem:** The `only_if_not_terminal` guard is the single mechanism preventing double-writes on a terminal node. If two `task/cancelled` events race to update the same node (which happens in diamond DAG failure cascades where multiple parents can independently trigger cancellation), one write must win and the other must be silently discarded. If the guard has a bug, the WAL gets a spurious second mutation and downstream assertions about node count and status are corrupted. - -This is the hardest test to write and the one most likely to catch real bugs. It requires the full Inngest + Postgres stack. - -**What to test:** Construct a DAG where a single leaf node has two parents, and both parents fail at approximately the same time. The leaf should receive two independent `task/cancelled` events nearly simultaneously. Assert: - -- The leaf has exactly one `CANCELLED` `RunGraphNode` row -- The WAL for the leaf has exactly one `CANCELLED` mutation (no duplicate entries) -- The `RunRecord` reaches `FAILED` (not stuck) -- The `RunTaskExecution` for the leaf (if any was started) has a single terminal row - -This test should be marked `pytest.mark.slow` and `pytest.mark.flaky_risk` with a note that it is exercising a race condition — it should be run with `--count=5` (using `pytest-repeat`) in CI to increase the chance of hitting the race. - ---- - -## Priority Order - -The current `tests/integration/` has one real integration test that checks HTTP response codes on a narrow happy path. The nine critical control flows above — propagation, failure cascade, subtask spawning, cancellation, restart, communication — have never had their Postgres state asserted at any tier. - -The Postgres state after any interesting event sequence is the system's ground truth. That is what the integration tier should be testing. - -The nine additional categories above protect structural correctness: events that are defined but never handled, duplicate tool names, schema drift, and concurrency races. Several of these (categories 10–15) are pure static analysis and belong in the unit tier — they require no running infrastructure and can fail fast in CI. Category 16 needs Postgres but not Inngest. Category 17 needs Docker and should run nightly. Category 18 needs the full stack and should run on every feature branch. - -**Priority order for implementation:** - -| Priority | Category | Tier | Value | -|----------|----------|------|-------| -| 1 | Fix 3 uncollected tests (`testresolve_*`) | Integration | Critical bug — tests have never run | -| 2 | Event subscriber coverage | Unit | Catches live `criterion/evaluate` orphan | -| 3 | Nine control flows | Integration | Core runtime correctness, zero coverage today | -| 4 | Inngest catalog + registry integrity | Unit | Cheap, high signal | -| 5 | DB schema reflection | Integration | Catches migration drift | -| 6 | Event payload round-trips | Unit | Catches serialisation bugs before they hit wire | -| 7 | Toolkit tool name uniqueness | Unit | Prevents silent shadowing | -| 8 | ABC compliance | Unit | Catches broken registrations at test time | -| 9 | Concurrency race | Full-stack | Validates the system's critical idempotency guard | -| 10 | Container builds | Docker/nightly | Catches broken sandbox images before E2E | diff --git a/docs/integration-spec/4-violated-assumptions.md b/docs/integration-spec/4-violated-assumptions.md deleted file mode 100644 index 069c9f2dc..000000000 --- a/docs/integration-spec/4-violated-assumptions.md +++ /dev/null @@ -1,212 +0,0 @@ -# Violated Assumptions - -Each entry below describes a behaviour the production code implements incorrectly relative to the intended domain model. Each has: what was assumed, what the code actually does, the fix required, and the integration test assertion that would catch it. Tests for these are marked `xfail(strict=True)` in the test suite — see `5-test-harness.md` for conventions. - ---- - -## Found Violated Assumptions - -Issues discovered by reading the actual production code during this session. Each entry states what we assumed, what the code actually does, the concrete fix needed, and the integration test assertion that would have caught it. - ---- - -### A. `run/cleanup` terminates one sandbox per run, not one per task execution - -**Assumed:** Container teardown happens per task-execution-attempt; `run/cleanup` kills all of them at run end. - -**Reality:** `run_cleanup.py:69` reads `run.parsed_summary().get("sandbox_id")` — a single string stored at run level. Only one sandbox ID is ever killed. If multiple tasks executed in parallel each had their own sandbox, all but the last one stored in the summary are silently leaked. - -**Fix:** Store sandbox IDs on `RunTaskExecution` rows (or in a separate table), not in `RunRecord.summary_json`. `run/cleanup` should collect all sandbox IDs across all `RunTaskExecution` rows for the run and terminate each one. - -**Integration test assertion:** After `run/cleanup` fires, query all `RunTaskExecution.sandbox_id` values for the run. For each non-stub ID, assert that `BaseSandboxManager.get(sandbox_id)` returns a terminated/not-found state. - ---- - -### B. Stub sandbox logic (`is_stub_sandbox_id`) leaks into four production files - -**Assumed:** Test infrastructure is confined to test code; the production path does not branch on whether a sandbox is "real" or a test stub. - -**Reality:** `is_stub_sandbox_id` and `StubSandboxManager` are defined in `manager.py` and imported into three production Inngest handlers: - -| File | Line | Usage | -|------|------|-------| -| `ergon_core/ergon_core/core/providers/sandbox/manager.py` | 578–613 | Definition of `_STUB_SANDBOX_PREFIX`, `is_stub_sandbox_id()`, `StubSandboxManager` | -| `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` | 72 | Guards sandbox termination | -| `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` | 96 | Guards evaluation dispatch | -| `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` | 102–116 | Creates stub sandbox ID for stub-worker path | - -**Fix:** Remove the stub concept from production code entirely. Tests should inject a real test-double at the `BaseSandboxManager` boundary (e.g. a subclass that returns a deterministic ID and no-ops on terminate) rather than relying on a magic string prefix to short-circuit production logic. - -**Static analysis test assertion:** `is_stub_sandbox_id` must not be imported in any module outside of `tests/`. This can be a simple grep-based unit test. - ---- - -### C. `evaluate_task_run` constructs `BenchmarkTask` with empty strings - -**Assumed:** Evaluation runs against the actual task data (description, instance key, task slug) fetched from the database. - -**Reality:** `evaluate_task_run.py:99–103` constructs `BenchmarkTask` with hardcoded empty strings: - -```python -task = BenchmarkTask( - task_slug="", - instance_key="", - description="", -) -``` - -The payload contains `task_id` and `execution_id` — enough to fetch the real data — but it is never queried. Criteria that inspect the task description or instance key for evaluation logic receive empty strings. - -**Fix:** Fetch the `RunGraphNode` (or definition task) using `payload.task_id` before constructing `BenchmarkTask`. Populate `task_slug`, `instance_key`, and `description` from the DB row. - -**Integration test assertion:** After evaluation completes, assert that `RunTaskEvaluation.summary_json` contains a non-empty `criterion_description` that matches the actual task description. - ---- - -### D. `criterion_description`, `feedback`, and `evaluation_input` use empty-string defaults instead of nullable - -**Assumed:** Missing evaluation fields are represented as `None`, not as empty strings. - -**Reality:** `CriterionResultEntry` (and the `_build_evaluation_summary` call at `evaluate_task_run.py:114`) uses `feedback=cr.feedback or ""` and `evaluation_input=evaluation_input` (passed as `""`). These are suppressed by the `# slopcop: ignore[no-str-empty-default]` comments rather than fixed. - -**Fix:** Change the fields to `str | None = None`. Update all callers to pass `None` instead of `""`. Remove the `slopcop: ignore` suppressions. - -**Unit test assertion:** Construct a `CriterionResultEntry` with no feedback provided and assert `entry.feedback is None`, not `entry.feedback == ""`. - ---- - -### E. `InngestCriterionExecutor` sandbox connection is unverified — may spin up its own container - -**Assumed:** The evaluator reuses the still-standing task sandbox (the one that was kept alive specifically for rubric evaluation) by connecting to it via `payload.sandbox_id`. - -**Concern:** `evaluate_task_run.py:82–90` creates `sandbox_manager = manager_cls()` and passes `payload.sandbox_id` into `TaskEvaluationContext`. Whether `InngestCriterionExecutor` actually connects to the existing sandbox via that ID — or ignores it and provisions a new one — is not verified by any test. - -**File to investigate:** `ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py` - -**Fix (pending investigation):** If `InngestCriterionExecutor` does not use `task_context.sandbox_id` to reconnect to the existing sandbox, it must be fixed to do so. Spinning up a second sandbox per evaluation doubles cost and breaks criteria that depend on inspecting the state the agent left behind in the original sandbox. - -**Integration test assertion:** Run a task that leaves a known artefact in the sandbox, then run evaluation. Assert the criterion received access to that artefact (i.e. it executed against the same sandbox, not a fresh one). - ---- - -### F. `restart_task` does not cancel stale subtask children from the prior execution - -**Assumed:** Restarting a task invalidates all outputs from its prior execution, including subtask children it previously spawned via `plan_subtasks`. - -**Reality:** `task_management_service.py::_invalidate_downstream` traverses outgoing `RunGraphEdge` only (the dependency DAG). It does not traverse the containment tree (`parent_node_id`). Subtask children from the prior execution are not touched. - -**Consequence:** A restarted task that previously called `plan_subtasks` leaves its old children completed in the graph. When the task re-executes and calls `plan_subtasks` again, duplicate child nodes are inserted — one set from the old execution (status `COMPLETED`) and one from the new (status `PENDING`). The graph is now corrupt: two generations of children coexist under the same parent. - -**Fix:** Before re-dispatching `task/ready` in `restart_task`, cancel all non-terminal descendants of the node (traversing `parent_node_id`) and reset any already-`COMPLETED` children to `CANCELLED` so the new execution starts from a clean subtask slate. This is the containment-axis equivalent of `_invalidate_downstream`. - -**Integration test assertion (part of test 8 — interrupt and restart):** After restarting a task that previously spawned two subtasks, assert that those old subtask nodes have status `CANCELLED`. Assert that after the restarted task completes and spawns subtasks again, there are exactly two nodes with `parent_node_id == restarted_node.id` and both are in a non-stale terminal state — not four. - ---- - -### G. `criterion/evaluate` event is defined but has no handler; `task/evaluate` is the actual evaluation trigger - -**Assumed (from earlier in this audit):** `criterion/evaluate` / `CriterionEvaluationEvent` is the event that drives per-criterion evaluation and is simply missing its handler. - -**Reality:** The actual evaluation handler (`evaluate-task-run`) is triggered by `task/evaluate`, not `criterion/evaluate`. `CriterionEvaluationEvent` in `evaluation_events.py` is a separate, orphaned event type with no handler anywhere in `ALL_FUNCTIONS`. It is unclear whether it is intended future infrastructure, dead code, or a vestige of a prior design. - -**Fix:** Either wire a handler to `criterion/evaluate` if per-criterion fan-out is the intended design, or delete `CriterionEvaluationEvent` from `evaluation_events.py`. The call graph fixture in category 10 should be updated to reflect whichever path is chosen. - -**Static analysis test assertion:** `criterion/evaluate` must either appear in `ALL_FUNCTIONS` as a trigger or must not exist as an event model class. The current state — defined but unhandled — must not pass the call graph test. - ---- - -### H. Failed-attempt sandbox is not killed before the restart container starts; prior container leaks - -**Assumed:** When a task is restarted, its failed attempt's sandbox is cleaned up before the new attempt begins. - -**Reality:** Every `task/ready` event goes through `sandbox-setup` (confirmed in `execute_task.py:140`), which provisions a fresh container. But when a task fails, `TaskFailedEvent` carries the sandbox_id and nothing kills that container at that point — teardown only happens at `run/cleanup`, which fires at run end. So between `restart_task` firing and the run eventually ending, two live containers exist for the same task simultaneously: the one from the failed attempt and the one from the current attempt. - -This compounds issue A: `run/cleanup` kills whichever sandbox_id was last written to `RunRecord.summary_json`, so the prior-attempt container leaks permanently. - -**Fix:** When `task/failed` fires (or when `restart_task` is called), kill the failed execution's sandbox immediately rather than waiting for run-level cleanup. The `execution_id` is available in both events and uniquely identifies which sandbox to terminate. - -**Integration test assertion:** After `restart_task` is called on a failed node, assert that the sandbox_id from the failed `RunTaskExecution` is no longer live. Assert that only one sandbox (from the new attempt) is live while the restart is in progress. - ---- - -### I. A completed run with evaluators configured may have zero `RunTaskEvaluation` rows with no published signal - -**Assumed:** If evaluation criteria are configured for a run, a completed run will always have a corresponding `RunTaskEvaluation` row per task per evaluator. If evaluation fails or is skipped, something is published to make this observable. - -**Reality:** If `check_evaluators` fires but dispatches no `task/evaluate` events (due to the `criterion/evaluate` orphan, a registry miss, or any silent failure in the dispatch path), the run reaches `RunRecord.status == COMPLETED` with zero `RunTaskEvaluation` rows. No event is emitted, no error is logged at run level, and the training loop receives a completed run with no scores. The absence of evaluation is indistinguishable from a run where evaluation genuinely returned a zero score. - -**Fix:** At `workflow/completed` time (or in `check_evaluators`), assert that the number of `RunTaskEvaluation` rows matches the expected count derived from the experiment definition (tasks × evaluator bindings). If the count is wrong, emit a distinct event or set a flag on `RunRecord` indicating evaluation was incomplete. Do not allow `RunRecord.status == COMPLETED` to coexist with missing evaluations silently. - -**Integration test assertion:** For any run where the experiment definition includes at least one evaluator binding, assert after terminal state that `COUNT(RunTaskEvaluation WHERE run_id = X) == expected_evaluation_count`. Assert that `RunRecord.summary_json` contains a non-null, non-empty scores field. A run that completes with zero evaluations must either have no evaluators configured or must have an explicit `evaluation_skipped` marker — never silent absence. - ---- - -### J. When a parent task fails, non-terminal containment children must become FAILED, not CANCELLED, recursively through all sublayers - -**Assumed:** When a parent task fails, its non-terminal containment children (all nodes reachable via `parent_node_id`, recursively) are marked `CANCELLED`. - -**Correct semantics (per domain model):** `CANCELLED` means "stopped intentionally before it got a chance to run" — an operator decision or a deliberate dependency invalidation. When a parent's execution context collapses (the container breaks, the worker raises), children whose work is now unreachable are not cancelled — they failed because their execution environment no longer exists. The correct status is `FAILED` with cause `parent_failed`. - -This distinction matters for operators: `CANCELLED` subtasks are treated as expected cleanup and get little scrutiny. `FAILED` subtasks surface in failure dashboards and signal "the scope of this failure was N nodes deep." - -**Scope:** Only non-terminal nodes are affected. Already-`COMPLETED` nodes represent genuinely finished work and must not be overwritten. Already-`FAILED` or `CANCELLED` nodes are left as-is. - -**Guard:** A node with `parent_node_id != NULL` should not be individually restartable — it can only be reset as part of a parent restart. Enforcing this prevents operators from retrying individual subtasks before their parent context is restored. - -**Reality (current code):** The current cascade code marks containment children `CANCELLED` with cause `parent_terminal`. This is the wrong status and the wrong cause string. - -**Fix:** Change the containment cascade to emit `FAILED` with cause `parent_failed`. Separate this from the existing `dep_invalidated` cascade (which operates on `RunGraphEdge` and correctly uses `CANCELLED`). - -**Integration test assertion:** See test 7 above. Assert that after a parent fails, every non-terminal containment descendant has status `FAILED` and no node in the subtree has status `CANCELLED` as a result of the parent's failure. - ---- - -### K. `restart_task` must reset the full containment subtree recursively, including COMPLETED children - -**Assumed:** Restarting a task only resets the task itself and invalidates its DAG-level dependency successors. - -**Correct semantics (per domain model):** A restart is a full reset of the node's entire containment subtree — every node reachable via `parent_node_id` recursively, including `COMPLETED` ones. The new execution starts in a fresh container; outputs from prior-attempt subtasks were written into the old container's context and cannot be assumed valid. Preserving COMPLETED subtask children from a prior run creates a mixed-generation problem where some outputs are from attempt N and some from attempt N+1. - -The restart should: -1. Reset every containment descendant to `PENDING` regardless of its current status -2. Dispatch `task/ready` for the subtree roots only (nodes with no in-edges within the subtask group) -3. Let normal propagation drive the rest as roots complete -4. Recurse: grandchildren and deeper sublayers reset alongside direct children - -**Reality (current code):** `_invalidate_downstream` in `task_management_service.py` traverses outgoing `RunGraphEdge` only (the dependency DAG). It does not traverse the containment tree (`parent_node_id`). Children from prior executions are left in their old states entirely — see issue F above. - -**Fix:** Add a containment-subtree reset pass to `restart_task` that runs before re-dispatching `task/ready`. Walk all nodes with `parent_node_id == node_id` recursively, reset each to `PENDING`, and dispatch `task/ready` for roots. This is distinct from and in addition to `_invalidate_downstream` (which handles DAG-level successors separately). - -**Integration test assertion:** See test 8 above. After `restart_task`, assert no `RunGraphNode` with `parent_node_id == restarted_node.id` (or deeper) retains `COMPLETED` or `FAILED` from the prior run. - -**Design clarification — `plan_subtasks` on a restarted execution:** Resetting prior children to `PENDING` (rather than `CANCELLED`) creates a conflict: the new parent execution will call `plan_subtasks` again, producing a second generation of child nodes alongside the reset first generation. The correct design is to **cancel** all prior containment children as part of `restart_task` (not reset to `PENDING`), then let the new execution call `plan_subtasks` fresh to create new nodes. This is cleaner: the worker's new decomposition may differ from the prior attempt's (especially after `refine_task`), and reusing old node rows assumes the plan is identical across attempts. The full-reset described above applies to the containment subtree reset at restart time; `plan_subtasks` in the new execution then builds the next generation from scratch. - ---- - -### L. `RunGraphMutation` has no causal lineage field — cascades are undebuggable - -**Assumed:** The WAL records enough information to reconstruct *why* any given state transition happened, including which upstream event triggered it. - -**Reality:** `RunGraphMutation` records `actor` and `reason` for each individual mutation, but has no pointer to the mutation that caused it. A cascade that sets 15 nodes to `FAILED` produces 15 independent WAL entries with the same `reason="parent_failed"` string. There is no way to query "what single event caused all of these?" or "what triggered this specific restart?" The WAL shows *what* happened but not *why* in a traceable, machine-queryable form. - -**Proposed fix — self-referential `triggered_by_mutation_id`:** - -Add `triggered_by_mutation_id: UUID | None` to `RunGraphMutation`. Root-level mutations (operator actions, external events) have `NULL`. Every mutation produced by a cascade points to the mutation that triggered it. The result is a causal tree: - -``` -mutation 1: node A → FAILED triggered_by=NULL reason="worker_error" -mutation 2: node B → FAILED triggered_by=1 reason="parent_failed" -mutation 3: node C → FAILED triggered_by=1 reason="parent_failed" -mutation 4: node D → FAILED triggered_by=2 reason="parent_failed" ← grandchild -mutation 5: node A → PENDING triggered_by=NULL reason="operator_restart" -mutation 6: node B → PENDING triggered_by=5 reason="parent_restart" -mutation 7: node C → PENDING triggered_by=5 reason="parent_restart" -mutation 8: node D → PENDING triggered_by=6 reason="parent_restart" -``` - -This enables: -- **Root cause query:** "Why is node D FAILED?" → mutation 4 → triggered by 2 → triggered by 1 → worker error on A. -- **Blast radius query:** "What did restarting A cause?" → all mutations with `triggered_by` in the transitive closure of mutation 5. -- **Orphan detection:** Any cascade mutation with `triggered_by=NULL` is a bug — something set a node's status without a traceable cause. - -**Integration test assertion (cross-cutting invariant):** Add to the shared assertion helpers: for any run, every `RunGraphMutation` whose `reason` is in `{parent_failed, parent_restart, dep_invalidated, downstream_invalidation}` must have a non-null `triggered_by_mutation_id`. A cascade mutation with no `triggered_by` is a test failure. diff --git a/docs/integration-spec/5-test-harness.md b/docs/integration-spec/5-test-harness.md deleted file mode 100644 index 6d9814f53..000000000 --- a/docs/integration-spec/5-test-harness.md +++ /dev/null @@ -1,102 +0,0 @@ -# Test Harness Design - -This document defines the shared infrastructure all integration tests use — stub workers, assertion helpers, fixture pattern, and xfail conventions. - -## Harness Components - -Every test needs four ingredients: - -**1. Stub workers** — workers that deterministically succeed, fail, or spawn subtasks with no E2B or LLM dependency. The system already has `training-stub` and the smoke worker fixtures. The integration tier needs its own small named set: -- `StubSuccessWorker` — returns `WorkerOutput(success=True)` immediately -- `StubFailWorker` — raises a controlled exception -- `StubSubtaskWorker(plan)` — calls `plan_subtasks()` with a provided spec, then succeeds - -**2. A run submitter** — posts a `benchmark/run-request` event to the local Inngest dev server via the existing `inngest_client` and returns the `run_id`. - -**3. A run poller** — polls `RunRecord.status` directly against Postgres until terminal. Times out with a clear message if convergence doesn't happen within a threshold (suggested: 30s for stub-only runs). - -**4. Shared assertion helpers** — functions that take a `run_id` and assert the cross-cutting invariants above. These live in `tests/integration/helpers/` and are imported by every test, not copy-pasted. - -The pattern for every test is then: - -```python -run_id = submit_run(worker="stub-success", tasks=[...]) -await wait_for_terminal(run_id, timeout=30) - -assert_run_completed(run_id) # RunRecord -assert_all_nodes_terminal(run_id) # graph -assert_wal_complete(run_id) # mutations -assert_executions_have_outputs(run_id) # telemetry -# ...plus test-specific assertions -``` - -## Fixture Layout - -``` -tests/integration/ -├── conftest.py ← Inngest + Postgres session fixtures (scoped only to tests that need live infra) -├── helpers/ -│ ├── __init__.py -│ ├── run_factory.py ← submit_run(), wait_for_terminal() -│ ├── assertions.py ← assert_run_completed(), assert_wal_complete(), assert_all_nodes_terminal(), etc. -│ └── stubs.py ← StubSuccessWorker, StubFailWorker, StubSubtaskWorker -├── smokes/ -├── swebench_verified/ -└── minif2f/ -``` - -Note that `conftest.py` should be split: the Inngest preflight (TCP probe + `pytest.exit()`) should be in a sub-conftest scoped only to the directories that actually need live Inngest, not session-wide. - -## Cross-Cutting Invariant Helpers - -| Invariant | What to assert | -|-----------|---------------| -| WAL completeness | Every `RunGraphNode` in the run has at least one `RunGraphMutation` entry | -| Execution coverage | Every node that reached `COMPLETED` or `FAILED` has a `RunTaskExecution` with non-null `completed_at` | -| No orphaned executions | Every `RunTaskExecution.node_id` references an existing `RunGraphNode` in the same run | -| RunRecord terminal consistency | `COMPLETED` implies all nodes are in `{COMPLETED, CANCELLED}`; `FAILED` implies at least one node is `FAILED` | -| Edge–node consistency | Every `EDGE_SATISFIED` edge has a source node with `status == COMPLETED`; every `EDGE_INVALIDATED` edge does not | -| Level consistency | Every node with `parent_node_id != NULL` has `level == parent.level + 1` | -| Append-only WAL | Mutation count for any given node only ever increases; no mutation rows are deleted or updated | -| Timeline consistency | `started_at ≤ completed_at` on every `RunTaskExecution` | - -These helpers must be called at the end of every control-flow test, not copy-pasted. They live in `tests/integration/helpers/assertions.py`. Each takes `run_id: UUID` and a database session and raises `AssertionError` with a descriptive message on failure. - -Once violated assumption L is resolved (`triggered_by_mutation_id` added to `RunGraphMutation`), add a ninth invariant: - -| Cascade lineage | Every `RunGraphMutation` with `reason` in `{parent_failed, parent_restart, dep_invalidated, downstream_invalidation}` has a non-null `triggered_by_mutation_id` | - -## xfail Conventions - -Tests for known-broken behaviours (violated assumptions A–L documented in `4-violated-assumptions.md`) are marked with: - -```python -@pytest.mark.xfail( - strict=True, - reason="violated assumption X: <one-line description>", -) -def test_name(): ... -``` - -Rules: -- `strict=True` is required. Without it, an xfail test that unexpectedly passes is silently ignored; with it, CI fails until the marker is explicitly removed. -- The `reason` string must cite the assumption letter so it traces directly to `4-violated-assumptions.md`. -- When the production fix is merged and the test passes, remove the `xfail` marker in the same PR as the fix. -- Never use `xfail` for tests that are merely slow or environment-dependent — use `pytest.mark.slow` or `pytest.mark.docker_build` instead. - -Reference table of current xfail targets: - -| Assumption | Short description | Marker reason string | -|---|---|---| -| A | run/cleanup kills one sandbox per run | `"violated assumption A: run/cleanup uses single sandbox_id from RunRecord.summary_json"` | -| B | is_stub_sandbox_id in prod path | `"violated assumption B: stub sandbox logic leaks into production handlers"` | -| C | evaluate_task_run uses empty BenchmarkTask | `"violated assumption C: evaluate_task_run constructs BenchmarkTask with empty strings"` | -| D | empty-string defaults instead of nullable | `"violated assumption D: criterion fields use empty-string defaults not None"` | -| E | InngestCriterionExecutor sandbox reuse unverified | `"violated assumption E: evaluator may spin up new sandbox instead of reusing task sandbox"` | -| F | restart_task ignores stale containment children | `"violated assumption F: restart_task does not cancel prior-execution subtask children"` | -| G | criterion/evaluate orphaned event | `"violated assumption G: CriterionEvaluationEvent has no registered handler"` | -| H | prior-attempt sandbox leaks on restart | `"violated assumption H: failed-attempt sandbox not killed before restart container starts"` | -| I | silent missing evaluations on completed run | `"violated assumption I: completed run with missing RunTaskEvaluation rows emits no signal"` | -| J | children marked CANCELLED not FAILED on parent failure | `"violated assumption J: parent failure cascade sets CANCELLED not FAILED on containment children"` | -| K | restart does not reset containment subtree | `"violated assumption K: restart_task does not reset containment subtree recursively"` | -| L | no causal lineage on RunGraphMutation | `"violated assumption L: RunGraphMutation has no triggered_by_mutation_id field"` | diff --git a/docs/integration-spec/6-implementation-plan.md b/docs/integration-spec/6-implementation-plan.md deleted file mode 100644 index d9843e848..000000000 --- a/docs/integration-spec/6-implementation-plan.md +++ /dev/null @@ -1,712 +0,0 @@ -# Six-Step Task Propagation — Implementation Plan - -> **Precondition:** `0-task-status-design.md` is agreed. All code changes here implement its FSM and schema requirements. Integration tests are the acceptance criterion; core logic changes must make the tests pass, not the other way around. - ---- - -## Execution order - -``` -Step 1: Schema migrations — Postgres DDL + Alembic + Python models -Step 2: Constants & enums — status_conventions.py, enums.py -Step 3: Write integration tests — all marked xfail; watch them fail -Step 4: Fix failure propagation — CANCELLED → BLOCKED, horizontal + vertical -Step 5: Fix terminal detection — RunRecord never auto-FAILED from propagation -Step 6: Fix restart + management — restart unblocks, new operator_unblock endpoint -→ Re-run integration tests — watch them pass -→ E2E reconciliation — align E2E tests to new semantics -``` - -Steps 1–2 have no logic changes — they are prerequisites. Write the tests in Step 3 *before* fixing the code in Steps 4–6. The red/green cycle is the signal. - ---- - -## Step 1: Schema migrations - -Four Alembic migrations, in dependency order. Each follows the pattern in `ergon_core/migrations/versions/84519b3f8431_add_cancelled_to_taskexecutionstatus_enum.py`. - -### Migration A — Add `BLOCKED` to `taskexecutionstatus` enum - -```bash -cd ergon_core -uv run alembic revision --autogenerate -m "add_blocked_to_taskexecutionstatus_enum" -# Then hand-edit the generated file to contain: -``` - -```python -# ergon_core/migrations/versions/<hash>_add_blocked_to_taskexecutionstatus_enum.py - -def upgrade() -> None: - if op.get_context().dialect.name != "postgresql": - return - with op.get_context().autocommit_block(): - op.execute( - sa.text("ALTER TYPE taskexecutionstatus ADD VALUE IF NOT EXISTS 'blocked'") - ) - -def downgrade() -> None: - pass # Postgres cannot remove enum values without rewriting the type -``` - -### Migration B — `triggered_by_mutation_id` on `RunGraphMutation` - -```python -# ergon_core/migrations/versions/<hash>_add_triggered_by_mutation_id.py - -def upgrade() -> None: - op.add_column( - "run_graph_mutations", - sa.Column("triggered_by_mutation_id", sa.UUID(), nullable=True), - ) - op.create_foreign_key( - "fk_run_graph_mutations_triggered_by", - "run_graph_mutations", "run_graph_mutations", - ["triggered_by_mutation_id"], ["id"], - ondelete="SET NULL", - ) - -def downgrade() -> None: - op.drop_constraint("fk_run_graph_mutations_triggered_by", "run_graph_mutations") - op.drop_column("run_graph_mutations", "triggered_by_mutation_id") -``` - -### Migration C — `batch_operation_id` on `RunGraphMutation` - -```python -# ergon_core/migrations/versions/<hash>_add_batch_operation_id.py - -def upgrade() -> None: - op.add_column( - "run_graph_mutations", - sa.Column("batch_operation_id", sa.UUID(), nullable=True), - ) - op.create_index( - "ix_run_graph_mutations_batch_operation_id", - "run_graph_mutations", - ["batch_operation_id"], - postgresql_where=sa.text("batch_operation_id IS NOT NULL"), - ) - -def downgrade() -> None: - op.drop_index("ix_run_graph_mutations_batch_operation_id") - op.drop_column("run_graph_mutations", "batch_operation_id") -``` - -### Migration D — `sandbox_id` on `RunTaskExecution` - -```python -# ergon_core/migrations/versions/<hash>_add_sandbox_id_to_run_task_execution.py - -def upgrade() -> None: - op.add_column( - "run_task_executions", - sa.Column("sandbox_id", sa.String(), nullable=True), - ) - -def downgrade() -> None: - op.drop_column("run_task_executions", "sandbox_id") -``` - -Run all four in sequence: -```bash -cd ergon_core && uv run alembic upgrade head -``` - ---- - -## Step 2: Constants, enums, and models - -Three files. Pure additions — no existing values change. - -### `ergon_core/ergon_core/core/persistence/graph/status_conventions.py` - -```python -# Add after CANCELLED: -BLOCKED = "blocked" - -# Update: -TERMINAL_STATUSES = frozenset({COMPLETED, FAILED, CANCELLED}) -# BLOCKED is intentionally absent — it is non-terminal - -# Update: -NodeStatus = Literal["pending", "ready", "running", "completed", "failed", "cancelled", "blocked"] -``` - -### `ergon_core/ergon_core/core/persistence/shared/enums.py` - -```python -class TaskExecutionStatus(StrEnum): - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - SKIPPED = "skipped" - CANCELLED = "cancelled" - BLOCKED = "blocked" # ← add -``` - -### `ergon_core/ergon_core/core/persistence/graph/models.py` - -Add two fields to `RunGraphMutation` (line ~185): - -```python -class RunGraphMutation(SQLModel, table=True): - # ... existing fields ... - triggered_by_mutation_id: UUID | None = Field( - default=None, - foreign_key="run_graph_mutations.id", - sa_column_kwargs={"ondelete": "SET NULL"}, - ) - batch_operation_id: UUID | None = Field(default=None, index=False) -``` - ---- - -## Step 3: Write the integration tests (all xfail) - -Create one file per logical group. All tests start as `xfail(strict=True)` — the code changes in Steps 4–6 should make them pass one by one. Remove `xfail` as each test goes green. - -``` -tests/integration/ - propagation/ - __init__.py - test_propagation_happy.py # Tests 1, 2, 4, 5, 9 - test_propagation_blocked.py # Tests 3, 7, 10, 11, 12, 13 - test_propagation_restart.py # Test 8 - test_propagation_cancel.py # Test 6 - test_propagation_edge_cases.py # EC-1 through EC-7 - test_propagation_bulk.py # B-1 through B-5 - test_propagation_races.py # R-1 through R-4 - _helpers.py # Shared polling, assertion helpers -``` - -### `_helpers.py` scaffold - -```python -import time -from uuid import UUID -from sqlmodel import Session, select -from ergon_core.core.persistence.graph.models import RunGraphNode, RunGraphMutation -from ergon_core.core.persistence.graph.status_conventions import TERMINAL_STATUSES - -def poll_until(condition, *, timeout=30, interval=0.5): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if condition(): - return - time.sleep(interval) - raise TimeoutError("poll_until timed out") - -def get_node_status(session: Session, run_id: UUID, node_id: UUID) -> str: - return session.exec( - select(RunGraphNode.status).where( - RunGraphNode.id == node_id, - RunGraphNode.run_id == run_id, - ) - ).one() - -def assert_wal_entry(session: Session, node_id: UUID, expected_status: str, cause: str | None = None): - """Assert at least one mutation row for node with new_value containing expected_status.""" - mutations = session.exec( - select(RunGraphMutation).where(RunGraphMutation.target_id == node_id) - ).all() - matching = [m for m in mutations if m.new_value.get("status") == expected_status] - assert matching, f"No WAL entry with status={expected_status!r} for node {node_id}" - if cause is not None: - assert any(m.reason and cause in m.reason for m in matching), \ - f"No WAL entry with cause={cause!r} for node {node_id}" - -def assert_cross_cutting_invariants(session: Session, run_id: UUID): - """Call from every test after the run reaches terminal or stuck state.""" - nodes = session.exec(select(RunGraphNode).where(RunGraphNode.run_id == run_id)).all() - # WAL completeness - for node in nodes: - mutations = session.exec( - select(RunGraphMutation).where(RunGraphMutation.target_id == node.id) - ).all() - assert mutations, f"Node {node.id} has no WAL entries" - # FAILED nodes must have RunTaskExecution with error_json - # (check via RunTaskExecution join — omitted here for brevity, add in implementation) - # No BLOCKED RunRecord status (RunRecord.status is never "blocked") - # ... etc -``` - -### First tests to write (highest signal, lowest complexity) - -Write these first — they exercise the core new semantics and will fail most obviously: - -```python -# test_propagation_blocked.py - -import pytest - -@pytest.mark.xfail(strict=True, reason="BLOCKED not implemented; successors become CANCELLED") -async def test_3_failure_cascade_successor_blocked(run_factory, db_session): - """A→B→C. B fails. C must be BLOCKED, not CANCELLED. RunRecord stays RUNNING.""" - run = await run_factory.linear_chain(steps=3) # A→B→C, B configured to fail - poll_until(lambda: get_node_status(db_session, run.id, run.nodes["B"]) == "failed") - poll_until(lambda: get_node_status(db_session, run.id, run.nodes["C"]) == "blocked", - timeout=5) # short timeout — should arrive quickly after B fails - - assert get_node_status(db_session, run.id, run.nodes["A"]) == "completed" - assert get_node_status(db_session, run.id, run.nodes["B"]) == "failed" - assert get_node_status(db_session, run.id, run.nodes["C"]) == "blocked" - assert run.record.status == "executing" # RunRecord stays RUNNING - - assert_wal_entry(db_session, run.nodes["C"], "blocked", cause="dep_failed") - assert_cross_cutting_invariants(db_session, run.id) - - -@pytest.mark.xfail(strict=True, reason="parent failure cascades CANCELLED not BLOCKED to children") -async def test_7_parent_failure_children_blocked(run_factory, db_session): - """Parent FAILED → PENDING/READY children BLOCKED. RUNNING children untouched.""" - run = await run_factory.parent_with_subtree(...) # parent fails; child-A PENDING, child-B RUNNING - poll_until(lambda: get_node_status(db_session, run.id, run.nodes["parent"]) == "failed") - poll_until(lambda: get_node_status(db_session, run.id, run.nodes["child_a"]) in ("blocked", "failed", "cancelled")) - - assert get_node_status(db_session, run.id, run.nodes["child_a"]) == "blocked" - assert get_node_status(db_session, run.id, run.nodes["child_b"]) in ("running", "completed", "failed") # not interrupted - assert run.record.status == "executing" # NOT failed - - assert_wal_entry(db_session, run.nodes["child_a"], "blocked", cause="parent_failed") -``` - -Write all 13 primary tests + 7 ECs + 5 Bs + 4 Rs before touching any production code. The full set of failures is the spec compliance baseline. - ---- - -## Step 4: Fix failure propagation — CANCELLED → BLOCKED - -Two files. This is the core change. - -### `ergon_core/ergon_core/core/runtime/execution/propagation.py` - -**Change 1 — `on_task_completed_or_failed`: horizontal BLOCKED** - -In the failure path (around line where `new_status=CANCELLED` appears): - -```python -# BEFORE: -if not is_success: - if candidate_node.parent_node_id is not None: - continue # Dynamic subtasks — leave for manager - await graph_repo.update_node_status( - session, run_id=run_id, node_id=candidate_id, - new_status=CANCELLED, - meta=MutationMeta(actor="system:propagation", reason=f"dependency {node_id} failed"), - only_if_not_terminal=True, - ) - invalidated.append(candidate_id) - continue - -# AFTER: -if not is_success: - # All successors (static and dynamic) become BLOCKED. - # BLOCKED means "predecessor failed; operator action required." - # Do NOT emit task/cancelled — BLOCKED is a DB write only. - await graph_repo.update_node_status( - session, run_id=run_id, node_id=candidate_id, - new_status=BLOCKED, - meta=MutationMeta( - actor="system:propagation", - reason=f"dependency {node_id} failed", - ), - only_if_not_terminal=True, - ) - # Do not append to invalidated — no events to emit for BLOCKED transitions - continue -``` - -**Change 2 — `is_workflow_failed_v2`: only terminal when ALL nodes done** - -```python -# BEFORE: -def is_workflow_failed_v2(session: Session, run_id: UUID) -> bool: - statuses = list(session.exec(select(RunGraphNode.status).where(...)).all()) - return any(s == TaskExecutionStatus.FAILED for s in statuses) - -# AFTER: -def is_workflow_failed_v2(session: Session, run_id: UUID) -> bool: - """All nodes terminal AND at least one FAILED. - - A run with BLOCKED tasks is stuck (RUNNING), not failed. Only when every - node is terminal (including any BLOCKED ones being resolved to CANCELLED - by the operator) and at least one is FAILED does the run finalise as FAILED. - """ - statuses = list(session.exec(select(RunGraphNode.status).where(...)).all()) - if not statuses: - return False - all_terminal = all(s in TERMINAL_STATUSES for s in statuses) - return all_terminal and any(s == FAILED for s in statuses) -``` - -Note: `BLOCKED ∉ TERMINAL_STATUSES`, so `all_terminal` is False whenever any node is BLOCKED. A stuck run with BLOCKED tasks will never trigger this function. Only when the operator has resolved all BLOCKED nodes (by restarting, cancelling, or unblocking them) and a FAILED node remains will this return True. - -### `ergon_core/ergon_core/core/runtime/inngest/cancel_orphan_subtasks.py` - -**Change — `cancel-orphans-on-failed`: cascade BLOCKED to descendants, not CANCELLED** - -The `cancel_orphans_on_failed_fn` function currently cascades CANCELLED to all non-terminal children of a FAILED parent. Under the new design, children of a FAILED parent must become BLOCKED (not CANCELLED — that's reserved for intentional stops). - -```python -# BEFORE: cancel-orphans-on-failed calls _cancel_orphans_for with cause="parent_terminal" -# which sets status=CANCELLED - -# AFTER: replace with block-descendants-on-failed: -@inngest_client.create_function( - fn_id="block-descendants-on-failed", - trigger=inngest.TriggerEvent(event="task/failed"), - cancel=RUN_CANCEL, - retries=1, -) -async def block_descendants_on_failed_fn(ctx: inngest.Context) -> int: - """When a parent fails, PENDING/READY containment descendants become BLOCKED. - - RUNNING descendants are not interrupted — they continue to their own terminal. - This function only walks the vertical (containment) axis via parent_node_id. - Horizontal (dependency) BLOCKED propagation is handled in propagation.py. - """ - payload = TaskFailedEvent.model_validate(ctx.event.data) - - async def _block_descendants() -> list[str]: - svc = SubtaskBlockingService() # new service — see below - with get_session() as session: - blocked_ids = await svc.block_pending_descendants( - session, - run_id=payload.run_id, - parent_node_id=payload.node_id, - cause="parent_failed", - ) - return [str(nid) for nid in blocked_ids] - - blocked = await ctx.step.run("block-pending-descendants", _block_descendants) - return len(blocked) -``` - -**New service: `SubtaskBlockingService`** in `ergon_core/ergon_core/core/runtime/services/subtask_blocking_service.py`: - -```python -"""Block PENDING/READY containment descendants when a parent fails.""" - -from uuid import UUID -from sqlmodel import Session, select -from ergon_core.core.persistence.graph.models import RunGraphNode -from ergon_core.core.persistence.graph.status_conventions import BLOCKED, TERMINAL_STATUSES, RUNNING -from ergon_core.core.runtime.services.graph_dto import MutationMeta -from ergon_core.core.runtime.services.graph_repository import WorkflowGraphRepository - - -class SubtaskBlockingService: - async def block_pending_descendants( - self, - session: Session, - *, - run_id: UUID, - parent_node_id: UUID, - cause: str, - ) -> list[UUID]: - """Recursively BLOCK all PENDING/READY descendants of parent_node_id. - - RUNNING descendants are skipped — live executions continue to their - own terminal state. Already-terminal descendants are skipped via - only_if_not_terminal=True. - - Returns IDs of nodes that were transitioned to BLOCKED. - """ - graph_repo = WorkflowGraphRepository() - blocked: list[UUID] = [] - queue = [parent_node_id] - - while queue: - current_parent = queue.pop() - children = session.exec( - select(RunGraphNode).where( - RunGraphNode.run_id == run_id, - RunGraphNode.parent_node_id == current_parent, - ) - ).all() - - for child in children: - queue.append(child.id) # recurse into grandchildren - - if child.status == RUNNING or child.status in TERMINAL_STATUSES: - continue # RUNNING: not interrupted; terminal: guard handles it - - await graph_repo.update_node_status( - session, - run_id=run_id, - node_id=child.id, - new_status=BLOCKED, - meta=MutationMeta(actor="system:propagation", reason=cause), - only_if_not_terminal=True, - ) - blocked.append(child.id) - - session.commit() - return blocked -``` - -### `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` - -**Change — `propagate_task_failure_fn`: stop emitting `task/cancelled` events** - -```python -# BEFORE: -failure_events: list[inngest.Event] = [ - inngest.Event( - name=TaskCancelledEvent.name, - data=TaskCancelledEvent( - ..., cause="dep_invalidated", - ).model_dump(mode="json"), - ) - for inv_node_id in propagation.invalidated_targets -] - -# AFTER: -# BLOCKED successors are a DB write only — no task/cancelled events. -# invalidated_targets is now empty (on_task_completed_or_failed no longer -# appends to it on the failure path). -failure_events: list[inngest.Event] = [] -``` - -Also remove the `WorkflowFailedEvent` emission from `propagate_task_failure_fn`. The workflow only fails when ALL nodes are terminal and any is FAILED — `is_workflow_failed_v2` now handles this correctly: - -```python -# BEFORE: -if propagation.workflow_terminal_state == WorkflowTerminalState.FAILED: - failure_events.append(inngest.Event(name=WorkflowFailedEvent.name, ...)) - -# AFTER: keep — but now is_workflow_failed_v2 only returns True when ALL -# nodes are terminal. A stuck run never reaches this branch. -# No change to the conditional itself; the fix is in is_workflow_failed_v2. -``` - ---- - -## Step 5: Fix workflow terminal detection - -### `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` - -The `propagate()` (success) path currently checks `is_workflow_failed_v2` and can set `WorkflowTerminalState.FAILED` when a task completes but there are pre-existing FAILED nodes. Under the new design, that is wrong — completing task A does not make the workflow enter a FAILED terminal state. - -```python -# BEFORE (in propagate()): -terminal = WorkflowTerminalState.NONE -if is_workflow_complete_v2(session, command.run_id): - terminal = WorkflowTerminalState.COMPLETED -elif is_workflow_failed_v2(session, command.run_id): - terminal = WorkflowTerminalState.FAILED - -# AFTER: unchanged — is_workflow_failed_v2 now returns False unless all -# nodes are terminal AND any is FAILED. A run with BLOCKED tasks will -# fall through to NONE correctly. -# If all nodes are terminal (operator resolved all BLOCKED to CANCELLED) -# and one is FAILED, WorkflowTerminalState.FAILED fires correctly here too. -``` - -The fix in `is_workflow_failed_v2` (Step 4) makes `propagate()` behave correctly without changing `task_propagation_service.py`. Verify with Test 3: after B fails and C is BLOCKED, `propagate()` completing A should see WorkflowTerminalState.NONE. - ---- - -## Step 6: Fix restart and add operator_unblock - -### `ergon_core/ergon_core/core/runtime/services/task_management_service.py` - -**Change — `restart_task._invalidate_downstream`** - -Currently traverses `RunGraphEdge` and emits `task/cancelled` for all downstream nodes. Under the new design: -- BLOCKED successors → PENDING (not CANCELLED — predecessor is being retried) -- COMPLETED successors → CANCELLED with cause `dep_invalidated` -- PENDING/READY successors — do NOT exist when restarting a FAILED node (they'd be BLOCKED already) - -```python -async def _invalidate_downstream(self, session, *, run_id, node_id, graph_repo): - """Reset outgoing edges; unblock BLOCKED successors; invalidate COMPLETED ones.""" - outgoing_edges = session.exec( - select(RunGraphEdge).where( - RunGraphEdge.run_id == run_id, - RunGraphEdge.source_node_id == node_id, - ) - ).all() - - for edge in outgoing_edges: - await graph_repo.update_edge_status( - session, run_id=run_id, edge_id=edge.id, - new_status=EDGE_PENDING, - meta=MutationMeta(actor="system:restart"), - ) - - successor = session.get(RunGraphNode, edge.target_node_id) - if successor is None: - continue - - if successor.status == BLOCKED: - # Predecessor is being retried — successor re-evaluates when it completes. - await graph_repo.update_node_status( - session, run_id=run_id, node_id=successor.id, - new_status=PENDING, - meta=MutationMeta(actor="system:restart", reason="predecessor_restarted"), - only_if_not_terminal=False, # BLOCKED is non-terminal; guard would pass anyway - ) - elif successor.status == COMPLETED: - # Prior outputs are stale — invalidate. - await graph_repo.update_node_status( - session, run_id=run_id, node_id=successor.id, - new_status=CANCELLED, - meta=MutationMeta(actor="system:restart", reason="dep_invalidated"), - only_if_not_terminal=True, - ) - # RUNNING successors: not interrupted. PENDING/READY: shouldn't exist - # (they'd be BLOCKED if this node was FAILED), but if somehow present, leave them. -``` - -**Change — containment descendants on restart** - -Currently `restart_task` does not cancel containment descendants (the existing `_invalidate_downstream` only walks `RunGraphEdge`). The design requires: all containment descendants → CANCELLED before the new execution starts. - -Add a call to `SubtaskCancellationService.cancel_orphans(parent_node_id=node_id)` (the existing service) before resetting the node itself: - -```python -async def restart_task(self, run_id, node_id, ...): - # 1. Cancel all containment descendants first (new execution creates fresh ones) - svc = SubtaskCancellationService() - await svc.cancel_orphans(session, run_id=run_id, parent_node_id=node_id, cause="operator_restart") - - # 2. Reset own outgoing edges + unblock/invalidate successors - await self._invalidate_downstream(session, run_id=run_id, node_id=node_id, ...) - - # 3. Reset own status to PENDING - await graph_repo.update_node_status(session, run_id=run_id, node_id=node_id, new_status=PENDING, ...) - - # 4. Emit task/ready - await inngest_client.send(inngest.Event(name="task/ready", data=...)) -``` - -**New method — `operator_unblock`** - -```python -async def operator_unblock(self, session, *, run_id: UUID, node_id: UUID) -> None: - """Transition a BLOCKED node to PENDING without restarting its predecessor. - - The operator is asserting "proceed despite the failed predecessor." - The predecessor stays FAILED. The node re-evaluates when predecessors - complete; it will stay PENDING until all predecessors complete or - the operator takes further action. - - This transition MUST NOT fire from propagation logic — operator-only. - """ - node = session.get(RunGraphNode, node_id) - if node is None or node.run_id != run_id: - raise NotFoundError(f"node {node_id} not found in run {run_id}") - if node.status != BLOCKED: - raise InvalidTransitionError( - f"operator_unblock requires BLOCKED; got {node.status!r}" - ) - - # Reset all incoming edges to EDGE_PENDING so the node re-evaluates - incoming_edges = session.exec( - select(RunGraphEdge).where( - RunGraphEdge.run_id == run_id, - RunGraphEdge.target_node_id == node_id, - ) - ).all() - for edge in incoming_edges: - await graph_repo.update_edge_status( - session, run_id=run_id, edge_id=edge.id, - new_status=EDGE_PENDING, - meta=MutationMeta(actor="operator", reason="operator_unblock"), - ) - - await graph_repo.update_node_status( - session, run_id=run_id, node_id=node_id, - new_status=PENDING, - meta=MutationMeta(actor="operator", reason="operator_unblock"), - only_if_not_terminal=False, - ) - session.commit() -``` - -Expose via a new API endpoint: `POST /runs/{run_id}/tasks/{node_id}/unblock`. - ---- - -## Run the integration tests - -```bash -uv run pytest tests/integration/propagation/ -v --tb=short -``` - -Expected sequence as steps land: -- After Step 4: Tests 3, 7, 10, 12 go green -- After Step 5: Test 3's RunRecord assertion goes green; `is_workflow_failed_v2` fix validated -- After Step 6 (restart): Tests 8, 11 go green -- After Step 6 (operator_unblock): Test 13 goes green -- EC-1 (fan-in under failure → BLOCKED): goes green after Step 4 -- EC-6 (multi-predecessor BLOCKED): goes green after Step 6 -- B and R tests: require bulk endpoint and locking model — implement after the above core tests pass - -Tests 1, 2, 4, 5, 6, 9 should be green before any changes (they test paths the existing code handles correctly). If they red, that's a regression — fix before proceeding. - ---- - -## E2E reconciliation - -After the integration tests pass, run the E2E suite: - -```bash -uv run pytest tests/e2e/ -v --tb=short -``` - -E2E tests currently assert `RunRecord.status == "failed"` when any task fails. Under the new design, `RunRecord` stays `"executing"` when tasks are BLOCKED. Two likely failure modes: - -**Failure mode 1 — E2E poll times out waiting for RunRecord.FAILED** - -The E2E assertion helper (`tests/e2e/_asserts.py`) polls for a terminal RunRecord status. If the run is stuck (BLOCKED tasks), it will poll until timeout. Fix: update the helper to also accept `"executing"` as a valid "settled" state for failure scenarios, or add a `poll_until_stuck` helper that checks for BLOCKED tasks. - -**Failure mode 2 — E2E expected final score is wrong** - -If a run that previously ended with `RunRecord.FAILED` (auto-propagated) now ends with `RunRecord.EXECUTING` (stuck), score aggregation never runs. The E2E test asserting a score of 0 (or any score) will fail because `summary_json` is never written. - -For each failing E2E test, determine which scenario it exercises: -1. Task genuinely fails and the run should be stuck → update assertion to check BLOCKED node exists, RunRecord is EXECUTING -2. Task fails and all others cancel → now all others become BLOCKED → run stays EXECUTING → test needs rethinking -3. E2E has an explicit cancellation step → behavior unchanged - -Specific files to check: -- `tests/e2e/_asserts.py` — update `assert_run_completed` / `assert_run_failed` helpers -- `tests/e2e/test_smoke_harness.py` (integration) — check what terminal state it polls for -- `tests/e2e/test_minif2f_smoke.py`, `test_swebench_smoke.py`, `test_researchrubrics_smoke.py` — these use real LLM; only run if a failure scenario is deliberately triggered - -Do not change E2E test behavior speculatively. Run them, read the failures, update the specific assertions that conflict with the new semantics. The goal is that the E2E tests still accurately describe what the system does — not that they all pass by lowering the bar. - ---- - -## What is NOT changing in this plan - -- `cancel-orphans-on-completed` — still cascades CANCELLED to orphaned subtasks when parent COMPLETES (correct behavior unchanged) -- `cancel-orphans-on-cancelled` — still cascades CANCELLED when parent is CANCELLED (correct behavior unchanged) -- `StubSandboxManager` / `is_stub_sandbox_id` removal — separate cleanup (violated assumption B), not part of this plan -- Bulk operation locking model (Section 9 of status design doc) — `SELECT FOR UPDATE` implementation is a separate pass after core tests pass -- `triggered_by_mutation_id` population — schema is added in Step 1; populating it in every mutation call is a follow-up once the core paths are stable - ---- - -## Checklist - -``` -[ ] Step 1: 4 Alembic migrations written and applied (uv run alembic upgrade head) -[ ] Step 2: BLOCKED in status_conventions.py, enums.py, RunGraphMutation model -[ ] Step 3: All integration tests written and failing (xfail strict) -[ ] Step 4a: on_task_completed_or_failed failure path → BLOCKED (not CANCELLED) -[ ] Step 4b: is_workflow_failed_v2 → only True when ALL nodes terminal -[ ] Step 4c: SubtaskBlockingService + block-descendants-on-failed Inngest fn -[ ] Step 4d: propagate_task_failure_fn stops emitting task/cancelled -[ ] Step 5: Verify propagate() (success path) WorkflowTerminalState.NONE when BLOCKED present -[ ] Step 6a: restart_task cancels containment descendants before resetting -[ ] Step 6b: restart_task unblocks BLOCKED successors → PENDING (not cancel them) -[ ] Step 6c: operator_unblock method + API endpoint -[ ] Integration tests: remove xfail from passing tests -[ ] E2E: run, read failures, update assertions that conflict with new semantics -[ ] pnpm run check:fast passes -[ ] Architecture docs updated (docs/architecture/) per CLAUDE.md requirement -``` diff --git a/docs/integration-spec/README.md b/docs/integration-spec/README.md deleted file mode 100644 index 82e7e4ea3..000000000 --- a/docs/integration-spec/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Integration Test Specification - -This folder defines the integration tier's intended behaviour as the authoritative source of truth for what the system must do. The workflow is: (1) violated assumptions and coverage gaps are identified here; (2) integration tests are written to match this spec — tests for known-broken behaviour are marked `xfail(strict=True)`; (3) production code is fixed until the xfail tests go green and the markers are explicitly removed; (4) E2E tests are updated to assert the same invariants in higher-fidelity environments. - -## Folder Contents - -| File | Description | -|------|-------------| -| `1-audit.md` | Audit of the current `tests/integration/` directory: what exists, what it actually tests, and its structural defects. | -| `2-control-flow-spec.md` | System model and the complete control-flow test specification: nine primary flows plus edge cases, with exact Postgres state assertions. | -| `3-structural-checks.md` | Static analysis, schema, registry, and infrastructure tests (categories 10–18) that verify structural correctness without running the full event pipeline. | -| `4-violated-assumptions.md` | Findings A–L: behaviours the production code implements incorrectly relative to the intended domain model, each with a fix and the integration test assertion that would catch it. | -| `5-test-harness.md` | Test harness design: stub workers, shared assertion helpers, fixture pattern, and xfail conventions. | - -## Priority Order - -| Priority | Category | Tier | Value | -|----------|----------|------|-------| -| 1 | Fix 3 uncollected tests (`testresolve_*`) | Integration | Critical bug — tests have never run | -| 2 | Event subscriber coverage | Unit | Catches live `criterion/evaluate` orphan | -| 3 | Nine control flows | Integration | Core runtime correctness, zero coverage today | -| 4 | Inngest catalog + registry integrity | Unit | Cheap, high signal | -| 5 | DB schema reflection | Integration | Catches migration drift | -| 6 | Event payload round-trips | Unit | Catches serialisation bugs before they hit wire | -| 7 | Toolkit tool name uniqueness | Unit | Prevents silent shadowing | -| 8 | ABC compliance | Unit | Catches broken registrations at test time | -| 9 | Concurrency race | Full-stack | Validates the system's critical idempotency guard | -| 10 | Container builds | Docker/nightly | Catches broken sandbox images before E2E | - -## xfail Conventions - -Tests for known-broken behaviours (violated assumptions) are marked: - -```python -@pytest.mark.xfail(strict=True, reason="violated assumption X: <description>") -``` - -`strict=True` means if the test accidentally passes, CI turns it into an error — the marker must be explicitly removed once the fix is confirmed and the test is observed green on a clean run. - -The reason string must cite the assumption letter (A–L) so it is traceable to `4-violated-assumptions.md`. - -Example: - -```python -@pytest.mark.xfail(strict=True, reason="violated assumption K: restart_task does not reset containment subtree") -def test_restart_resets_all_containment_children(): ... -``` - -## Current Status - -- Real integration tests today: 1 (`test_smoke_harness` — HTTP status codes only, no Postgres state) -- Violated assumptions identified: 12 (A–L), all requiring either a production code fix, a schema migration, or both diff --git a/docs/integration-test-audit-2026-04-23.md b/docs/integration-test-audit-2026-04-23.md deleted file mode 100644 index a18bbf315..000000000 --- a/docs/integration-test-audit-2026-04-23.md +++ /dev/null @@ -1,1112 +0,0 @@ -# Integration Tier Audit — 2026-04-23 - -Scope: `tests/integration/` only. - ---- - -## Directory Structure - -``` -tests/integration/ -├── conftest.py -├── smokes/ -│ └── test_smoke_harness.py -├── swebench_verified/ -│ ├── test_benchmark.py -│ ├── test_criterion.py -│ ├── test_rubric.py -│ ├── test_sandbox_manager.py -│ ├── test_smoke_e2e.py -│ ├── test_task_schemas.py -│ └── test_toolkit.py -└── minif2f/ - ├── test_sandbox_manager.py - └── test_verification_integration.py -``` - ---- - -## What Is Actually Covered - -| File | What it tests | Live infra required | -|------|--------------|-------------------| -| `smokes/test_smoke_harness.py` | seed → read → reset HTTP round-trip against a real server + Postgres | Server + Postgres | -| `swebench_verified/test_criterion.py` | `SWEBenchTestCriterion.evaluate()` with a mock `CriterionRuntime` | None | -| `swebench_verified/test_benchmark.py` | `build_instances()` with mocked `_load_rows` | None | -| `swebench_verified/test_task_schemas.py` | `SWEBenchInstance`, `SWEBenchTaskPayload` parsing, `_parse_test_list` | None | -| `swebench_verified/test_toolkit.py` | bash and str_replace_editor tools with a mock sandbox | None | -| `swebench_verified/test_rubric.py` | Rubric has one criterion named "test-resolution" with weight 1.0 | None | -| `swebench_verified/test_sandbox_manager.py` | Template resolution + `AsyncSandbox.create` call shape (mocked) | None | -| `swebench_verified/test_smoke_e2e.py` | Dockerfile and `e2b.toml.template` exist on disk | None | -| `minif2f/test_sandbox_manager.py` | Template resolution + sandbox create/verify lifecycle (mocked) — **3 tests uncollected** | None | -| `minif2f/test_verification_integration.py` | `ProofVerificationCriterion` — live test skipped without E2B key; static 'sorry' rejection test | Optional E2B | - -**There are no Postgres persistence round-trip tests for `RunRecord`, `RunTaskExecution`, `RunResource`, or `RunGraphNode`. There are no Inngest event schema or propagation tests. The single test that exercises real infrastructure is `test_smoke_harness.py::test_seed_then_read_then_reset_roundtrip`, and it only asserts HTTP response codes — not Postgres state.** - ---- - -## Existing Issues - -### Critical - -#### 1. Three tests have never been collected — `minif2f/test_sandbox_manager.py` - -Three functions are named `testresolve_*` instead of `test_*`. pytest never collects them. They have passed in CI since they were written only because they are invisible to the runner. - -```python -# Current — never collected: -def testresolve_template_falls_back_to_name_when_registry_missing(...): ... -def testresolve_template_prefers_registry_template_id(...): ... -def testresolve_template_falls_back_on_malformed_registry(...): ... - -# Should be: -def test_resolve_template_falls_back_to_name_when_registry_missing(...): ... -def test_resolve_template_prefers_registry_template_id(...): ... -def test_resolve_template_falls_back_on_malformed_registry(...): ... -``` - -**Fix:** Rename the three functions. Zero-risk, one commit. - ---- - -### High - -#### 2. The Inngest preflight gates tests that do not use Inngest - -`conftest.py` probes Inngest TCP connectivity at session start and calls `pytest.exit()` if it is unreachable. This applies to every file in `tests/integration/` regardless of whether the test needs Inngest. - -Eight of the ten test files require no live infrastructure at all — they are fully mocked unit tests sitting behind the integration preflight for no reason. A ten-line rubric test that checks a list length cannot run in any environment without a running Inngest server. - -**Fix:** Move the fully-mocked files to `tests/unit/` where they belong, or introduce a sub-conftest scoped only to the files that actually need Inngest. - -#### 3. Eight of ten files are misclassified unit tests - -Every file below needs no live infrastructure and is currently blocked behind the integration preflight: - -- `swebench_verified/test_benchmark.py` — mocks `_load_rows` -- `swebench_verified/test_criterion.py` — mocks `CriterionRuntime` -- `swebench_verified/test_rubric.py` — instantiates a local object -- `swebench_verified/test_sandbox_manager.py` — mocks `AsyncSandbox` -- `swebench_verified/test_smoke_e2e.py` — filesystem stat only -- `swebench_verified/test_task_schemas.py` — pure data parsing -- `swebench_verified/test_toolkit.py` — mocks `AsyncSandbox` -- `minif2f/test_sandbox_manager.py` — mocks `AsyncSandbox` - -These should live in `tests/unit/` and run under `pnpm run test:be:fast`. - ---- - -### Medium - -#### 4. `_reset_sandbox_singleton` fixture is duplicated - -The fixture that resets `BaseSandboxManager` class-level state is defined independently in both `minif2f/test_sandbox_manager.py` and `minif2f/test_verification_integration.py`. It is identical in both files. - -**Fix:** Extract to `tests/integration/minif2f/conftest.py`. - -#### 5. The one real integration test covers a narrow happy path - -`test_smoke_harness.py` has a single test: seed → read → reset, ending in a 404. It asserts HTTP status codes only. It does not assert Postgres state, does not test non-happy-path statuses, and does not verify that reset actually cascades to child rows. - ---- - -## What the System Actually Is (and What Integration Tests Should Assert) - -The integration tier is testing the wrong things because it is not oriented around the actual system model. Understanding that model is a prerequisite for knowing what to test. - -### The system model - -Ergon is an Inngest-driven, append-only workflow engine. Every meaningful invariant lives in Postgres. The unit of truth is **the database state after events settle**, not HTTP response codes. - -The four persistence tables that record all meaningful state: - -| Table | Purpose | -|-------|---------| -| `RunRecord` | One row per benchmark run; owns the run-level lifecycle status | -| `RunGraphNode` / `RunGraphEdge` / `RunGraphMutation` | DAG execution state; the WAL is the ground truth | -| `RunTaskExecution` | One row per execution attempt; records start/end, output, error | -| `RunResource` / `ThreadMessage` | Task outputs and agent-to-agent communication | - -### The state machine - -**Node statuses:** `PENDING → READY → RUNNING → {COMPLETED | FAILED | CANCELLED}` - -Valid transitions only. The `only_if_not_terminal` guard prevents any second write once a node reaches `COMPLETED`, `FAILED`, or `CANCELLED`. This guard is the critical idempotency mechanism for concurrent event delivery. - -**RunRecord statuses:** `CREATED → RUNNING → {COMPLETED | FAILED}` - -`RunRecord.status == COMPLETED` is only valid if all nodes in the run are in `{COMPLETED, CANCELLED}`. - -**Edge statuses:** `EDGE_PENDING → EDGE_SATISFIED | EDGE_INVALIDATED` - -An edge flips to `EDGE_SATISFIED` when its source node reaches `COMPLETED`. It flips to `EDGE_INVALIDATED` when a downstream task is restarted and its prior completion is no longer valid. - -### The Inngest event flow - -``` -benchmark/run-request - → [benchmark_run_start_fn] - persist ExperimentDefinition - INSERT RunRecord (status=CREATED) - emit workflow/started - -workflow/started - → [start_workflow_fn] - create RunGraphNode for each definition task - create RunGraphEdge for each dependency - mark root nodes READY - emit task/ready (fanout to all roots) - -task/ready - → [execute_task_fn] - INSERT RunTaskExecution (status=PENDING) - UPDATE RunGraphNode → RUNNING - [sandbox_setup_fn] - [worker_execute_fn] ← worker code runs here - optionally: plan_subtasks() → inserts child RunGraphNodes, emits task/ready for roots - optionally: save_message() → inserts Thread + ThreadMessage - [persist_outputs_fn] → INSERT RunResource rows - UPDATE RunTaskExecution (status=COMPLETED) - UPDATE RunGraphNode → COMPLETED - INSERT RunGraphMutation (audit entry) - emit task/completed - - [on any failure] - UPDATE RunTaskExecution (status=FAILED, error_json=...) - UPDATE RunGraphNode → FAILED - INSERT RunGraphMutation - emit task/failed - -task/completed → [propagate_execution] - flip source edges → EDGE_SATISFIED - check each dependent: if all incoming edges EDGE_SATISFIED → mark READY, emit task/ready - if all nodes terminal → emit workflow/completed - -task/failed → [propagate_execution] - find all non-terminal dependents (direct + transitive) - emit task/cancelled for each (cause=dep_invalidated) - -task/cancelled → [cancel_orphans_fn] - cascade CANCELLED to all non-terminal children (cause=parent_terminal) - UPDATE RunTaskExecution → CANCELLED - UPDATE RunGraphNode → CANCELLED - INSERT RunGraphMutation - -workflow/completed - → [complete_workflow_fn] - aggregate scores from RunTaskExecution outputs - UPDATE RunRecord (status=COMPLETED, completed_at=now, summary_json={scores}) - emit run/cleanup - -workflow/failed - → [fail_workflow_fn] - UPDATE RunRecord (status=FAILED, error_message=...) - emit run/cleanup -``` - -### Subtask spawning (dynamic DAG mutation) - -A worker can call `plan_subtasks()` or `add_subtask()` during execution. This inserts child `RunGraphNode` rows under the currently-executing node and immediately emits `task/ready` for the new roots. The containment tree is recorded via `parent_node_id` (self-referential FK) and `level` (precomputed depth: `parent.level + 1`). - -**Key invariants of the subtask tree:** -- Every node with `parent_node_id != NULL` has `level == parent.level + 1` -- Cycle detection runs at `plan_subtasks` time; any cycle is rejected before any DB write -- The parent task does not finalize until all its children are terminal (the propagation loop handles this) - -### The cancellation causes - -When a `task/cancelled` event is emitted, the `cause` field records why: - -| Cause | Trigger | -|-------|---------| -| `manager_decision` | Agent explicitly called `cancel_task(node_id)` via the toolkit | -| `parent_terminal` | Parent node reached COMPLETED, FAILED, or CANCELLED | -| `dep_invalidated` | A dependency failed; this task's inputs are no longer valid | -| `run_cancelled` | Run-level cancellation broadcast | - -The WAL (`RunGraphMutation`) records the cause for every cancellation. This is the only place it is permanently stored. - -### The restart / interrupt flow - -`refine_task(node_id, new_description)` → updates description, allowed on any non-RUNNING node. - -`restart_task(node_id)` → resets a terminal node back to `PENDING`. Also: -- Flips all outgoing `EDGE_SATISFIED` edges back to `EDGE_INVALIDATED` -- Emits `task/cancelled` for all non-terminal downstream nodes (their inputs are now stale) -- A new `task/ready` is emitted, incrementing `attempt_number` on the new `RunTaskExecution` - ---- - -## The Nine Control Flows Integration Tests Must Cover - -Each of these should use stub/mock workers (no E2B, no real LLM), submit a real event to the local Inngest dev server, poll until `RunRecord.status` is terminal, then assert the exact Postgres state. - -### 1. Single-task happy path - -Submit a run with one task and a worker that returns `WorkerOutput(success=True)`. - -Assert: -- `RunRecord.status == COMPLETED`, `completed_at` is set -- `RunGraphNode.status == COMPLETED` -- `RunTaskExecution.status == COMPLETED`, `started_at ≤ completed_at` -- At least one `RunResource` row with correct `run_id` and `task_execution_id` -- `RunGraphMutation` WAL contains entries for `PENDING → RUNNING → COMPLETED` - -### 2. Linear dependency chain — propagation - -Three tasks A → B → C. All workers succeed. - -Assert: -- Tasks reach `COMPLETED` in topological order (`completed_at` ordering: A before B before C) -- `RunGraphEdge` rows: all `EDGE_SATISFIED` -- After A completes, B transitions to `RUNNING` before C does (WAL timestamps) -- `RunRecord.status == COMPLETED` -- WAL has a `READY` transition for B only after A's `COMPLETED` entry, ditto C after B - -### 3. Failure cascade — dep_invalidated - -Tasks A → B → C. A succeeds. B fails. C never ran. - -Assert: -- A: `COMPLETED` -- B: `FAILED`, `RunTaskExecution.error_json` non-null -- C: `CANCELLED`, no `RunTaskExecution` with `status == COMPLETED` or `RUNNING` -- WAL entry for C contains `reason` reflecting `dep_invalidated` -- `RunRecord.status == FAILED` -- No `RunResource` rows owned by C's execution - -### 4. Diamond DAG — propagation convergence - -Four tasks: root → left, root → right, left → sink, right → sink. All succeed. - -Assert: -- `left` and `right` both reach `RUNNING` before `sink` becomes `READY` -- `sink` transitions to `READY` only after both `left` AND `right` are `COMPLETED` (both edges `EDGE_SATISFIED`) -- `sink` transitions to `READY` exactly once (the `only_if_not_terminal` idempotency guard is exercised — sink receives two "dep satisfied" signals) -- Final: all four `COMPLETED`, `RunRecord.status == COMPLETED` - -### 5. Subtask spawning — dynamic DAG - -A parent worker that calls `plan_subtasks()` with this spec: - -``` -root_child (no deps within the subtask group) - ↓ -leaf_child (depends on root_child) -``` - -Both `root_child` and `leaf_child` are **direct children of the parent node** — they share the same `parent_node_id = parent.id` and the same `level = parent.level + 1`. The `→` is a **dependency edge between two siblings**, not a nesting level. The sublayer is flat: a set of new `RunGraphNode` rows all at the same depth, with their own `RunGraphEdge` rows connecting them internally. `plan_subtasks()` must insert both the containment rows (`parent_node_id`) and the inter-sibling edges in the same operation. - -Assert: -- Two `RunGraphNode` rows exist with `parent_node_id == parent_node.id` -- Both have `level == parent.level + 1` -- `root_child` was the only node to transition to `READY` immediately after subtask insertion (`leaf_child` was `PENDING` at that point — its edge was not yet satisfied) -- `leaf_child` transitions to `READY` only after `root_child` reaches `COMPLETED` (WAL timestamps) -- `leaf_child.status == COMPLETED` -- Parent node reaches `COMPLETED` only after both children are terminal -- WAL has a `PENDING` entry for `root_child` timestamped after parent reached `RUNNING` -- No `RunGraphEdge` exists pointing directly from parent to either child — the parent/child relationship is expressed by `parent_node_id`, not by graph edges - -### 6. Cancellation: manager_decision - -A run with two sibling tasks (`target` and `sibling`). `target` itself has a two-level subtask tree spawned during execution: - -``` -target (cancelled via manager_decision) -├── target-child-A -│ ├── target-grandchild-A1 -│ └── target-grandchild-A2 -└── target-child-B - -sibling (independent; must NOT be affected) -└── sibling-child -``` - -Call `cancel_task(target.node_id)` while `target` is `RUNNING` and its subtree is live. - -Assert — cancellation target: -- `target.status == CANCELLED` -- WAL entry for `target` has cause `manager_decision` -- `target`'s `RunTaskExecution.status == CANCELLED` - -Assert — full subtree cascade (all descendants, not just direct children): -- `target-child-A`, `target-child-B`: `CANCELLED` with cause `parent_terminal` -- `target-grandchild-A1`, `target-grandchild-A2`: `CANCELLED` with cause `parent_terminal` -- WAL entries for each grandchild are timestamped after the corresponding child's `CANCELLED` mutation (the cascade propagates level by level, not all at once) -- No `RunTaskExecution` with `status IN (RUNNING, COMPLETED)` exists for any descendant - -Assert — sibling isolation (the cancellation is contained to the subtree): -- `sibling.status` is unchanged (`RUNNING` or `COMPLETED`) -- `sibling-child.status` is unchanged -- WAL for `sibling` and `sibling-child` contains no `CANCELLED` entries - -Assert — idempotency: -- Calling `cancel_task(target.node_id)` a second time returns an error -- WAL mutation count for `target` does not increase after the second call - -### 7. Parent-failure cascade — FAILED propagates through the full containment tree - -**Note:** The original framing (parent COMPLETED while children PENDING) is architecturally impossible — the propagation loop prevents it. The correct scenario is parent FAILED. - -**Correct semantics:** When a parent fails, all non-terminal nodes in its containment subtree (all nodes reachable via `parent_node_id`, recursively through every sublayer) must become `FAILED` with cause `parent_failed`. `CANCELLED` is wrong here — it means "stopped intentionally before getting a chance to run." `FAILED` correctly signals "this execution context collapsed and this work did not complete." - -The test must use a multi-level tree to verify the cascade goes all the way down: - -``` -parent (raises controlled exception during execution) -├── child-A (PENDING when parent fails) -│ ├── grandchild-A1 (PENDING) -│ └── grandchild-A2 (RUNNING) -└── child-B (RUNNING when parent fails) -└── child-C (COMPLETED before parent fails — must survive) -``` - -Assert: -- Parent: `FAILED`, `RunTaskExecution.error_json` non-null -- `child-A`, `child-B`: `FAILED` with cause `parent_failed` — **not** `CANCELLED` -- `grandchild-A1`, `grandchild-A2`: `FAILED` with cause `parent_failed` — cascade reaches grandchildren -- WAL entries for grandchildren are timestamped after their respective parent's `FAILED` mutation (cascade propagates level by level) -- `child-C`: remains `COMPLETED` — already-terminal nodes are not overwritten -- No non-terminal node in the containment subtree has any status other than `FAILED` -- `RunRecord.status == FAILED` - -Assert the positive invariant (COMPLETED is unreachable while children are live): -- In tests 1 and 5, assert `RunRecord.status == COMPLETED` is only set after every `RunGraphNode` in the run is in `{COMPLETED, FAILED, CANCELLED}` — directly tests the propagation loop's wait-for-children guarantee - -**Depth parametrisation:** Run this test parametrised over sublayer depths N=1, N=3, N=10. At N=10, every one of the 10 levels of containment descendants must be `FAILED` — not just direct children. This catches iterative vs recursive implementation bugs in the cascade where only the first level is processed. - -### 8. Interrupt and restart — full containment subtree resets recursively - -**Correct semantics:** Restarting a task is a full reset of its entire containment subtree — not just non-terminal nodes, but *all* nodes including previously `COMPLETED` ones, all the way down through every sublayer. The new execution starts in a fresh container; any work the old execution's subtasks produced is bound to that container's context and must not be assumed valid. The restart dispatches `task/ready` for the subtree roots only; propagation drives the rest. - -A node with `parent_node_id != NULL` cannot be restarted independently — it can only be reset as part of its parent restarting. This must be enforced as a guard. - -Use a task that previously ran and produced a two-level subtask tree: - -``` -parent (FAILED after prior execution) -├── child-root (COMPLETED in prior run — must be reset) -│ └── child-leaf (COMPLETED in prior run — must be reset) -└── child-standalone (FAILED in prior run — must be reset) -``` - -Also set up one DAG-level successor of `parent` (connected by a `RunGraphEdge`, not `parent_node_id`) that was `COMPLETED` in the prior run and must be invalidated. - -**Step 1 — refine:** - -`refine_task(parent.node_id, new_description)`: -- `parent.description` updated in `RunGraphNode` -- `parent.status` unchanged (still `FAILED`) - -**Step 2 — restart:** - -`restart_task(parent.node_id)`: - -Assert — parent reset: -- `parent.status == PENDING`, then transitions to `RUNNING` once `task/ready` fires -- WAL records `FAILED → PENDING` with cause `operator_restart` -- New `RunTaskExecution` row created with `attempt_number` incremented - -Assert — full containment subtree reset (all children, regardless of prior status): -- `child-root`, `child-leaf`, `child-standalone`: all reset to `PENDING` -- No node in the containment subtree retains `COMPLETED` or `FAILED` from the prior run -- Recursive: grandchildren reset alongside direct children - -Assert — subtree root dispatch: -- `task/ready` fired for `child-root` only (it has no in-edges within the subtask group) -- `child-leaf` is `PENDING` waiting for propagation — it does NOT receive `task/ready` until `child-root` completes -- `child-standalone` is `PENDING` waiting for propagation (or `READY` if it has no deps) - -Assert — DAG successor invalidated: -- The `RunGraphEdge` from `parent` to its DAG successor is reset to `EDGE_PENDING` -- The DAG successor transitions from `COMPLETED` to `CANCELLED` (its inputs are stale) - -Assert — independent guard: -- Calling `restart_task(child-root.node_id)` directly (without going through the parent) raises an error — nodes with `parent_node_id != NULL` cannot be restarted independently - -**Step 3 — restarted execution completes:** - -Assert: -- All containment children reach `COMPLETED` via normal propagation -- `parent.status == COMPLETED` -- DAG successor re-activates (edges re-satisfied) and reaches `COMPLETED` -- `RunRecord.status == COMPLETED` -- Exactly two `RunTaskExecution` rows exist for `parent` (attempt 1 and attempt 2) - -**Depth parametrisation:** Run this test parametrised over sublayer depths N=1, N=3, N=10. At N=10, every containment descendant at every level must be reset to `PENDING`, and `task/ready` must fire only for the roots at each level — not every node simultaneously. This validates that propagation, not bulk-dispatch, drives the non-root activations. - -### 9. Communication service — message routing - -A stub worker that calls `communication_service.save_message()` to send a message with `from_agent_id=leaf-X`, `to_agent_id=parent`, `thread_topic=smoke-completion`. - -Assert: -- A `Thread` row exists scoped to `run_id` with the correct `topic` -- A `ThreadMessage` row exists with the correct `from_agent_id`, `to_agent_id`, `run_id`, `task_execution_id` -- `sequence_num == 1` for the first message in the thread -- A second message to the same thread gets `sequence_num == 2` (ordering guarantee) - -### Edge Cases and Boundary Conditions - -The nine control flows above cover the primary happy and failure paths. The tests below cover timing races, idempotency, forbidden graph structures, and boundary conditions that the primary tests do not exercise. - -#### EC-1: Fan-in convergence race under failure - -Setup: Diamond DAG — `root → left`, `root → right`, `left → sink`, `right → sink`. Arrange for left to FAIL at the same moment right is completing successfully. Use a sleep/barrier in the stub workers to make the race reproducible. - -Two propagation events race to the `sink`: `dep_invalidated` (from left's failure) and `edge_satisfied` (from right's completion). Depending on which lands first, `sink` may briefly reach `READY` before being cancelled, or may be cancelled before it is ever activated. - -Assert: -- `sink.status == CANCELLED` regardless of event arrival order -- `RunRecord.status == FAILED` -- `sink` has exactly ONE terminal WAL mutation (`CANCELLED`) — no `COMPLETED` entry exists for `sink` -- The `only_if_not_terminal` guard prevented any post-cancellation write if right's completion arrived after the cancellation - -Mark `pytest.mark.slow` and run with `--count=5` to reliably trigger the race. - -#### EC-2: Duplicate `task/ready` delivery — idempotency at prepare - -Inngest guarantees at-least-once delivery. A duplicate `task/ready` event for a node that is already `RUNNING` must be a no-op. - -This is a unit-tier test — no Inngest required. Seed a `RunGraphNode` in `RUNNING` state and a corresponding `RunTaskExecution`, then call the `prepare-execution` logic directly. - -Assert: -- No second `RunTaskExecution` row is created -- No second sandbox is provisioned -- No duplicate `RUNNING` WAL entry exists -- The function returns without error (idempotent skip, not a crash) - -#### EC-3: Cross-containment dependency edges are forbidden - -A `RunGraphEdge` where `source.parent_node_id != target.parent_node_id` (i.e. the two endpoints live in different containment subtrees) must be rejected at graph construction time. Without this guard, cascade and restart logic would need to reason about cross-containment dependencies, which is undefined. - -This is a unit-tier test. - -Assert: -- `add_edge(source, target)` raises an error when `source.parent_node_id != target.parent_node_id` -- `add_edge` raises an error when one node has a `parent_node_id` and the other does not -- The error is raised before any DB write — no partial edge rows are created - -#### EC-4: Evaluation after restart — which attempt's score counts - -Setup: -1. Task completes (attempt 1) → `RunTaskEvaluation` row created with score X -2. `restart_task` is called → task re-runs (attempt 2) → second `RunTaskEvaluation` row created with score Y - -Assert: -- Exactly two `RunTaskEvaluation` rows exist for the same `definition_task_id` and `run_id` -- `RunRecord.summary_json` reflects score Y (the most recent attempt), not score X -- The attempt-1 `RunTaskEvaluation` is preserved (append-only audit) but not used in the summary -- The row used in the summary can be identified by matching `execution_id` to the most recent `RunTaskExecution` for the node - -If the system does not yet implement "use most recent attempt's score", mark as `pytest.mark.xfail(strict=True, reason="evaluation after restart: score selection across attempts not yet defined")`. - -#### EC-5: Concurrency queue + node cancellation while queued - -Context: `execute_task_fn` has `concurrency limit=15`. If a 16th `task/ready` fires, Inngest queues the invocation. - -Setup: Saturate the 15 slots with long-running stub tasks, then fire a 16th `task/ready`. Cancel the 16th node via `cancel_task` while it is queued (before Inngest dispatches it). - -Assert: -- When Inngest eventually dispatches the queued invocation, `prepare-execution` detects the node is `CANCELLED` and exits without creating a `RunTaskExecution` row -- No sandbox is provisioned for the cancelled node -- No `TaskCompletedEvent` or `TaskFailedEvent` is emitted for the cancelled node -- The node's WAL has a `CANCELLED` entry but no `RUNNING` entry - -Mark `pytest.mark.slow` — requires saturating the concurrency pool. - ---- - -## Cross-Cutting Invariants (shared assertion helpers) - -These are properties that can be written once as helper functions and called from every test above. - -| Invariant | What to assert | -|-----------|---------------| -| WAL completeness | Every `RunGraphNode` in the run has at least one `RunGraphMutation` entry | -| Execution coverage | Every node that reached `COMPLETED` or `FAILED` has a `RunTaskExecution` with non-null `completed_at` | -| No orphaned executions | Every `RunTaskExecution.node_id` references an existing `RunGraphNode` in the same run | -| RunRecord terminal consistency | `COMPLETED` implies all nodes are in `{COMPLETED, CANCELLED}`; `FAILED` implies at least one node is `FAILED` | -| Edge–node consistency | Every `EDGE_SATISFIED` edge has a source node with `status == COMPLETED`; every `EDGE_INVALIDATED` edge does not | -| Level consistency | Every node with `parent_node_id != NULL` has `level == parent.level + 1` | -| Append-only WAL | Mutation count for any given node only ever increases; no mutation rows are deleted or updated | -| Timeline consistency | `started_at ≤ completed_at` on every `RunTaskExecution` | - ---- - -## How the Test Harness Should Work - -Every test needs four ingredients: - -**1. Stub workers** — workers that deterministically succeed, fail, or spawn subtasks with no E2B or LLM dependency. The system already has `training-stub` and the smoke worker fixtures. The integration tier needs its own small named set: -- `StubSuccessWorker` — returns `WorkerOutput(success=True)` immediately -- `StubFailWorker` — raises a controlled exception -- `StubSubtaskWorker(plan)` — calls `plan_subtasks()` with a provided spec, then succeeds - -**2. A run submitter** — posts a `benchmark/run-request` event to the local Inngest dev server via the existing `inngest_client` and returns the `run_id`. - -**3. A run poller** — polls `RunRecord.status` directly against Postgres until terminal. Times out with a clear message if convergence doesn't happen within a threshold (suggested: 30s for stub-only runs). - -**4. Shared assertion helpers** — functions that take a `run_id` and assert the cross-cutting invariants above. These live in `tests/integration/helpers/` and are imported by every test, not copy-pasted. - -The pattern for every test is then: - -```python -run_id = submit_run(worker="stub-success", tasks=[...]) -await wait_for_terminal(run_id, timeout=30) - -assert_run_completed(run_id) # RunRecord -assert_all_nodes_terminal(run_id) # graph -assert_wal_complete(run_id) # mutations -assert_executions_have_outputs(run_id) # telemetry -# ...plus test-specific assertions -``` - ---- - ---- - -## Beyond the Nine Flows: Additional Bulletproofing Categories - -The nine control flows are about *runtime correctness* — does the system reach the right Postgres state after events settle? The categories below protect a different surface: *structural correctness* — does the codebase wire things up consistently in the first place? Some of these are best placed in the unit tier (no infra required), but they belong in the same TDD mandate. - ---- - -### 10. Event Pub→Sub Call Graph (static analysis — unit tier) - -**Problem:** A simple "every event name has a handler" check is one-directional. It catches orphaned event types (defined but nothing subscribes), but misses two equally dangerous failure modes: - -1. **Dead handlers** — a handler is registered in `ALL_FUNCTIONS` but nothing in the codebase ever emits the event it listens for. The handler is live but unreachable. -2. **Missing fan-out** — an event should trigger multiple handlers (Inngest supports N subscribers per event), but one is missing from `ALL_FUNCTIONS`. Publisher and one subscriber are fine; the second subscriber is silently absent. - -The fix is a bidirectional call graph: map every event to its expected publishers and expected subscribers, then assert both sides. - -**How to build the graph:** - -The subscriber side is trivially introspectable — `ALL_FUNCTIONS` is the source of truth: - -```python -from ergon_core.core.runtime.inngest_registry import ALL_FUNCTIONS - -def build_subscriber_map() -> dict[str, list[str]]: - """event_name → [handler_id, ...]""" - result: dict[str, list[str]] = {} - for fn in ALL_FUNCTIONS: - event = fn.trigger.event - result.setdefault(event, []).append(fn.id) - return result -``` - -The publisher side cannot be introspected automatically without AST analysis, so it is declared explicitly as a canonical fixture. This has a secondary benefit: the fixture **is the architecture document** for the event graph. Anyone reading it can trace the full flow. - -```python -# tests/unit/state/test_event_call_graph.py - -# Canonical pub→sub map. Each entry states: -# publishers: which Inngest function IDs emit this event (or "external" for CLI/API entrypoints) -# subscribers: which Inngest function IDs must handle it -EXPECTED_CALL_GRAPH: dict[str, dict[str, list[str]]] = { - "benchmark/run-request": { - "publishers": ["external:cli"], - "subscribers": ["benchmark-run-start"], - }, - "workflow/started": { - "publishers": ["benchmark-run-start"], - "subscribers": ["start-workflow"], - }, - "task/ready": { - # emitted by start-workflow (initial roots), by propagate-execution (after - # dep satisfied), and by execute-task (when plan_subtasks inserts new roots) - "publishers": ["start-workflow", "propagate-execution", "execute-task"], - "subscribers": ["execute-task"], - }, - "task/completed": { - "publishers": ["execute-task"], - "subscribers": ["propagate-execution"], - }, - "task/failed": { - "publishers": ["execute-task"], - "subscribers": ["propagate-execution"], - }, - "task/cancelled": { - "publishers": ["propagate-execution", "cancel-orphans"], - "subscribers": ["cancel-orphans"], - }, - "workflow/completed": { - "publishers": ["propagate-execution"], - "subscribers": ["complete-workflow"], - }, - "workflow/failed": { - "publishers": ["propagate-execution"], - "subscribers": ["fail-workflow"], - }, - "run/cancelled": { - "publishers": ["external:api"], - "subscribers": ["cancel-run"], - }, - "run/cleanup": { - "publishers": ["complete-workflow", "fail-workflow"], - "subscribers": ["cleanup-run"], - }, - # criterion/evaluate: publishers unknown, subscribers MISSING — this is the live bug - # this entry must be completed and a handler added before this test can pass - "criterion/evaluate": { - "publishers": [], # TODO: identify where this is emitted - "subscribers": [], # BUG: no handler registered in ALL_FUNCTIONS - }, -} -``` - -**Three assertions from one fixture:** - -```python -def test_every_declared_subscriber_is_registered(): - """All expected subscribers exist in ALL_FUNCTIONS.""" - registered = {fn.id for fn in ALL_FUNCTIONS} - for event, graph in EXPECTED_CALL_GRAPH.items(): - for expected_sub in graph["subscribers"]: - assert expected_sub in registered, ( - f"Event '{event}' expects handler '{expected_sub}' " - f"but it is not in ALL_FUNCTIONS" - ) - -def test_no_registered_handler_is_absent_from_call_graph(): - """Every handler in ALL_FUNCTIONS appears as a subscriber somewhere in the graph. - A handler that appears in no event's subscriber list is unreachable dead code.""" - declared_subs = {s for g in EXPECTED_CALL_GRAPH.values() for s in g["subscribers"]} - for fn in ALL_FUNCTIONS: - assert fn.id in declared_subs, ( - f"Handler '{fn.id}' is registered in ALL_FUNCTIONS but does not appear " - f"in EXPECTED_CALL_GRAPH — either add it or remove the handler" - ) - -def test_every_declared_event_type_has_a_model_class(): - """Every event slug in the call graph has a corresponding Pydantic event model. - Catches typos in the fixture and models that were deleted without updating the graph.""" - from ergon_core.core.runtime.events import all_event_models # hypothetical collector - known_slugs = {m.model_fields["name"].default for m in all_event_models()} - for event in EXPECTED_CALL_GRAPH: - assert event in known_slugs, ( - f"Event '{event}' in call graph has no corresponding event model class" - ) -``` - -**What this catches that the original one-liner missed:** - -| Failure mode | Simple slug check | Call graph | -|---|---|---| -| Event defined, no handler | ✅ | ✅ | -| Handler registered, nothing emits to it | ❌ | ✅ | -| Event should fan out to N handlers, only N-1 registered | ❌ | ✅ | -| Event slug in graph but no Pydantic model class | ❌ | ✅ | -| Handler slug typo in `ALL_FUNCTIONS` | ❌ | ✅ | - -**Concrete live example — `criterion/evaluate`:** The fixture above documents the bug explicitly. The test `test_every_declared_subscriber_is_registered` will fail on the `criterion/evaluate` entry the moment a subscriber is declared in the fixture but absent from `ALL_FUNCTIONS`. Until the fixture entry has a non-empty `subscribers` list AND a matching registered handler, the bug is recorded but the test doesn't crash the suite — which is the right behaviour during a fix. Add the handler, update the fixture, the test goes green. - ---- - -### 11. Inngest Function Catalog Integrity (static analysis — unit tier) - -**Problem:** `ALL_FUNCTIONS` could have duplicate slugs (two functions with the same name) or functions with no trigger at all. Inngest silently accepts duplicate registrations and last-write-wins, meaning one handler silently shadows another. - -**What to test:** - -```python -def test_no_duplicate_inngest_slugs(): - slugs = [fn.id for fn in inngest_registry.ALL_FUNCTIONS] - assert len(slugs) == len(set(slugs)), f"Duplicate slugs: {[s for s in slugs if slugs.count(s) > 1]}" - -def test_all_inngest_functions_have_event_trigger(): - for fn in inngest_registry.ALL_FUNCTIONS: - assert hasattr(fn.trigger, "event"), f"{fn.id} has no event trigger" -``` - ---- - -### 12. Registry Integrity (static analysis — unit tier) - -**Problem:** A slug can be registered in `BENCHMARKS`, `WORKERS`, `EVALUATORS`, or `CRITERIA` that points to a class that cannot be instantiated — wrong base class, missing required class vars, or import error at collection time. - -**What to test:** For each registry (`CORE_BENCHMARKS`, `WORKERS`, etc.), assert that every value is a class, that it is a subclass of the correct ABC, and that it can be constructed with minimal arguments (or at least that `__init__` doesn't immediately raise on a no-arg call where no args are required). The `onboarding_deps` benchmark contract in `test_benchmark_contract.py` is a partial model for this; extend the pattern to all registries. - -```python -@pytest.mark.parametrize("slug,cls", list(CORE_BENCHMARKS.items())) -def test_benchmark_registry_entries_are_valid_subclasses(slug, cls): - assert issubclass(cls, Benchmark), f"{slug} is not a Benchmark subclass" - assert hasattr(cls, "type_slug"), f"{slug} missing type_slug" -``` - ---- - -### 13. Event Payload Round-Trips (static analysis — unit tier) - -**Problem:** All Inngest events are Pydantic models. A broken `model_dump()` / `model_validate()` cycle (e.g. a UUID field that serialises to a non-string in one environment) means events cannot be deserialized by the Inngest dev server or the handler. - -**What to test:** For every concrete event class defined in `task_events.py`, `evaluation_events.py`, and `infrastructure_events.py`, construct a minimal valid instance, round-trip it through `model_dump()` + `model_validate()`, and assert equality. Use `mode="json"` on `model_dump` to catch UUID/datetime serialisation issues that only surface over the wire. - -```python -@pytest.mark.parametrize("event_cls,kwargs", [ - (TaskReadyEvent, {"run_id": uuid4(), "node_id": uuid4(), ...}), - ... -]) -def test_event_round_trips(event_cls, kwargs): - original = event_cls(**kwargs) - payload = original.model_dump(mode="json") - restored = event_cls.model_validate(payload) - assert original == restored -``` - ---- - -### 14. Toolkit Tool Name Uniqueness (static analysis — unit tier) - -**Problem:** The Inngest worker toolkit is assembled by combining tool lists from different providers. If two providers define a tool with the same `__name__`, the second silently shadows the first at the model-context level — the LLM sees duplicate names and the worker may call the wrong function. - -**What to test:** For each registered worker class, instantiate its toolkit (with a mock sandbox/runtime) and assert that all tool `__name__` values within that toolkit are unique. - -```python -def test_swebench_toolkit_tool_names_are_unique(): - tools = build_swebench_toolkit(sandbox=MockSandbox(), runtime=MockRuntime()) - names = [t.__name__ for t in tools] - assert len(names) == len(set(names)), f"Duplicate tool names: {set(n for n in names if names.count(n) > 1)}" -``` - ---- - -### 15. Worker / Benchmark / Evaluator ABC Compliance (static analysis — unit tier) - -**Problem:** Every concrete worker, benchmark, and evaluator must implement specific abstract methods. Python's ABC machinery only raises at *instantiation time*, not at import/collection time. A class that forgets to implement `execute()` passes all static checks until someone tries to run it. - -**What to test:** Attempt instantiation of every registered concrete class (with minimal stub arguments) and assert no `TypeError` is raised for missing abstract methods. This goes one step beyond the subclass check in category 12. - -```python -@pytest.mark.parametrize("slug,cls", list(WORKERS.items())) -def test_worker_is_fully_concrete(slug, cls): - # Should not raise TypeError: Can't instantiate abstract class - try: - instance = cls.__new__(cls) - except TypeError as e: - pytest.fail(f"Worker '{slug}' is not fully concrete: {e}") -``` - ---- - -### 16. DB Schema Reflection (Postgres only — integration tier, no Inngest) - -**Problem:** SQLModel generates table DDL from Python models. If a migration is applied in the wrong order or skipped, the live Postgres schema diverges from the models. This silently breaks writes to new columns or reads of renamed columns. - -**What to test:** Connect to the Postgres instance, reflect every table that `SQLModel.metadata` knows about, and assert that every column on every model exists in the reflected schema with the correct type and nullability. This test needs Postgres but not Inngest, so it should be gated separately from the event-flow tests. - -```python -def test_db_schema_matches_models(postgres_engine): - from sqlalchemy import inspect - inspector = inspect(postgres_engine) - for table_name, table in SQLModel.metadata.tables.items(): - reflected_cols = {c["name"] for c in inspector.get_columns(table_name)} - model_cols = {c.name for c in table.columns} - assert model_cols <= reflected_cols, ( - f"Table '{table_name}' missing columns: {model_cols - reflected_cols}" - ) -``` - ---- - -### 17. Sandbox Container Builds (Docker — slow tier) - -**Problem:** Each benchmark sandbox is built from a `Dockerfile` in the repo. A broken base image pin, a removed system package, or a pip install that fails silently produces a container that appears to build but crashes at runtime. This is only caught when someone actually tries to run a benchmark. - -**Two Dockerfiles to test:** -- `ergon_builtins/benchmarks/minif2f/Dockerfile` (Lean4 toolchain) -- `ergon_builtins/benchmarks/swebench_verified/Dockerfile` (Python dev environment) - -**What to test:** A `docker build` that must exit zero. These are slow (minutes each) and should be gated behind a `pytest.mark.docker_build` marker so they run in CI nightly but not on every PR push. A basic smoke: build succeeds, `docker run --rm <image> echo ok` exits zero. - -```python -@pytest.mark.docker_build -@pytest.mark.parametrize("dockerfile_path,context_dir", [ - ("ergon_builtins/benchmarks/minif2f/Dockerfile", "ergon_builtins/benchmarks/minif2f/"), - ("ergon_builtins/benchmarks/swebench_verified/Dockerfile", "ergon_builtins/benchmarks/swebench_verified/"), -]) -def test_sandbox_container_builds(dockerfile_path, context_dir, tmp_path): - result = subprocess.run( - ["docker", "build", "-f", dockerfile_path, "-t", f"ergon-test-{tmp_path.name}", context_dir], - capture_output=True, text=True, timeout=600, - ) - assert result.returncode == 0, f"Docker build failed:\n{result.stderr}" -``` - ---- - -### 18. Concurrency Race: `only_if_not_terminal` Under Concurrent Cancellation (full-stack integration) - -**Problem:** The `only_if_not_terminal` guard is the single mechanism preventing double-writes on a terminal node. If two `task/cancelled` events race to update the same node (which happens in diamond DAG failure cascades where multiple parents can independently trigger cancellation), one write must win and the other must be silently discarded. If the guard has a bug, the WAL gets a spurious second mutation and downstream assertions about node count and status are corrupted. - -This is the hardest test to write and the one most likely to catch real bugs. It requires the full Inngest + Postgres stack. - -**What to test:** Construct a DAG where a single leaf node has two parents, and both parents fail at approximately the same time. The leaf should receive two independent `task/cancelled` events nearly simultaneously. Assert: - -- The leaf has exactly one `CANCELLED` `RunGraphNode` row -- The WAL for the leaf has exactly one `CANCELLED` mutation (no duplicate entries) -- The `RunRecord` reaches `FAILED` (not stuck) -- The `RunTaskExecution` for the leaf (if any was started) has a single terminal row - -This test should be marked `pytest.mark.slow` and `pytest.mark.flaky_risk` with a note that it is exercising a race condition — it should be run with `--count=5` (using `pytest-repeat`) in CI to increase the chance of hitting the race. - ---- - -## Summary - -The current `tests/integration/` has one real integration test that checks HTTP response codes on a narrow happy path. The nine critical control flows above — propagation, failure cascade, subtask spawning, cancellation, restart, communication — have never had their Postgres state asserted at any tier. - -The Postgres state after any interesting event sequence is the system's ground truth. That is what the integration tier should be testing. - -The nine additional categories above protect structural correctness: events that are defined but never handled, duplicate tool names, schema drift, and concurrency races. Several of these (categories 10–15) are pure static analysis and belong in the unit tier — they require no running infrastructure and can fail fast in CI. Category 16 needs Postgres but not Inngest. Category 17 needs Docker and should run nightly. Category 18 needs the full stack and should run on every feature branch. - -**Priority order for implementation:** - -| Priority | Category | Tier | Value | -|----------|----------|------|-------| -| 1 | Fix 3 uncollected tests (`testresolve_*`) | Integration | Critical bug — tests have never run | -| 2 | Event subscriber coverage | Unit | Catches live `criterion/evaluate` orphan | -| 3 | Nine control flows | Integration | Core runtime correctness, zero coverage today | -| 4 | Inngest catalog + registry integrity | Unit | Cheap, high signal | -| 5 | DB schema reflection | Integration | Catches migration drift | -| 6 | Event payload round-trips | Unit | Catches serialisation bugs before they hit wire | -| 7 | Toolkit tool name uniqueness | Unit | Prevents silent shadowing | -| 8 | ABC compliance | Unit | Catches broken registrations at test time | -| 9 | Concurrency race | Full-stack | Validates the system's critical idempotency guard | -| 10 | Container builds | Docker/nightly | Catches broken sandbox images before E2E | - ---- - -## Found Violated Assumptions - -Issues discovered by reading the actual production code during this session. Each entry states what we assumed, what the code actually does, the concrete fix needed, and the integration test assertion that would have caught it. - ---- - -### A. `run/cleanup` terminates one sandbox per run, not one per task execution - -**Assumed:** Container teardown happens per task-execution-attempt; `run/cleanup` kills all of them at run end. - -**Reality:** `run_cleanup.py:69` reads `run.parsed_summary().get("sandbox_id")` — a single string stored at run level. Only one sandbox ID is ever killed. If multiple tasks executed in parallel each had their own sandbox, all but the last one stored in the summary are silently leaked. - -**Fix:** Store sandbox IDs on `RunTaskExecution` rows (or in a separate table), not in `RunRecord.summary_json`. `run/cleanup` should collect all sandbox IDs across all `RunTaskExecution` rows for the run and terminate each one. - -**Integration test assertion:** After `run/cleanup` fires, query all `RunTaskExecution.sandbox_id` values for the run. For each non-stub ID, assert that `BaseSandboxManager.get(sandbox_id)` returns a terminated/not-found state. - ---- - -### B. Stub sandbox logic (`is_stub_sandbox_id`) leaks into four production files - -**Assumed:** Test infrastructure is confined to test code; the production path does not branch on whether a sandbox is "real" or a test stub. - -**Reality:** `is_stub_sandbox_id` and `StubSandboxManager` are defined in `manager.py` and imported into three production Inngest handlers: - -| File | Line | Usage | -|------|------|-------| -| `ergon_core/ergon_core/core/providers/sandbox/manager.py` | 578–613 | Definition of `_STUB_SANDBOX_PREFIX`, `is_stub_sandbox_id()`, `StubSandboxManager` | -| `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` | 72 | Guards sandbox termination | -| `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` | 96 | Guards evaluation dispatch | -| `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` | 102–116 | Creates stub sandbox ID for stub-worker path | - -**Fix:** Remove the stub concept from production code entirely. Tests should inject a real test-double at the `BaseSandboxManager` boundary (e.g. a subclass that returns a deterministic ID and no-ops on terminate) rather than relying on a magic string prefix to short-circuit production logic. - -**Static analysis test assertion:** `is_stub_sandbox_id` must not be imported in any module outside of `tests/`. This can be a simple grep-based unit test. - ---- - -### C. `evaluate_task_run` constructs `BenchmarkTask` with empty strings - -**Assumed:** Evaluation runs against the actual task data (description, instance key, task slug) fetched from the database. - -**Reality:** `evaluate_task_run.py:99–103` constructs `BenchmarkTask` with hardcoded empty strings: - -```python -task = BenchmarkTask( - task_slug="", - instance_key="", - description="", -) -``` - -The payload contains `task_id` and `execution_id` — enough to fetch the real data — but it is never queried. Criteria that inspect the task description or instance key for evaluation logic receive empty strings. - -**Fix:** Fetch the `RunGraphNode` (or definition task) using `payload.task_id` before constructing `BenchmarkTask`. Populate `task_slug`, `instance_key`, and `description` from the DB row. - -**Integration test assertion:** After evaluation completes, assert that `RunTaskEvaluation.summary_json` contains a non-empty `criterion_description` that matches the actual task description. - ---- - -### D. `criterion_description`, `feedback`, and `evaluation_input` use empty-string defaults instead of nullable - -**Assumed:** Missing evaluation fields are represented as `None`, not as empty strings. - -**Reality:** `CriterionResultEntry` (and the `_build_evaluation_summary` call at `evaluate_task_run.py:114`) uses `feedback=cr.feedback or ""` and `evaluation_input=evaluation_input` (passed as `""`). These are suppressed by the `# slopcop: ignore[no-str-empty-default]` comments rather than fixed. - -**Fix:** Change the fields to `str | None = None`. Update all callers to pass `None` instead of `""`. Remove the `slopcop: ignore` suppressions. - -**Unit test assertion:** Construct a `CriterionResultEntry` with no feedback provided and assert `entry.feedback is None`, not `entry.feedback == ""`. - ---- - -### E. `InngestCriterionExecutor` sandbox connection is unverified — may spin up its own container - -**Assumed:** The evaluator reuses the still-standing task sandbox (the one that was kept alive specifically for rubric evaluation) by connecting to it via `payload.sandbox_id`. - -**Concern:** `evaluate_task_run.py:82–90` creates `sandbox_manager = manager_cls()` and passes `payload.sandbox_id` into `TaskEvaluationContext`. Whether `InngestCriterionExecutor` actually connects to the existing sandbox via that ID — or ignores it and provisions a new one — is not verified by any test. - -**File to investigate:** `ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py` - -**Fix (pending investigation):** If `InngestCriterionExecutor` does not use `task_context.sandbox_id` to reconnect to the existing sandbox, it must be fixed to do so. Spinning up a second sandbox per evaluation doubles cost and breaks criteria that depend on inspecting the state the agent left behind in the original sandbox. - -**Integration test assertion:** Run a task that leaves a known artefact in the sandbox, then run evaluation. Assert the criterion received access to that artefact (i.e. it executed against the same sandbox, not a fresh one). - ---- - -### F. `restart_task` does not cancel stale subtask children from the prior execution - -**Assumed:** Restarting a task invalidates all outputs from its prior execution, including subtask children it previously spawned via `plan_subtasks`. - -**Reality:** `task_management_service.py::_invalidate_downstream` traverses outgoing `RunGraphEdge` only (the dependency DAG). It does not traverse the containment tree (`parent_node_id`). Subtask children from the prior execution are not touched. - -**Consequence:** A restarted task that previously called `plan_subtasks` leaves its old children completed in the graph. When the task re-executes and calls `plan_subtasks` again, duplicate child nodes are inserted — one set from the old execution (status `COMPLETED`) and one from the new (status `PENDING`). The graph is now corrupt: two generations of children coexist under the same parent. - -**Fix:** Before re-dispatching `task/ready` in `restart_task`, cancel all non-terminal descendants of the node (traversing `parent_node_id`) and reset any already-`COMPLETED` children to `CANCELLED` so the new execution starts from a clean subtask slate. This is the containment-axis equivalent of `_invalidate_downstream`. - -**Integration test assertion (part of test 8 — interrupt and restart):** After restarting a task that previously spawned two subtasks, assert that those old subtask nodes have status `CANCELLED`. Assert that after the restarted task completes and spawns subtasks again, there are exactly two nodes with `parent_node_id == restarted_node.id` and both are in a non-stale terminal state — not four. - ---- - -### G. `criterion/evaluate` event is defined but has no handler; `task/evaluate` is the actual evaluation trigger - -**Assumed (from earlier in this audit):** `criterion/evaluate` / `CriterionEvaluationEvent` is the event that drives per-criterion evaluation and is simply missing its handler. - -**Reality:** The actual evaluation handler (`evaluate-task-run`) is triggered by `task/evaluate`, not `criterion/evaluate`. `CriterionEvaluationEvent` in `evaluation_events.py` is a separate, orphaned event type with no handler anywhere in `ALL_FUNCTIONS`. It is unclear whether it is intended future infrastructure, dead code, or a vestige of a prior design. - -**Fix:** Either wire a handler to `criterion/evaluate` if per-criterion fan-out is the intended design, or delete `CriterionEvaluationEvent` from `evaluation_events.py`. The call graph fixture in category 10 should be updated to reflect whichever path is chosen. - -**Static analysis test assertion:** `criterion/evaluate` must either appear in `ALL_FUNCTIONS` as a trigger or must not exist as an event model class. The current state — defined but unhandled — must not pass the call graph test. - ---- - -### H. Failed-attempt sandbox is not killed before the restart container starts; prior container leaks - -**Assumed:** When a task is restarted, its failed attempt's sandbox is cleaned up before the new attempt begins. - -**Reality:** `run_cleanup.py:69` reads `run.parsed_summary().get("sandbox_id")` — a single string stored at run level. Only one sandbox ID is ever killed. If multiple tasks executed in parallel each had their own sandbox, all but the last one stored in the summary are silently leaked. - -This compounds on restart: every `task/ready` event provisions a fresh container (confirmed in `execute_task.py:140`). When a task fails and is restarted, the failed attempt's container is not killed at `task/failed` time — it sits live until `run/cleanup`. At that point `run/cleanup` kills whichever sandbox ID was last written to the summary, and the prior attempt's container leaks permanently. - -**Fix:** Store sandbox IDs on `RunTaskExecution` rows, not in `RunRecord.summary_json`. `run/cleanup` should collect all sandbox IDs across all `RunTaskExecution` rows for the run and terminate each one. Additionally, kill the failed attempt's sandbox immediately when `task/failed` fires — don't defer to run-level cleanup. - -**Integration test assertion:** After `restart_task` is called on a failed node, assert that the sandbox_id from the failed `RunTaskExecution` is no longer live. After `run/cleanup` fires, assert that every `RunTaskExecution.sandbox_id` for the run is terminated. - ---- - -### I. A completed run with evaluators configured may have zero `RunTaskEvaluation` rows with no published signal - -**Assumed:** If evaluation criteria are configured for a run, a completed run will always have a corresponding `RunTaskEvaluation` row per task per evaluator. If evaluation fails or is skipped, something is published to make this observable. - -**Reality:** If `check_evaluators` fires but dispatches no `task/evaluate` events (due to the `criterion/evaluate` orphan, a registry miss, or any silent failure in the dispatch path), the run reaches `RunRecord.status == COMPLETED` with zero `RunTaskEvaluation` rows. No event is emitted, no error is logged at run level, and the training loop receives a completed run with no scores. The absence of evaluation is indistinguishable from a run where evaluation genuinely returned a zero score. - -**Fix:** At `workflow/completed` time (or in `check_evaluators`), assert that the number of `RunTaskEvaluation` rows matches the expected count derived from the experiment definition (tasks × evaluator bindings). If the count is wrong, emit a distinct event or set a flag on `RunRecord` indicating evaluation was incomplete. Do not allow `RunRecord.status == COMPLETED` to coexist with missing evaluations silently. - -**Integration test assertion:** For any run where the experiment definition includes at least one evaluator binding, assert after terminal state that `COUNT(RunTaskEvaluation WHERE run_id = X) == expected_evaluation_count`. Assert that `RunRecord.summary_json` contains a non-null, non-empty scores field. A run that completes with zero evaluations must either have no evaluators configured or must have an explicit `evaluation_skipped` marker — never silent absence. - ---- - -### J. When a parent task fails, non-terminal containment children must become FAILED, not CANCELLED, recursively through all sublayers - -**Assumed:** When a parent task fails, its non-terminal containment children (all nodes reachable via `parent_node_id`, recursively) are marked `CANCELLED`. - -**Correct semantics (per domain model):** `CANCELLED` means "stopped intentionally before it got a chance to run" — an operator decision or a deliberate dependency invalidation. When a parent's execution context collapses (the container breaks, the worker raises), children whose work is now unreachable are not cancelled — they failed because their execution environment no longer exists. The correct status is `FAILED` with cause `parent_failed`. - -This distinction matters for operators: `CANCELLED` subtasks are treated as expected cleanup and get little scrutiny. `FAILED` subtasks surface in failure dashboards and signal "the scope of this failure was N nodes deep." - -**Scope:** Only non-terminal nodes are affected. Already-`COMPLETED` nodes represent genuinely finished work and must not be overwritten. Already-`FAILED` or `CANCELLED` nodes are left as-is. - -**Guard:** A node with `parent_node_id != NULL` should not be individually restartable — it can only be reset as part of a parent restart. Enforcing this prevents operators from retrying individual subtasks before their parent context is restored. - -**Reality (current code):** The current cascade code marks containment children `CANCELLED` with cause `parent_terminal`. This is the wrong status and the wrong cause string. - -**Fix:** Change the containment cascade to emit `FAILED` with cause `parent_failed`. Separate this from the existing `dep_invalidated` cascade (which operates on `RunGraphEdge` and correctly uses `CANCELLED`). - -**Integration test assertion:** See test 7 above. Assert that after a parent fails, every non-terminal containment descendant has status `FAILED` and no node in the subtree has status `CANCELLED` as a result of the parent's failure. - ---- - -### K. `restart_task` must reset the full containment subtree recursively, including COMPLETED children - -**Assumed:** Restarting a task only resets the task itself and invalidates its DAG-level dependency successors. - -**Correct semantics (per domain model):** A restart is a full reset of the node's entire containment subtree — every node reachable via `parent_node_id` recursively, including `COMPLETED` ones. The new execution starts in a fresh container; outputs from prior-attempt subtasks were written into the old container's context and cannot be assumed valid. Preserving COMPLETED subtask children from a prior run creates a mixed-generation problem where some outputs are from attempt N and some from attempt N+1. - -The restart should: -1. Reset every containment descendant to `PENDING` regardless of its current status -2. Dispatch `task/ready` for the subtree roots only (nodes with no in-edges within the subtask group) -3. Let normal propagation drive the rest as roots complete -4. Recurse: grandchildren and deeper sublayers reset alongside direct children - -**Reality (current code):** `_invalidate_downstream` in `task_management_service.py` traverses outgoing `RunGraphEdge` only (the dependency DAG). It does not traverse the containment tree (`parent_node_id`). Children from prior executions are left in their old states entirely — see issue F above. - -**Fix:** Add a containment-subtree reset pass to `restart_task` that runs before re-dispatching `task/ready`. Walk all nodes with `parent_node_id == node_id` recursively, reset each to `PENDING`, and dispatch `task/ready` for roots. This is distinct from and in addition to `_invalidate_downstream` (which handles DAG-level successors separately). - -**Integration test assertion:** See test 8 above. After `restart_task`, assert no `RunGraphNode` with `parent_node_id == restarted_node.id` (or deeper) retains `COMPLETED` or `FAILED` from the prior run. - -**Design clarification — `plan_subtasks` on a restarted execution:** Resetting prior children to `PENDING` (rather than `CANCELLED`) creates a conflict: the new parent execution will call `plan_subtasks` again, producing a second generation of child nodes alongside the reset first generation. The correct design is to **cancel** all prior containment children as part of `restart_task` (not reset to `PENDING`), then let the new execution call `plan_subtasks` fresh to create new nodes. This is cleaner: the worker's new decomposition may differ from the prior attempt's (especially after `refine_task`), and reusing old node rows assumes the plan is identical across attempts. The full-reset described above applies to the containment subtree reset at restart time; `plan_subtasks` in the new execution then builds the next generation from scratch. - ---- - -### L. `RunGraphMutation` has no causal lineage field — cascades are undebuggable - -**Assumed:** The WAL records enough information to reconstruct *why* any given state transition happened, including which upstream event triggered it. - -**Reality:** `RunGraphMutation` records `actor` and `reason` for each individual mutation, but has no pointer to the mutation that caused it. A cascade that sets 15 nodes to `FAILED` produces 15 independent WAL entries with the same `reason="parent_failed"` string. There is no way to query "what single event caused all of these?" or "what triggered this specific restart?" The WAL shows *what* happened but not *why* in a traceable, machine-queryable form. - -**Proposed fix — self-referential `triggered_by_mutation_id`:** - -Add `triggered_by_mutation_id: UUID | None` to `RunGraphMutation`. Root-level mutations (operator actions, external events) have `NULL`. Every mutation produced by a cascade points to the mutation that triggered it. The result is a causal tree: - -``` -mutation 1: node A → FAILED triggered_by=NULL reason="worker_error" -mutation 2: node B → FAILED triggered_by=1 reason="parent_failed" -mutation 3: node C → FAILED triggered_by=1 reason="parent_failed" -mutation 4: node D → FAILED triggered_by=2 reason="parent_failed" ← grandchild -mutation 5: node A → PENDING triggered_by=NULL reason="operator_restart" -mutation 6: node B → PENDING triggered_by=5 reason="parent_restart" -mutation 7: node C → PENDING triggered_by=5 reason="parent_restart" -mutation 8: node D → PENDING triggered_by=6 reason="parent_restart" -``` - -This enables: -- **Root cause query:** "Why is node D FAILED?" → mutation 4 → triggered by 2 → triggered by 1 → worker error on A. -- **Blast radius query:** "What did restarting A cause?" → all mutations with `triggered_by` in the transitive closure of mutation 5. -- **Orphan detection:** Any cascade mutation with `triggered_by=NULL` is a bug — something set a node's status without a traceable cause. - -**Integration test assertion (cross-cutting invariant):** Add to the shared assertion helpers: for any run, every `RunGraphMutation` whose `reason` is in `{parent_failed, parent_restart, dep_invalidated, downstream_invalidation}` must have a non-null `triggered_by_mutation_id`. A cascade mutation with no `triggered_by` is a test failure. - ---- - -### H. Failed-attempt sandbox is not killed before the restart container starts; prior container leaks - -**Assumed:** When a task is restarted, its failed attempt's sandbox is cleaned up before the new attempt begins. - -**Reality:** Every `task/ready` event goes through `sandbox-setup` (confirmed in `execute_task.py:140`), which provisions a fresh container. But when a task fails, `TaskFailedEvent` carries the sandbox_id and nothing kills that container at that point — teardown only happens at `run/cleanup`, which fires at run end. So between `restart_task` firing and the run eventually ending, two live containers exist for the same task simultaneously: the one from the failed attempt and the one from the current attempt. - -This compounds issue A: `run/cleanup` kills whichever sandbox_id was last written to `RunRecord.summary_json`, so the prior-attempt container leaks permanently. - -**Fix:** When `task/failed` fires (or when `restart_task` is called), kill the failed execution's sandbox immediately rather than waiting for run-level cleanup. The `execution_id` is available in both events and uniquely identifies which sandbox to terminate. - -**Integration test assertion:** After `restart_task` is called on a failed node, assert that the sandbox_id from the failed `RunTaskExecution` is no longer live. Assert that only one sandbox (from the new attempt) is live while the restart is in progress. - ---- - -### I. A completed run with evaluators configured may have zero `RunTaskEvaluation` rows with no published signal - -**Assumed:** If evaluation criteria are configured for a run, a completed run will always have a corresponding `RunTaskEvaluation` row per task per evaluator. If evaluation fails or is skipped, something is published to make this observable. - -**Reality:** If `check_evaluators` fires but dispatches no `task/evaluate` events (due to the `criterion/evaluate` orphan, a registry miss, or any silent failure in the dispatch path), the run reaches `RunRecord.status == COMPLETED` with zero `RunTaskEvaluation` rows. No event is emitted, no error is logged at run level, and the training loop receives a completed run with no scores. The absence of evaluation is indistinguishable from a run where evaluation genuinely returned a zero score. - -**Fix:** At `workflow/completed` time (or in `check_evaluators`), assert that the number of `RunTaskEvaluation` rows matches the expected count derived from the experiment definition (tasks × evaluator bindings). If the count is wrong, emit a distinct event or set a flag on `RunRecord` indicating evaluation was incomplete. Do not allow `RunRecord.status == COMPLETED` to coexist with missing evaluations silently. - -**Integration test assertion:** For any run where the experiment definition includes at least one evaluator binding, assert after terminal state that `COUNT(RunTaskEvaluation WHERE run_id = X) == expected_evaluation_count`. Assert that `RunRecord.summary_json` contains a non-null, non-empty scores field. A run that completes with zero evaluations must either have no evaluators configured or must have an explicit `evaluation_skipped` marker — never silent absence. diff --git a/docs/karnas-experiment-log-2026-04-25.md b/docs/karnas-experiment-log-2026-04-25.md deleted file mode 100644 index e96385abd..000000000 --- a/docs/karnas-experiment-log-2026-04-25.md +++ /dev/null @@ -1,304 +0,0 @@ -# Karnas ResearchRubrics Rollout Experiment Log - -Date: 2026-04-25 -Branch: `experiments/karnas-researchrubrics-rollouts-main-20260425` -Base: `origin/main` at `4bb7be5` - -## Purpose - -Use the real-LLM ResearchRubrics rollout harness as a manual research/debug loop: -generate rollouts, read the persisted Postgres/dashboard state, debug Ergon core, -and extend the research agent action space where the evidence shows a real gap. - -## Log - -### 12:43 UTC+1 - Fresh branch setup - -- Confirmed `origin/main` and local `main` both point to `4bb7be5`. -- Created `experiments/karnas-researchrubrics-rollouts-main-20260425` from fresh `main`. -- Abandoned the earlier stale-base experiment branch for this workflow. -- Left pre-existing untracked `ergon_paper_overleaf_edit/` untouched. -- Created this log as the tracked record of changes, rollout results, and decisions. - -### 12:45 UTC+1 - Stack and harness preflight - -- Started the full Docker Compose stack with `TEST_HARNESS_SECRET=real-llm-secret docker compose up -d --wait`. -- Compose reported Postgres, API, Inngest, and dashboard as healthy. -- Host reachability: - - Dashboard is reachable at `http://127.0.0.1:3001/`. - - Inngest is reachable at `http://127.0.0.1:8289/`. - - API root returns `404`, which is expected for a FastAPI app with no `/` route once startup is complete. -- Credential preflight: - - `.env` contains `OPENAI_API_KEY`, `EXA_API_KEY`, `E2B_API_KEY`, and `HF_API_KEY`. - - `.env` does not contain `OPENROUTER_API_KEY` or `OPEN_ROUTER_API_KEY`. -- First canary attempt: - - Command: `ERGON_REAL_LLM=1 ERGON_DASHBOARD_URL=http://127.0.0.1:3001 ERGON_DATABASE_URL=postgresql://ergon:ergon_dev@127.0.0.1:5433/ergon uv run pytest tests/real_llm/benchmarks/test_smoke_stub.py -v -rs --assume-stack-up` - - Result: skipped with `OPENROUTER_API_KEY not set`. - - Root cause: `tests/real_llm/fixtures/openrouter_budget.py` autoused the OpenRouter budget gate for every real-LLM test, including the zero-cost stub canary and non-OpenRouter model runs. -- Harness fixes applied: - - Made `openrouter_budget` yield `None` when OpenRouter is not configured instead of skipping the whole tier. - - Made the budget gate a no-op when no OpenRouter budget is available. - - Changed Playwright's default real-LLM dashboard base URL from `3101` to Compose's actual `3001`. - - Exported `OPENAI_API_KEY` from `settings.openai_api_key` the same way the code already exported `OPENROUTER_API_KEY`. - - Made `test_researchrubrics_rollout` require provider-specific keys based on `ERGON_REAL_LLM_MODEL`, so this machine can run `openai:*` rollouts without OpenRouter. - -### 12:49 UTC+1 - OpenRouter key added and services recreated - -- User added `OPENROUTER_API_KEY` to `.env`. -- Verified through `uv run python` that settings now sees OpenRouter, OpenAI, Exa, E2B, and HF keys. -- Recreated API, dashboard, and Inngest containers with `TEST_HARNESS_SECRET=real-llm-secret docker compose up -d --wait --force-recreate api dashboard inngest-dev` so the API/Inngest runtime process receives the updated env. -- Normalized `.env.example` to include `OPENROUTER_API_KEY=` without a trailing space. - -### 12:54 UTC+1 - Canary hang root cause - -- Reran the real-LLM stub canary after installing Python Playwright and Chromium. -- Result: pytest hung for more than 180s before creating any `RunRecord`. -- Evidence: - - Postgres query showed `runs 0`. - - Process tree showed no `ergon benchmark` subprocess. - - Process tree did show Playwright driver + Chromium children under pytest. -- Root cause: the canary was wedged before the test body, inside the Python Playwright fixture path. This is a harness robustness bug: dashboard capture is useful rollout signal, but it must not prevent rollout generation or DB capture. -- Fix applied: - - Added bounded Playwright launch/context timeouts. - - Made Python Playwright fixtures yield `None` when unavailable or wedged. - - Made `capture_dashboard()` return an empty screenshot map when Playwright is unavailable. - - Made the stub canary skip only its dashboard-content assertion when Playwright is unavailable, while still exercising CLI -> Postgres -> harness polling. - -### 12:58 UTC+1 - Canary stale smoke slug - -- Canary reached its CLI subprocess after the Playwright timeout fix. -- Result: CLI failed before dispatch with `KeyError: 'smoke-test'`. -- Root cause: the old `smoke-test` benchmark was retired in the canonical-smoke refactor; deterministic smoke now works by importing `tests.e2e._fixtures` with `ENABLE_TEST_HARNESS=1`, which replaces real benchmark loaders with test-owned benchmark roots. -- Fix applied: - - `ergon_cli.composition.build_experiment()` now imports `tests.e2e._fixtures` only when `ENABLE_TEST_HARNESS=1`. - - Retargeted the real-LLM canary to `researchrubrics --worker researchrubrics-smoke-worker --evaluator researchrubrics-smoke-criterion --model stub:constant`. -- Verification: - - Command: `ERGON_REAL_LLM=1 ERGON_DATABASE_URL=postgresql://ergon:ergon_dev@127.0.0.1:5433/ergon uv run pytest tests/real_llm/benchmarks/test_smoke_stub.py -v -rs --assume-stack-up --timeout=240` - - Result: passed in 25.96s. - -### 13:02 UTC+1 - Research target - -- Read the paper framing around the ResearchRubrics experiment. -- Target for rollouts: preserve enough run structure to recover worker-level role specialisation or duplication that a scalar rubric score would discard. -- Relevant paper claim: long-horizon web research rollouts scored by scalar rubrics can be re-analysed as per-worker streams, measuring whether spawned subagents specialised or duplicated each other's work. - -### 13:06 UTC+1 - Real ResearchRubrics CLI pre-dispatch failure - -- First real rollout harness attempt failed before creating a `RunRecord`. -- Evidence: - - Harness failed at `_latest_run_id_since(started_at)` because no run existed after CLI invocation. - - Direct CLI reproduction failed with `TypeError: ResearchRubricsRubric.__init__() missing 1 required keyword-only argument: 'rubric_criteria'`. -- Root cause: - - The public evaluator contract instantiates evaluators as `evaluator_cls(name=...)` and then calls `criteria_for(task)`. - - `ResearchRubricsRubric` instead required task-specific dataset criteria at construction time, so both CLI composition and runtime evaluator reconstruction could not instantiate it. -- Fix applied: - - Made `ResearchRubricsRubric` constructible with only `name`. - - Moved task-payload-driven criterion construction into `criteria_for(task)`. - - Changed aggregation to use `CriterionResult.weight`, so dynamically constructed criteria and runtime result rows remain aligned. - - Added unit coverage for no-criteria construction and weighted aggregation. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -q` - - Result: `7 passed`. - -### 13:10 UTC+1 - Real rollout accidentally used smoke fixtures - -- Reran the real ResearchRubrics harness after fixing evaluator construction. -- Result: harness passed, but the generated rollout is not useful for the paper experiment. -- Artifact path: `tests/real_llm/.rollouts/20260425T115729Z-8da9abdf-d1cd-4bcd-99ee-a768d48ef669/`. -- Evidence: - - `RunRecord` completed, but had `evaluators_count: 0`. - - DB rows: 1 task execution, 0 generation turns, 0 task evaluations, 0 resources, 1 sandbox event. - - Sandbox ID was `smoke-sandbox-...`, proving execution used the smoke sandbox manager instead of the real ResearchRubrics E2B manager. -- Root cause: - - `ENABLE_TEST_HARNESS=1` had two meanings inside the API container: expose `/api/test/read/*` endpoints and import `tests.e2e._fixtures`. - - Importing `tests.e2e._fixtures` replaces the production `researchrubrics` benchmark/sandbox registry entries with deterministic smoke fixtures. - - Real-LLM rollouts need the read-only harness endpoints, but must not replace production benchmark registries. -- Fix applied: - - Added `ENABLE_SMOKE_FIXTURES` as a separate flag. - - API includes the test harness router when `ENABLE_TEST_HARNESS=1`. - - API imports smoke fixtures only when `ENABLE_SMOKE_FIXTURES=1` (defaulting to the old `ENABLE_TEST_HARNESS` value for existing e2e behavior). - - CLI composition uses the same smoke-fixture flag. - - The stub canary sets both flags; real rollouts will set `ENABLE_TEST_HARNESS=1 ENABLE_SMOKE_FIXTURES=0`. -- Recreated API/dashboard/Inngest with `ENABLE_TEST_HARNESS=1 ENABLE_SMOKE_FIXTURES=0 TEST_HARNESS_SECRET=real-llm-secret docker compose up -d --wait --force-recreate api dashboard inngest-dev`. -- Verified: - - API harness endpoint still returns `200`. - - Dashboard root returns `200`. - - API container registry resolves `researchrubrics` to `ergon_builtins.benchmarks.researchrubrics.benchmark`. - -### 13:17 UTC+1 - Real rollout card still missing outputs/evaluation - -- Corrected-stack rollout completed and spent OpenRouter budget, but the card was incomplete. -- Artifact path: `tests/real_llm/.rollouts/20260425T120137Z-ad3ecb2d-e4b6-49a1-bb85-e214520ec7f0/`. -- Evidence: - - 8 `run_context_events` captured the real agent's tool calls (`exa_search`, `exa_qa`, `exa_get_content`, `write_report_draft`, `final_result`). - - 0 `run_resources`, 0 `run_task_evaluations`, 0 `run_generation_turns`. - - `persist-outputs` had no live sandbox to publish because sandbox setup used the default manager, while `ResearchRubricsResearcherWorker` uses `ResearchRubricsSandboxManager` for `publisher_sync()`. - - `evaluate_task_run()` reconstructed an empty `BenchmarkTask`, so task-payload-driven ResearchRubrics criteria had no rubrics. -- Fix applied: - - Registered `ResearchRubricsSandboxManager` for `researchrubrics`, `researchrubrics-ablated`, and `researchrubrics-vanilla`. - - Merged data registry sandbox managers into the composed builtin registry. - - Reconstructed the real `BenchmarkTask` in `evaluate_task_run()` from `ExperimentDefinitionTask` and `ExperimentDefinitionInstance`, preserving description and `task_payload`. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py tests/unit/smoke_base/test_registry_smoke_entries.py -q` - - Result: `13 passed`. - - Command: `uv run python - <<'PY' ... print(SANDBOX_MANAGERS['researchrubrics'].__name__)` - - Result: `ResearchRubricsSandboxManager`. - -### 13:24 UTC+1 - Report tool still simulated, not sandbox-backed - -- Ran another corrected-stack rollout (`6f454cb7-bbc8-460f-9572-9bce620c6976`). -- Result: harness passed, but still produced 0 resources and 0 evaluations. -- Evidence: - - Context events captured `write_report_draft` and `final_result`. - - No `file.write` sandbox events and no `RunResource` rows. -- Root cause: - - `_run_skill.py` is explicitly a stub skill runner: it asks the model to fabricate structured responses instead of calling real tools. - - Therefore `write_report_draft` could return `kind='success'` without actually writing `/workspace/final_output/report.md`. -- Fix applied: - - `ResearchRubricsResearcherWorker` now intercepts `write_report_draft`, `edit_report_draft`, and `read_report_draft` and executes them against the live E2B sandbox. - - The existing model-backed skill runner remains in place for Exa-style research calls for now. - - Real rollout harness now waits up to 300s after terminal status for at least one resource and evaluation row before dumping artifacts, so post-completion evaluation/resource writes are included when they land. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_workers.py tests/unit/state/test_research_rubrics_benchmark.py -q` - - Result: `9 passed`. - -### 13:18 UTC+1 - First sandbox-backed report resource captured - -- Ran real rollout `b4bb5efd-0900-4fd6-a01a-5b9daee010ea`. -- Result: - - Harness passed: `1 passed, 6 warnings in 484.44s`. - - Artifact directory: `tests/real_llm/.rollouts/20260425T121827Z-b4bb5efd-0900-4fd6-a01a-5b9daee010ea/`. - - `RunResource` now contains `report.md`, `kind='report'`, `size_bytes=12476`, with `metadata_json.sandbox_origin='/workspace/final_output/report.md'`. -- Remaining issue: - - `evaluate-task-run` was invoked but failed before persisting `RunTaskEvaluation`. - - Because the error was only visible as an Inngest `function.failed` event, the rollout had no durable evaluator failure details. -- Fix applied: - - Simplified `InngestCriterionExecutor` from `ctx.group.parallel(...)` to sequential `ctx.step.run(...)` execution, avoiding an unverified parallel step path while debugging. - - `evaluate_task_run` now logs evaluator exceptions and persists a failed `RunTaskEvaluation` row containing the error type/message, so future rollout artifacts preserve evaluator failures instead of dropping them. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py tests/unit/state/test_research_rubrics_workers.py -q` - - Result: `9 passed`. - -### 13:25 UTC+1 - Evaluator registry and final output extraction fixed - -- Root cause for missing `RunTaskEvaluation`: - - Experiment definitions persist the evaluator class `type_slug`, `researchrubrics-rubric`. - - The registry only exposed the CLI slug, `research-rubric`, so `evaluate-task-run` failed during evaluator lookup. -- Fix applied: - - Added `researchrubrics-rubric` as an alias in `registry_data.EVALUATORS`. - - Added regression coverage that both the CLI slug and persisted type slug resolve to `ResearchRubricsRubric`. -- Manual verification: - - Re-sent `task/evaluate` for run `b4bb5efd-0900-4fd6-a01a-5b9daee010ea`. - - Result: one `RunTaskEvaluation` row persisted and dashboard emitted `task.evaluation_updated`. -- New observation: - - The evaluation scored 0 because `RunTaskExecution.final_assistant_message` was empty. - - The ReAct agent emitted the final answer as a structured `final_result` tool call, not as an `assistant_text` event. -- Fix applied: - - `ReActWorker.get_output()` now falls back to the latest `final_result.args.final_assistant_message` when no assistant-text event exists. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py tests/unit/state/test_research_rubrics_workers.py -q` - - Result: `10 passed`. - -### 16:55 UTC+1 - Strongly typed benchmark task payloads - -- Change requested after PR creation: replace weak `dict[str, Any]` task payloads with benchmark-owned Pydantic payload models. -- Fix applied: - - `BenchmarkTask` is now generic over a `BaseModel` payload and defaults to `EmptyTaskPayload`. - - `Benchmark` classes now declare `task_payload_model` and use it to rehydrate persisted JSON payloads. - - ResearchRubrics, SWE-Bench Verified, MiniF2F, and GDPEval now build tasks with typed payload objects instead of raw dictionaries. - - Runtime persistence still stores JSON via `model_dump(mode="json")`, while worker/evaluator execution reconstructs the typed payload from the benchmark registry. -- Verification: - - Command: `uv run pytest tests/unit/state/test_benchmark_contract.py tests/unit/state/test_research_rubrics_benchmark.py tests/integration/swebench_verified/test_benchmark.py tests/integration/swebench_verified/test_criterion.py tests/unit/benchmarks/test_swebench_criterion_patch_source.py tests/unit/benchmarks/test_minif2f_proof_verification.py -q` - - Result: `32 passed`. - - Command: `uv run ruff check ...changed Python files...` - - Result: `All checks passed`. - -### 16:58 UTC+1 - Typed dataset loader boundaries - -- Follow-up cleanup: - - ResearchRubrics `_load_rows()` now returns `list[ResearchRubricsTaskPayload]` instead of `list[dict[str, Any]]`. - - SWE-Bench Verified `_load_rows()` now returns `list[SWEBenchInstance]` instead of raw HuggingFace row dictionaries. - - GDPEval now has `_load_task_configs()` returning `list[GDPTaskConfig]` before task materialization. - - MiniF2F was already typed at the loader boundary with `list[MiniF2FProblem]`. -- Verification: - - Command: `uv run pytest tests/unit/state/test_gdpeval_benchmark.py tests/unit/state/test_research_rubrics_benchmark.py tests/integration/swebench_verified/test_benchmark.py tests/integration/swebench_verified/test_smoke_e2e.py -q` - - Result: `19 passed`. - - Command: `uv run ruff check ...typed loader files...` - - Result: `All checks passed`. - -### 17:05 UTC+1 - PR review triage fixes - -- Review feedback applied: - - Removed settings-time mutation of `os.environ` for API keys. - - Added `ENABLE_TEST_HARNESS` / `ENABLE_SMOKE_FIXTURES` to the settings object and routed API feature gates through settings. - - Restored OpenRouter as a required dependency for real-LLM rollouts and budget gating. - - Moved definition task + instance hydration behind the persistence query layer and added a boundary test for Inngest runtime modules. - - Moved run evaluation summary refresh into `TelemetryRepository`. - - Restored Inngest criterion execution through `ctx.group.parallel(...)`. - - Removed nullable `rubric_criteria` from `ResearchRubricsRubric` constructor. - - Extracted the `final_result` fallback parsing in `ReActWorker` into a helper. -- Verification: - - Command: `uv run pytest tests/unit/test_app_mounts_harness_conditionally.py tests/unit/runtime/test_definition_lookup_boundaries.py tests/unit/state/test_research_rubrics_benchmark.py tests/unit/state/test_benchmark_contract.py tests/unit/benchmarks/test_swebench_criterion_patch_source.py tests/integration/swebench_verified/test_criterion.py -q` - - Result: `30 passed`. - - Command: `uv run ruff check ...review-fix files...` - - Result: `All checks passed`. - -### 17:18 UTC+1 - Broad review refactors - -- Follow-up on broad review questions: - - Added TODO comments at the temporary smoke-fixture imports in the CLI and API app, marking that the fixtures should move out of `tests`. - - Added public `read_report_file` / `write_report_file` methods to `ResearchRubricsSandboxManager`; `ResearchRubricsResearcherWorker` now calls those instead of reaching into `_get_raw_sandbox`, `_emit_wal_entry`, and file registries directly. - - Extracted evaluation persistence, summary construction, and dashboard DTO construction into `EvaluationPersistenceService`. - - Tightened dashboard evaluation events to carry `RunTaskEvaluationDto` instead of an untyped dict and regenerated dashboard event contracts. - - Updated the graph mutation contract fixture to use valid UUIDs now that generated schemas validate UUID formats. -- Verification: - - Command: `uv run pytest tests/unit/runtime/test_definition_lookup_boundaries.py tests/unit/state/test_research_rubrics_benchmark.py tests/unit/state/test_research_rubrics_workers.py tests/unit/state/test_benchmark_contract.py tests/unit/benchmarks/test_swebench_criterion_patch_source.py tests/integration/swebench_verified/test_criterion.py -q` - - Result: `32 passed`. - - Command: `pnpm -C ergon-dashboard run generate:contracts` - - Result: succeeded. - - Command: `pnpm -C ergon-dashboard run test:contracts` - - Result: passed after fixing the test fixture UUIDs. - - Command: `uv run ruff check ...broad refactor files...` - - Result: `All checks passed`. - -### 13:35 UTC+1 - Final telemetry verification rollout - -- Ran final full rollout `b395b9e7-b111-4f2d-9df2-49829d5dff75`. -- Result: - - Harness passed: `1 passed, 6 warnings in 211.61s`. - - Run summary now refreshes after evaluation: - - `final_score=1.0` - - `normalized_score=1.0` - - `evaluators_count=1` - - `RunResource` rows include: - - `report.md`, `kind='report'`, `size_bytes=12896`, `sandbox_origin='/workspace/final_output/report.md'` - - `worker_output`, `kind='output'`, `size_bytes=2189` - - `RunTaskEvaluation` persisted: - - score `1.0`, passed `True` - - criterion 0 (`Response is non-empty`) passed with score `1.0` - - criterion 1 (`Response is relevant`) passed with score `0.5` - - Sandbox WAL now captures report file I/O: - - `sandbox.created` - - `files.write /workspace/final_output/report.md` - - `sandbox.closed: completed` -- This is now a usable manual rollout artifact for debugging: context events, report resources, worker output, evaluation, run summary, and sandbox file-write telemetry all land in Postgres/artifacts. - -### 13:30 UTC+1 - Complete rollout artifact verified - -- Ran fresh full rollout `3c3cb9de-4c26-4624-9b6c-771e1a93b7f2`. -- Result: - - Harness passed: `1 passed, 6 warnings in 201.77s`. - - Artifact directory: `tests/real_llm/.rollouts/20260425T122920Z-3c3cb9de-4c26-4624-9b6c-771e1a93b7f2/`. - - `RunTaskExecution.final_assistant_message` is populated (`2497` chars). - - `RunResource` rows include: - - `report.md`, `kind='report'`, `size_bytes=12383`, `sandbox_origin='/workspace/final_output/report.md'`. - - `worker_output`, `kind='output'`, `size_bytes=2537`. - - `RunTaskEvaluation` row persisted with score `1.0`, passed `True`, normalized score `0.6666666667`. -- Remaining polish: - - `RunRecord.summary_json` is finalized before async evaluation lands, so the live row still reported `evaluators_count=0` after the run. -- Fix applied: - - `evaluate_task_run` now refreshes `RunRecord.summary_json` after persisting successful or failed evaluator rows. - - Sandbox-backed report read/write tools now emit WAL entries (`files.read ...`, `files.write ...`) and register created files, so future rollouts should show actual report I/O in sandbox command telemetry. -- Verification: - - Command: `uv run pytest tests/unit/state/test_research_rubrics_benchmark.py tests/unit/state/test_research_rubrics_workers.py -q` - - Result: `10 passed`. diff --git a/docs/real-llm-rollout-harness.md b/docs/real-llm-rollout-harness.md deleted file mode 100644 index 902c4706c..000000000 --- a/docs/real-llm-rollout-harness.md +++ /dev/null @@ -1,374 +0,0 @@ -# Real-LLM rollout harness for researchrubrics - -Date: 2026-04-24 -Status: design sketch (ignores prior RFCs); spikes complete, one -pre-work PR identified before the harness itself can land. - -## Intent - -Build a **rollout harness**, not a TDD tier. The test is a trigger; the -artifacts are the product. Running the harness produces a rich, -inspectable snapshot of a real-LLM run against the `researchrubrics` -benchmark that a future agent session (or a human) can read and reason -about in order to iterate on either the agent or the simulator. - -This supports the methodological contribution we want to express in the -paper: we are debugging the simulator (ergon_core) and growing a stronger -agent by running real LLM rollouts end-to-end and reading them back. - -## Reframe vs. a test tier - -The previous framing was a per-benchmark pytest tier with hard-gate -assertions on DB state and UI state. We drop that. The harness asserts -**only** that the benchmark reached a terminal state (`completed`, -`failed`, or `cancelled`) within budget/timeout. `failed` is still a -successful rollout from the harness's perspective — it is a data point. - -What drops out: - -- No shared `assertions.py` helper (rubric parsing, tool-call - inspection). -- No per-benchmark assertion knowledge in the test body. -- No "which fields to verify" question at harness-authoring time. - -The single assertion is: - -```python -assert state["status"] in {"completed", "failed", "cancelled"} -``` - -## Current state of `tests/real_llm/` - -Shipped (PR 1): - -- `conftest.py` — marker-gated (`ERGON_REAL_LLM=1`), `--assume-stack-up` - flag, session fixtures wired. -- `fixtures/stack.py` — docker-compose up/wait/down against the unified - `docker-compose.yml`. -- `openrouter_budget.py` + `fixtures/openrouter_budget.py` - — live spend check against `/api/v1/auth/key`. -- `fixtures/harness_client.py` — polls - `/api/test/read/run/{id}/state` for terminal status. -- `fixtures/playwright_client.py` — browser/context session fixtures. -- `benchmarks/test_smoke_stub.py` — zero-cost canary proving the pipe - (CLI → Postgres → harness → Playwright) with stub workers. - -Missing for a real-LLM researchrubrics rollout: - -- **Pre-work**: per-benchmark sandbox env-injection + an integration - test that asserts every required key for every benchmark's sandbox - is actually present inside the sandbox at provision time. See - §"Pre-work PR" below. -- `tests/real_llm/rollout.py` — artifact dump helpers. -- `tests/real_llm/benchmarks/test_researchrubrics.py` — the trigger. - -## Researchrubrics moving parts (already shipped) - -- **Benchmark**: `ResearchRubricsBenchmark` - (`ergon_builtins/benchmarks/researchrubrics/benchmark.py:32`), plus - `researchrubrics-vanilla` and `researchrubrics-ablated`. Loads HF - dataset, honours `--limit N`. -- **Worker**: `researchrubrics-researcher` - (`ergon_builtins/workers/research_rubrics/researcher_worker.py:53`). - Concrete `ReActWorker` subclass. At `execute()` time builds a 9-tool - surface (Exa search / QA / get_content + write/edit/read report - drafts + `ResearchGraphToolkit` observability tools). Uses - `pydantic_ai.Agent`, `max_iterations=25`. -- **Evaluator**: `research-rubric` (normalised weighted-criteria - scorer), wired in `registry_data.py:28`. -- **Sandbox**: `ResearchRubricsSandboxManager` — blank E2B sandbox with - workspace dirs provisioned. No template file needed. -- **CLI**: `ergon experiment define researchrubrics --worker - researchrubrics-researcher --evaluator research-rubric --model <X> - --limit 1` followed by `ergon experiment run <experiment-id>`. - -## Keys come from `settings` - -All required keys are already surfaced on -`ergon_core/core/settings.py`: - -```python -openrouter_api_key # aliases: OPENROUTER_API_KEY, OPEN_ROUTER_API_KEY -openrouter_base_url # default: https://openrouter.ai/api/v1 -openai_api_key -e2b_api_key -exa_api_key -hf_api_key # researchrubrics dataset on HF hub -database_url # for rollout dump -``` - -`Settings.missing_values(names: list[str]) -> list[str]` -(`settings.py:65`) is the idiomatic preflight. - -Fixture-level preflight replaces the old key-plumbed fixtures: - -```python -from ergon_core.core.settings import settings - -@pytest.fixture(scope="session") -def _required_keys(): - missing = settings.missing_values( - ["openrouter_api_key", "exa_api_key", "e2b_api_key"], - ) - if missing: - pytest.skip(f"real-llm rollout requires {missing}") -``` - -Budget becomes **soft**: record remaining OpenRouter spend in the -manifest, skip only if already ≤0 at session start. - -## Artifact layout - -One directory per rollout, named with timestamp and run_id: - -``` -tests/real_llm/.rollouts/<timestamp>-<run_id>/ -├── manifest.json # run metadata -├── db/ -│ ├── run_record.json -│ ├── run_graph_nodes.jsonl -│ ├── run_graph_edges.jsonl -│ ├── run_graph_mutations.jsonl -│ ├── run_generation_turns.jsonl -│ ├── run_context_events.jsonl -│ ├── run_resources.jsonl -│ ├── run_task_evaluations.json -│ └── sandbox_events.jsonl -├── screenshots/ -│ ├── cohort_index.png -│ ├── run_detail.png -│ └── node_<id>.png # optional per-node -└── report.md # stitched summary for quick skim -``` - -`manifest.json` contents: - -- `run_id`, `benchmark`, `worker`, `evaluator`, `model` -- wall-clock start/end -- terminal status -- OpenRouter budget snapshot (baseline, post-run, delta) -- key fingerprints (hashes), never raw values -- harness version / git sha - -`report.md` is the first thing a reviewing agent should read: prompt, -turn count, tools called (counts), final report excerpt, score if the -evaluator ran, links into the `db/` jsonl files. - -## Rollout dump surface - -SQLModel tables to snapshot (all present in -`ergon_core/core/persistence/`): - -- `RunRecord` (`telemetry/models.py:61`) -- `RunTaskExecution` (`telemetry/models.py:110`) -- `RunResource` (`telemetry/models.py:217`) -- `RunTaskEvaluation` (`telemetry/models.py:267`) -- `RunGenerationTurn` (`telemetry/models.py:415`) -- `SandboxEvent` (`telemetry/models.py:646`) -- `RunGraphNode` (`graph/models.py:44`) -- `RunGraphEdge` (`graph/models.py:96`) -- `RunGraphMutation` (`graph/models.py:172`) -- `RunContextEvent` (`context/models.py:25`) - -A single `dump_rollout(run_id, out_dir)` helper: one `get_session()`, -~10 queries, `.model_dump_json()` per row. - -## The test itself - -Roughly 30 lines: - -```python -pytestmark = [pytest.mark.real_llm, pytest.mark.asyncio] - -async def test_researchrubrics_rollout( - real_llm_stack, - _required_keys, - playwright_context, - harness_client, -): - before = datetime.now(timezone.utc) - rc = subprocess.run( - [ - "uv", "run", "ergon", "benchmark", "run", "researchrubrics", - "--worker", "researchrubrics-researcher", - "--evaluator", "research-rubric", - "--model", os.environ.get( - "ERGON_REAL_LLM_MODEL", - "anthropic:claude-sonnet-4.6", - ), - "--limit", "1", - ], - timeout=900, - ).returncode - - run_id = _latest_run_id_since(before) - state = harness_client.wait_for_terminal(run_id, timeout_s=900) - - out_dir = _rollout_dir(run_id) - dump_rollout(run_id, out_dir) - await capture_dashboard(run_id, playwright_context, out_dir) - write_manifest(out_dir, rc=rc, state=state, ...) - - assert state["status"] in {"completed", "failed", "cancelled"} -``` - -## Spike results - -**1. OpenRouter model routing.** -`resolve_model_target("anthropic:claude-sonnet-4.6")` resolves to a -PydanticAI chat model backed by -`pydantic_ai.providers.openrouter.OpenRouterProvider`. Cloud provider -prefixes (`openai:`, `anthropic:`, `google:`) are OpenRouter-hosted in -Ergon; use `OPENROUTER_API_KEY` in the process env and do not route -through direct provider APIs. - -**2. Exa inside the sandbox — confirmed not wired.** The plumbing -exists but nothing populates it: - -- `BaseSandboxManager.create(envs=...)` → `AsyncSandbox.create(envs=...)` - is wired (`manager.py:280`). -- `SandboxSetupRequest.envs: dict[str, str] = {}` is declared - (`child_function_payloads.py:23`). -- `execute_task.py:141` constructs the `SandboxSetupRequest` **without - ever setting `envs`** — the field is always the default empty dict. -- `ResearchRubricsSandboxManager` does not override `create()` to - inject its own envs; it only overrides `_install_dependencies` to - `pip install exa-py` and scaffold `/workspace/*` dirs. - -Net effect: every researchrubrics rollout today would complete with -Exa tools failing auth inside the sandbox. A rollout harness built on -top of this would produce a degenerate distribution of failure modes -(every run is "agent can't search") instead of useful data. - -**And the test gap is real.** Neither -`tests/integration/minif2f/test_sandbox_manager.py` nor -`tests/integration/swebench_verified/test_sandbox_manager.py` asserts -anything about sandbox env injection. There is no -`tests/integration/researchrubrics/test_sandbox_manager.py` at all -(only a stale `.pyc` left over — the `.py` was deleted). This needs a -fix *before* the rollout harness, and the fix should be generic across -all benchmarks. - -## Pre-work PR: sandbox env-injection + integration tests - -Sandbox env injection is a cross-cutting concern — every benchmark's -sandbox manager should declare exactly which keys its in-sandbox tools -need, and an integration test should assert each declared key actually -arrives inside the sandbox at provision time. This covers -researchrubrics today and anything we add tomorrow. - -### Declaration site - -Each `Benchmark` subclass already declares -`onboarding_deps: BenchmarkDeps` with an `optional_keys` tuple. For -example: - -```python -class ResearchRubricsBenchmark(Benchmark): - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps( - extras=("ergon-builtins[data]",), - optional_keys=("EXA_API_KEY",), - ) -``` - -Treat `optional_keys` as the **canonical list of env keys the in-sandbox -tools will read**. The sandbox manager for that benchmark must forward -every key in that list into the sandbox process env. - -### Implementation - -The cleanest locus is the per-benchmark manager — it already owns -"what this benchmark needs in its sandbox" (packages, directories, -templates). Extending that to env vars keeps benchmark-specific -knowledge out of upstream dispatch code. - -Two reasonable patterns — pick one and apply uniformly: - -- **a. Manager-composed envs** (preferred): each manager overrides - `create()` (or a new `_compose_envs()` hook) to read the keys named - in its benchmark's `BenchmarkDeps.optional_keys` from `settings`, - merge them into the caller-supplied `envs` dict, and call - `super().create()`. Missing keys in `settings` raise a clear error - at `create()` time so misconfigured rollouts fail fast instead of - producing Exa-401 soup. -- **b. Upstream composer populates `SandboxSetupRequest.envs`** based - on the benchmark's `BenchmarkDeps` — more generic but spreads - benchmark-specific knowledge upstream. - -### Integration test - -One shared parametrised test, one file per benchmark with a sandbox -manager. Mirrors the minif2f/swebench `test_sandbox_manager.py` shape. - -For each benchmark: provision the sandbox with a dummy value for every -declared key (e.g. `{"EXA_API_KEY": "test-dummy-12345"}`), then for -each key run `echo $<KEY_NAME>` inside the sandbox and assert stdout -equals the dummy value. - -Also assert the negative: if a required key is missing from `settings`, -`manager.create()` raises a clear message. - -Concrete files: - -- `tests/integration/researchrubrics/test_sandbox_manager.py` (new) -- `tests/integration/minif2f/test_sandbox_manager.py` (extend with env - assertions — even if minif2f declares no optional keys today, the - test confirms the declared-set size matches observed-set size and - future-proofs additions) -- `tests/integration/swebench_verified/test_sandbox_manager.py` (same) -- `tests/integration/gdpeval/test_sandbox_manager.py` (if/when gdpeval - grows a sandbox manager) - -### Acceptance - -- `tests/integration/**/test_sandbox_manager.py` green with a dummy - `EXA_API_KEY` exported by the researchrubrics manager. -- A manually-run researchrubrics smoke rollout no longer emits Exa 401s - from inside the sandbox. -- `pnpm run check:fast` + `pnpm run test:be:fast` green. - -Estimated effort: **half to one day**, unconditional of the rollout -harness — this lands as a standalone fix. - -## Minimum shippable harness (after pre-work PR lands) - -Files: - -1. `tests/real_llm/rollout.py` — `dump_rollout`, `capture_dashboard`, - `write_manifest`, `_rollout_dir`. -2. `tests/real_llm/benchmarks/test_researchrubrics.py` — the 30-line - trigger above. -3. Model targets resolve centrally in `resolve_model_target`; use - provider-prefixed targets such as `anthropic:claude-sonnet-4.6`. - Cloud provider prefixes route through OpenRouter. - -Estimated effort: **half a day** on top of the pre-work PR. - -## The loop - -After shipping: - -```bash -pnpm run test:be:real-llm -k researchrubrics -open tests/real_llm/.rollouts/<latest>/report.md -``` - -The agent (me, or a future session) reads `report.md`, drills into the -jsonl files or screenshots as needed, and reasons about whether the -task ran successfully — and what to tweak in either the model or the -environment to iterate on the agent. That is the paper-worthy loop: -rollout → read → tweak → rollout. - -No unit-test-shaped discipline; rollout-shaped discipline. - -## Extensions (deliberately not in v1) - -- Parametrise over N seeded instances. -- Soft gates: "at least 1 of N produced a non-zero rubric score". -- Per-instance screenshot of the run graph expanded. -- Auto-opening a `docs/bugs/open/` entry when a rollout surfaces a - simulator bug (as opposed to a model-quality issue). -- Nightly cron running the harness against all three benchmarks with a - fresh artifact directory each time. - -Keep v1 to one benchmark, one instance, one rollout directory. diff --git a/docs/release.md b/docs/release.md deleted file mode 100644 index 5b92e7743..000000000 --- a/docs/release.md +++ /dev/null @@ -1,36 +0,0 @@ -# Release Process - -Ergon uses a lightweight release-train model. - -## Branches - -- `main` is the stable release branch. Every merge to `main` is expected to be releasable. -- `dev` is the integration branch for normal feature and fix work. -- `feature/*`, `fix/*`, and `codex/*` branches should target `dev` by default. -- `release/vX.Y.Z` branches are optional short-lived stabilization branches created from `dev`. -- `hotfix/vX.Y.Z` branches are created from `main` for urgent production fixes and must be backmerged to `dev`. - -## Normal Flow - -1. Open feature and fix PRs against `dev`. -2. Keep `dev` green. -3. When `dev` is ready to release, update `project.version` in `pyproject.toml`. -4. Open a release PR from `dev` to `main`. -5. Merge the release PR after required checks pass. -6. The release tagger creates `vX.Y.Z` from the version in `pyproject.toml` and publishes a GitHub Release. -7. If release-only changes landed on `main`, merge `main` back into `dev`. - -## Hotfix Flow - -1. Branch from `main` as `hotfix/vX.Y.Z`. -2. Apply the smallest safe fix and bump `project.version`. -3. Open the hotfix PR against `main`. -4. Merge after checks pass; the release tagger publishes the patch release. -5. Backmerge or cherry-pick the hotfix into `dev`. - -## Release Tagger Rules - -- The tag name is `v` plus `project.version` from the root `pyproject.toml`. -- `project.version` must look like SemVer, for example `0.1.0` or `0.1.1`. -- If the tag already exists at the same commit, the workflow exits cleanly. -- If the tag already exists at a different commit, the workflow fails. diff --git a/docs/rfcs/TEMPLATE.md b/docs/rfcs/TEMPLATE.md deleted file mode 100644 index fbcf81e03..000000000 --- a/docs/rfcs/TEMPLATE.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -status: active # active | accepted | rejected -opened: YYYY-MM-DD -author: <human-or-agent> -architecture_refs: [] # e.g. [docs/architecture/02_runtime_lifecycle.md#task-state-machine] -supersedes: [] -superseded_by: null ---- - -# RFC: <title> - -## Problem - -Why are we changing something? What invariant is this fixing, which gap is this -closing, or which capability is this adding? One or two paragraphs. - -## Proposal - -The change in one paragraph, then concrete specifics. Include code sketches, -type signatures, or SQL where it sharpens the proposal. - -## Invariants affected - -Which architecture-doc invariants does this introduce, change, or break? Cite -exact sections. If a new invariant is introduced, state it precisely. - -## Migration - -What breaks and must change in parallel? Data migrations? Alembic revision? -Test rewrites? Downstream consumers? - -## Alternatives considered - -Options we rejected and why. Be honest — if a simpler option exists, say why -we aren't taking it. - -## Open questions - -Things we're deferring or need steer on. Mark with `@<person>` if a specific -person should decide. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Update the cited `architecture_refs` sections if invariants changed. - - Link the implementation plan in `docs/superpowers/plans/`. - - Close any related bug files in `docs/bugs/open/`. diff --git a/docs/rfcs/accepted/2026-04-17-criterion-runtime-di-container.md b/docs/rfcs/accepted/2026-04-17-criterion-runtime-di-container.md deleted file mode 100644 index ffedae2da..000000000 --- a/docs/rfcs/accepted/2026-04-17-criterion-runtime-di-container.md +++ /dev/null @@ -1,1416 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/01_public_api.md#criterionruntime, docs/architecture/06_builtins.md#evaluator-criterion-and-rubric-layout] -supersedes: [] -superseded_by: null ---- - -# RFC: Expand `CriterionRuntime` into a dependency-injection container - -## Problem - -`CriterionRuntime` is defined at -`ergon_core/ergon_core/api/criterion_runtime.py:48–63` as a seven-method -Protocol: - -```python -class CriterionRuntime(Protocol): - async def ensure_sandbox(self) -> None: ... - async def upload_files(self, files: list[dict]) -> None: ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def run_command(self, command: str, timeout: int = 30) -> CommandResult: ... - async def execute_code(self, code: str) -> SandboxResult: ... - async def call_llm_judge(self, messages: list, response_type: type[T]) -> T: ... - async def cleanup(self) -> None: ... -``` - -The concrete implementation is `DefaultCriterionRuntime` at -`ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py:32`. The -Protocol covers sandbox access and one-shot LLM judging. Three gaps remain: - -1. **No resource I/O.** Criteria that need to read worker-published blobs - (`RunResource` rows) have no `read_resource(name)` or `list_resources()` - accessor on the Protocol. They must open their own DB session, query - `RunResource` by `(run_id, name)`, and read from the blob path on disk — - bypassing the runtime entirely. - -2. **No read-only DB session.** Criteria that query run state (prior attempts, - sibling task results, task metadata) must call - `ergon_core.core.persistence.shared.db.get_session()` directly. There is no - `db_read_session()` on the Protocol, so the session-management contract is - implicit and unchecked. - -3. **No event-emission surface.** Agentic criteria that want to surface - progress to the dashboard have no path to the `DashboardEmitter` at - `ergon_core/ergon_core/core/dashboard/emitter.py:51`. The runtime does not - expose an `event_sink()` accessor. - -4. **Anti-pattern evidence.** The SWE-Bench criterion at - `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py:66–78` - defines `_spawn_eval_sandbox(run_id)` which instantiates - `SWEBenchSandboxManager()` directly at line 72, allocates a fresh - `sandbox_key` at line 73, calls `manager.create(...)` at line 74, and - returns the new sandbox. Every SWE-bench eval invocation pays an extra - sandbox allocation. - - Root cause: when `SWEBenchTestCriterion` was written, there was no way to - pull worker-published artifacts via the runtime. The simplest available path - was to bring up a fresh sandbox. Tracked as P2 bug - `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md`, asserted - (xfail) at `tests/state/test_criteria_do_not_spawn_sandboxes.py:19–32`. - - Closing the resource I/O gap removes the last legitimate reason a criterion - would construct a sandbox manager itself. After this RFC, `_spawn_eval_sandbox` - is replaced by `await runtime.ensure_sandbox()` plus - `await runtime.read_resource(name)` for any worker-produced artifacts. - - Note: the bug report's proposed fix references `runtime.get_sandbox()`, but - the RFC (§ Proposal below) explicitly rejects adding `get_sandbox()`. - The bug-report fix is superseded by the `ensure_sandbox()` + - `read_resource()` path. - -## Proposal - -Extend `CriterionRuntime` with four new methods, implement them on -`DefaultCriterionRuntime`, and migrate the swebench offender. - -**Option chosen: extend the existing Protocol (Option A).** - -The four new methods are: - -```python -# ergon_core/ergon_core/api/criterion_runtime.py — additions - -async def read_resource(self, name: str) -> bytes: ... -async def list_resources(self) -> list[RunResourceView]: ... -def db_read_session(self) -> Session: ... -def event_sink(self) -> SandboxEventSink: ... -``` - -`RunResourceView` is the frozen DTO at -`ergon_core/ergon_core/api/run_resource.py:22`. -`SandboxEventSink` is the Protocol at -`ergon_core/ergon_core/core/providers/sandbox/event_sink.py:7`. -`Session` is `sqlmodel.Session`. - -`DefaultCriterionRuntime` is constructed with `run_id` and `task_id` from the -`EvaluationContext` already in scope at the call site -(`inngest_executor.py:73–76`). The `run_id` is used to scope resource and DB -queries; `task_id` is stored for tracing. - -### What does NOT change - -- The seven existing Protocol methods — signatures unchanged. -- `EvaluationContext` — no new fields. -- `CriterionContext` / `TaskEvaluationContext` — no changes. -- `InngestCriterionExecutor` constructor — only `DefaultCriterionRuntime` - construction inside it changes. -- Alembic migrations — no schema change; `RunResource` table already exists. -- Criterion base class (`ergon_core/ergon_core/api/criterion.py`) — unchanged. - -### Option B (rejected): separate agentic and non-agentic Protocols - -Split into `CriterionRuntime` (sandbox/resource) and `AgentCriterionRuntime` -(adds `event_sink`). Rejected — every criterion benefits uniformly from -resource and DB access. A split doubles inheritance surface with no concrete -benefit until there is a criterion that truly cannot tolerate the extra -methods. - -### Option C (rejected): add `get_sandbox()` accessor - -An earlier draft proposed `get_sandbox() -> AsyncSandbox | None`. Rejected — -`ensure_sandbox()` already covers "provision or reconnect and hand back the -task sandbox." A separate raw accessor splits responsibility without benefit -and would expose the sandbox provider type across the Protocol boundary. - -## Architecture overview - -### Current data flow (swebench criterion) - -``` -inngest_executor.py - └── InngestCriterionExecutor.execute_all() - └── DefaultCriterionRuntime(context, sandbox_manager) - └── criterion.evaluate(eval_ctx) ← EvaluationContext.runtime set - └── _spawn_eval_sandbox(run_id) ← ANTI-PATTERN - └── SWEBenchSandboxManager() ← fresh sandbox manager - └── manager.create(...) ← new sandbox allocation -``` - -### After this RFC - -``` -inngest_executor.py - └── InngestCriterionExecutor.execute_all() - └── DefaultCriterionRuntime( - context=criterion_context, - sandbox_manager=self.sandbox_manager, - run_id=eval_ctx.run_id, ← NEW - task_id=eval_ctx.task_id, ← NEW - ) - └── criterion.evaluate(eval_ctx) - ├── runtime.ensure_sandbox() ← reuses task sandbox - ├── runtime.read_resource("patch") ← queries RunResource table - ├── runtime.list_resources() ← lists run resources - ├── runtime.db_read_session() ← read-only session - └── runtime.event_sink() ← DashboardEmitterSandboxEventSink -``` - -### `DefaultCriterionRuntime.__init__` before / after - -Before (line 38–50 of `criterion_runtime.py`): - -```python -def __init__( - self, - context: CriterionContext, - sandbox_manager: "BaseSandboxManager", - llm_model: str = "gpt-4o", - llm_max_tokens: int = 1024, - llm_temperature: float = 0.0, -) -> None: -``` - -After: - -```python -def __init__( - self, - context: CriterionContext, - sandbox_manager: "BaseSandboxManager", - run_id: UUID | None = None, - task_id: UUID | None = None, - llm_model: str = "gpt-4o", - llm_max_tokens: int = 1024, - llm_temperature: float = 0.0, -) -> None: -``` - -`run_id` defaults to `context.run_id` if `None` is passed; `task_id` is -optional and used only in trace attributes. Both are keyword-only in practice -(positional slots after `sandbox_manager` — callers that pass only -`context` and `sandbox_manager` are unaffected). - -## Type / interface definitions - -### Updated `CriterionRuntime` Protocol - -```python -# ergon_core/ergon_core/api/criterion_runtime.py - -"""Public Protocol for the criterion runtime + its small result DTOs. - -``CriterionRuntime`` is the capabilities surface criteria use to interact -with the sandbox and LLM judge while they evaluate. Lives in ``api/`` so -that ``EvaluationContext`` (also in ``api/``) can type it without dragging -in the core runtime package (which would cause a circular import). -""" - -from typing import TYPE_CHECKING, Protocol, TypeVar - -from pydantic import BaseModel, Field -from sqlmodel import Session - -T = TypeVar("T", bound=BaseModel) - -if TYPE_CHECKING: - from ergon_core.api.run_resource import RunResourceView - from ergon_core.core.providers.sandbox.event_sink import SandboxEventSink - -__all__ = ["CommandResult", "CriterionRuntime", "SandboxResult"] - - -class SandboxResult(BaseModel): - """Result from sandbox code execution.""" - - stdout: list[str] = Field(default_factory=list) - stderr: list[str] = Field(default_factory=list) - - -class CommandResult(BaseModel): - """Result from command execution in a sandbox.""" - - stdout: str | None = Field(default=None) - stderr: str | None = Field(default=None) - exit_code: int | None = Field(default=None) - - -class CriterionRuntime(Protocol): - """Execution surface injected into a ``Criterion`` at evaluation time. - - The runtime owns the sandbox lifecycle (create / reset timeout / - cleanup) on behalf of the criterion and exposes a small set of - primitives the criterion calls to gather evidence. A criterion that - doesn't need sandbox access or a judge simply ignores it. - - Surface-area constraint: this Protocol is narrowly scoped to sandbox - lifecycle, resource I/O, and event emission. It should not grow into - a generic service locator. ``call_llm_judge`` is the one method not - strictly about I/O; if the Protocol keeps expanding it is a candidate - for extraction into an ``LLMJudgeMixin``. - """ - - # ── sandbox lifecycle (existing) ────────────────────────────────── - async def ensure_sandbox(self) -> None: ... - async def upload_files(self, files: list[dict]) -> None: ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def run_command(self, command: str, timeout: int = 30) -> CommandResult: ... - async def execute_code(self, code: str) -> SandboxResult: ... - async def call_llm_judge(self, messages: list, response_type: type[T]) -> T: ... - async def cleanup(self) -> None: ... - - # ── resource I/O (new) ──────────────────────────────────────────── - async def read_resource(self, name: str) -> bytes: ... - """Read a worker-published blob by name for this run. - - Queries ``run_resources`` for the latest row matching ``(run_id, name)`` - and reads the content-addressed blob from disk. - - Raises ``ResourceNotFoundError`` if no row matches. - Raises ``OSError`` if the blob file is missing or unreadable. - """ - - async def list_resources(self) -> list["RunResourceView"]: ... - """Return all ``RunResourceView`` DTOs for this run, newest first.""" - - # ── DB access (new) ─────────────────────────────────────────────── - def db_read_session(self) -> Session: ... - """Return a ``sqlmodel.Session`` for read-only queries. - - The caller owns the session lifecycle (open / close). Use as a - context manager: ``with runtime.db_read_session() as s: ...`` - Mutating writes via this session violate the contract but are not - enforced at runtime in v1. - """ - - # ── event emission (new) ────────────────────────────────────────── - def event_sink(self) -> "SandboxEventSink": ... - """Return the ``SandboxEventSink`` wired to the dashboard emitter. - - Agentic criteria may call ``await sink.sandbox_command(...)`` to stream - progress events to the real-time dashboard. Returns a ``NoopSandboxEventSink`` - if dashboard streaming is disabled. - """ -``` - -## Full implementations - -### `ergon_core/ergon_core/api/criterion_runtime.py` (complete updated file) - -```python -# ergon_core/ergon_core/api/criterion_runtime.py - -"""Public Protocol for the criterion runtime + its small result DTOs.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol, TypeVar - -from pydantic import BaseModel, Field - -T = TypeVar("T", bound=BaseModel) - -if TYPE_CHECKING: - from sqlmodel import Session - - from ergon_core.api.run_resource import RunResourceView - from ergon_core.core.providers.sandbox.event_sink import SandboxEventSink - -__all__ = ["CommandResult", "CriterionRuntime", "SandboxResult"] - - -class SandboxResult(BaseModel): - """Result from sandbox code execution.""" - - stdout: list[str] = Field( - default_factory=list, - description="Captured stdout lines from the sandbox process.", - ) - stderr: list[str] = Field( - default_factory=list, - description="Captured stderr lines from the sandbox process.", - ) - - -class CommandResult(BaseModel): - """Result from command execution in a sandbox.""" - - stdout: str | None = Field( - default=None, - description="Captured stdout; ``None`` if the command never produced any.", - ) - stderr: str | None = Field( - default=None, - description="Captured stderr; ``None`` if the command never produced any.", - ) - exit_code: int | None = Field( - default=None, - description="Process exit code; ``None`` if the command could not be started.", - ) - - -class CriterionRuntime(Protocol): - """Execution surface injected into a ``Criterion`` at evaluation time. - - The runtime owns the sandbox lifecycle (create / reset timeout / - cleanup) on behalf of the criterion and exposes a small set of - primitives the criterion calls to gather evidence. A criterion that - doesn't need sandbox access or a judge simply ignores it. - - Surface-area constraint: this Protocol is narrowly scoped to sandbox - lifecycle, resource I/O, and event emission. It should not grow into - a generic service locator. - """ - - # ── sandbox lifecycle ───────────────────────────────────────────── - async def ensure_sandbox(self) -> None: ... - async def upload_files(self, files: list[dict]) -> None: ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def run_command(self, command: str, timeout: int = 30) -> CommandResult: ... - async def execute_code(self, code: str) -> SandboxResult: ... - async def call_llm_judge(self, messages: list, response_type: type[T]) -> T: ... - async def cleanup(self) -> None: ... - - # ── resource I/O ────────────────────────────────────────────────── - async def read_resource(self, name: str) -> bytes: ... - async def list_resources(self) -> list[RunResourceView]: ... - - # ── DB access ───────────────────────────────────────────────────── - def db_read_session(self) -> Session: ... - - # ── event emission ──────────────────────────────────────────────── - def event_sink(self) -> SandboxEventSink: ... -``` - -### `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` (complete updated file) - -```python -# ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py - -"""Default concrete implementation of ``CriterionRuntime``.""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import TYPE_CHECKING, TypeVar -from uuid import UUID - -from ergon_core.api.criterion_runtime import ( - CommandResult, - CriterionRuntime, - SandboxResult, -) -from ergon_core.api.run_resource import RunResourceView -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.telemetry.models import RunResource -from ergon_core.core.providers.sandbox.event_sink import ( - DashboardEmitterSandboxEventSink, - NoopSandboxEventSink, - SandboxEventSink, -) -from ergon_core.core.runtime.evaluation.evaluation_schemas import CriterionContext -from ergon_core.core.settings import settings -from openai import AsyncOpenAI -from pydantic import BaseModel -from sqlmodel import Session, select - -if TYPE_CHECKING: - from ergon_core.core.providers.sandbox.manager import BaseSandboxManager - -T = TypeVar("T", bound=BaseModel) -logger = logging.getLogger(__name__) - -__all__ = ["CriterionRuntime", "DefaultCriterionRuntime", "ResourceNotFoundError"] - - -class ResourceNotFoundError(LookupError): - """Raised by ``read_resource`` when no ``RunResource`` row matches the name.""" - - -class DefaultCriterionRuntime: - """Real criterion runtime backed by sandbox manager + OpenAI + DB. - - Parameters - ---------- - context: - ``CriterionContext`` passed by the executor. ``context.run_id`` is - the default ``run_id`` for resource and DB queries if ``run_id`` is - not provided explicitly. - sandbox_manager: - The ``BaseSandboxManager`` that owns the task sandbox. - run_id: - Explicit run UUID for resource/DB scoping. Defaults to - ``context.run_id`` if ``None``. - task_id: - Task UUID used in trace attributes; optional. - llm_model: - OpenAI model name for ``call_llm_judge``. - llm_max_tokens: - Token limit for judge responses. - llm_temperature: - Sampling temperature for judge calls. - event_sink: - Pre-constructed ``SandboxEventSink``. If ``None`` a - ``NoopSandboxEventSink`` is used. - """ - - def __init__( - self, - context: CriterionContext, - sandbox_manager: BaseSandboxManager, - run_id: UUID | None = None, - task_id: UUID | None = None, - llm_model: str = "gpt-4o", - llm_max_tokens: int = 1024, - llm_temperature: float = 0.0, - event_sink: SandboxEventSink | None = None, - ) -> None: - self.context = context - self.sandbox_manager: BaseSandboxManager = sandbox_manager - self._run_id: UUID = run_id if run_id is not None else context.run_id - self._task_id: UUID | None = task_id - self._owns_sandbox = False - self._llm_model = llm_model - self._llm_max_tokens = llm_max_tokens - self._llm_temperature = llm_temperature - self._event_sink: SandboxEventSink = event_sink or NoopSandboxEventSink() - - # ── sandbox lifecycle ───────────────────────────────────────────── - - async def ensure_sandbox(self) -> None: - sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - await self.sandbox_manager.create( - self._run_id, - run_id=self._run_id, - timeout_minutes=30, - ) - self._owns_sandbox = True - return - await self.sandbox_manager.reset_timeout(self._run_id, timeout_minutes=30) - - async def upload_files(self, files: list[dict]) -> None: - sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - raise RuntimeError("Sandbox not created - call ensure_sandbox first") - for resource in files: - name = resource.get("name", "unknown") - sandbox_path = f"/evaluation/{name}" - content = resource.get("content", b"") - if isinstance(content, str): - content = content.encode("utf-8") - try: - await sandbox.files.write(sandbox_path, content) - except Exception as exc: # slopcop: ignore[no-broad-except] - logger.warning("Failed to upload %s: %s", name, exc) - - async def write_file(self, path: str, content: bytes) -> None: - sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - raise RuntimeError("Sandbox not created - call ensure_sandbox first") - await sandbox.files.write(path, content) - - async def run_command(self, command: str, timeout: int = 30) -> CommandResult: - sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - raise RuntimeError("Sandbox not created - call ensure_sandbox first") - try: - result = await sandbox.commands.run(command, timeout=timeout) - return CommandResult( - stdout=result.stdout, - stderr=result.stderr, - exit_code=result.exit_code, - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - return CommandResult(stdout="", stderr=str(exc), exit_code=1) - - async def execute_code(self, code: str) -> SandboxResult: - sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - raise RuntimeError("Sandbox not created") - try: - execution = await sandbox.run_code(code, language="python", timeout=30) - return SandboxResult( - stdout=list(execution.logs.stdout), - stderr=list(execution.logs.stderr), - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - error_msg = str(exc) - if "timeout" in error_msg.lower() or "sandbox was not found" in error_msg.lower(): - raise RuntimeError( - f"Sandbox execution failed (likely timeout): {error_msg}. " - "Code criterion may have taken too long (>30s)." - ) from exc - raise - - async def call_llm_judge(self, messages: list, response_type: type[T]) -> T: - client = AsyncOpenAI(api_key=settings.openai_api_key) - response = await client.beta.chat.completions.parse( - model=self._llm_model, - messages=messages, - max_tokens=self._llm_max_tokens, - temperature=self._llm_temperature, - response_format=response_type, - ) - message = response.choices[0].message - if message.parsed is None: - raise ValueError("No parsed response from LLM judge") - return message.parsed - - async def cleanup(self) -> None: - if self._owns_sandbox: - await self.sandbox_manager.terminate(self._run_id) - self._owns_sandbox = False - - # ── resource I/O (new) ──────────────────────────────────────────── - - async def read_resource(self, name: str) -> bytes: - """Read the latest worker-published blob for ``name`` in this run. - - Queries ``run_resources`` for the most-recently-created row matching - ``(run_id, name)``, then reads bytes from ``file_path`` on disk. - - Raises - ------ - ResourceNotFoundError - No ``run_resources`` row matches ``(run_id, name)``. - OSError - The blob file is missing or unreadable. - """ - with get_session() as session: - stmt = ( - select(RunResource) - .where(RunResource.run_id == self._run_id) - .where(RunResource.name == name) - .order_by(RunResource.created_at.desc()) # type: ignore[arg-type] - .limit(1) - ) - row = session.exec(stmt).first() - - if row is None: - raise ResourceNotFoundError( - f"No run_resource named {name!r} for run {self._run_id}" - ) - - return Path(row.file_path).read_bytes() - - async def list_resources(self) -> list[RunResourceView]: - """Return all ``RunResourceView`` DTOs for this run, newest first.""" - with get_session() as session: - stmt = ( - select(RunResource) - .where(RunResource.run_id == self._run_id) - .order_by(RunResource.created_at.desc()) # type: ignore[arg-type] - ) - rows = list(session.exec(stmt).all()) - return [RunResourceView.from_row(r) for r in rows] - - # ── DB access (new) ─────────────────────────────────────────────── - - def db_read_session(self) -> Session: - """Return a ``sqlmodel.Session`` for read-only queries. - - The caller owns the session lifecycle. Use as a context manager: - - with runtime.db_read_session() as s: - result = s.exec(select(RunRecord).where(...)).first() - - Mutating writes via this session violate the intent but are not - blocked at runtime in v1. - """ - return get_session() - - # ── event emission (new) ────────────────────────────────────────── - - def event_sink(self) -> SandboxEventSink: - """Return the ``SandboxEventSink`` wired to the dashboard emitter.""" - return self._event_sink -``` - -### Migrated `swebench_verified/criterion.py` (Task 0 — `_spawn_eval_sandbox` removed) - -Only the changed region is shown; the rest of the file is unchanged. - -```python -# ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py -# Task 0 diff target: remove _spawn_eval_sandbox, use runtime.ensure_sandbox() - -# REMOVED imports: -# from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import SWEBenchSandboxManager -# import uuid (no longer needed for sandbox_key allocation) - -# REMOVED function: -# async def _spawn_eval_sandbox(run_id: UUID) -> Any: ... - -# CHANGED in SWEBenchTestCriterion.evaluate(): - - async def evaluate(self, context: EvaluationContext) -> CriterionResult: - worker = context.worker_result - patch_text = (worker.artifacts or {}).get("patch") or worker.output or "" - if not patch_text.strip(): - return CriterionResult( - name=self.name, - score=0.0, - passed=False, - weight=self.weight, - feedback="Empty patch — agent did not produce any edits.", - metadata={}, - ) - - payload = context.task.task_payload - row = _payload_to_swebench_row(payload) - spec = make_test_spec(row) - - if context.runtime is None: - raise RuntimeError( - "SWEBenchTestCriterion requires a CriterionRuntime; " - "none was injected into EvaluationContext." - ) - - await context.runtime.ensure_sandbox() - sandbox = context.runtime.sandbox_manager.get_sandbox(context.run_id) - if sandbox is None: - return CriterionResult( - name=self.name, - score=0.0, - passed=False, - weight=self.weight, - feedback="Sandbox unavailable after ensure_sandbox().", - metadata={"error": "sandbox_unavailable"}, - ) - - try: - return await self._run_and_grade( - sandbox=sandbox, spec=spec, payload=payload, patch_text=patch_text - ) - finally: - # Runtime owns cleanup; do not kill here. - pass -``` - -**Note:** `context.runtime.sandbox_manager` is accessed directly because -`CriterionRuntime` does not expose the raw sandbox object — by design. This -is an acceptable internal coupling within `ergon_builtins` because -`DefaultCriterionRuntime` is the only concrete runtime in production and -`sandbox_manager` is a public attribute. If the Protocol later adds a -`run_command`-based harness path, this coupling can be removed. - -Alternatively: replace the `sandbox.commands.run(...)` calls in -`_run_and_grade` with `await context.runtime.run_command(...)` calls. That -would make `_run_and_grade` fully Protocol-bound. That migration is Phase 3 -scope. - -### Updated `InngestCriterionExecutor` (construction site) - -```python -# ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py -# Changed region only — construct DefaultCriterionRuntime with run_id + task_id - - runtime = DefaultCriterionRuntime( - context=criterion_context, - sandbox_manager=self.sandbox_manager, - run_id=task_context.run_id, # NEW - task_id=self.task_id, # NEW - ) -``` - -`self.task_id` is already stored on `InngestCriterionExecutor` at line 37 -(`self.task_id = task_id`). No other changes to that file. - -## Exact diffs - -### `ergon_core/ergon_core/api/criterion_runtime.py` - -```diff --from typing import Protocol, TypeVar -+from __future__ import annotations -+ -+from typing import TYPE_CHECKING, Protocol, TypeVar - --from pydantic import BaseModel, Field -+from pydantic import BaseModel, Field - - T = TypeVar("T", bound=BaseModel) - --__all__ = ["CommandResult", "CriterionRuntime", "SandboxResult"] -+if TYPE_CHECKING: -+ from sqlmodel import Session -+ -+ from ergon_core.api.run_resource import RunResourceView -+ from ergon_core.core.providers.sandbox.event_sink import SandboxEventSink -+ -+__all__ = ["CommandResult", "CriterionRuntime", "SandboxResult"] - - class CriterionRuntime(Protocol): - """...""" -+ """ -+ Surface-area constraint: narrowly scoped to sandbox lifecycle, -+ resource I/O, and event emission. -+ """ - - async def ensure_sandbox(self) -> None: ... - async def upload_files(self, files: list[dict]) -> None: ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def run_command(self, command: str, timeout: int = 30) -> CommandResult: ... - async def execute_code(self, code: str) -> SandboxResult: ... - async def call_llm_judge(self, messages: list, response_type: type[T]) -> T: ... - async def cleanup(self) -> None: ... -+ -+ async def read_resource(self, name: str) -> bytes: ... -+ async def list_resources(self) -> list[RunResourceView]: ... -+ def db_read_session(self) -> Session: ... -+ def event_sink(self) -> SandboxEventSink: ... -``` - -### `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` - -```diff -+from __future__ import annotations -+ - import logging -+from pathlib import Path - from typing import TYPE_CHECKING, TypeVar --from uuid import UUID -+from uuid import UUID - -+from ergon_core.api.run_resource import RunResourceView -+from ergon_core.core.persistence.shared.db import get_session -+from ergon_core.core.persistence.telemetry.models import RunResource -+from ergon_core.core.providers.sandbox.event_sink import ( -+ DashboardEmitterSandboxEventSink, -+ NoopSandboxEventSink, -+ SandboxEventSink, -+) - from ergon_core.core.runtime.evaluation.evaluation_schemas import CriterionContext -+from sqlmodel import Session, select - --__all__ = ["CriterionRuntime", "DefaultCriterionRuntime"] -+__all__ = ["CriterionRuntime", "DefaultCriterionRuntime", "ResourceNotFoundError"] -+ -+ -+class ResourceNotFoundError(LookupError): -+ """Raised by ``read_resource`` when no ``RunResource`` row matches the name.""" - - - class DefaultCriterionRuntime: - def __init__( - self, - context: CriterionContext, - sandbox_manager: "BaseSandboxManager", -+ run_id: UUID | None = None, -+ task_id: UUID | None = None, - llm_model: str = "gpt-4o", - llm_max_tokens: int = 1024, - llm_temperature: float = 0.0, -+ event_sink: SandboxEventSink | None = None, - ) -> None: - self.context = context - self.sandbox_manager: BaseSandboxManager = sandbox_manager -+ self._run_id: UUID = run_id if run_id is not None else context.run_id -+ self._task_id: UUID | None = task_id - self._owns_sandbox = False - self._llm_model = llm_model - self._llm_max_tokens = llm_max_tokens - self._llm_temperature = llm_temperature -+ self._event_sink: SandboxEventSink = event_sink or NoopSandboxEventSink() - -- async def ensure_sandbox(self) -> None: -- sandbox = self.sandbox_manager.get_sandbox(self.context.run_id) -+ async def ensure_sandbox(self) -> None: -+ sandbox = self.sandbox_manager.get_sandbox(self._run_id) - if sandbox is None: - await self.sandbox_manager.create( -- self.context.run_id, -- run_id=self.context.run_id, -+ self._run_id, -+ run_id=self._run_id, - timeout_minutes=30, - ) - ... -- await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=30) -+ await self.sandbox_manager.reset_timeout(self._run_id, timeout_minutes=30) - -- # [upload_files, write_file, run_command, execute_code: same change self.context.run_id → self._run_id] -+ # ... [all get_sandbox/reset_timeout calls updated analogously] - -- async def cleanup(self) -> None: -- if self._owns_sandbox: -- await self.sandbox_manager.terminate(self.context.run_id) -+ async def cleanup(self) -> None: -+ if self._owns_sandbox: -+ await self.sandbox_manager.terminate(self._run_id) - self._owns_sandbox = False - -+ async def read_resource(self, name: str) -> bytes: -+ with get_session() as session: -+ stmt = ( -+ select(RunResource) -+ .where(RunResource.run_id == self._run_id) -+ .where(RunResource.name == name) -+ .order_by(RunResource.created_at.desc()) -+ .limit(1) -+ ) -+ row = session.exec(stmt).first() -+ if row is None: -+ raise ResourceNotFoundError( -+ f"No run_resource named {name!r} for run {self._run_id}" -+ ) -+ return Path(row.file_path).read_bytes() -+ -+ async def list_resources(self) -> list[RunResourceView]: -+ with get_session() as session: -+ stmt = ( -+ select(RunResource) -+ .where(RunResource.run_id == self._run_id) -+ .order_by(RunResource.created_at.desc()) -+ ) -+ rows = list(session.exec(stmt).all()) -+ return [RunResourceView.from_row(r) for r in rows] -+ -+ def db_read_session(self) -> Session: -+ return get_session() -+ -+ def event_sink(self) -> SandboxEventSink: -+ return self._event_sink -``` - -### `ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py` - -```diff - runtime = DefaultCriterionRuntime( - context=criterion_context, - sandbox_manager=self.sandbox_manager, -+ run_id=task_context.run_id, -+ task_id=self.task_id, - ) -``` - -### `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` - -```diff --import uuid - import logging - import shlex - import tempfile -+from pathlib import Path - from typing import Any, ClassVar - from uuid import UUID - --from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( -- SWEBenchSandboxManager, --) - from ergon_builtins.workers.baselines.swebench_worker import _payload_to_swebench_row - --async def _spawn_eval_sandbox(run_id: UUID) -> Any: -- manager = SWEBenchSandboxManager() -- sandbox_key = uuid.uuid4() -- await manager.create(sandbox_key=sandbox_key, run_id=run_id) -- sandbox = manager.get_sandbox(sandbox_key) -- if sandbox is None: -- raise RuntimeError("Failed to acquire eval sandbox after create()") -- return sandbox -- - async def evaluate(self, context: EvaluationContext) -> CriterionResult: - ... -- sandbox = await _spawn_eval_sandbox(context.run_id) -- try: -- return await self._run_and_grade(...) -- finally: -- await _safe_kill(sandbox) -+ if context.runtime is None: -+ raise RuntimeError( -+ "SWEBenchTestCriterion requires a CriterionRuntime; " -+ "none was injected into EvaluationContext." -+ ) -+ await context.runtime.ensure_sandbox() -+ sandbox = context.runtime.sandbox_manager.get_sandbox(context.run_id) -+ if sandbox is None: -+ return CriterionResult( -+ name=self.name, score=0.0, passed=False, weight=self.weight, -+ feedback="Sandbox unavailable after ensure_sandbox().", -+ metadata={"error": "sandbox_unavailable"}, -+ ) -+ return await self._run_and_grade( -+ sandbox=sandbox, spec=spec, payload=payload, patch_text=patch_text -+ ) - --async def _safe_kill(sandbox: Any) -> None: ... # removed — runtime owns cleanup -``` - -### `tests/state/test_criteria_do_not_spawn_sandboxes.py` - -```diff --@pytest.mark.xfail( -- reason=( -- "swebench criterion still spawns its own sandbox; tracked in " -- "docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md" -- ), -- strict=False, --) - def test_criteria_do_not_instantiate_sandbox_managers() -> None: - offenders: list[str] = [] - for path in CRITERION_DIR.rglob("criterion.py"): - content = path.read_text() - if SANDBOX_MANAGER_PATTERN in content: - offenders.append(str(path)) - assert not offenders, f"Criterion files directly instantiate SandboxManager: {offenders}" -``` - -## Package structure - -No new packages are introduced. All changes are modifications within existing -packages. Updated `__all__` exports: - -`ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py`: - -```python -__all__ = ["CriterionRuntime", "DefaultCriterionRuntime", "ResourceNotFoundError"] -``` - -## Implementation order - -Phased into two PRs. - -### PR 1 — Protocol + `DefaultCriterionRuntime` extension (additive, no criterion changes) - -| Step | What | Files touched | -|---|---|---| -| 1 | Add `read_resource`, `list_resources`, `db_read_session`, `event_sink` signatures to `CriterionRuntime` Protocol | MODIFY `ergon_core/ergon_core/api/criterion_runtime.py` | -| 2 | Add `ResourceNotFoundError` to `criterion_runtime.py`; add `run_id`, `task_id`, `event_sink` params to `DefaultCriterionRuntime.__init__`; migrate all `self.context.run_id` → `self._run_id` calls | MODIFY `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | -| 3 | Pass `run_id=task_context.run_id`, `task_id=self.task_id` at `DefaultCriterionRuntime` construction | MODIFY `ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py` | -| 4 | Implement `read_resource` + `list_resources` (DB query + blob read) on `DefaultCriterionRuntime` | MODIFY `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | -| 5 | Implement `db_read_session` (returns `get_session()`) | MODIFY `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | -| 6 | Implement `event_sink` (returns stored `_event_sink`); wire `DashboardEmitterSandboxEventSink` in executor | MODIFY `criterion_runtime.py`, MODIFY `inngest_executor.py` | -| 7 | Unit tests for all four new methods | ADD `tests/unit/test_criterion_runtime_di.py` | - -### PR 2 — Task 0: swebench migration + xfail removal - -| Step | What | Files touched | -|---|---|---| -| 8 | Remove `_spawn_eval_sandbox`, remove `SWEBenchSandboxManager` import, replace criterion body with `runtime.ensure_sandbox()` path | MODIFY `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` | -| 9 | Remove `_safe_kill` (no longer needed — runtime owns sandbox cleanup) | MODIFY `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` | -| 10 | Remove `@pytest.mark.xfail` from state test | MODIFY `tests/state/test_criteria_do_not_spawn_sandboxes.py` | -| 11 | Integration test: `SWEBenchTestCriterion.evaluate` with a mock runtime, assert no `SWEBenchSandboxManager` instantiation | ADD `tests/unit/test_swebench_criterion_no_sandbox.py` | -| 12 | Close bug `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md` — move to `docs/bugs/fixed/` with `fixed_pr` set | MOVE (out of scope for this RFC; done when PR 2 merges) | - -### PR 3 — Phase 3 audit sweep (future, not blocked) - -Walk every `Criterion` subclass under `ergon_builtins/`; convert ad-hoc DB -reads, `RunResource` queries, and emitter access to the new Protocol methods. -Estimated scope: single-digit files. Not required for this RFC to be accepted. - -## File map - -### MODIFY - -| File | Change | -|---|---| -| `ergon_core/ergon_core/api/criterion_runtime.py` | Add 4 method signatures; add `TYPE_CHECKING` imports for `RunResourceView`, `SandboxEventSink`, `Session`; add docstring note on surface constraint | -| `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | Add `ResourceNotFoundError`; add `run_id`, `task_id`, `event_sink` params to `__init__`; migrate `self.context.run_id` → `self._run_id` throughout; implement `read_resource`, `list_resources`, `db_read_session`, `event_sink`; update `__all__` | -| `ergon_core/ergon_core/core/runtime/evaluation/inngest_executor.py` | Pass `run_id=task_context.run_id`, `task_id=self.task_id` to `DefaultCriterionRuntime` at line 73 | -| `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` | Remove `_spawn_eval_sandbox`, `_safe_kill`, `SWEBenchSandboxManager` import, `uuid` import; replace `evaluate` body with `runtime.ensure_sandbox()` path | -| `tests/state/test_criteria_do_not_spawn_sandboxes.py` | Remove `@pytest.mark.xfail` decorator (lines 19–25) | - -### ADD - -| File | Purpose | -|---|---| -| `tests/unit/test_criterion_runtime_di.py` | Unit tests: `read_resource` (found / not-found), `list_resources`, `db_read_session`, `event_sink` on `DefaultCriterionRuntime` | -| `tests/unit/test_swebench_criterion_no_sandbox.py` | Integration test: `SWEBenchTestCriterion.evaluate` with mock runtime; asserts `SWEBenchSandboxManager` never constructed | - -## Testing approach - -### Unit tests — `tests/unit/test_criterion_runtime_di.py` - -```python -# tests/unit/test_criterion_runtime_di.py - -from __future__ import annotations - -from pathlib import Path -from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import pytest - -from ergon_core.core.runtime.evaluation.criterion_runtime import ( - DefaultCriterionRuntime, - ResourceNotFoundError, -) -from ergon_core.core.runtime.evaluation.evaluation_schemas import CriterionContext - - -def _make_runtime(**overrides: Any) -> DefaultCriterionRuntime: - context = CriterionContext(run_id=uuid4()) - sandbox_manager = MagicMock() - kwargs: dict[str, Any] = { - "context": context, - "sandbox_manager": sandbox_manager, - } - kwargs.update(overrides) - return DefaultCriterionRuntime(**kwargs) - - -class TestReadResource: - def test_found_reads_blob(self, tmp_path: Path) -> None: - """read_resource returns bytes from file_path on disk.""" - blob = tmp_path / "abc" - blob.write_bytes(b"hello-world") - - from ergon_core.core.persistence.telemetry.models import RunResource - from datetime import datetime, UTC - - run_id = uuid4() - row = MagicMock(spec=RunResource) - row.file_path = str(blob) - row.run_id = run_id - row.name = "patch" - - runtime = _make_runtime(run_id=run_id) - - with patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session" - ) as mock_sess: - mock_ctx = MagicMock() - mock_ctx.__enter__ = MagicMock(return_value=mock_ctx) - mock_ctx.__exit__ = MagicMock(return_value=False) - mock_ctx.exec.return_value.first.return_value = row - mock_sess.return_value = mock_ctx - - import asyncio - result = asyncio.get_event_loop().run_until_complete( - runtime.read_resource("patch") - ) - - assert result == b"hello-world" - - def test_not_found_raises(self) -> None: - """read_resource raises ResourceNotFoundError when no row matches.""" - runtime = _make_runtime() - - with patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session" - ) as mock_sess: - mock_ctx = MagicMock() - mock_ctx.__enter__ = MagicMock(return_value=mock_ctx) - mock_ctx.__exit__ = MagicMock(return_value=False) - mock_ctx.exec.return_value.first.return_value = None - mock_sess.return_value = mock_ctx - - import asyncio - with pytest.raises(ResourceNotFoundError, match="no_such_resource"): - asyncio.get_event_loop().run_until_complete( - runtime.read_resource("no_such_resource") - ) - - -class TestListResources: - def test_returns_dtos_newest_first(self) -> None: - """list_resources maps ORM rows to RunResourceView DTOs.""" - from ergon_core.api.run_resource import RunResourceView - - runtime = _make_runtime() - mock_row = MagicMock() - - with ( - patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session" - ) as mock_sess, - patch.object(RunResourceView, "from_row", return_value=MagicMock()) as mock_from_row, - ): - mock_ctx = MagicMock() - mock_ctx.__enter__ = MagicMock(return_value=mock_ctx) - mock_ctx.__exit__ = MagicMock(return_value=False) - mock_ctx.exec.return_value.all.return_value = [mock_row] - mock_sess.return_value = mock_ctx - - import asyncio - result = asyncio.get_event_loop().run_until_complete(runtime.list_resources()) - - assert len(result) == 1 - mock_from_row.assert_called_once_with(mock_row) - - -class TestDbReadSession: - def test_returns_session(self) -> None: - """db_read_session returns a sqlmodel Session.""" - from sqlmodel import Session - - runtime = _make_runtime() - with patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session" - ) as mock_get: - mock_get.return_value = MagicMock(spec=Session) - sess = runtime.db_read_session() - assert sess is mock_get.return_value - - -class TestEventSink: - def test_returns_noop_by_default(self) -> None: - from ergon_core.core.providers.sandbox.event_sink import NoopSandboxEventSink - - runtime = _make_runtime() - assert isinstance(runtime.event_sink(), NoopSandboxEventSink) - - def test_returns_injected_sink(self) -> None: - from ergon_core.core.providers.sandbox.event_sink import DashboardEmitterSandboxEventSink - - emitter = MagicMock() - sink = DashboardEmitterSandboxEventSink(emitter) - runtime = _make_runtime(event_sink=sink) - assert runtime.event_sink() is sink - - -class TestRunIdResolution: - def test_explicit_run_id_overrides_context(self) -> None: - context = CriterionContext(run_id=uuid4()) - explicit_id = uuid4() - runtime = DefaultCriterionRuntime( - context=context, - sandbox_manager=MagicMock(), - run_id=explicit_id, - ) - assert runtime._run_id == explicit_id - - def test_default_falls_back_to_context(self) -> None: - context = CriterionContext(run_id=uuid4()) - runtime = DefaultCriterionRuntime( - context=context, - sandbox_manager=MagicMock(), - ) - assert runtime._run_id == context.run_id -``` - -### Unit test — `tests/unit/test_swebench_criterion_no_sandbox.py` - -```python -# tests/unit/test_swebench_criterion_no_sandbox.py - -"""Verify SWEBenchTestCriterion no longer constructs SWEBenchSandboxManager.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import pytest - -from ergon_builtins.benchmarks.swebench_verified.criterion import SWEBenchTestCriterion -from ergon_core.api.evaluation_context import EvaluationContext -from ergon_core.api.results import WorkerOutput -from ergon_core.api.task_types import BenchmarkTask - - -@pytest.mark.asyncio -async def test_evaluate_uses_runtime_ensure_sandbox() -> None: - """After Task 0: criterion calls runtime.ensure_sandbox(), not SWEBenchSandboxManager.""" - criterion = SWEBenchTestCriterion() - - mock_sandbox = MagicMock() - mock_sandbox.commands.run = AsyncMock(return_value=MagicMock(exit_code=0, stdout="")) - mock_sandbox.files.write = AsyncMock() - - mock_runtime = MagicMock() - mock_runtime.ensure_sandbox = AsyncMock() - mock_runtime.sandbox_manager.get_sandbox.return_value = mock_sandbox - - context = EvaluationContext( - run_id=uuid4(), - task_id=uuid4(), - execution_id=uuid4(), - task=BenchmarkTask( - task_key="swe-001", - instance_key="inst-001", - description="fix the bug", - task_payload={ - "instance_id": "swe-001", - "test_patch": "", - "base_commit": "abc123", - }, - ), - worker_result=WorkerOutput(output="diff --git a/foo.py b/foo.py\n+pass"), - sandbox_id="sbx-test", - runtime=mock_runtime, - ) - - with patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.make_test_spec" - ) as mock_spec, patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.get_eval_report" - ) as mock_report: - mock_spec.return_value = MagicMock( - install_repo_script="echo ok", - eval_script="echo ok", - ) - mock_report.return_value = { - "swe-001": {"resolved": True, "tests_status": {}} - } - await criterion.evaluate(context) - - mock_runtime.ensure_sandbox.assert_awaited_once() - - # Confirm SWEBenchSandboxManager was NOT instantiated. - import ergon_builtins.benchmarks.swebench_verified.criterion as m - assert not hasattr(m, "_spawn_eval_sandbox"), ( - "_spawn_eval_sandbox must be removed after Task 0" - ) -``` - -### State test (after xfail removal) — existing file - -`tests/state/test_criteria_do_not_spawn_sandboxes.py` — unchanged except -removal of the `@pytest.mark.xfail` decorator. After Task 0 merges, the test -passes unconditionally and serves as a hard lint-style enforcement that -no criterion re-introduces direct sandbox construction. - -## Trace / observability impact - -### Existing span unchanged - -`InngestCriterionExecutor` emits a `"evaluation.criterion"` span per criterion -at `inngest_executor.py:101–126`. No new span attributes are added by this RFC. - -### New span attribute: `resource_reads` - -When `read_resource` is called, add an INFO log with resource name and size: - -```python -logger.info( - "criterion read_resource run_id=%s name=%s size_bytes=%d", - self._run_id, - name, - len(result), -) -``` - -This is a log, not a span attribute mutation, so no trace-schema migration is -required. If observability for resource reads becomes important, a counter -metric or span event can be added in a follow-up. - -### No dashboard event from resource reads - -`event_sink()` is surfaced to criteria for progress streaming (e.g. -`sandbox_command` events during agentic evaluation). The runtime itself does -not emit events on `read_resource` / `list_resources` calls — those are -transparent to the dashboard. - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| `read_resource` reads a stale or incomplete blob if two tasks publish the same name | Criterion gets wrong data | Query orders by `created_at DESC`, returns latest row. Document that name collisions across task executions are caller's responsibility. | -| `db_read_session()` used for writes (convention not enforced) | Unintended DB mutations inside criterion | Add "read-only" to docstring and naming; v2 can enforce via a read-only engine URL or a wrapper that raises on `session.add()`. | -| `context.runtime.sandbox_manager` accessed in swebench criterion (internal coupling) | Breaks if `DefaultCriterionRuntime.sandbox_manager` is renamed | Attribute is public in `DefaultCriterionRuntime`; rename would be caught by `ty`. Long-term: replace `sandbox.commands.run(...)` calls with `runtime.run_command(...)`. | -| `_spawn_eval_sandbox` removal breaks a criterion caller that was not identified | `SWEBenchTestCriterion.evaluate` fails with `AttributeError` | `grep` confirms `_spawn_eval_sandbox` is only called from `evaluate` at `criterion.py:156`; one callsite. | -| `NoopSandboxEventSink` injected at construction but criterion calls `await sink.sandbox_command(...)` synchronously | `NoopSandboxEventSink` is async-compatible — all methods are `async def` at `event_sink.py:11–31`. No risk. | Already handled. | -| Protocol structural compliance check misses the four new methods for existing mock runtimes in tests | Tests silently pass with incomplete mocks | Add a `TestCriterionRuntimeProtocolCompliance` test class that instantiates a `MagicMock(spec=CriterionRuntime)` and asserts all eleven methods exist. | - -## Invariants affected - -### `docs/architecture/01_public_api.md#criterionruntime` - -Current text (lines 48–54): - -> **`CriterionRuntime`** — Protocol. The execution context an agentic -> criterion uses to reach into its environment. **Surface-area constraint:** -> this Protocol is narrowly scoped to sandbox lifecycle and resource I/O; it -> should not grow into a generic service locator. The one current method that -> is not about sandbox/I/O is a candidate for extraction if the surface -> continues to accumulate capabilities. Expansion is in flight — see -> follow-ups. - -After this RFC: update to reflect seven existing + four new methods. Surface -now includes sandbox lifecycle (7 methods), resource I/O (`read_resource`, -`list_resources`), DB read access (`db_read_session`), and event emission -(`event_sink`). Follow-up remains: if the surface keeps growing, -`call_llm_judge` is a candidate for extraction into `LLMJudgeMixin`. - -Code-map table: add `ResourceNotFoundError | ergon_core/core/runtime/evaluation/criterion_runtime.py`. - -### `docs/architecture/01_public_api.md` — anti-patterns - -Current text (lines 213–214): - -> - **Criteria that allocate their own sandbox.** Agentic criteria must run in -> the task's existing sandbox via the `CriterionRuntime` seam. Current -> offender: `ergon_builtins/benchmarks/swebench_verified/criterion.py:72` -> constructs its own sandbox manager — tracked in the follow-up below. - -After Phase 2 (Task 0) merges: remove the callout to the swebench offender. -The anti-pattern statement remains ("criteria that allocate their own sandbox -MUST NOT"). The follow-up note moves from `docs/architecture/01_public_api.md` -to the accepted RFC. - -### `docs/architecture/06_builtins.md#anti-patterns` - -Current text (lines 149–151): - -> - **A Criterion spawning its own sandbox.** Current known offender: -> `ergon_builtins/benchmarks/swebench_verified/criterion.py:72`; tracked at -> `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md`. - -After Phase 2: remove the offender callout. The invariant statement at -`docs/architecture/06_builtins.md` line 104 (`Criteria MUST NOT spawn their -own sandboxes`) is strengthened to: "Enforced by -`tests/state/test_criteria_do_not_spawn_sandboxes.py` (hard-pass after -2026-04-17 RFC)." - -## Alternatives considered - -- **Keep `call_llm_judge()` as a first-class Protocol method.** Tentatively - accepted. It is the only method not about sandbox/resource/event I/O — a - convenience wrapper. If the Protocol keeps growing, extract it into an - `LLMJudgeMixin` or helper module. Flagged as future cleanup, not a blocker. - -- **Add `get_sandbox()` as the earlier draft of this RFC proposed.** Rejected: - `ensure_sandbox()` at `criterion_runtime.py:57` already covers "provision - or reconnect and hand back the task sandbox." A separate accessor would - split the responsibility without benefit. Note: the bug report at - `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md` references - `runtime.get_sandbox()` — that proposed fix is superseded by the - `ensure_sandbox()` + direct `sandbox_manager.get_sandbox()` path adopted here. - -- **Move all Protocol methods into an abstract base class.** Rejected. - Protocols stay structural so criterion authors do not need to inherit. - `DefaultCriterionRuntime` does not subclass the Protocol and doesn't need to. - -- **Separate Protocols for agentic vs. non-agentic criteria.** Rejected. - Every criterion benefits uniformly from resource and DB access. - -- **`db_read_session()`: read-only via session-factory split.** Deferred to v2. - A separate read-only engine URL (pointing to a replica or using the same - engine with `execution_options(no_autocommit=True)`) is architecturally - cleaner but adds infra complexity. v1 uses the same `get_session()` factory - with a documented convention ("do not write"). - -## Open questions - -- Should `event_sink()` return a narrow interface (e.g. "emit criterion - progress event" only) or the full `SandboxEventSink`? Narrow is safer and - keeps criteria out of unrelated emitter surface; broad is more flexible for - agentic criteria that want to stream their own structured updates. Decision - deferred to implementation review. - -- `db_read_session()`: read-only via session-factory split, or a regular - session with a "please don't write" convention? Strict is safer; convention - is cheaper. Deferred to PR 1 implementation review. - -- Phase 3 audit scope: are there non-swebench criteria in `ergon_builtins/` - that already open DB sessions directly? A one-time `grep` before PR 2 - merges will confirm whether the audit has zero or nonzero work. - -## On acceptance - -- [ ] Move this file from `docs/rfcs/active/` to `docs/rfcs/accepted/`. -- [ ] Update `docs/architecture/01_public_api.md#criterionruntime` to reflect - the expanded surface (seven existing + four new methods); remove offender - callout from anti-patterns after Phase 2. -- [ ] Update `docs/architecture/06_builtins.md` — strengthen "Criteria MUST - NOT spawn their own sandboxes" invariant to reference hard-enforcement - by state test; remove swebench-verified callout. -- [ ] Close `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md` - once Task 0 lands — move to `docs/bugs/fixed/` with `fixed_pr` set. diff --git a/docs/rfcs/accepted/2026-04-17-delete-run-task-state-event.md b/docs/rfcs/accepted/2026-04-17-delete-run-task-state-event.md deleted file mode 100644 index b08737f63..000000000 --- a/docs/rfcs/accepted/2026-04-17-delete-run-task-state-event.md +++ /dev/null @@ -1,861 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/02_runtime_lifecycle.md#invariants, docs/architecture/04_persistence.md#core-abstractions] -supersedes: [] -superseded_by: null ---- - -# RFC: Delete `RunTaskStateEvent` and its legacy queries - -## Problem - -`RunTaskStateEvent` (`ergon_core/ergon_core/core/persistence/telemetry/models.py:243–278`) -is the legacy append-only WAL for task status transitions. It was superseded by -`RunGraphMutation` (`ergon_core/ergon_core/core/persistence/graph/models.py:174–187`), -which is a strict superset of the same data plus monotonic `sequence` per run. Two -sources of truth is a maintenance hazard: every contributor reading the persistence -layer must trace the abandoned table, confirm it is abandoned, and then internalise -"ignore this". - -**What exists today:** - -- `RunTaskStateEvent` is defined at `telemetry/models.py:243` with five columns: - `run_id`, `definition_task_id`, `task_execution_id`, `event_type`, - `old_status`, `new_status`, `event_metadata`, `created_at`. A `model_validator` - (`_validate_event_metadata`) exists at line 276. -- `propagation.py` (line 7) declares: `"RunTaskStateEvent is no longer written or - read by this module."` No write site exists anywhere in the codebase. -- `StateEventsQueries` at `persistence/queries.py:266–307` is the last reader. - It exposes four methods (`list_by_run`, `get_by_task`, `get_latest_status`, - `get_by_event_type`) wired into the `Queries` singleton at line 475/483. Nothing - calls `queries.state_events.*` anywhere in the runtime (verified by grep). -- The `Queries` singleton at `queries.py:469–488` exposes `state_events: - StateEventsQueries` as a named attribute, so any contributor who reads the - singleton's fields will see this as a live entry point. -- Three test files import or reference `RunTaskStateEvent`: - - `tests/state/test_type_invariants.py:25` — imports and constructs - `RunTaskStateEvent` to verify enum constraints. - - `tests/state/test_type_invariants.py:53–71` — `TestRunTaskStateEventTypes` - class with two test methods. - - `tests/state/test_propagation.py:225` — docstring mentions - `RunTaskStateEvent` in a test class verifying writes go to `RunGraphNode`, - not `RunTaskStateEvent`. -- `docs/event-wal/STATE_UNIFICATION_PLAN.md` documents the migration to graph WAL - and lists `RunTaskStateEvent` write sites (now all gone). -- `docs/TY_PASS_PLAN.md` lists `RunTaskStateEvent.task_execution_id` in the - nullable-field audit (Category G.4, correctly optional). -- `docs/architecture/04_persistence.md` §4 has the invariant: "**`RunTaskStateEvent` - is frozen.** No new writes. Reads permitted only for legacy-run rehydration." - §7 has the follow-up: "An RFC is in flight to remove the legacy table outright." -- `docs/architecture/02_runtime_lifecycle.md` §4.1 (Known limits) has: - "`RunTaskStateEvent` is deprecated and unread. [...] `StateEventsQueries` is the - last reader and goes away with the table in this RFC." -- The table was created in the initial schema migration - `5f01559f2bc3_initial_schema_v2.py` (revision `5f01559f2bc3`). The current - head of the migration chain is `b5b36e45e5e6_add_containment_and_cancelled.py` - (`down_revision = "f9075c2ddbc9"`). - -The cost of leaving this in place is real. The `Queries` singleton surfaces -`state_events` as a live entry point; `test_type_invariants.py` keeps the class -exercised; the architecture docs carry two clauses that are strictly false (there -is no rehydration use of this table). The longer this sits, the more likely a -contributor wires a new reader against it. - -## Proposal - -Six changes, ordered to guarantee no races: - -1. **Data export** — before the drop, stream any remaining rows to - `exports/run_task_state_events_<timestamp>.jsonl.gz` via a checked-in - one-shot script (`scripts/export_run_task_state_events.py`). The script - runs as part of the deploy that applies the Alembic migration. -2. **Alembic revision** — new revision dropping `run_task_state_events` and - its three indexes (`ix_run_task_state_events_run_id`, - `ix_run_task_state_events_event_type`, - `ix_run_task_state_events_definition_task_id`). The revision also drops the - `experimentcohortstatus`-family Postgres enum if still owned by this table - (verify in migration autogen — `old_status`/`new_status` are `TaskExecutionStatus`, - not a named enum). `down_revision` must point to `b5b36e45e5e6`. -3. **Delete `RunTaskStateEvent` model** and `_validate_event_metadata` validator - (`telemetry/models.py:239–278`). Drop the import of `RunTaskStateEvent` from - `queries.py:30`. -4. **Delete `StateEventsQueries`** (`queries.py:266–307`) and remove - `state_events: StateEventsQueries` from the `Queries` class (lines 475/483). -5. **Purge test references** — remove `TestRunTaskStateEventTypes` and the - `RunTaskStateEvent` import from `test_type_invariants.py`; update the - `TestGraphStateVerification` docstring in `test_propagation.py`. -6. **Update docs** — delete `docs/event-wal/STATE_UNIFICATION_PLAN.md`, - prune `RunTaskStateEvent` references from `docs/event-wal/01_AUDIT.md` and - `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md`, remove the nullable audit row - from `docs/TY_PASS_PLAN.md`, and update architecture docs (see §Invariants - affected). -7. **Sentinel test** — add `tests/state/test_legacy_wal_absent.py` asserting the - table does not appear in `pg_tables`. - -The Alembic drop must precede the model deletion commit so that `alembic -autogenerate` does not race against a missing SQLModel class. In practice the -drop revision and model deletion land in the same PR, with the migration script -committed first. - -## Architecture overview - -### Before - -``` -persistence/queries.py - Queries.state_events: StateEventsQueries ← orphaned; nothing calls it - StateEventsQueries ← wraps RunTaskStateEvent - list_by_run() - get_by_task() - get_latest_status() - get_by_event_type() - -persistence/telemetry/models.py - RunTaskStateEvent ← table: run_task_state_events - id, run_id, definition_task_id, - task_execution_id, event_type, - old_status, new_status, - event_metadata, created_at - _validate_event_metadata ← model_validator - -DB schema - run_task_state_events (0 new writes since propagation.py refactor) - ix_run_task_state_events_run_id - ix_run_task_state_events_event_type - ix_run_task_state_events_definition_task_id - -Single source of truth for task state: - RunGraphMutation ← append-only, monotonic sequence - RunGraphNode ← mutable status cache -``` - -### After - -``` -persistence/queries.py - Queries (no state_events attribute) ← leaner singleton - -persistence/telemetry/models.py - (RunTaskStateEvent deleted) - -DB schema - (run_task_state_events table gone) - (three indexes gone) - -Single source of truth for task state: - RunGraphMutation ← unchanged, still canonical - RunGraphNode ← unchanged -``` - -### Data flow (no change to runtime) - -The runtime never wrote to `run_task_state_events` after the propagation -refactor. The data flow through `WorkflowGraphRepository` → -`RunGraphMutation` / `RunGraphNode` is unchanged by this RFC. Removing the -table removes dead weight, not any live path. - -## Type / interface definitions - -No new types. The following types are **deleted**: - -```python -# DELETED FROM: ergon_core/ergon_core/core/persistence/telemetry/models.py -# Lines 239–278 - -# --------------------------------------------------------------------------- -# RunTaskStateEvent -# --------------------------------------------------------------------------- - -class RunTaskStateEvent(SQLModel, table=True): - __tablename__ = "run_task_state_events" - - id: UUID = Field(default_factory=uuid4, primary_key=True) - run_id: UUID = Field(foreign_key="runs.id", index=True) - definition_task_id: UUID = Field( - foreign_key="experiment_definition_tasks.id", - index=True, - ) - task_execution_id: UUID | None = Field( - default=None, - foreign_key="run_task_executions.id", - ) - event_type: str = Field( - default="state_change", index=True - ) - old_status: TaskExecutionStatus | None = None - new_status: TaskExecutionStatus - event_metadata: dict = Field(default_factory=dict, sa_column=Column(JSON)) - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - - def parsed_event_metadata(self) -> dict[str, object]: ... - @classmethod - def _parse_event_metadata(cls, data: dict) -> dict[str, object]: ... - @model_validator(mode="after") - def _validate_event_metadata(self) -> "RunTaskStateEvent": ... -``` - -```python -# DELETED FROM: ergon_core/ergon_core/core/persistence/queries.py -# Lines 266–307, and the state_events attribute at 475/483 - -class StateEventsQueries(BaseQueries[RunTaskStateEvent]): - def __init__(self) -> None: ... - def list_by_run(self, run_id: UUID) -> list[RunTaskStateEvent]: ... - def get_by_task(self, run_id: UUID, definition_task_id: UUID) -> list[RunTaskStateEvent]: ... - def get_latest_status(self, run_id: UUID, definition_task_id: UUID) -> str | None: ... - def get_by_event_type(self, run_id: UUID, event_type: str) -> list[RunTaskStateEvent]: ... -``` - -## New files - -### `scripts/export_run_task_state_events.py` - -```python -# scripts/export_run_task_state_events.py -"""One-shot export of run_task_state_events rows to JSONL.gz. - -Run BEFORE applying the Alembic migration that drops the table. - -Usage: - uv run python scripts/export_run_task_state_events.py - -Output: exports/run_task_state_events_<ISO8601_timestamp>.jsonl.gz - -The archive is insurance. Most systems have 0 rows since propagation.py -stopped writing to this table. The script is idempotent — running it twice -produces two archive files; the table is not modified. -""" - -from __future__ import annotations - -import gzip -import json -import os -import sys -from datetime import UTC, datetime -from pathlib import Path - -# Ensure ergon_core is importable when run from the repo root. -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.telemetry.models import RunTaskStateEvent -from sqlmodel import select - - -def main() -> None: - timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") - exports_dir = Path(__file__).parent.parent / "exports" - exports_dir.mkdir(exist_ok=True) - out_path = exports_dir / f"run_task_state_events_{timestamp}.jsonl.gz" - - row_count = 0 - with gzip.open(out_path, "wt", encoding="utf-8") as fh: - with get_session() as session: - stmt = select(RunTaskStateEvent).order_by(RunTaskStateEvent.created_at) - for row in session.exec(stmt): - fh.write( - json.dumps( - { - "id": str(row.id), - "run_id": str(row.run_id), - "definition_task_id": str(row.definition_task_id), - "task_execution_id": str(row.task_execution_id) - if row.task_execution_id - else None, - "event_type": row.event_type, - "old_status": row.old_status, - "new_status": row.new_status, - "event_metadata": row.event_metadata, - "created_at": row.created_at.isoformat(), - } - ) - + "\n" - ) - row_count += 1 - - print(f"Exported {row_count} rows to {out_path}") - - -if __name__ == "__main__": - main() -``` - -### `ergon_core/migrations/versions/<revision_id>_drop_run_task_state_events.py` - -The revision ID is generated by `alembic revision --autogenerate`. The template -below shows the expected structure; the implementer must run autogenerate (or -author by hand) after deleting the `RunTaskStateEvent` model so that Alembic -detects the table removal. - -```python -# ergon_core/migrations/versions/<rev>_drop_run_task_state_events.py -"""drop run_task_state_events table - -Revision ID: <generated> -Revises: b5b36e45e5e6 -Create Date: <generated> - -Drops the legacy RunTaskStateEvent table and its three indexes. -Data was exported to exports/run_task_state_events_<timestamp>.jsonl.gz -before this migration ran. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -import sqlmodel -from alembic import op - -revision: str = "<generated>" -down_revision: Union[str, None] = "b5b36e45e5e6" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_index( - op.f("ix_run_task_state_events_definition_task_id"), - table_name="run_task_state_events", - ) - op.drop_index( - op.f("ix_run_task_state_events_event_type"), - table_name="run_task_state_events", - ) - op.drop_index( - op.f("ix_run_task_state_events_run_id"), - table_name="run_task_state_events", - ) - op.drop_table("run_task_state_events") - - -def downgrade() -> None: - op.create_table( - "run_task_state_events", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("run_id", sa.Uuid(), nullable=False), - sa.Column("definition_task_id", sa.Uuid(), nullable=False), - sa.Column("task_execution_id", sa.Uuid(), nullable=True), - sa.Column("event_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column("old_status", sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column("new_status", sqlmodel.sql.sqltypes.AutoString(), nullable=False), - sa.Column("event_metadata", sa.JSON(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["definition_task_id"], ["experiment_definition_tasks.id"]), - sa.ForeignKeyConstraint(["run_id"], ["runs.id"]), - sa.ForeignKeyConstraint(["task_execution_id"], ["run_task_executions.id"]), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - op.f("ix_run_task_state_events_run_id"), - "run_task_state_events", - ["run_id"], - unique=False, - ) - op.create_index( - op.f("ix_run_task_state_events_event_type"), - "run_task_state_events", - ["event_type"], - unique=False, - ) - op.create_index( - op.f("ix_run_task_state_events_definition_task_id"), - "run_task_state_events", - ["definition_task_id"], - unique=False, - ) -``` - -### `tests/state/test_legacy_wal_absent.py` - -```python -# tests/state/test_legacy_wal_absent.py -"""Sentinel: run_task_state_events must not exist in the schema. - -If someone re-introduces the table by copy-paste, this test fails immediately -in CI. The check runs against the same test DB used by all other state tests -(SQLite in tests, Postgres in integration). -""" - -from sqlmodel import Session, text - - -def test_run_task_state_events_table_absent(session: Session) -> None: - """The legacy table must not exist in the current schema.""" - # Works for both Postgres and SQLite. - # Postgres: pg_tables is a system catalog view. - # SQLite: sqlite_master is the schema table. - try: - # Postgres path - result = session.exec( - text( - "SELECT tablename FROM pg_tables " - "WHERE schemaname = 'public' " - "AND tablename = 'run_task_state_events'" - ) - ).all() - assert result == [], ( - "run_task_state_events still exists in pg_tables — " - "migration not applied or table re-created." - ) - except Exception: - # SQLite path (test environment) - result = session.exec( - text( - "SELECT name FROM sqlite_master " - "WHERE type='table' AND name='run_task_state_events'" - ) - ).all() - assert result == [], ( - "run_task_state_events still exists in sqlite_master — " - "SQLModel still has table=True for RunTaskStateEvent." - ) -``` - -## Exact diffs for modified files - -### `ergon_core/ergon_core/core/persistence/telemetry/models.py` - -```diff --# --------------------------------------------------------------------------- --# RunTaskStateEvent --# --------------------------------------------------------------------------- -- -- --class RunTaskStateEvent(SQLModel, table=True): -- __tablename__ = "run_task_state_events" -- -- id: UUID = Field(default_factory=uuid4, primary_key=True) -- run_id: UUID = Field(foreign_key="runs.id", index=True) -- definition_task_id: UUID = Field( -- foreign_key="experiment_definition_tasks.id", -- index=True, -- ) -- task_execution_id: UUID | None = Field( -- default=None, -- foreign_key="run_task_executions.id", -- ) -- event_type: str = Field( -- default="state_change", index=True -- ) # Literal["state_change"] — str for SQLModel compat -- old_status: TaskExecutionStatus | None = None -- new_status: TaskExecutionStatus -- event_metadata: dict = Field(default_factory=dict, sa_column=Column(JSON)) -- created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) -- -- # -- JSON accessor: event_metadata -- -- -- def parsed_event_metadata(self) -> dict[str, object]: -- return self.__class__._parse_event_metadata(self.event_metadata) -- -- @classmethod -- def _parse_event_metadata(cls, data: dict) -> dict[str, object]: -- if not isinstance(data, dict): -- raise ValueError(f"event_metadata must be a dict, got {type(data).__name__}") -- return data -- -- @model_validator(mode="after") -- def _validate_event_metadata(self) -> "RunTaskStateEvent": -- self.__class__._parse_event_metadata(self.event_metadata) -- return self -- -- - # --------------------------------------------------------------------------- - # RunTaskEvaluation - # --------------------------------------------------------------------------- -``` - -The `model_validator` import at the top of `models.py` (`from pydantic import -model_validator`) must be retained — other models in the same file use it -(`RunRecord._validate_summary_json`, `RunTaskExecution._validate_output_json`, -etc.). No import change needed. - -### `ergon_core/ergon_core/core/persistence/queries.py` - -```diff - from ergon_core.core.persistence.telemetry.models import ( - RunRecord, - RunResource, - RunTaskEvaluation, - RunTaskExecution, -- RunTaskStateEvent, - ) -``` - -```diff --# --------------------------------------------------------------------------- --# State Events --# --------------------------------------------------------------------------- -- -- --class StateEventsQueries(BaseQueries[RunTaskStateEvent]): -- def __init__(self) -> None: -- super().__init__(RunTaskStateEvent) -- -- def list_by_run(self, run_id: UUID) -> list[RunTaskStateEvent]: -- with get_session() as session: -- stmt = ( -- select(RunTaskStateEvent) -- .where(RunTaskStateEvent.run_id == run_id) -- .order_by(RunTaskStateEvent.created_at) -- ) -- return list(session.exec(stmt).all()) -- -- def get_by_task(self, run_id: UUID, definition_task_id: UUID) -> list[RunTaskStateEvent]: -- with get_session() as session: -- stmt = ( -- select(RunTaskStateEvent) -- .where( -- RunTaskStateEvent.run_id == run_id, -- RunTaskStateEvent.definition_task_id == definition_task_id, -- ) -- .order_by(RunTaskStateEvent.created_at) -- ) -- return list(session.exec(stmt).all()) -- -- def get_latest_status(self, run_id: UUID, definition_task_id: UUID) -> str | None: -- events = self.get_by_task(run_id, definition_task_id) -- if not events: -- return None -- return events[-1].new_status -- -- def get_by_event_type(self, run_id: UUID, event_type: str) -> list[RunTaskStateEvent]: -- with get_session() as session: -- stmt = ( -- select(RunTaskStateEvent) -- .where( -- RunTaskStateEvent.run_id == run_id, -- RunTaskStateEvent.event_type == event_type, -- ) -- .order_by(RunTaskStateEvent.created_at) -- ) -- return list(session.exec(stmt).all()) -- -- - # --------------------------------------------------------------------------- - # Evaluations - # --------------------------------------------------------------------------- -``` - -```diff - class Queries: - """Namespace singleton providing typed query methods for all tables.""" - - runs: RunsQueries - definitions: DefinitionsQueries - task_executions: TaskExecutionsQueries -- state_events: StateEventsQueries - evaluations: EvaluationsQueries - resources: ResourcesQueries - - def __init__(self) -> None: - self.runs = RunsQueries() - self.definitions = DefinitionsQueries() - self.task_executions = TaskExecutionsQueries() -- self.state_events = StateEventsQueries() - self.evaluations = EvaluationsQueries() - self.resources = ResourcesQueries() -``` - -### `tests/state/test_type_invariants.py` - -```diff - from ergon_core.core.persistence.telemetry.models import ( - ExperimentCohort, - ExperimentCohortStatus, - RunGenerationTurn, - RunRecord, - RunResource, - RunTaskExecution, -- RunTaskStateEvent, - TrainingSession, - ) -``` - -```diff --class TestRunTaskStateEventTypes: -- def test_accepts_valid_event_type(self): -- event = RunTaskStateEvent( -- run_id=uuid4(), -- definition_task_id=uuid4(), -- event_type="state_change", -- new_status=TaskExecutionStatus.COMPLETED, -- ) -- assert event.event_type == "state_change" -- -- def test_accepts_valid_old_status(self): -- event = RunTaskStateEvent( -- run_id=uuid4(), -- definition_task_id=uuid4(), -- event_type="state_change", -- old_status=TaskExecutionStatus.PENDING, -- new_status=TaskExecutionStatus.RUNNING, -- ) -- assert event.old_status == TaskExecutionStatus.PENDING -- -- - class TestExperimentCohortStatus: -``` - -### `tests/state/test_propagation.py` (docstring only) - -```diff - class TestGraphStateVerification: -- """Verify that state is written to RunGraphNode and RunGraphMutation, -- not to RunTaskStateEvent.""" -+ """Verify that state is written to RunGraphNode and RunGraphMutation.""" -``` - -## Implementation order - -### Phase 1 — Pre-flight and data export (single PR) - -| Step | What | Files touched | -|---|---|---| -| 1 | Write and commit `scripts/export_run_task_state_events.py` | ADD `scripts/export_run_task_state_events.py` | -| 2 | Run the export script locally and commit any resulting archive to `exports/` (or document that it produced 0 rows) | ADD `exports/run_task_state_events_<ts>.jsonl.gz` (if non-empty) | -| 3 | Write the pre-migration audit script (inline in PR description) checking that every `run_task_state_events` row has a corresponding `run_graph_mutations` entry for the same `run_id` / `definition_task_id` status transition. Note result. | (no file; inline check) | - -### Phase 2 — Schema drop (single PR, merges after Phase 1) - -| Step | What | Files touched | -|---|---|---| -| 4 | Delete `RunTaskStateEvent` class from `telemetry/models.py` (lines 239–278) | MODIFY `ergon_core/ergon_core/core/persistence/telemetry/models.py` | -| 5 | Run `alembic revision --autogenerate -m "drop_run_task_state_events"` to produce the migration; verify it contains only `drop_table` + three `drop_index`; set `down_revision = "b5b36e45e5e6"` | ADD `ergon_core/migrations/versions/<rev>_drop_run_task_state_events.py` | -| 6 | Delete `RunTaskStateEvent` import and `StateEventsQueries` class from `queries.py`; remove `state_events` from `Queries.__init__` | MODIFY `ergon_core/ergon_core/core/persistence/queries.py` | -| 7 | Remove `TestRunTaskStateEventTypes` and `RunTaskStateEvent` import from `test_type_invariants.py` | MODIFY `tests/state/test_type_invariants.py` | -| 8 | Update docstring in `test_propagation.py:225` | MODIFY `tests/state/test_propagation.py` | -| 9 | Add sentinel test `test_legacy_wal_absent.py` | ADD `tests/state/test_legacy_wal_absent.py` | - -### Phase 3 — Docs cleanup (same PR as Phase 2 or immediate follow-up) - -| Step | What | Files touched | -|---|---|---| -| 10 | Delete `docs/event-wal/STATE_UNIFICATION_PLAN.md` | DELETE `docs/event-wal/STATE_UNIFICATION_PLAN.md` | -| 11 | Prune `RunTaskStateEvent` references in `docs/event-wal/01_AUDIT.md` (§2 tables, §4.2 fix options) and `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md` (§6.3 type-tightening list, §14 MODIFY block) | MODIFY `docs/event-wal/01_AUDIT.md`, `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md` | -| 12 | Remove `RunTaskStateEvent.task_execution_id` row from `docs/TY_PASS_PLAN.md` §G.4 table | MODIFY `docs/TY_PASS_PLAN.md` | -| 13 | Update `docs/architecture/02_runtime_lifecycle.md` and `docs/architecture/04_persistence.md` (see §Invariants affected) | MODIFY both architecture docs | - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `scripts/export_run_task_state_events.py` | One-shot JSONL.gz export before table drop | -| `exports/run_task_state_events_<ts>.jsonl.gz` | Archive of any remaining rows (may be 0 bytes) | -| `ergon_core/migrations/versions/<rev>_drop_run_task_state_events.py` | Alembic revision dropping `run_task_state_events` and its three indexes; `down_revision = "b5b36e45e5e6"` | -| `tests/state/test_legacy_wal_absent.py` | Sentinel asserting `run_task_state_events` is absent from `pg_tables` / `sqlite_master` | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/persistence/telemetry/models.py` | Delete lines 239–278: `RunTaskStateEvent` class and `_validate_event_metadata` validator | -| `ergon_core/ergon_core/core/persistence/queries.py` | Delete import of `RunTaskStateEvent` (line 30); delete `StateEventsQueries` class (lines 266–307); remove `state_events` attribute from `Queries` class (lines 475/483) | -| `tests/state/test_type_invariants.py` | Remove `RunTaskStateEvent` import (line 25); remove `TestRunTaskStateEventTypes` class (lines 53–71) | -| `tests/state/test_propagation.py` | Update `TestGraphStateVerification` docstring at line 225 | -| `docs/event-wal/01_AUDIT.md` | Prune §2 tables and §4.2 describing `RunTaskStateEvent` write sites (now historical) | -| `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md` | Remove `RunTaskStateEvent` from §6.3 type-tightening list and §14 MODIFY block | -| `docs/TY_PASS_PLAN.md` | Remove `RunTaskStateEvent.task_execution_id` row from §G.4 table | -| `docs/architecture/02_runtime_lifecycle.md` | See §Invariants affected | -| `docs/architecture/04_persistence.md` | See §Invariants affected | - -### DELETE - -| File | Reason | -|---|---| -| `docs/event-wal/STATE_UNIFICATION_PLAN.md` | The migration plan it describes is now complete; keeping it creates false impression of in-flight work | - -## Testing approach - -### Unit — `test_type_invariants.py` (post-removal) - -After removing `TestRunTaskStateEventTypes`, the remaining test coverage in -`test_type_invariants.py` verifies `RunRecord`, `RunTaskExecution`, -`ExperimentCohort`, `RunGenerationTurn`, etc. No regression to existing tests. - -No replacement test needed for `RunTaskStateEvent` construction — the class -no longer exists. Any attempt to import or instantiate it after removal will -fail at import time (module error), not silently. - -### Sentinel — `test_legacy_wal_absent.py` - -```python -# tests/state/test_legacy_wal_absent.py - -def test_run_task_state_events_table_absent(session: Session) -> None: - """Locks the removal: if the table is re-created, CI fails immediately.""" - try: - result = session.exec( - text( - "SELECT tablename FROM pg_tables " - "WHERE schemaname = 'public' " - "AND tablename = 'run_task_state_events'" - ) - ).all() - assert result == [] - except Exception: - result = session.exec( - text( - "SELECT name FROM sqlite_master " - "WHERE type='table' AND name='run_task_state_events'" - ) - ).all() - assert result == [] -``` - -**Note on SQLite test environment:** The sentinel uses a `try/except` to handle -both Postgres (CI/integration) and SQLite (fast unit tests). Verify that -`conftest.py` in `tests/state/` provides a `session` fixture compatible with -this approach. If the test session uses a SQLite backend (likely, given the -fast test suite design), the `except` branch handles it. - -### Integration — migration correctness - -After applying the migration against a Postgres instance seeded with the -initial schema, confirm: - -1. `\d run_task_state_events` in psql returns "did not find any relation named". -2. `alembic current` reports the new revision as head. -3. `alembic downgrade -1` restores the table (the `downgrade()` function is - provided in the revision). - -### Pre-migration audit (one-time, not automated) - -Before running in any environment with real data, run: - -```python -# Inline check — run in a migration script or psql session -SELECT - COUNT(*) AS state_event_rows, - COUNT(DISTINCT run_id) AS affected_runs -FROM run_task_state_events; -``` - -If this returns non-zero, run the export script first and verify the archive -before dropping. If zero, proceed directly to the migration. - -## Trace / observability impact - -None. `run_task_state_events` is not referenced by any span, metric, or log -emitter. No dashboard query reads from it. No trace attribute is sourced from -it. Removing the table and its queries is invisible to all observability paths. - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| Unreconciled rows in `run_task_state_events` | Historical state lost without archive | Export script runs before migration; pre-migration row-count audit catches non-zero cases | -| Another module imports `RunTaskStateEvent` (missed by grep) | Import error at runtime after model deletion | `ruff check` + `ty check` catch all import references at CI time; grep confirms no callers today | -| `Queries.state_events` referenced by a downstream consumer (e.g. dashboard, CLI) | `AttributeError` at runtime | Grep confirms no callers of `queries.state_events.*` anywhere; sentinel test locks the removal | -| `alembic downgrade` needed after deploy | Table re-created but model class gone | `downgrade()` in the revision re-creates the table DDL without SQLModel; the class is recoverable from git history | -| SQLite schema for fast tests still registers the `RunTaskStateEvent` table | `test_legacy_wal_absent.py` passes for wrong reason | The sentinel uses `sqlite_master` for SQLite environments; removing `table=True` from the class prevents SQLite table creation entirely | -| `model_validator` import becomes unused after deletion | Ruff `F401` error in CI | Verified: `RunRecord`, `RunTaskExecution`, `RunResource`, `ExperimentCohort` all use `@model_validator`; import remains live | - -## Invariants affected - -### `docs/architecture/02_runtime_lifecycle.md` - -**§4 Invariants (Known limits):** - -Remove the following bullet entirely: - -> "**`RunTaskStateEvent` is deprecated and unread.** Propagation no longer -> writes to it (`propagation.py:7-8`). `StateEventsQueries` is the last reader -> and goes away with the table in `docs/rfcs/active/2026-04-17-delete-run-task-state-event.md`. -> New code must read state from `RunGraphNode` via `GraphNodeLookup`." - -After merge, the table is gone; the sentence becomes a dead reference. - -**§7 Follow-ups:** - -Remove the line: - -> "`docs/rfcs/active/2026-04-17-delete-run-task-state-event.md` — drop the -> deprecated `RunTaskStateEvent` table and the last reader in `StateEventsQueries`." - -**§6 Anti-patterns:** - -The existing anti-pattern entry "Direct DB writes to `RunGraphNode.status`" -and related entries are unaffected. No new anti-pattern entry needed — the -deletion is the enforcement. - -### `docs/architecture/04_persistence.md` - -**§2 Core abstractions:** - -Remove or update the following paragraph: - -> "**`RunTaskStateEvent` is legacy.** The table is frozen. New code MUST NOT -> read or write it; rehydration of legacy runs is the only permitted read." - -After merge, replace with: - -> "**`RunTaskStateEvent` is deleted.** The table was dropped in migration -> `<rev>_drop_run_task_state_events`. Any historical data was archived to -> `exports/run_task_state_events_<ts>.jsonl.gz` before the drop." - -**§4 Invariants:** - -Remove the bullet: - -> "**`RunTaskStateEvent` is frozen.** No new writes. Reads permitted only -> for legacy-run rehydration. New code uses the mutation log plus node -> status instead." - -**§6 Anti-patterns:** - -Remove the bullet: - -> "**Writing to `RunTaskStateEvent`.** The table is frozen legacy. Any new -> write is a regression; use the mutation log plus node status." - -**§7 Follow-ups:** - -Remove the entry: - -> "**`RunTaskStateEvent` deletion.** An RFC is in flight to remove the legacy -> table outright. Until it lands, the frozen-table invariant stands; on merge, -> drop the legacy entry from §2 and the matching anti-pattern bullet." - -## Alternatives considered - -- **Keep the table, make the queries harder to reach.** Rejected: dead weight - in the schema. Every contributor still has to understand why it's there. -- **Keep the model as a historical data-read adapter.** Rejected: nobody reads - it today; if a later need arises, we can bring back a read-only JSON archive - consumer against the `.jsonl.gz` file. -- **Migrate existing rows into `RunGraphMutation` instead of archiving.** - Rejected: the mutation log has different invariants (monotonic sequence per - run, actor, reason); fabricating those for legacy rows pollutes the WAL. -- **Soft-delete by setting `table=False` on the SQLModel class.** Rejected: - leaves the table in the DB schema indefinitely; Alembic autogenerate would - detect and re-propose the drop on every subsequent migration run. - -## Open questions - -- Who owns verifying that no production Postgres instance has - `run_task_state_events` rows that have NOT been reconciled into - `run_graph_mutations`? A pre-migration audit script is probably worth 30 - minutes (see §Testing approach — pre-migration audit). -- Naming for the archive file — `exports/` directory (as proposed) or a - dedicated `archives/` directory? -- Should the sentinel test `test_legacy_wal_absent.py` live in `tests/state/` - (alongside other state tests) or in a dedicated `tests/schema/` directory - to group all schema-invariant tests together? - -## On acceptance - -- [ ] Update `docs/architecture/02_runtime_lifecycle.md#invariants` — remove - the `RunTaskStateEvent` deprecation note from §4.1 Known limits and the RFC - link from §7. -- [ ] Update `docs/architecture/04_persistence.md#core-abstractions` — remove - the frozen-table invariant from §4 and the anti-pattern bullet from §6; update - §2 and §7 as described in §Invariants affected. -- [ ] Link the implementation plan at - `docs/superpowers/plans/2026-04-??-delete-run-task-state-event.md` (if - created separately; otherwise this RFC serves as the plan). -- [ ] Move this file to `docs/rfcs/accepted/`. diff --git a/docs/rfcs/accepted/2026-04-17-sandbox-event-sink-activation.md b/docs/rfcs/accepted/2026-04-17-sandbox-event-sink-activation.md deleted file mode 100644 index ef2f69541..000000000 --- a/docs/rfcs/accepted/2026-04-17-sandbox-event-sink-activation.md +++ /dev/null @@ -1,881 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/03_providers.md#sandboxeventsink, docs/architecture/05_dashboard.md#invariants] -supersedes: [docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md] -superseded_by: null ---- - -# RFC: Activate `SandboxEventSink` via a process-level class setter - -## Problem - -`SandboxEventSink` is defined at -`ergon_core/ergon_core/core/providers/sandbox/event_sink.py:7-36` as the -Protocol through which sandbox managers emit lifecycle events -(`sandbox_created`, `sandbox_command`, `sandbox_closed`). Two implementations -exist — `NoopSandboxEventSink` (default, lines 39-71) and -`DashboardEmitterSandboxEventSink` (forwards to `dashboard_emitter`, lines -74-129). The three emitter methods it delegates to -(`DashboardEmitter.sandbox_created/sandbox_command/sandbox_closed`) are defined -at `ergon_core/ergon_core/core/dashboard/emitter.py:246`, `:271`, and `:302`. - -Zero production call sites construct a manager with a non-noop sink. Every -instantiation of a `BaseSandboxManager` subclass omits `event_sink=`: - -| File | Line | Manager class | -|---|---|---| -| `ergon_builtins/ergon_builtins/workers/baselines/minif2f_react_worker.py` | 111 | `MiniF2FSandboxManager()` | -| `ergon_builtins/ergon_builtins/workers/baselines/swebench_worker.py` | 123 | `SWEBenchSandboxManager()` | -| `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` | 72 | `SWEBenchSandboxManager()` | -| `ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` | 74 | `ResearchRubricsSandboxManager()` | -| `ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py` | 78 | `ResearchRubricsSandboxManager()` | - -The `DefaultSandboxManager` (used by the smoke-test path via -`ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py:56`) has no direct -construction sites in `ergon_builtins` but is acquired by the fallback -`SANDBOX_MANAGERS.get(benchmark_type, DefaultSandboxManager)` in three Inngest -functions. - -`dashboard_emitter` is a module-level singleton at -`ergon_core/ergon_core/core/dashboard/emitter.py:451`. There is only ever one -emitter in the process. - -### Why passing `event_sink=` at five call sites is wrong - -The original draft of this RFC proposed making `event_sink=` a required -constructor parameter at every site. Re-reading the code revealed that approach -is brittle for three reasons: - -1. `BaseSandboxManager` uses a `__new__`-based singleton-per-subclass pattern - at `manager.py:78-81`. The class-level `_event_sink` attribute at - `manager.py:83` is shared by every "instance" of a given subclass. -2. `__init__` at `manager.py:85-87` conditionally overwrites `_event_sink` - when a non-None value is passed — a last-write-wins stomp on shared state. - Any late re-construction with a non-None sink silently replaces the active - sink for all in-flight tasks. Tracked separately at - `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`. -3. Passing `DashboardEmitterSandboxEventSink(dashboard_emitter)` at five sites - is a behavioral no-op (the same process-wide emitter either way) while - inviting a test-isolation hazard: a `RecordingSink` passed at one site stomps - a live sink held by another via the class-level attribute. - -**Consequence of the current state:** sandbox lifecycle is not visible in the -dashboard during a live run. `dashboard_emitter.sandbox_created/command/closed` -have zero direct callers anywhere in the tree (confirmed by -`grep -rn "dashboard_emitter.sandbox_" ergon_core/ ergon_builtins/`). The -dashboard's sandbox view populates only on cold-start via the REST snapshot in -`build_run_snapshot()` at `ergon_core/ergon_core/core/api/runs.py:343`. While a -run is live, sandbox lifecycle is invisible in the UI. - -## Proposal - -Wire the sink **once, at the process boundary**, via a class-level setter on -`BaseSandboxManager`. All five production construction sites remain unchanged. - -### Design - -**Option chosen: class-level `set_event_sink` classmethod (Option A)** - -Add `@classmethod set_event_sink(cls, sink: SandboxEventSink) -> None` to -`BaseSandboxManager`. It assigns `cls._event_sink = sink` directly on the -concrete subclass (not the base), so each subclass gets its own class-attribute -value. During FastAPI `lifespan` startup — the single place in the process where -all managers are known to be idle — iterate the known subclasses and call -`Manager.set_event_sink(DashboardEmitterSandboxEventSink(dashboard_emitter))` -for each. - -The existing `event_sink: SandboxEventSink | None = None` parameter is removed -from `BaseSandboxManager.__init__`. Tests use `set_event_sink` in fixture setup -and reset to `NoopSandboxEventSink()` in teardown. This is the correct -inversion: the setter is the one sanctioned mutation point; `__init__` no longer -touches sink state. - -### Subclasses to wire at app init - -These are every concrete `BaseSandboxManager` subclass in the codebase today: - -| Class | Module | -|---|---| -| `DefaultSandboxManager` | `ergon_core/ergon_core/core/providers/sandbox/manager.py` | -| `MiniF2FSandboxManager` | `ergon_builtins/ergon_builtins/benchmarks/minif2f/sandbox_manager.py` | -| `SWEBenchSandboxManager` | `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py` | -| `ResearchRubricsSandboxManager` | `ergon_core/ergon_core/core/providers/sandbox/research_rubrics_manager.py` | -| `GDPEvalSandboxManager` | `ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox.py` | - -`DefaultSandboxManager` is in `ergon_core` and always available. The four -benchmark-specific subclasses are in `ergon_builtins`. `app.py` already imports -from `ergon_builtins` (via `ALL_FUNCTIONS`), so the import boundary is not -violated. However, the `lifespan` block should use the `SANDBOX_MANAGERS` -registry from `ergon_builtins.registry` rather than enumerating subclasses by -name, so new subclasses are picked up automatically. - -## Architecture overview - -### Before (current state) - -``` -app startup - └─ lifespan: ensure_db, init_rollout_service - (no sink configuration) - -Worker.execute() / Criterion.evaluate() - └─ SomeManager() # __new__ returns cached singleton - └─ __init__(event_sink=None) - # _event_sink = NoopSandboxEventSink() — unchanged - └─ manager.create(...) - └─ _event_sink.sandbox_created(...) # NOOP — never reaches emitter - └─ manager.terminate(...) - └─ _event_sink.sandbox_closed(...) # NOOP -``` - -### After (this RFC) - -``` -app startup - └─ lifespan: ensure_db, init_rollout_service - └─ for Manager in [DefaultSandboxManager] + list(SANDBOX_MANAGERS.values()): - Manager.set_event_sink(DashboardEmitterSandboxEventSink(dashboard_emitter)) - # Each subclass._event_sink = DashboardEmitterSandboxEventSink instance - -Worker.execute() / Criterion.evaluate() - └─ SomeManager() # __new__ returns cached singleton; __init__ no longer touches sink - └─ manager.create(...) - └─ _event_sink.sandbox_created(...) - └─ DashboardEmitterSandboxEventSink.sandbox_created(...) - └─ dashboard_emitter.sandbox_created(...) - └─ inngest_client.send(DashboardSandboxCreatedEvent) - └─ manager.terminate(...) - └─ _event_sink.sandbox_closed(...) - └─ DashboardEmitterSandboxEventSink.sandbox_closed(...) - └─ dashboard_emitter.sandbox_closed(...) - └─ inngest_client.send(DashboardSandboxClosedEvent) -``` - -### Event pipeline (after) - -``` -BaseSandboxManager._emit_wal_entry / .create / .terminate - │ - ▼ -SandboxEventSink.sandbox_{created,command,closed} - │ (DashboardEmitterSandboxEventSink — set once at lifespan) - ▼ -DashboardEmitter.sandbox_{created,command,closed} - │ (emitter.py:246, :271, :302) - ▼ -inngest_client.send(DashboardSandbox{Created,Command,Closed}Event) - │ - ▼ -Next.js Inngest handler → DashboardStore reducer → Socket.io room run:<id> - │ - ▼ -Browser — live sandbox lifecycle events -``` - -## Type / interface definitions - -### `SandboxEventSink` (no change — shown for reference) - -```python -# ergon_core/ergon_core/core/providers/sandbox/event_sink.py (lines 7-36, unchanged) - -class SandboxEventSink(Protocol): - """Observer for sandbox lifecycle and append-only WAL events.""" - - async def sandbox_created( - self, - run_id: UUID, - task_id: UUID, - sandbox_id: str, - timeout_minutes: int, - template: str | None = None, - ) -> None: ... - - async def sandbox_command( - self, - run_id: UUID, - task_id: UUID, - sandbox_id: str, - command: str, - stdout: str | None = None, - stderr: str | None = None, - exit_code: int | None = None, - duration_ms: int | None = None, - ) -> None: ... - - async def sandbox_closed( - self, - task_id: UUID, - sandbox_id: str, - reason: str, - ) -> None: ... -``` - -### `RecordingSandboxEventSink` (test fixture — new) - -```python -# tests/state/fixtures/recording_event_sink.py (new file) - -from __future__ import annotations - -from dataclasses import dataclass, field -from uuid import UUID - - -@dataclass -class SandboxCreatedCall: - run_id: UUID - task_id: UUID - sandbox_id: str - timeout_minutes: int - template: str | None - - -@dataclass -class SandboxCommandCall: - run_id: UUID - task_id: UUID - sandbox_id: str - command: str - stdout: str | None - stderr: str | None - exit_code: int | None - duration_ms: int | None - - -@dataclass -class SandboxClosedCall: - task_id: UUID - sandbox_id: str - reason: str - - -@dataclass -class RecordingSandboxEventSink: - """Test double that records every sink call for assertion.""" - - created: list[SandboxCreatedCall] = field(default_factory=list) - commands: list[SandboxCommandCall] = field(default_factory=list) - closed: list[SandboxClosedCall] = field(default_factory=list) - - async def sandbox_created( - self, - run_id: UUID, - task_id: UUID, - sandbox_id: str, - timeout_minutes: int, - template: str | None = None, - ) -> None: - self.created.append( - SandboxCreatedCall(run_id, task_id, sandbox_id, timeout_minutes, template) - ) - - async def sandbox_command( - self, - run_id: UUID, - task_id: UUID, - sandbox_id: str, - command: str, - stdout: str | None = None, - stderr: str | None = None, - exit_code: int | None = None, - duration_ms: int | None = None, - ) -> None: - self.commands.append( - SandboxCommandCall(run_id, task_id, sandbox_id, command, stdout, stderr, exit_code, duration_ms) - ) - - async def sandbox_closed( - self, - task_id: UUID, - sandbox_id: str, - reason: str, - ) -> None: - self.closed.append(SandboxClosedCall(task_id, sandbox_id, reason)) -``` - -## Full implementations - -### 1. `set_event_sink` classmethod on `BaseSandboxManager` - -Complete replacement for `BaseSandboxManager.__init__` and addition of -`set_event_sink`. All other methods of `BaseSandboxManager` are unchanged. - -```python -# ergon_core/ergon_core/core/providers/sandbox/manager.py -# Replace __init__ (lines 85-87) and add set_event_sink after it. - - _event_sink: SandboxEventSink = NoopSandboxEventSink() - - def __init__(self) -> None: - # Sink is configured process-wide via set_event_sink() in app lifespan. - # Do not accept event_sink= here; the singleton pattern (see __new__ above) - # makes constructor-level sink assignment a last-write-wins stomp on shared - # class state. Tests must use set_event_sink() in fixture setup. - pass - - @classmethod - def set_event_sink(cls, sink: SandboxEventSink) -> None: - """Install a process-level event sink on this manager subclass. - - Called once during FastAPI lifespan startup for each concrete subclass. - Tests may call this in fixture setup and reset with - ``NoopSandboxEventSink()`` in teardown. - - Assigns directly to ``cls._event_sink`` (not to the base class - attribute), so each subclass carries its own sink and subclasses can - be individually targeted in tests. - - Production callers MUST NOT call this after startup. The only - sanctioned call site is inside the ``lifespan`` context manager in - ``ergon_core/ergon_core/core/api/app.py``. - """ - cls._event_sink = sink -``` - -### 2. `lifespan` block in `app.py` - -```python -# ergon_core/ergon_core/core/api/app.py -# Full file after changes: - -"""FastAPI application with Inngest webhook registration.""" - -import logging -from contextlib import asynccontextmanager - -import inngest.fast_api -from ergon_core.core.api.cohorts import router as cohorts_router -from ergon_core.core.api.rollouts import init_service as init_rollout_service -from ergon_core.core.api.rollouts import router as rollouts_router -from ergon_core.core.api.runs import router as runs_router -from ergon_core.core.dashboard.emitter import dashboard_emitter -from ergon_core.core.persistence.shared.db import ensure_db, get_session -from ergon_core.core.providers.sandbox.event_sink import DashboardEmitterSandboxEventSink -from ergon_core.core.providers.sandbox.manager import DefaultSandboxManager -from ergon_core.core.rl.rollout_service import RolloutService -from ergon_core.core.runtime.inngest_client import inngest_client -from ergon_core.core.runtime.inngest_registry import ALL_FUNCTIONS -from ergon_core.core.settings import Settings -from fastapi import FastAPI - -logger = logging.getLogger(__name__) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - logger.info("starting ensure_db...") - ensure_db() - logger.info("ensure_db done, initializing RolloutService...") - settings = Settings() - init_rollout_service( - RolloutService( - session_factory=get_session, - inngest_send=inngest_client.send_sync, - tokenizer_name=settings.default_tokenizer, - ) - ) - - # Wire the dashboard event sink on every sandbox manager subclass. - # Import ergon_builtins here (deferred) to avoid a circular import at - # module level; ergon_builtins imports ergon_core, not the reverse. - from ergon_builtins.registry import SANDBOX_MANAGERS # noqa: PLC0415 - - sink = DashboardEmitterSandboxEventSink(dashboard_emitter) - # DefaultSandboxManager is the fallback; it is not in SANDBOX_MANAGERS. - DefaultSandboxManager.set_event_sink(sink) - for manager_cls in SANDBOX_MANAGERS.values(): - manager_cls.set_event_sink(sink) - logger.info( - "sandbox event sink wired on %d manager subclass(es)", - 1 + len(SANDBOX_MANAGERS), - ) - - logger.info("ready") - yield - - -app = FastAPI( - title="Ergon Core", - description="Ergon experiment orchestration API", - version="0.1.0", - lifespan=lifespan, -) - -app.include_router(runs_router) -app.include_router(cohorts_router) -app.include_router(rollouts_router) - -inngest.fast_api.serve(app, inngest_client, ALL_FUNCTIONS) -``` - -## Exact diffs for modified files - -### `manager.py` — `__init__` change and `set_event_sink` addition - -```diff ---- a/ergon_core/ergon_core/core/providers/sandbox/manager.py -+++ b/ergon_core/ergon_core/core/providers/sandbox/manager.py -@@ -83,7 +83,22 @@ class BaseSandboxManager(ABC): - _event_sink: SandboxEventSink = NoopSandboxEventSink() - -- def __init__(self, event_sink: SandboxEventSink | None = None): -- if event_sink is not None: -- self._event_sink = event_sink -+ def __init__(self) -> None: -+ # Sink is configured process-wide via set_event_sink() in app lifespan. -+ # Do not accept event_sink= here; the singleton pattern (see __new__ above) -+ # makes constructor-level sink assignment a last-write-wins stomp on shared -+ # class state. Tests must use set_event_sink() in fixture setup. -+ pass -+ -+ @classmethod -+ def set_event_sink(cls, sink: SandboxEventSink) -> None: -+ """Install a process-level event sink on this manager subclass. -+ -+ Called once during FastAPI lifespan startup for each concrete subclass. -+ Tests may call this in fixture setup and reset with -+ ``NoopSandboxEventSink()`` in teardown. -+ -+ Assigns directly to ``cls._event_sink`` (not to the base class -+ attribute), so each subclass carries its own sink and subclasses can -+ be individually targeted in tests. -+ -+ Production callers MUST NOT call this after startup. The only -+ sanctioned call site is inside the ``lifespan`` context manager in -+ ``ergon_core/ergon_core/core/api/app.py``. -+ """ -+ cls._event_sink = sink -``` - -### `app.py` — `lifespan` block additions - -```diff ---- a/ergon_core/ergon_core/core/api/app.py -+++ b/ergon_core/ergon_core/core/api/app.py -@@ -6,6 +6,8 @@ import inngest.fast_api - from ergon_core.core.api.cohorts import router as cohorts_router - from ergon_core.core.api.rollouts import init_service as init_rollout_service - from ergon_core.core.api.rollouts import router as rollouts_router - from ergon_core.core.api.runs import router as runs_router -+from ergon_core.core.dashboard.emitter import dashboard_emitter - from ergon_core.core.persistence.shared.db import ensure_db, get_session -+from ergon_core.core.providers.sandbox.event_sink import DashboardEmitterSandboxEventSink -+from ergon_core.core.providers.sandbox.manager import DefaultSandboxManager - from ergon_core.core.rl.rollout_service import RolloutService - from ergon_core.core.runtime.inngest_client import inngest_client - from ergon_core.core.runtime.inngest_registry import ALL_FUNCTIONS -@@ -22,6 +25,17 @@ async def lifespan(app: FastAPI): - logger.info("ensure_db done, initializing RolloutService...") - settings = Settings() - init_rollout_service( - RolloutService( - session_factory=get_session, - inngest_send=inngest_client.send_sync, - tokenizer_name=settings.default_tokenizer, - ) - ) -+ -+ from ergon_builtins.registry import SANDBOX_MANAGERS # noqa: PLC0415 -+ -+ sink = DashboardEmitterSandboxEventSink(dashboard_emitter) -+ DefaultSandboxManager.set_event_sink(sink) -+ for manager_cls in SANDBOX_MANAGERS.values(): -+ manager_cls.set_event_sink(sink) -+ logger.info( -+ "sandbox event sink wired on %d manager subclass(es)", -+ 1 + len(SANDBOX_MANAGERS), -+ ) -+ - logger.info("ready") - yield -``` - -## Package structure - -No new packages are introduced. The `RecordingSandboxEventSink` test fixture -lives in the existing `tests/` tree. If a `tests/state/fixtures/` directory does -not yet exist, add an empty `__init__.py`. - -``` -tests/ - state/ - fixtures/ - __init__.py # empty — new if directory is new - recording_event_sink.py # new: RecordingSandboxEventSink and call dataclasses - test_sandbox_event_sink_activation.py # new: unit + integration tests -``` - -## Implementation order - -| Step | What | Files touched | PR | -|---|---|---|---| -| **1** | Add `set_event_sink` classmethod to `BaseSandboxManager`; replace `__init__` (remove `event_sink=` param); update `_event_sink` class-attr declaration so it remains typed `SandboxEventSink` | MODIFY `ergon_core/ergon_core/core/providers/sandbox/manager.py` | PR 1 | -| **2** | Write `RecordingSandboxEventSink` fixture and unit tests for `set_event_sink` (isolated subclass, verify class-attr assignment, verify base class unaffected) | ADD `tests/state/fixtures/recording_event_sink.py`, ADD `tests/state/test_sandbox_event_sink_activation.py` | PR 1 | -| **3** | Wire `lifespan` in `app.py`: import `dashboard_emitter`, `DashboardEmitterSandboxEventSink`, `DefaultSandboxManager`, `SANDBOX_MANAGERS`; call `set_event_sink` on each subclass before `yield` | MODIFY `ergon_core/ergon_core/core/api/app.py` | PR 1 | -| **4** | Integration test: simulate `lifespan` startup, install a `RecordingSandboxEventSink` via `set_event_sink`, assert that `DefaultSandboxManager` produces `sandbox_created` and `sandbox_closed` events on `create`/`terminate` round-trip | ADD test to `tests/state/test_sandbox_event_sink_activation.py` | PR 1 | -| **5** | Update `docs/architecture/03_providers.md`: remove "activation path in flux" hedges from the `SandboxEventSink` entry and Section 2.4; state the process-level-setter rule; update Section 5.2 ("Add a new sandbox manager") to include `Manager.set_event_sink(sink)` in `lifespan` | MODIFY `docs/architecture/03_providers.md` | PR 1 (same commit as implementation) | - -All five steps ship in a single PR. No phasing is needed: the change is -self-contained (two modified files, one new test fixture, one new test module) -and does not depend on any other in-flight RFC to function correctly. - -**Dependency note.** `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md` -proposes replacing the singleton-per-subclass pattern and class-level state dicts -with instance-owned state. That RFC is larger in scope and not required for this -fix. The `set_event_sink` classmethod introduced here is **forward-compatible** -with the process-state RFC: when class-level state is moved to instance-level -state, `set_event_sink` becomes an instance-attribute wire-up instead. The -classmethod itself can be retained as-is or converted; no call-site changes -are needed at that point. - -The latent shared-state race (a late `__init__` call with `event_sink!=None` -stomps the live sink for all in-flight tasks) is resolved by this RFC because -`__init__` no longer accepts `event_sink=`. The underlying class-dict stomp -risk that the process-state RFC targets remains, but this RFC eliminates one -of its concrete manifestations. - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `tests/state/fixtures/__init__.py` | Empty package init (add only if directory is new) | -| `tests/state/fixtures/recording_event_sink.py` | `RecordingSandboxEventSink` and associated call-record dataclasses for use in unit and integration tests | -| `tests/state/test_sandbox_event_sink_activation.py` | Unit and integration tests for `set_event_sink`, lifecycle event emission, and fixture-based reset | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/providers/sandbox/manager.py` | Remove `event_sink: SandboxEventSink \| None = None` from `__init__`; replace body with `pass`; add `set_event_sink` classmethod after `__init__` | -| `ergon_core/ergon_core/core/api/app.py` | Add three imports (`dashboard_emitter`, `DashboardEmitterSandboxEventSink`, `DefaultSandboxManager`); add deferred `SANDBOX_MANAGERS` import inside `lifespan`; add sink-wiring block before `yield` | - -No changes to: the five construction sites, `event_sink.py`, `emitter.py`, -`registry.py`, or any benchmark/worker file. - -## Testing approach - -### Unit tests - -```python -# tests/state/test_sandbox_event_sink_activation.py - -from __future__ import annotations - -import pytest - -from ergon_core.core.providers.sandbox.event_sink import NoopSandboxEventSink -from ergon_core.core.providers.sandbox.manager import BaseSandboxManager, DefaultSandboxManager -from tests.state.fixtures.recording_event_sink import RecordingSandboxEventSink - - -class _TestManagerA(BaseSandboxManager): - """Isolated subclass for testing — never constructed outside this module.""" - - async def _install_dependencies(self, sandbox, task_id): # type: ignore[override] - pass - - -class _TestManagerB(BaseSandboxManager): - """Second isolated subclass — verifies per-class independence.""" - - async def _install_dependencies(self, sandbox, task_id): # type: ignore[override] - pass - - -@pytest.fixture(autouse=True) -def reset_test_manager_sinks(): - """Restore noop sink on test managers after each test.""" - noop = NoopSandboxEventSink() - yield - _TestManagerA.set_event_sink(noop) - _TestManagerB.set_event_sink(noop) - - -class TestSetEventSink: - def test_set_event_sink_assigns_to_subclass(self) -> None: - sink = RecordingSandboxEventSink() - _TestManagerA.set_event_sink(sink) - assert _TestManagerA._event_sink is sink - - def test_set_event_sink_does_not_affect_other_subclass(self) -> None: - sink_a = RecordingSandboxEventSink() - _TestManagerA.set_event_sink(sink_a) - # _TestManagerB should still hold its own noop - assert _TestManagerB._event_sink is not sink_a - - def test_set_event_sink_does_not_affect_base_class(self) -> None: - sink = RecordingSandboxEventSink() - _TestManagerA.set_event_sink(sink) - # BaseSandboxManager._event_sink is the class-level default NoopSandboxEventSink - # set_event_sink writes to _TestManagerA, not to BaseSandboxManager - assert BaseSandboxManager._event_sink is not sink - - def test_default_sink_is_noop(self) -> None: - assert isinstance(_TestManagerB._event_sink, NoopSandboxEventSink) - - def test_init_no_longer_accepts_event_sink_kwarg(self) -> None: - """__init__ removing event_sink= means passing it is a TypeError.""" - with pytest.raises(TypeError): - _TestManagerA(event_sink=RecordingSandboxEventSink()) # type: ignore[call-arg] -``` - -### Integration test — lifecycle events flow to sink - -```python -# tests/state/test_sandbox_event_sink_activation.py (continued) - -import asyncio -from uuid import uuid4 - -import pytest - -from ergon_core.core.providers.sandbox.event_sink import NoopSandboxEventSink -from ergon_core.core.providers.sandbox.manager import DefaultSandboxManager -from tests.state.fixtures.recording_event_sink import RecordingSandboxEventSink - - -@pytest.fixture() -def recording_default_manager(): - """Install a RecordingSandboxEventSink on DefaultSandboxManager for the test.""" - sink = RecordingSandboxEventSink() - DefaultSandboxManager.set_event_sink(sink) - yield DefaultSandboxManager(), sink - DefaultSandboxManager.set_event_sink(NoopSandboxEventSink()) - # Reset singleton state so task-id doesn't leak between tests - DefaultSandboxManager._sandboxes.clear() - DefaultSandboxManager._run_ids.clear() - DefaultSandboxManager._display_task_ids.clear() - DefaultSandboxManager._file_registries.clear() - DefaultSandboxManager._created_files_registry.clear() - DefaultSandboxManager._creation_locks.clear() - - -@pytest.mark.asyncio -async def test_sandbox_created_emits_to_sink( - recording_default_manager, - monkeypatch, -) -> None: - """DefaultSandboxManager.create() calls sink.sandbox_created exactly once.""" - manager, sink = recording_default_manager - task_id = uuid4() - run_id = uuid4() - - # Stub out the E2B call so no real sandbox is provisioned - class _FakeSandbox: - sandbox_id = "sbx-test-123" - - async def run_code(self, *a, **kw): - class R: - error = None - logs = None - return R() - - async def files(self): - pass - - async def commands(self): - pass - - from unittest.mock import AsyncMock, MagicMock, patch - - fake_sandbox = _FakeSandbox() - with patch( - "ergon_core.core.providers.sandbox.manager.AsyncSandbox" - ) as mock_cls: - mock_cls.create = AsyncMock(return_value=fake_sandbox) - # Skip directory structure for unit speed - monkeypatch.setattr(manager, "_create_directory_structure", AsyncMock()) - - await manager.create( - sandbox_key=task_id, - run_id=run_id, - timeout_minutes=5, - ) - - assert len(sink.created) == 1 - assert sink.created[0].sandbox_id == "sbx-test-123" - assert sink.created[0].run_id == run_id - assert sink.created[0].task_id == task_id - - -@pytest.mark.asyncio -async def test_sandbox_closed_emits_to_sink( - recording_default_manager, - monkeypatch, -) -> None: - """DefaultSandboxManager.terminate() calls sink.sandbox_closed exactly once.""" - manager, sink = recording_default_manager - task_id = uuid4() - - from unittest.mock import AsyncMock, MagicMock - - class _FakeSandbox: - sandbox_id = "sbx-test-456" - kill = AsyncMock() - - manager._sandboxes[task_id] = _FakeSandbox() - manager._run_ids[task_id] = uuid4() - manager._display_task_ids[task_id] = task_id - - await manager.terminate(task_id, reason="completed") - - assert len(sink.closed) == 1 - assert sink.closed[0].sandbox_id == "sbx-test-456" - assert sink.closed[0].reason == "completed" -``` - -### Contract test — `lifespan` wires sink on all managers - -```python -# tests/state/test_sandbox_event_sink_activation.py (continued) - -def test_lifespan_wires_all_known_managers() -> None: - """Every entry in SANDBOX_MANAGERS + DefaultSandboxManager must have - set_event_sink called during lifespan. Verify the call surface exists - and the method is a classmethod (not instance method). - """ - from ergon_builtins.registry import SANDBOX_MANAGERS - from ergon_core.core.providers.sandbox.manager import DefaultSandboxManager - - all_managers = [DefaultSandboxManager, *SANDBOX_MANAGERS.values()] - for mgr_cls in all_managers: - assert hasattr(mgr_cls, "set_event_sink"), ( - f"{mgr_cls.__name__} missing set_event_sink — " - "did it accidentally shadow BaseSandboxManager?" - ) - sink = RecordingSandboxEventSink() - mgr_cls.set_event_sink(sink) - assert mgr_cls._event_sink is sink, ( - f"{mgr_cls.__name__}._event_sink was not updated by set_event_sink" - ) - # Restore noop to avoid polluting other tests - mgr_cls.set_event_sink(NoopSandboxEventSink()) -``` - -## Trace / observability impact - -No new spans or metrics are introduced by this RFC. The effect is that three -Inngest events — `dashboard/sandbox.created`, `dashboard/sandbox.command`, and -`dashboard/sandbox.closed` — now fire on the live path where previously they -were suppressed by the noop sink. - -Logfire / existing Inngest event tracing picks up these new event types -automatically via the existing `inngest_client.send()` instrumentation. - -One log line is added at startup in `lifespan`: - -``` -INFO ergon_core.core.api.app: sandbox event sink wired on 5 manager subclass(es) -``` - -(Count reflects `DefaultSandboxManager` + the four entries in `SANDBOX_MANAGERS` -today: `gdpeval`, `minif2f`, `swebench-verified`, plus `ResearchRubricsSandboxManager` -if registered — verify actual count against `SANDBOX_MANAGERS` at time of -implementation.) - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| A new `BaseSandboxManager` subclass lands without being added to `SANDBOX_MANAGERS` | Its `_event_sink` stays `NoopSandboxEventSink`; events silently dropped | Architecture doc Section 5.2 mandates adding the subclass to `SANDBOX_MANAGERS` before use. `test_lifespan_wires_all_known_managers` will catch registered-but-unwired classes automatically. | -| `SANDBOX_MANAGERS` is imported deferred (inside `lifespan`) and the optional `[data]` extra is not installed | `ergon_builtins.registry` import succeeds; `SANDBOX_MANAGERS` contains only the always-available entries; benchmark-specific managers won't be wired | The sink assignment is best-effort — if a manager class is not imported, it can't be wired. In practice the app process that serves API requests also imports `ergon_builtins` in full (via `ALL_FUNCTIONS`). Document this assumption. | -| `set_event_sink` called multiple times during lifespan (e.g. hot-reload) | Last call wins; benign if same sink type | `lifespan` runs once per process startup; ASGI hot-reload restarts the process. Not a real risk. | -| `set_event_sink` called after startup by test code that forgets the teardown | Pollutes sink state for subsequent tests in the session | `recording_default_manager` fixture (above) resets to noop in teardown. CI autouse fixtures should enforce isolation. | -| `DashboardEmitter.sandbox_*` calls fail (Inngest unreachable) | Exception caught inside `DashboardEmitter`; logged at WARNING; never propagates to sandbox lifecycle | Existing `except Exception: logger.warning(...)` blocks in `emitter.py:268`, `:299`, `:319` already handle this. No change needed. | -| Shared-state race (the `__init__` stomp) re-introduced by a future contributor adding `event_sink=` back | Silent override of production sink mid-run | Removing the param from `__init__` is the fix. The docstring on `set_event_sink` explicitly prohibits production callers. Slopcop or review discipline enforces. | -| `ResearchRubricsSandboxManager` lives in `ergon_core` but `researchrubrics` slug is only available under `ergon_builtins[data]`; it may not appear in `SANDBOX_MANAGERS` in all environments | Manager class exists but is not wired | It is not currently in `SANDBOX_MANAGERS`; it is wired via direct instantiation in `researcher_worker.py` and `stub_worker.py`. Those files will pick up the sink because `set_event_sink` writes to the class attribute. Adding `ResearchRubricsSandboxManager` to `SANDBOX_MANAGERS` is a follow-up. | - -## Invariants affected - -From `docs/architecture/03_providers.md`: - -- **Section 2.4 `SandboxEventSink`:** Status line "unwired on the live path" - and the "Sandbox event sink unwired" bullet in Section 4.1 are removed. The - invariant stated in the bug (`"every sandbox event flows through the sink"`) - becomes true and must be stated without the "partially wired" softener. - -- **New rule (add to Section 4 or 4.1):** Sinks are process-level, set once at - app init, never swapped at runtime in production code. Tests may swap via - fixture scope; production callers MUST NOT call `set_event_sink` after - startup. - -- **Section 5.2 "Add a new sandbox manager":** Replace the existing step - "Do NOT pass `event_sink=` at construction today" (which is marked as - temporary) with: "Add the new subclass to `SANDBOX_MANAGERS` in - `ergon_builtins/ergon_builtins/registry_core.py`. The `lifespan` block - wires the dashboard sink automatically for every entry in that registry." - -- **Section 6 "Anti-patterns":** Replace the anti-pattern - "Passing `event_sink=` at manager construction" with: - "Calling `set_event_sink` outside of `lifespan` or test fixtures. Production - callers must not call it after startup." - -From `docs/architecture/05_dashboard.md`: - -- **Section 4 invariant 1** ("The dashboard is event-driven end-to-end for the - surfaces that are wired") now applies to the sandbox surface. No wording - change needed; the sandbox entries in the Follow-ups section are resolved. - -- **Section 7 Follow-ups:** Remove - `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md` from the list. - The `DashboardEmitterSandboxEventSink has no constructor site today` note in - Section 2 is also removed. - -## Alternatives considered - -- **Make `event_sink` a required constructor kwarg** (original proposal in the - superseded draft of this RFC). Rejected: no behavioral win since - `dashboard_emitter` is itself a process-wide singleton, and the - singleton-per-subclass pattern at `manager.py:78-81` with a last-write-wins - `__init__` stomp at `manager.py:85-87` creates a real test-isolation hazard. - Documented separately as the latent race in - `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`. - -- **Drop the singleton entirely, make managers instance-owned.** Correctly - resolves the underlying class-state sharing problem but depends on the broader - process-state reform in - `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md`, which is - bigger scope. Revisit after that lands; the setter approach here is - forward-compatible with an eventual instance-owned world (the classmethod - becomes an instance attribute wire-up). - -- **Module-level global sink.** Rejected: implicit dependency; worse than the - classmethod for testability and for subclass-scoped overrides. Adds a new - singleton without gaining any benefit over the classmethod. - -## Open questions - -- Should `ResearchRubricsSandboxManager` be added to `SANDBOX_MANAGERS` in - `registry_core.py` so it is picked up automatically by `lifespan`? Currently - it is absent from that registry (wired only by direct import in worker files). - If not added, it still receives the sink because the class attribute is written - by `set_event_sink` when the `data` extra is installed — but only if someone - explicitly calls `ResearchRubricsSandboxManager.set_event_sink()` before the - manager is first constructed. A follow-up PR should either add it to the - registry or add an explicit call in `lifespan`. - -- Whether to keep `set_event_sink` on `BaseSandboxManager` as a classmethod - after the process-state RFC lands and the singleton is removed. If managers - become proper instances, the setter can move to `__init__` as an optional - kwarg — but with the DI container approach rather than the stomp approach. - Defer this decision to that RFC. - -## On acceptance - -- [ ] Move this file to `docs/rfcs/accepted/`. -- [ ] Update `docs/architecture/03_providers.md` — remove "intended path" hedges - in Section 2.4; remove "unwired" and "in flux" status notes; state the - process-level-setter rule explicitly in Section 4; update Section 5.2 to - include `SANDBOX_MANAGERS` registration as a step; update Section 6 - anti-pattern wording. -- [ ] Update `docs/architecture/05_dashboard.md` Section 7 Follow-ups — remove - the sandbox sink bullet. -- [ ] Move `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md` → - `docs/bugs/fixed/` with `fixed_pr` set. -- [ ] Link the implementation plan at - `docs/superpowers/plans/2026-04-??-sandbox-event-sink-activation.md`. diff --git a/docs/rfcs/accepted/2026-04-18-dashboard-event-wiring-enforcement.md b/docs/rfcs/accepted/2026-04-18-dashboard-event-wiring-enforcement.md deleted file mode 100644 index 645b77a4e..000000000 --- a/docs/rfcs/accepted/2026-04-18-dashboard-event-wiring-enforcement.md +++ /dev/null @@ -1,761 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: agent -architecture_refs: [docs/architecture/05_dashboard.md#invariants] -supersedes: [] -superseded_by: null ---- - -# RFC: Contract test enforcing dashboard event wiring - -## Problem - -The Layer 5 invariant "every persistent backend state change has a corresponding -`dashboard/*` event" (`docs/architecture/05_dashboard.md#invariants`) has no -enforcement today. It is review-only, and review has failed: 9 of 12 -`DashboardEmitter` methods are defined but never invoked from runtime code (bug -`docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md`). The gap -was invisible until a grep. - -### Current wiring state - -`DashboardEmitter` is defined at -`ergon_core/ergon_core/core/dashboard/emitter.py:51`. It exposes 12 public -methods: - -| Method | Status | Call site | -|---|---|---| -| `graph_mutation` | **WIRED** | `task_management_service.py:113` — `self._graph_repo.add_mutation_listener(dashboard_emitter.graph_mutation)` | -| `on_context_event` | **WIRED** | `worker_execute.py:81` — `context_event_repo.add_listener(dashboard_emitter.on_context_event)` | -| `cohort_updated` | **WIRED (indirect)** | `emitter.py:467` via `emit_cohort_updated_for_run`, called from `complete_workflow.py:48` | -| `register_execution` | WIRED (not a dashboard event; called at `worker_execute.py:82`) | — | -| `workflow_started` | **UNWIRED** | zero call sites in `ergon_core/`, `ergon_builtins/`, `ergon_infra/` | -| `workflow_completed` | **UNWIRED** | zero call sites | -| `task_status_changed` | **UNWIRED** | zero call sites | -| `task_evaluation_updated` | **UNWIRED** | zero call sites | -| `task_cancelled` | **UNWIRED** | zero call sites | -| `resource_published` | **UNWIRED** | zero call sites | -| `thread_message_created` | **UNWIRED** | zero call sites | -| `sandbox_created` | **UNWIRED** — `DashboardEmitterSandboxEventSink` calls it (`event_sink.py:88`) but the sink has no constructor site (tracked in `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md`) | -| `sandbox_command` | **UNWIRED** — same reason (`event_sink.py:107`) | -| `sandbox_closed` | **UNWIRED** — same reason (`event_sink.py:124`) | - -**Confirmed by grep:** - -``` -rg -n '\.(task_status_changed|workflow_started|workflow_completed|resource_published| \ - thread_message_created|task_evaluation_updated|task_cancelled)\(' \ - ergon_core ergon_builtins ergon_infra -``` - -Zero matches outside `emitter.py` and `event_contracts.py`. - -A parallel gap exists on the frontend: every `mutation_type` variant in -`MutationTypeSchema` (`ergon-dashboard/src/features/graph/contracts/graphMutations.ts:6`) -must have a matching `case` arm in `applyGraphMutation` -(`ergon-dashboard/src/features/graph/state/graphMutationReducer.ts:122`). The -reducer is currently exhaustive and has a `never`-typed default (`graphMutationReducer.ts:188`), -but there is no automated assertion that it stays that way when a new -`mutation_type` is added on the Python side. - -## Proposal - -Three pieces, in order of implementation cost: - -1. **Backend contract test** (`tests/contract/test_dashboard_emitter_wiring.py`). - Use `inspect` on `DashboardEmitter` to enumerate public async methods. For - each method name, scan `ergon_core/`, `ergon_builtins/`, and `ergon_infra/` - for a call pattern matching `\.{name}\(` or - `add_listener\(.*\.{name}\b`. Zero call sites fails with a message pointing - at the method and this RFC. - - Skip list: `register_execution` (not a dashboard event — internal mapping - helper). The sandbox trio (`sandbox_created`, `sandbox_command`, - `sandbox_closed`) count `DashboardEmitterSandboxEventSink` as their - proof-of-wiring because the sink forwards to them; the separate - sink-activation bug is tracked independently. - -2. **Frontend mutation-kind coverage test** - (`ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts`). - Import the `MutationTypeSchema` enum values and `applyGraphMutation`. Assert - that passing a synthetic mutation of each kind into `applyGraphMutation` does - not fall through to the `default` branch (i.e., the mutation does not appear - in `state.unhandledMutations`). Missing branch fails the build. - -3. **Architecture doc table** — inline in `docs/architecture/05_dashboard.md`, - listing each emitter method and its call site(s), updated whenever the - contract test enumeration changes. - -Wire test (1) into `pnpm run test:be:state` (the `tests/state/` suite runs under -`pnpm run test:be:fast`); test (2) into `pnpm -C ergon-dashboard run test:contracts`. - -## Architecture overview - -### Before (today) - -``` -DashboardEmitter methods (12) - ├── graph_mutation ← wired via listener - ├── on_context_event ← wired via listener - ├── cohort_updated ← wired (indirect) via emit_cohort_updated_for_run - ├── register_execution ← wired (internal helper, not a dashboard event) - └── 8 remaining methods ← DEAD — defined, never called - CI does not catch this -``` - -### After (this RFC, post bug fix) - -``` -DashboardEmitter methods (12) - ├── graph_mutation ← wired: task_management_service.py:113 - ├── on_context_event ← wired: worker_execute.py:81 - ├── cohort_updated ← wired: complete_workflow.py:48 - ├── register_execution ← wired (exempt from contract test) - ├── workflow_started ← wired: start_workflow.py (after bug fix) - ├── workflow_completed ← wired: complete_workflow.py (after bug fix) - ├── task_status_changed ← wired: task_management_service.py (after bug fix) - ├── task_evaluation_updated ← wired: evaluate_task_run.py (after bug fix) - ├── task_cancelled ← wired: cleanup_cancelled_task.py (after bug fix) - ├── resource_published ← wired: sandbox_resource_publisher (after bug fix) - ├── thread_message_created ← wired: messaging service (after bug fix) - ├── sandbox_created ← wired via DashboardEmitterSandboxEventSink - ├── sandbox_command ← wired via DashboardEmitterSandboxEventSink - └── sandbox_closed ← wired via DashboardEmitterSandboxEventSink - -contract test runs in CI - → zero-call-site emitter method → test FAILS with actionable message - → new mutation_type without reducer → TS test FAILS -``` - -### Contract test data flow - -``` -test_dashboard_emitter_wiring.py - 1. inspect.getmembers(DashboardEmitter) → list of public async method names - 2. For each name not in EXEMPT_METHODS: - scan ergon_core/, ergon_builtins/, ergon_infra/ for - \.{name}\( OR add_listener\(.*\.{name}\b - 3. If no hits → pytest.fail() with method name + file path + RFC reference -``` - -## Type / interface definitions - -No new types are introduced. The contract test operates entirely through -`inspect` and `ast`-free text scanning. For clarity, the skip list is defined -as a module-level constant: - -```python -# ergon/tests/contract/test_dashboard_emitter_wiring.py (top of file) - -# Methods exempt from the call-site requirement. -# register_execution: internal mapping helper, not a dashboard event. -# Sandbox trio: DashboardEmitterSandboxEventSink (event_sink.py) serves as -# their wiring proof; the sink-activation gap is tracked separately. -_EXEMPT_METHODS: frozenset[str] = frozenset( - { - "register_execution", - "sandbox_created", - "sandbox_command", - "sandbox_closed", - } -) -``` - -## Full implementations - -### Backend contract test - -```python -# ergon/tests/contract/test_dashboard_emitter_wiring.py -"""Contract test: every DashboardEmitter public method must have at least one -call site in ergon_core/, ergon_builtins/, or ergon_infra/. - -Fails CI when an emitter method is added without a corresponding call site. -See docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md. -""" - -from __future__ import annotations - -import inspect -import re -import subprocess -from pathlib import Path -from typing import Final - -import pytest - -from ergon_core.core.dashboard.emitter import DashboardEmitter - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- - -# Paths to scan (relative to the repo root, which is determined at runtime). -_SCAN_PACKAGES: Final[tuple[str, ...]] = ( - "ergon_core", - "ergon_builtins", - "ergon_infra", -) - -# Files that DEFINE the emitter — excluded from the scan so self-references -# inside emitter.py don't count as call sites. -_DEFINITION_FILES: Final[frozenset[str]] = frozenset( - { - "ergon_core/ergon_core/core/dashboard/emitter.py", - "ergon_core/ergon_core/core/dashboard/event_contracts.py", - "ergon_core/ergon_core/core/providers/sandbox/event_sink.py", - } -) - -# Methods exempt from the zero-call-sites check. -# register_execution: internal mapping helper, not a dashboard event. -# Sandbox trio: DashboardEmitterSandboxEventSink (event_sink.py) is their -# proof-of-wiring; the sink-activation gap is a separate bug. -_EXEMPT_METHODS: Final[frozenset[str]] = frozenset( - { - "register_execution", - "sandbox_created", - "sandbox_command", - "sandbox_closed", - } -) - - -def _repo_root() -> Path: - """Find the repo root by walking up from this file until pyproject.toml - is found, or fall back to the tests/ grandparent.""" - p = Path(__file__).resolve() - for parent in p.parents: - if (parent / "pyproject.toml").exists(): - return parent - return p.parent.parent.parent - - -def _emitter_method_names() -> list[str]: - """Return public async method names on DashboardEmitter.""" - return [ - name - for name, member in inspect.getmembers(DashboardEmitter, predicate=inspect.isfunction) - if not name.startswith("_") and inspect.iscoroutinefunction(member) - ] - - -def _call_patterns(method_name: str) -> list[str]: - """Return the grep patterns that count as proof-of-wiring for a method. - - Accepts: - - Direct call: .method_name( - - Listener wiring: add_listener(...emitter.method_name (no open paren needed) - - Listener wiring: add_mutation_listener(...emitter.method_name - """ - return [ - rf"\.{re.escape(method_name)}\(", - rf"add_listener\(.*\.{re.escape(method_name)}\b", - rf"add_mutation_listener\(.*\.{re.escape(method_name)}\b", - ] - - -def _has_call_site(method_name: str, repo_root: Path) -> bool: - """Return True if at least one non-definition file contains a call site.""" - for package in _SCAN_PACKAGES: - package_path = repo_root / package - if not package_path.exists(): - continue - for py_file in package_path.rglob("*.py"): - rel = str(py_file.relative_to(repo_root)) - if rel in _DEFINITION_FILES: - continue - try: - source = py_file.read_text(encoding="utf-8") - except OSError: - continue - for pattern in _call_patterns(method_name): - if re.search(pattern, source): - return True - return False - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -class TestDashboardEmitterWiring: - """Every public DashboardEmitter method must have at least one call site.""" - - def test_no_unwired_methods(self) -> None: - """Enumerate all public async methods; fail if any have zero call sites. - - The full list of unwired methods is reported in a single failure so the - developer sees all gaps at once rather than fixing one at a time. - """ - repo_root = _repo_root() - methods = _emitter_method_names() - - unwired: list[str] = [] - for method_name in methods: - if method_name in _EXEMPT_METHODS: - continue - if not _has_call_site(method_name, repo_root): - unwired.append(method_name) - - if unwired: - pytest.fail( - "DashboardEmitter methods with zero call sites in " - "ergon_core/, ergon_builtins/, ergon_infra/:\n" - + "\n".join(f" - {m}" for m in sorted(unwired)) - + "\n\nFor each method, add a call site at the point of the " - "corresponding state mutation. See:\n" - " docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md\n" - " docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md", - ) - - @pytest.mark.parametrize("method_name", _emitter_method_names()) - def test_method_is_async(self, method_name: str) -> None: - """Guard against accidentally making an emitter method sync. - - All DashboardEmitter public methods must be async — they call - inngest_client.send(), which is a coroutine. - """ - member = getattr(DashboardEmitter, method_name) - assert inspect.iscoroutinefunction(member), ( - f"DashboardEmitter.{method_name} is not async. " - "All emitter methods must be async coroutines." - ) - - def test_exempt_methods_still_exist(self) -> None: - """The exempt list must not drift from the actual class definition. - - If an exempt method is renamed or removed, this test catches it so the - skip list is kept honest. - """ - all_methods = set(_emitter_method_names()) - # Exempt methods that are NOT async (register_execution) are excluded - # from the async method list; check all members including sync ones. - all_public = { - name - for name, _ in inspect.getmembers(DashboardEmitter, predicate=inspect.isfunction) - if not name.startswith("_") - } - missing_from_class = _EXEMPT_METHODS - all_public - assert not missing_from_class, ( - f"Exempt methods not found on DashboardEmitter: {missing_from_class}. " - "Update _EXEMPT_METHODS in this file." - ) -``` - -### Frontend mutation-kind coverage test - -```typescript -// ergon/ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts -/** - * Contract test: every MutationType variant must have a matching case arm in - * applyGraphMutation, i.e. the mutation must not appear in unhandledMutations - * after being applied to an empty state. - * - * See docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md. - */ - -import { describe, expect, it } from "vitest"; -import { MutationTypeSchema } from "./graphMutations"; -import { applyGraphMutation } from "../state/graphMutationReducer"; -import type { WorkflowRunState } from "@/lib/types"; -import type { DashboardGraphMutationData } from "@/lib/contracts/events"; - -function emptyState(): WorkflowRunState { - return { - id: "run-test", - name: "test", - status: "running", - tasks: new Map(), - startedAt: new Date().toISOString(), - completedAt: null, - durationSeconds: null, - finalScore: null, - error: null, - totalTasks: 0, - totalLeafTasks: 0, - completedTasks: 0, - runningTasks: 0, - failedTasks: 0, - generations: [], - resources: [], - sandboxes: new Map(), - threads: [], - evaluations: new Map(), - }; -} - -/** - * Build a minimal synthetic mutation for a given mutation_type. - * new_value contents must satisfy the schema for that type. - */ -function syntheticMutation( - mutationType: string, -): DashboardGraphMutationData { - const nodeId = "00000000-0000-0000-0000-000000000001"; - const newValueByType: Record<string, Record<string, unknown>> = { - "node.added": { - mutation_type: "node.added", - task_key: "test-task", - instance_key: "inst-1", - description: "test", - status: "pending", - assigned_worker_key: null, - }, - "node.removed": { mutation_type: "node.removed", status: "cancelled" }, - "node.status_changed": { - mutation_type: "node.status_changed", - status: "running", - }, - "node.field_changed": { - mutation_type: "node.field_changed", - field: "description", - value: "updated", - }, - "edge.added": { - mutation_type: "edge.added", - source_node_id: nodeId, - target_node_id: "00000000-0000-0000-0000-000000000002", - status: "pending", - }, - "edge.removed": { mutation_type: "edge.removed" }, - "edge.status_changed": { - mutation_type: "edge.status_changed", - status: "satisfied", - }, - "annotation.set": { - mutation_type: "annotation.set", - namespace: "test", - payload: {}, - }, - "annotation.deleted": { - mutation_type: "annotation.deleted", - namespace: "test", - payload: {}, - }, - }; - - return { - run_id: "00000000-0000-0000-0000-000000000000", - sequence: 1, - mutation_type: mutationType as DashboardGraphMutationData["mutation_type"], - target_type: "node", - target_id: nodeId, - actor: "test", - new_value: newValueByType[mutationType] ?? {}, - old_value: null, - reason: null, - timestamp: new Date().toISOString(), - }; -} - -const ALL_MUTATION_TYPES = MutationTypeSchema.options; - -describe("graphMutationReducer — mutation-kind coverage", () => { - it.each(ALL_MUTATION_TYPES)( - "mutation_type '%s' is handled (does not fall through to unhandledMutations)", - (mutationType) => { - const state = emptyState(); - const mutation = syntheticMutation(mutationType); - const next = applyGraphMutation(state, mutation); - const unhandled = next.unhandledMutations ?? []; - const fell = unhandled.some((u) => u.mutationType === mutationType); - expect(fell).toBe(false); - }, - ); - - it("ALL_MUTATION_TYPES matches MutationTypeSchema.options (no stale snapshot)", () => { - // This test fails if the schema drifts from the test's local snapshot. - expect(ALL_MUTATION_TYPES).toEqual(MutationTypeSchema.options); - }); -}); -``` - -### `tests/contract/__init__.py` - -```python -# ergon/tests/contract/__init__.py -``` - -Empty — required for pytest discovery. - -## Exact diffs for modified files - -The only modification to existing files in this RFC is the `package.json` -`test:contracts` script already exists at -`ergon-dashboard/package.json` (`"test:contracts": "tsx --test tests/contracts/contracts.test.ts"`). -No changes needed there; the frontend test file uses `describe`/`it` from vitest, -so the invocation command must be updated to use vitest instead of the node test -runner used by the existing `contracts.test.ts`. - -```diff ---- a/ergon-dashboard/package.json -+++ b/ergon-dashboard/package.json -@@ ... scripts section ... -- "test:contracts": "tsx --test tests/contracts/contracts.test.ts", -+ "test:contracts": "vitest run tests/contracts/ tests/graph/", -``` - -**Note:** The existing `tests/contracts/contracts.test.ts` uses the Node built-in -test runner (`import test from "node:test"`), not vitest. The new -`graphMutations.test.ts` uses vitest. Both can coexist if `test:contracts` is -split or if the existing test is migrated to vitest. The simplest change is to -run them with separate commands: - -```diff ---- a/ergon-dashboard/package.json -+++ b/ergon-dashboard/package.json -@@ ... scripts ... -- "test:contracts": "tsx --test tests/contracts/contracts.test.ts", -+ "test:contracts": "tsx --test tests/contracts/contracts.test.ts && vitest run tests/graph/", -``` - -Alternatively, migrate `contracts.test.ts` to vitest entirely (out of scope for -this RFC). For now the diff above is the minimal change. - -The backend test lives in a new `tests/contract/` directory (note: singular, to -distinguish from the `ergon-dashboard/tests/contracts/` directory). No changes -to existing Python files are required. - -## Package structure - -``` -ergon/ - tests/ - contract/ ← NEW directory - __init__.py ← NEW (empty) - test_dashboard_emitter_wiring.py ← NEW - - ergon-dashboard/ - src/ - features/graph/ - contracts/ - graphMutations.test.ts ← NEW -``` - -The `tests/contract/` directory sits alongside `tests/state/` and -`tests/integration/`. It is a pytest-discovered package; `__init__.py` is -required because other test packages under `tests/` have it. - -## Implementation order - -| Step | What | Files touched | PR | -|---|---|---|---| -| **1** | Land the per-method wiring fix for the 9 dead emitter methods (`docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md`). Without this, the contract test is red on arrival. | `ergon_core/ergon_core/core/runtime/inngest/start_workflow.py`, `complete_workflow.py`, `task_management_service.py`, `evaluate_task_run.py`, `cleanup_cancelled_task.py`, and wherever `resource_published` and `thread_message_created` belong | PR 1 — bug fix | -| **2** | Create `tests/contract/__init__.py` and `tests/contract/test_dashboard_emitter_wiring.py` | ADD 2 files | PR 2 — contract test | -| **3** | Verify the backend contract test is green locally (`uv run pytest tests/contract -v`), then add `tests/contract` to the `test:be:state` invocation in `package.json` | MODIFY `package.json` | PR 2 | -| **4** | Create `ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts` | ADD 1 file | PR 3 — FE contract test | -| **5** | Update `ergon-dashboard/package.json` `test:contracts` script to include vitest for the new test | MODIFY `ergon-dashboard/package.json` | PR 3 | -| **6** | Update `docs/architecture/05_dashboard.md#invariants` to cite the contract tests; add emitter coverage table inline | MODIFY `docs/architecture/05_dashboard.md` | PR 2 or PR 3 (alongside whichever closes last) | - -Steps 2–3 and 4–5 are independent and can land in parallel PRs once Step 1 is -merged. - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `ergon/tests/contract/__init__.py` | Empty init; required for pytest discovery of the new `contract/` package | -| `ergon/tests/contract/test_dashboard_emitter_wiring.py` | Backend contract test: enumerates `DashboardEmitter` public methods via `inspect` and asserts each has at least one call site in the runtime packages | -| `ergon/ergon-dashboard/src/features/graph/contracts/graphMutations.test.ts` | Frontend coverage test: asserts every `MutationType` variant in `MutationTypeSchema` is handled by `applyGraphMutation` without falling through to `unhandledMutations` | - -### MODIFY - -| File | Change | -|---|---| -| `ergon/ergon-dashboard/package.json` | Extend `test:contracts` script to invoke vitest for the new TS test file | -| `ergon/docs/architecture/05_dashboard.md` | Update `#invariants` section to cite the two new contract tests; add emitter-method coverage table | - -**Note:** `package.json` (repo root) does not need modification if `test:be:state` -already runs all `tests/state/` content. The `tests/contract/` directory must be -added to the pytest invocation explicitly if `test:be:state` uses a directory -glob rather than `discover`. Check `pnpm run test:be:state` in `package.json`: - -```json -"test:be:state": "uv run pytest tests/state -q" -``` - -This targets `tests/state/` explicitly. The contract tests must be added either -by changing the command to `uv run pytest tests/state tests/contract -q` or by -adding a dedicated `test:be:contract` script. The simpler path is: - -```diff -- "test:be:state": "uv run pytest tests/state -q", -+ "test:be:state": "uv run pytest tests/state tests/contract -q", -``` - -This is also a MODIFY to `package.json` (repo root). - -Updated MODIFY table: - -| File | Change | -|---|---| -| `ergon/package.json` | Add `tests/contract` to `test:be:state` pytest invocation | -| `ergon/ergon-dashboard/package.json` | Extend `test:contracts` script to include vitest for new TS test | -| `ergon/docs/architecture/05_dashboard.md` | Update invariants section + add emitter coverage table | - -## Testing approach - -### Unit / state tests (backend) - -The contract test itself is both unit and contract: - -- `test_no_unwired_methods` — runs the full scan; intended to be the CI gate. - In isolation it is fast (pure filesystem + regex, no DB, no Inngest). -- `test_method_is_async` — parametrized over all methods; guards against async - regression. -- `test_exempt_methods_still_exist` — guards against stale `_EXEMPT_METHODS` - if a method is renamed. - -Representative invocation: - -```bash -uv run pytest tests/contract/test_dashboard_emitter_wiring.py -v -``` - -Expected passing output (post bug fix): - -``` -PASSED tests/contract/test_dashboard_emitter_wiring.py::TestDashboardEmitterWiring::test_no_unwired_methods -PASSED tests/contract/test_dashboard_emitter_wiring.py::TestDashboardEmitterWiring::test_exempt_methods_still_exist -PASSED tests/contract/test_dashboard_emitter_wiring.py::TestDashboardEmitterWiring::test_method_is_async[workflow_started] -PASSED tests/contract/test_dashboard_emitter_wiring.py::TestDashboardEmitterWiring::test_method_is_async[workflow_completed] -... -``` - -Expected failing output (before bug fix or after adding an unwired method): - -``` -FAILED tests/contract/test_dashboard_emitter_wiring.py::TestDashboardEmitterWiring::test_no_unwired_methods - DashboardEmitter methods with zero call sites in ergon_core/, ergon_builtins/, ergon_infra/: - - task_status_changed - - workflow_started - ... - For each method, add a call site at the point of the corresponding state mutation. - See: docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md - docs/rfcs/active/2026-04-18-dashboard-event-wiring-enforcement.md -``` - -### Integration tests - -No integration tests are added by this RFC. The contract test is deliberately -free of I/O so it runs in `test:be:fast`. The wiring itself (calling the emitter -from the right place) is exercised by the existing state and integration tests -once the bug fix lands. - -### Frontend contract test - -```bash -pnpm -C ergon-dashboard run test:contracts -``` - -Uses vitest with `it.each` over `MutationTypeSchema.options`. Any new variant -added to `MutationTypeSchema` without a handler in `applyGraphMutation` causes: - -``` -FAIL src/features/graph/contracts/graphMutations.test.ts - ✗ mutation_type 'new.kind' is handled (does not fall through to unhandledMutations) - AssertionError: expected true to be false -``` - -### Coverage of all 12 emitter methods - -The contract test verifies method-level wiring. Individual method correctness -(correct arguments at the right moment) is covered by existing state tests once -the bug-fix PR lands: - -- `workflow_started` / `workflow_completed` — `test_workflow_finalization.py` -- `task_status_changed` — `test_graph_repository.py`, `test_plan_subtasks.py` -- `task_evaluation_updated` — `test_type_invariants.py` -- `task_cancelled` — `test_subtask_cancellation_service.py` -- `graph_mutation` / `on_context_event` — `test_graph_mutation_listener.py`, - `test_context_event_repository.py` - -## Trace / observability impact - -No new spans, metrics, or log lines. The contract test runs offline; it touches -no Inngest client or trace sink. The architecture doc update (Step 6) adds the -emitter coverage table as a developer-visible artifact — no runtime change. - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| Contract test runs before the bug-fix PR merges | Test is red in CI on first merge | Sequence: land bug-fix PR first, then the contract test PR. CI pipeline enforces ordering via PR dependencies or manual gating. | -| `inspect.getmembers` misses dynamically-added methods | Silent gap in coverage | All `DashboardEmitter` methods are statically defined; no `__getattr__` or dynamic method addition. Acceptable risk. | -| Regex scan produces false positives (e.g. a comment containing `.workflow_started(`) | Method counted as wired when it is not | False positives would cause the test to *not* fail when it should. Acceptable — the test is a floor, not a proof. Code review remains the correctness gate; the test catches complete blindness. | -| `_EXEMPT_METHODS` grows silently | New methods added to the exempt list with no justification | `test_exempt_methods_still_exist` ensures all exempt methods still exist; but it does not prevent adding new entries. Reviewed in PR. | -| TS test uses `vitest` but existing `contracts.test.ts` uses node test runner | `test:contracts` command conflict | Split the commands or migrate the existing test to vitest in the same PR. Explicit note in Implementation Order Step 5. | -| Frontend test checks `unhandledMutations` presence but that field is optional on `WorkflowRunState` | Test false-negative if field is undefined | `const unhandled = next.unhandledMutations ?? []` — the `?? []` default is in the test; safe. | -| Sandbox emitter methods remain wired only through `DashboardEmitterSandboxEventSink`, which has no constructor site | Three methods stay effectively dead at runtime | Tracked separately as `docs/bugs/open/2026-04-17-sandbox-event-sink-unactivated.md`. The contract test exempts them explicitly with a comment explaining the separate tracker. | - -## Invariants affected - -From `docs/architecture/05_dashboard.md#invariants`: - -> Every persistent backend state change on a wired surface must have a -> corresponding `dashboard/*` event; a state change without an emit is a bug -> in the emitter layer. **Enforcement is not automated today (see Follow-ups).** - -This RFC replaces "Enforcement is not automated today" with machine enforcement. -The updated invariant text (to be applied in Step 6): - -> Every persistent backend state change on a wired surface must have a -> corresponding `dashboard/*` event. A `DashboardEmitter` method with zero -> call sites in `ergon_core/`, `ergon_builtins/`, or `ergon_infra/` fails -> `pnpm run test:be:fast` via `tests/contract/test_dashboard_emitter_wiring.py`. -> A `RunGraphMutation.kind` with no frontend reducer branch fails -> `pnpm -C ergon-dashboard run test:contracts` via -> `src/features/graph/contracts/graphMutations.test.ts`. - -No other invariants are changed. The "emitter calls are best-effort" and -"DashboardStore is a cache" invariants are untouched. - -## Alternatives considered - -- **Review-only discipline.** Status quo; has demonstrably failed — 9 of 12 - methods were dead for an unknown period before discovery via grep. -- **Runtime assertion on first emit.** Rejected: false positives on emitter - paths that are legitimately quiescent during a given run (e.g. a run with - zero sandboxes would never trigger `sandbox_created`), and fails-open in - tests that don't exercise the path. -- **Type-level encoding (requiring each method to be referenced at compile - time via a sentinel).** Rejected: more invasive than a pytest, and Python's - import graph doesn't give useful reachability without extra machinery. -- **AST-based scan instead of regex.** More robust (no false positives from - comments); considerably more implementation complexity for marginal gain. - The regex approach is acceptable because false positives cause the test to - pass when it should fail (not fail when it should pass), and code review - remains the correctness gate. -- **Generated emitter-coverage table as a separate file.** Inline table in - `05_dashboard.md` preferred: one fewer file, no separate sync step, stays - under ~30 rows per the open-question resolution below. - -## Open questions - -- Should the backend test tolerate listener-style wiring - (`repo.add_listener(emitter.foo)`) as proof-of-wiring, or require a - direct call site? **Resolution:** Allow both. The pattern list is - `\.{name}\(` OR `add_listener\(.*\.{name}\b` OR - `add_mutation_listener\(.*\.{name}\b`. The current wiring of - `graph_mutation` and `on_context_event` uses listener registration; both - patterns are required for the test to pass without exempting them. -- Where should the generated emitter-coverage table live — inline in - `05_dashboard.md` or a sibling `05_dashboard_emitter_coverage.md`? - **Resolution:** Inline in `05_dashboard.md`. 12 emitter methods fit well - under the 30-row threshold; a sibling file adds navigation friction without - benefit. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`: -- Update `docs/architecture/05_dashboard.md#invariants` to cite the contract - tests (per Implementation Order Step 6). -- Close `docs/bugs/open/2026-04-18-dashboard-emitter-methods-not-wired.md` - once the bug-fix PR (Step 1) lands and the contract test is green. -- No separate plan file is needed — the implementation order table above is - the plan. diff --git a/docs/rfcs/accepted/2026-04-18-onboarding-deps-on-benchmark-abc.md b/docs/rfcs/accepted/2026-04-18-onboarding-deps-on-benchmark-abc.md deleted file mode 100644 index 549625990..000000000 --- a/docs/rfcs/accepted/2026-04-18-onboarding-deps-on-benchmark-abc.md +++ /dev/null @@ -1,928 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: architecture-qa -architecture_refs: [docs/architecture/06_builtins.md#invariants, docs/architecture/01_public_api.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Move `BENCHMARK_DEPS` onto the `Benchmark` ABC - -## Problem - -`BENCHMARK_DEPS` at `ergon_cli/ergon_cli/onboarding/profile.py:48–56` is a -hand-maintained dict parallel to the benchmark registry. Every benchmark -registered in `ergon_builtins/ergon_builtins/registry_core.py` and -`ergon_builtins/ergon_builtins/registry_data.py` must also appear in this -dict or `ergon onboard` silently omits the benchmark's required keys and -extras. The failure mode is silent: the prompt never fires and the first error -is a sandbox-provisioning failure deep in an actual run. - -**The regression has already happened once.** `swebench-verified` was absent -from `BENCHMARK_DEPS` until 2026-04-17; users onboarding for SWE-Bench were -never prompted for `E2B_API_KEY` or `ergon-builtins[data]`. Symptom: opaque -sandbox failure at runtime, not at `ergon onboard`. - -**Current gap is larger than the dict shows.** Nine benchmark slugs are -registered (five core, four data-extra); `BENCHMARK_DEPS` covers only five. -The four missing slugs — `delegation-smoke`, `researchrubrics-smoke`, -`researchrubrics-ablated`, `researchrubrics-vanilla` — never surface in the -onboarding wizard at all (the wizard iterates `BENCHMARK_DEPS` keys at -`ergon_cli/ergon_cli/commands/onboard.py:22`). - -Key file:line anchors for the problem: - -- `ergon_cli/ergon_cli/onboarding/profile.py:40–55` — `BenchmarkDeps` class - and `BENCHMARK_DEPS` dict (5 entries; 4 registered benchmarks absent) -- `ergon_cli/ergon_cli/onboarding/profile.py:77` — `required_keys()` reads - `BENCHMARK_DEPS.get(b, BenchmarkDeps())` so missing benchmarks silently - return empty deps -- `ergon_cli/ergon_cli/commands/onboard.py:22` — benchmark list offered to - user is `[(slug, slug) for slug in BENCHMARK_DEPS]` — only 5 of 9 - registered benchmarks appear -- `ergon_builtins/ergon_builtins/registry_core.py:66–72` — 5 core benchmarks -- `ergon_builtins/ergon_builtins/registry_data.py:20–26` — 4 data-extra - benchmarks -- `ergon_core/ergon_core/api/benchmark.py:18–65` — `Benchmark` ABC (no - `onboarding_deps` field today) - -The root cause is having two registries of the same fact. Validation tests -on top of two dicts catch regressions after the fact; collapsing to a single -source of truth prevents them. - -## Proposal - -Add `onboarding_deps: ClassVar[BenchmarkDeps]` as a required class attribute -on the `Benchmark` ABC. Move `BenchmarkDeps` from `ergon_cli/` to -`ergon_core/api/` so it is a public API type importable by benchmark authors -without pulling in CLI internals. Every concrete `Benchmark` subclass declares -its own deps at the class body. `ergon_cli/onboarding/profile.py` derives -deps by iterating the registry instead of a separate dict. - -### Option chosen: class-level `ClassVar` on `Benchmark` - -Benchmark authors set `onboarding_deps` at the class body, co-located with -`type_slug`. The ABC declares the attribute without a default so omitting it -is a `TypeError` at class instantiation time (Python raises when a concrete -class inherits an abstract `ClassVar` annotation that has no value and is -enforced via `__init_subclass__`). A class-body `__init_subclass__` hook on -`Benchmark` enforces presence at import time, not at runtime. - -**Why not `Optional[BenchmarkDeps] = None`?** An implicit opt-out is the -failure mode this RFC exists to prevent. `None` as a default means a new -benchmark author can silently skip the declaration. Enforcement must be at -class-definition time, not runtime. - -**Why move `BenchmarkDeps` to `ergon_core/api/`?** Benchmark classes live in -`ergon_builtins/`; they must not import from `ergon_cli/`. Today they don't — -but they will need to import `BenchmarkDeps` after this change. The type must -live in `ergon_core/api/` (the layer both `ergon_builtins/` and `ergon_cli/` -already depend on). - -## Architecture overview - -### Before - -``` -ergon_cli/onboarding/profile.py - BENCHMARK_DEPS: dict[str, BenchmarkDeps] ← hand-maintained, drifts - "smoke-test" → BenchmarkDeps(e2b=True) - "minif2f" → BenchmarkDeps(e2b=True) - "gdpeval" → BenchmarkDeps(e2b=True, extras=["ergon-builtins[data]"]) - "researchrubrics" → BenchmarkDeps(extras=[...], optional_keys=["EXA_API_KEY"]) - "swebench-verified" → BenchmarkDeps(e2b=True, extras=["ergon-builtins[data]"]) - # delegation-smoke, researchrubrics-smoke, -ablated, -vanilla: ABSENT - - OnboardProfile.required_keys() - ↓ BENCHMARK_DEPS.get(b, BenchmarkDeps()) ← silent empty for unknown slugs - -ergon_builtins/benchmarks/<slug>/benchmark.py - type_slug: ClassVar[str] = "..." ← only link to onboarding - -ergon_cli/commands/onboard.py - [(slug, slug) for slug in BENCHMARK_DEPS] ← 5 of 9 benchmarks visible -``` - -### After - -``` -ergon_core/api/benchmark_deps.py - BenchmarkDeps(e2b, extras, optional_keys) ← frozen Pydantic model, public API - -ergon_core/api/benchmark.py - class Benchmark(ABC): - type_slug: ClassVar[str] - onboarding_deps: ClassVar[BenchmarkDeps] ← enforced by __init_subclass__ - -ergon_builtins/benchmarks/<slug>/benchmark.py - class SomeBenchmark(Benchmark): - type_slug = "some-slug" - onboarding_deps = BenchmarkDeps(...) ← declared once, here - -ergon_cli/onboarding/profile.py - # BenchmarkDeps imported from ergon_core.api (re-exported for CLI convenience) - # BENCHMARK_DEPS dict DELETED - - OnboardProfile.required_keys() - ↓ BENCHMARKS[slug].onboarding_deps ← single source of truth - -ergon_cli/commands/onboard.py - [(slug, slug) for slug in BENCHMARKS] ← all 9 registered benchmarks visible -``` - -### Data-flow change in `ergon onboard` - -``` -ergon onboard - │ - ├─ wizard: benchmarks = select_multiple([slug for slug in BENCHMARKS]) - │ ^-- was BENCHMARK_DEPS keys, now BENCHMARKS keys (full set) - │ - ├─ OnboardProfile.required_keys() - │ for b in self.benchmarks: - │ deps = BENCHMARKS[b].onboarding_deps ← registry lookup, not dict - │ if deps.e2b: result["E2B_API_KEY"] = ... - │ for k in deps.optional_keys: result.setdefault(k, ...) - │ - └─ OnboardProfile.required_extras() - for b in self.benchmarks: - for e in BENCHMARKS[b].onboarding_deps.extras: extras.add(e) -``` - -## Type / interface definitions - -### `BenchmarkDeps` — new file in `ergon_core/api/` - -```python -# ergon_core/ergon_core/api/benchmark_deps.py - -from pydantic import BaseModel, Field - - -class BenchmarkDeps(BaseModel, frozen=True): - """Onboarding requirements for a single benchmark. - - Declared as a ClassVar on every Benchmark subclass. The onboarding - wizard reads these to determine which API keys to prompt for and - which pip extras to install. - - This is the single source of truth for a benchmark's onboarding - requirements. Do not add a corresponding entry in any dict elsewhere. - """ - - e2b: bool = False - """True if this benchmark requires an E2B sandbox (implies E2B_API_KEY).""" - - extras: tuple[str, ...] = () - """Pip extras that must be installed to use this benchmark, - e.g. ``("ergon-builtins[data]",)``.""" - - optional_keys: tuple[str, ...] = () - """API keys to prompt for during onboarding that are not strictly required - but enhance benchmark functionality (e.g. ``("EXA_API_KEY",)``).""" -``` - -**Note:** `extras` and `optional_keys` change from `list[str]` to -`tuple[str, ...]` to satisfy `frozen=True`. The CLI callers iterate them — -no behavior change. - -### `Benchmark` ABC changes — `ergon_core/api/benchmark.py` - -```python -# ergon_core/ergon_core/api/benchmark.py (modified) - -from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence -from typing import Any, ClassVar - -from ergon_core.api.benchmark_deps import BenchmarkDeps -from ergon_core.api.dependencies import check_packages -from ergon_core.api.errors import DependencyError -from ergon_core.api.task_types import BenchmarkTask - - -class Benchmark(ABC): - """Base class for all benchmarks. - - Subclasses MUST set ``type_slug`` and ``onboarding_deps`` and implement - ``build_instances``. Omitting ``onboarding_deps`` raises ``TypeError`` - at class definition time. - """ - - type_slug: ClassVar[str] - onboarding_deps: ClassVar[BenchmarkDeps] - required_packages: ClassVar[list[str]] = [] - install_hint: ClassVar[str] = "" - - def __init_subclass__(cls, **kwargs: Any) -> None: # slopcop: ignore[no-typing-any] - super().__init_subclass__(**kwargs) - # Only enforce on concrete classes (those that also define type_slug). - # Abstract intermediate subclasses (e.g. ResearchRubricsBenchmark before - # ResearchRubricsVanillaBenchmark) may omit the field. - if not getattr(cls, "__abstractmethods__", None) and hasattr(cls, "type_slug"): - if not hasattr(cls, "onboarding_deps") or not isinstance( - cls.__dict__.get("onboarding_deps") or getattr(cls, "onboarding_deps", None), - BenchmarkDeps, - ): - raise TypeError( - f"{cls.__qualname__} must declare " - f"'onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(...)'. " - f"See ergon_core/api/benchmark_deps.py." - ) - - def __init__( - self, - *, - name: str | None = None, - description: str | None = None, - metadata: Mapping[str, Any] | None = None, # slopcop: ignore[no-typing-any] - ) -> None: - self.name = name or self.__class__.__name__ - self.description = description or "" - self.metadata: dict[str, Any] = dict(metadata or {}) # slopcop: ignore[no-typing-any] - - @abstractmethod - def build_instances(self) -> Mapping[str, Sequence[BenchmarkTask]]: - """Materialize benchmark instances. - - Returns a mapping of instance_key -> tasks for that instance. - """ - ... - - def evaluator_requirements(self) -> Sequence[str]: - """Declare evaluator slot names required by this benchmark.""" - return ("default",) - - def validate(self) -> None: - """Check that runtime dependencies are available.""" - errors = check_packages( - self.required_packages, - f"Benchmark '{self.type_slug}'", - ) - if errors: - parts = [*errors] - if self.install_hint: - parts.append(f"Install with: {self.install_hint}") - raise DependencyError("\n".join(parts)) -``` - -**Enforcement logic:** `__init_subclass__` fires at class-definition time (at -import). It skips classes that still have `__abstractmethods__` (abstract -intermediates) and skips classes without `type_slug` (avoids false positives -on internal base classes). For concrete registered benchmarks this fires the -moment the module is imported — well before any runtime operation. - -**Inheritance case (`ResearchRubricsVanillaBenchmark`):** The parent -`ResearchRubricsBenchmark` declares `onboarding_deps`; the child -`ResearchRubricsVanillaBenchmark` inherits it. The check uses `getattr` (not -`cls.__dict__`) so inheritance works. If the child needs different deps it -overrides the field; if not, it inherits. Both cases pass. - -## Full implementations - -### New file: `ergon_core/ergon_core/api/benchmark_deps.py` - -```python -# ergon_core/ergon_core/api/benchmark_deps.py - -from pydantic import BaseModel - - -class BenchmarkDeps(BaseModel, frozen=True): - """Onboarding requirements for a single benchmark. - - Declared as a ClassVar on every Benchmark subclass. The onboarding - wizard reads these to determine which API keys to prompt for and - which pip extras to install. - - This is the single source of truth for a benchmark's onboarding - requirements. Do not add a corresponding entry in any dict elsewhere. - """ - - e2b: bool = False - extras: tuple[str, ...] = () - optional_keys: tuple[str, ...] = () -``` - -### Modified benchmark classes - -Each benchmark gains one line: `onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(...)`. - -**`SmokeTestBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/smoke_test/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class SmokeTestBenchmark(Benchmark): - type_slug: ClassVar[str] = "smoke-test" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(e2b=True) - # ... rest unchanged -``` - -**`MiniF2FBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class MiniF2FBenchmark(Benchmark): - type_slug: ClassVar[str] = "minif2f" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(e2b=True) - # ... rest unchanged -``` - -**`DelegationSmokeBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/delegation_smoke/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class DelegationSmokeBenchmark(Benchmark): - type_slug: ClassVar[str] = "delegation-smoke" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps() - # ... rest unchanged -``` - -**`ResearchRubricsSmokeTestBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/smoke.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class ResearchRubricsSmokeTestBenchmark(Benchmark): - type_slug: ClassVar[str] = "researchrubrics-smoke" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps() - # ... rest unchanged -``` - -**`SweBenchVerifiedBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class SweBenchVerifiedBenchmark(Benchmark): - type_slug: ClassVar[str] = "swebench-verified" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps( - e2b=True, - extras=("ergon-builtins[data]",), - ) - # ... rest unchanged -``` - -**`GDPEvalBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/gdpeval/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class GDPEvalBenchmark(Benchmark): - type_slug: ClassVar[str] = "gdpeval" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps( - e2b=True, - extras=("ergon-builtins[data]",), - ) - # ... rest unchanged -``` - -**`ResearchRubricsBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` - -```python -from ergon_core.api.benchmark_deps import BenchmarkDeps - -class ResearchRubricsBenchmark(Benchmark): - type_slug: ClassVar[str] = "researchrubrics" - onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps( - extras=("ergon-builtins[data]",), - optional_keys=("EXA_API_KEY",), - ) - # ... rest unchanged -``` - -**`ResearchRubricsVanillaBenchmark`** — `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/vanilla.py` - -`ResearchRubricsVanillaBenchmark` subclasses `ResearchRubricsBenchmark`. Since -both `researchrubrics` and `researchrubrics-vanilla` have the same deps, -the child inherits `onboarding_deps` from the parent. No declaration needed. -However, `researchrubrics-ablated` is registered under a separate key but -reuses `ResearchRubricsBenchmark` directly (same class, two registry keys) — -see registry_data.py. The `__init_subclass__` check uses `getattr` on the class -so inheritance satisfies the check. - -**`ResearchRubricsAblated`** — no separate class exists; `registry_data.py:23` -maps `"researchrubrics-ablated"` to `ResearchRubricsBenchmark` directly. -The inherited `onboarding_deps` from `ResearchRubricsBenchmark` applies; no -new class file change needed. - -## Exact diffs for modified files - -### `ergon_core/ergon_core/api/benchmark.py` - -```diff ---- a/ergon_core/ergon_core/api/benchmark.py -+++ b/ergon_core/ergon_core/api/benchmark.py -@@ -7,12 +7,15 @@ - from abc import ABC, abstractmethod - from collections.abc import Mapping, Sequence --from typing import Any, ClassVar -+from typing import Any, ClassVar - -+from ergon_core.api.benchmark_deps import BenchmarkDeps - from ergon_core.api.dependencies import check_packages - from ergon_core.api.errors import DependencyError - from ergon_core.api.task_types import BenchmarkTask - - - class Benchmark(ABC): - """Base class for all benchmarks. - -- Subclasses must set ``type_slug`` and implement ``build_instances``. -+ Subclasses MUST set ``type_slug`` and ``onboarding_deps`` and implement -+ ``build_instances``. Omitting ``onboarding_deps`` raises ``TypeError`` -+ at class definition time. - """ - - type_slug: ClassVar[str] -+ onboarding_deps: ClassVar[BenchmarkDeps] - required_packages: ClassVar[list[str]] = [] - install_hint: ClassVar[str] = "" - -+ def __init_subclass__(cls, **kwargs: Any) -> None: # slopcop: ignore[no-typing-any] -+ super().__init_subclass__(**kwargs) -+ if not getattr(cls, "__abstractmethods__", None) and hasattr(cls, "type_slug"): -+ if not hasattr(cls, "onboarding_deps") or not isinstance( -+ cls.__dict__.get("onboarding_deps") or getattr(cls, "onboarding_deps", None), -+ BenchmarkDeps, -+ ): -+ raise TypeError( -+ f"{cls.__qualname__} must declare " -+ f"'onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(...)'. " -+ f"See ergon_core/api/benchmark_deps.py." -+ ) -+ - def __init__( -``` - -### `ergon_core/ergon_core/api/__init__.py` - -```diff ---- a/ergon_core/ergon_core/api/__init__.py -+++ b/ergon_core/ergon_core/api/__init__.py -@@ -1,6 +1,7 @@ - """Object-first Ergon public API surface.""" - - from ergon_core.api.benchmark import Benchmark -+from ergon_core.api.benchmark_deps import BenchmarkDeps - from ergon_core.api.criterion import Criterion - ... - - __all__ = [ - "Benchmark", -+ "BenchmarkDeps", - "BenchmarkTask", - ... - ] -``` - -### `ergon_cli/ergon_cli/onboarding/profile.py` - -```diff ---- a/ergon_cli/ergon_cli/onboarding/profile.py -+++ b/ergon_cli/ergon_cli/onboarding/profile.py -@@ -1,10 +1,10 @@ --"""OnboardProfile: user choices -> required keys and pip extras. -- --The BENCHMARK_DEPS dict is the single source of truth for what each benchmark --needs. Keep it aligned with registry_core.py / registry.py. --""" -+"""OnboardProfile: user choices -> required keys and pip extras.""" - - from enum import Enum - --from pydantic import BaseModel, Field -+from pydantic import BaseModel, Field -+ -+from ergon_core.api.benchmark_deps import BenchmarkDeps # noqa: F401 — re-exported for CLI callers - - - class LLMProvider(str, Enum): -@@ -36,18 +36,6 @@ - GPUProvider.RUNPOD: "RUNPOD_API_KEY", - } - -- --class BenchmarkDeps(BaseModel): -- """What a single benchmark requires beyond the base install.""" -- -- e2b: bool = False -- extras: list[str] = Field(default_factory=list) -- optional_keys: list[str] = Field(default_factory=list) -- -- --BENCHMARK_DEPS: dict[str, BenchmarkDeps] = { -- "smoke-test": BenchmarkDeps(e2b=True), -- "minif2f": BenchmarkDeps(e2b=True), -- "gdpeval": BenchmarkDeps(e2b=True, extras=["ergon-builtins[data]"]), -- "researchrubrics": BenchmarkDeps( -- extras=["ergon-builtins[data]"], optional_keys=["EXA_API_KEY"] -- ), -- "swebench-verified": BenchmarkDeps(e2b=True, extras=["ergon-builtins[data]"]), --} -- - - class OnboardProfile(BaseModel): - """Captures every user choice made during onboarding.""" -@@ -60,24 +48,24 @@ - def required_keys(self) -> dict[str, str]: - """Return {env_var: human_reason} derived purely from user choices.""" -+ # reason: deferred import avoids circular dep at CLI startup; registry -+ # depends on ergon_builtins which depends on ergon_core. -+ from ergon_builtins.registry import BENCHMARKS -+ - result: dict[str, str] = {} - - for provider in self.llm_providers: - env_var = PROVIDER_KEY_MAP[provider] - result[env_var] = f"{provider.value} API access" - -- if any(BENCHMARK_DEPS.get(b, BenchmarkDeps()).e2b for b in self.benchmarks): -+ if any(BENCHMARKS[b].onboarding_deps.e2b for b in self.benchmarks if b in BENCHMARKS): - result["E2B_API_KEY"] = "Sandboxed code execution for selected benchmarks" - - for b in self.benchmarks: -- for k in BENCHMARK_DEPS.get(b, BenchmarkDeps()).optional_keys: -- result.setdefault(k, f"Optional for {b}") -+ if b in BENCHMARKS: -+ for k in BENCHMARKS[b].onboarding_deps.optional_keys: -+ result.setdefault(k, f"Optional for {b}") - - if self.gpu_provider and self.gpu_provider != GPUProvider.LOCAL: - env_var = GPU_PROVIDER_KEY_MAP[self.gpu_provider] - result[env_var] = f"GPU provisioning via {self.gpu_provider.value}" - - return result - - def required_extras(self) -> list[str]: - """Pip extras to install based on choices.""" -+ from ergon_builtins.registry import BENCHMARKS -+ - extras: set[str] = set() - for b in self.benchmarks: -- for e in BENCHMARK_DEPS.get(b, BenchmarkDeps()).extras: -- extras.add(e) -+ if b in BENCHMARKS: -+ for e in BENCHMARKS[b].onboarding_deps.extras: -+ extras.add(e) - if self.training: - extras.add("ergon-infra[training]") - if self.gpu_provider and self.gpu_provider != GPUProvider.LOCAL: - extras.add("ergon-infra[skypilot]") - return sorted(extras) -``` - -### `ergon_cli/ergon_cli/commands/onboard.py` - -```diff ---- a/ergon_cli/ergon_cli/commands/onboard.py -+++ b/ergon_cli/ergon_cli/commands/onboard.py -@@ -5,8 +5,6 @@ - from ergon_cli.onboarding.env_writer import write_env - from ergon_cli.onboarding.installer import install_extras - from ergon_cli.onboarding.profile import ( -- BENCHMARK_DEPS, - GPUProvider, - LLMProvider, - OnboardProfile, -@@ -17,9 +15,13 @@ - def handle_onboard(args: Namespace) -> int: # noqa: ARG001 - print("\nWelcome to Ergon! Let's get your environment set up.\n") - -+ # reason: deferred import avoids pulling heavy ergon_builtins deps at CLI startup. -+ from ergon_builtins.registry import BENCHMARKS -+ - profile = OnboardProfile() - - # --- Q1: benchmarks ------------------------------------------------------- - profile.benchmarks = select_multiple( - "Which benchmarks do you want to run?", -- [(slug, slug) for slug in BENCHMARK_DEPS], -+ [(slug, slug) for slug in sorted(BENCHMARKS)], - ) -``` - -## Package structure - -### New file: `ergon_core/ergon_core/api/benchmark_deps.py` - -The file stands alone; no new package is created. It is imported by -`ergon_core/api/benchmark.py` and re-exported via `ergon_core/api/__init__.py`. - -### `ergon_core/ergon_core/api/__init__.py` additions - -```python -# Add to existing __init__.py: -from ergon_core.api.benchmark_deps import BenchmarkDeps - -# Add to __all__: -"BenchmarkDeps", -``` - -## Implementation order - -| Step | PR | What | Files touched | -|---|---|---|---| -| **1** | PR 1 | Add `BenchmarkDeps` to `ergon_core/api/benchmark_deps.py`; export from `ergon_core/api/__init__.py` | ADD `ergon_core/ergon_core/api/benchmark_deps.py`; MODIFY `ergon_core/ergon_core/api/__init__.py` | -| **2** | PR 1 | Add `onboarding_deps: ClassVar[BenchmarkDeps]` annotation and `__init_subclass__` enforcement to `Benchmark`; import `BenchmarkDeps` | MODIFY `ergon_core/ergon_core/api/benchmark.py` | -| **3** | PR 1 | Populate `onboarding_deps` on all five core benchmarks: `SmokeTestBenchmark`, `MiniF2FBenchmark`, `DelegationSmokeBenchmark`, `ResearchRubricsSmokeTestBenchmark`, `SweBenchVerifiedBenchmark` | MODIFY 5 files | -| **4** | PR 1 | Populate `onboarding_deps` on all four data-extra benchmarks: `GDPEvalBenchmark`, `ResearchRubricsBenchmark`, `ResearchRubricsVanillaBenchmark`; confirm `researchrubrics-ablated` inherits from `ResearchRubricsBenchmark` and needs no separate class change | MODIFY 3 files | -| **5** | PR 1 | Rewrite `OnboardProfile.required_keys()` and `required_extras()` to read `BENCHMARKS[b].onboarding_deps`; delete `BENCHMARK_DEPS` dict and `BenchmarkDeps` class from `profile.py`; add re-export import for `BenchmarkDeps` from `ergon_core.api` | MODIFY `ergon_cli/ergon_cli/onboarding/profile.py` | -| **6** | PR 1 | Update `ergon onboard` wizard: replace `BENCHMARK_DEPS` key iteration with `BENCHMARKS` key iteration; remove `BENCHMARK_DEPS` import | MODIFY `ergon_cli/ergon_cli/commands/onboard.py` | -| **7** | PR 2 | Add contract test asserting every value in `BENCHMARKS` (core + data) has a `BenchmarkDeps`-typed `onboarding_deps` | ADD `tests/state/test_benchmark_contract.py` | -| **8** | PR 2 | Update existing `tests/state/test_onboard_profile.py` to remove assertions that reference `BENCHMARK_DEPS` directly; add assertions for the four previously-absent benchmarks | MODIFY `tests/state/test_onboard_profile.py` | -| **9** | PR 2 | Update `docs/architecture/06_builtins.md` and `docs/architecture/01_public_api.md` per the "On acceptance" section | MODIFY 2 docs | - -Steps 1–6 land as a single PR. Steps 7–9 land as a follow-on PR. The contract -test (step 7) can be written against the completed implementation from PR 1 -and acts as a regression guard. - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `ergon_core/ergon_core/api/benchmark_deps.py` | `BenchmarkDeps` frozen Pydantic model; single source of truth type | -| `tests/state/test_benchmark_contract.py` | Contract test: every registered benchmark has a valid `onboarding_deps` | - -### MODIFY - -| File | Change | -|---|---| -| `ergon_core/ergon_core/api/benchmark.py` | Add `onboarding_deps: ClassVar[BenchmarkDeps]` and `__init_subclass__` enforcement; import `BenchmarkDeps` | -| `ergon_core/ergon_core/api/__init__.py` | Export `BenchmarkDeps` | -| `ergon_builtins/ergon_builtins/benchmarks/smoke_test/benchmark.py` | Add `onboarding_deps = BenchmarkDeps(e2b=True)` | -| `ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py` | Add `onboarding_deps = BenchmarkDeps(e2b=True)` | -| `ergon_builtins/ergon_builtins/benchmarks/delegation_smoke/benchmark.py` | Add `onboarding_deps = BenchmarkDeps()` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/smoke.py` | Add `onboarding_deps = BenchmarkDeps()` | -| `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py` | Add `onboarding_deps = BenchmarkDeps(e2b=True, extras=("ergon-builtins[data]",))` | -| `ergon_builtins/ergon_builtins/benchmarks/gdpeval/benchmark.py` | Add `onboarding_deps = BenchmarkDeps(e2b=True, extras=("ergon-builtins[data]",))` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` | Add `onboarding_deps = BenchmarkDeps(extras=("ergon-builtins[data]",), optional_keys=("EXA_API_KEY",))` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/vanilla.py` | No class-body change required — inherits from `ResearchRubricsBenchmark`; verify via contract test | -| `ergon_cli/ergon_cli/onboarding/profile.py` | Delete `BenchmarkDeps` class and `BENCHMARK_DEPS` dict; rewrite `required_keys()` and `required_extras()` to use registry; add `BenchmarkDeps` re-export import | -| `ergon_cli/ergon_cli/commands/onboard.py` | Replace `BENCHMARK_DEPS` import and key iteration with `BENCHMARKS`; deferred import | -| `tests/state/test_onboard_profile.py` | Remove `BENCHMARK_DEPS` references; add assertions for `delegation-smoke`, `researchrubrics-smoke`, `researchrubrics-ablated`, `researchrubrics-vanilla` | - -## Testing approach - -### Contract test: `tests/state/test_benchmark_contract.py` - -```python -# tests/state/test_benchmark_contract.py - -"""Contract: every registered benchmark declares onboarding_deps.""" - -from __future__ import annotations - -import pytest - -from ergon_core.api.benchmark_deps import BenchmarkDeps - - -class TestBenchmarkOnboardingDepsContract: - """Every benchmark in both registries must declare onboarding_deps.""" - - def test_core_benchmarks_have_onboarding_deps(self) -> None: - from ergon_builtins.registry_core import BENCHMARKS - - for slug, cls in BENCHMARKS.items(): - assert hasattr(cls, "onboarding_deps"), ( - f"Benchmark '{slug}' ({cls.__qualname__}) is missing 'onboarding_deps'. " - f"Add 'onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(...)' " - f"to the class body." - ) - assert isinstance(cls.onboarding_deps, BenchmarkDeps), ( - f"Benchmark '{slug}' ({cls.__qualname__}).onboarding_deps is not a " - f"BenchmarkDeps instance; got {type(cls.onboarding_deps)!r}." - ) - - def test_data_benchmarks_have_onboarding_deps(self) -> None: - pytest.importorskip("datasets", reason="ergon-builtins[data] not installed") - from ergon_builtins.registry_data import BENCHMARKS - - for slug, cls in BENCHMARKS.items(): - assert hasattr(cls, "onboarding_deps"), ( - f"Benchmark '{slug}' ({cls.__qualname__}) is missing 'onboarding_deps'." - ) - assert isinstance(cls.onboarding_deps, BenchmarkDeps) - - def test_onboarding_deps_is_frozen(self) -> None: - """BenchmarkDeps instances must be immutable (frozen=True).""" - from ergon_builtins.registry_core import BENCHMARKS - - for slug, cls in BENCHMARKS.items(): - deps = cls.onboarding_deps - with pytest.raises(Exception): # Pydantic ValidationError or TypeError - deps.__dict__["e2b"] = not deps.e2b # type: ignore[index] - - def test_known_e2b_benchmarks(self) -> None: - from ergon_builtins.registry_core import BENCHMARKS - - assert BENCHMARKS["smoke-test"].onboarding_deps.e2b is True - assert BENCHMARKS["minif2f"].onboarding_deps.e2b is True - assert BENCHMARKS["swebench-verified"].onboarding_deps.e2b is True - assert BENCHMARKS["delegation-smoke"].onboarding_deps.e2b is False - assert BENCHMARKS["researchrubrics-smoke"].onboarding_deps.e2b is False -``` - -### Unit tests: update `tests/state/test_onboard_profile.py` - -The existing tests in `tests/state/test_onboard_profile.py` pass through -`OnboardProfile` and do not import `BENCHMARK_DEPS` directly — they will -continue to pass after the dict is deleted. Additions needed: - -```python -# Additional tests to add to tests/state/test_onboard_profile.py - -class TestPreviouslyMissingBenchmarks: - """Regression: delegation-smoke and researchrubrics-smoke were absent - from BENCHMARK_DEPS before this RFC. Verify they now appear in the - onboarding wizard choices and produce correct deps.""" - - def test_delegation_smoke_has_no_e2b(self) -> None: - p = OnboardProfile(benchmarks=["delegation-smoke"]) - assert "E2B_API_KEY" not in p.required_keys() - assert p.required_extras() == [] - - def test_researchrubrics_smoke_has_no_e2b(self) -> None: - p = OnboardProfile(benchmarks=["researchrubrics-smoke"]) - assert "E2B_API_KEY" not in p.required_keys() - assert p.required_extras() == [] - - def test_researchrubrics_ablated_needs_data_extra(self) -> None: - p = OnboardProfile(benchmarks=["researchrubrics-ablated"]) - assert "ergon-builtins[data]" in p.required_extras() - - def test_researchrubrics_vanilla_needs_data_extra(self) -> None: - p = OnboardProfile(benchmarks=["researchrubrics-vanilla"]) - assert "ergon-builtins[data]" in p.required_extras() - - -class TestOnboardingWizardSeesAllBenchmarks: - """The wizard must offer all registered benchmarks, not just the old 5.""" - - def test_wizard_sees_nine_slugs(self) -> None: - from ergon_builtins.registry import BENCHMARKS - - # All nine registered slugs must be visible - expected = { - "smoke-test", "minif2f", "delegation-smoke", "researchrubrics-smoke", - "swebench-verified", "gdpeval", "researchrubrics", "researchrubrics-ablated", - "researchrubrics-vanilla", - } - assert expected <= set(BENCHMARKS.keys()) -``` - -### `__init_subclass__` enforcement test - -```python -# Can live in tests/state/test_benchmark_contract.py - -class TestBenchmarkSubclassEnforcement: - def test_missing_onboarding_deps_raises_at_class_definition(self) -> None: - from ergon_core.api.benchmark import Benchmark - from ergon_core.api.task_types import BenchmarkTask - - with pytest.raises(TypeError, match="onboarding_deps"): - class BadBenchmark(Benchmark): - type_slug = "bad-test" - - def build_instances(self): - return {} - - def test_valid_declaration_does_not_raise(self) -> None: - from ergon_core.api.benchmark import Benchmark - from ergon_core.api.benchmark_deps import BenchmarkDeps - - class GoodBenchmark(Benchmark): - type_slug = "good-test" - onboarding_deps = BenchmarkDeps() - - def build_instances(self): - return {} - # No exception raised -``` - -## Trace / observability impact - -No span, log, or metric changes. This RFC is source-only and touches no -runtime hot paths. `ergon onboard` is a CLI wizard (synchronous, no spans). -The `__init_subclass__` check fires at import time and is not traced. - -If a benchmark author ships a class without `onboarding_deps`, the `TypeError` -appears in the import traceback — visible in logs during module load. This is -intentional: fail fast, fail loudly, at import rather than at runtime. - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| `__init_subclass__` fires on abstract intermediate subclasses | `TypeError` for legitimate abstract base classes that don't declare `type_slug` | Enforcement skips classes with `__abstractmethods__` set and skips classes without `type_slug`. `ResearchRubricsBenchmark` has `type_slug` and must declare `onboarding_deps`. | -| `ergon_cli/` imports `ergon_builtins/` (circular potential) | Import error if `profile.py` top-level imports from `ergon_builtins` | Imports of `BENCHMARKS` in `required_keys()`, `required_extras()`, and `handle_onboard()` are deferred inside function bodies (same pattern used elsewhere in CLI). | -| `BenchmarkDeps` field type change (`list` → `tuple`) breaks callers | Existing callers iterating `extras` or `optional_keys` expect sequences | Both `list` and `tuple` are iterable — the change is backward-compatible for iteration. `BENCHMARK_DEPS` is deleted so there are no external dict callers. | -| `researchrubrics-ablated` has no class of its own | `onboarding_deps` not visible from the registry value | `researchrubrics-ablated` maps to `ResearchRubricsBenchmark` in `registry_data.py:23`. `ResearchRubricsBenchmark.onboarding_deps` is set; the contract test passes on the class, not the slug. | -| `ResearchRubricsVanillaBenchmark` inherits but doesn't override `onboarding_deps` | `__init_subclass__` passes because it checks `getattr`, not `cls.__dict__` | Desired behavior. The contract test verifies the value is the right type regardless of whether it was inherited or declared. | -| Deleting `BENCHMARK_DEPS` breaks external code that imports it | API break for any caller outside the monorepo | No external package can register benchmarks (see `docs/architecture/06_builtins.md#extension-points`); the dict is not a documented public API. Worktrees have their own copy — they will need the same change applied. | -| `ergon onboard` wizard benchmark list order changes | UX difference; alphabetical sort vs. insertion order | The diff uses `sorted(BENCHMARKS)` — deterministic alphabetical order, cleaner than implicit insertion order. | - -## Invariants affected - -### `docs/architecture/06_builtins.md#invariants` (section 4) - -- **REPLACE** the invariant "Every registered benchmark MUST have a matching - onboarding deps entry [in `BENCHMARK_DEPS`]" with: "Every concrete - `Benchmark` subclass MUST declare `onboarding_deps: ClassVar[BenchmarkDeps]` - at the class body. The ABC's `__init_subclass__` enforces this at import - time. The parallel `BENCHMARK_DEPS` dict no longer exists." -- **REMOVE** the `docs/architecture/06_builtins.md#anti-patterns` bullet - "Forgetting the onboarding deps entry" (replaced by the enforced invariant). - -### `docs/architecture/06_builtins.md` section 3 (Control flow — adding a benchmark) - -- **REPLACE** step 3 "Declare the benchmark's onboarding deps so `ergon onboard` - prompts correctly (see invariants)." → "Set `onboarding_deps: - ClassVar[BenchmarkDeps] = BenchmarkDeps(...)` on the class. The ABC enforces - this at import; the onboarding wizard reads it from the registry." - -### `docs/architecture/06_builtins.md` Code map (section at end) - -- **CHANGE** row "Onboarding deps dict" from `ergon_cli/onboarding/profile.py` - to `ergon_core/api/benchmark_deps.py` (type declaration) and "each - `Benchmark` subclass" (values). - -### `docs/architecture/01_public_api.md` (core abstractions) - -- **EXTEND** the `Benchmark` bullet: "Carries a `type_slug: ClassVar[str]` ... - and an `onboarding_deps: ClassVar[BenchmarkDeps]` that the onboarding wizard - reads to determine required API keys and pip extras." -- **ADD** a `BenchmarkDeps` bullet in the core abstractions list. -- **UPDATE** `add a new benchmark` extension-point step 6: delete the note - about `ergon_cli/onboarding/profile.py::BENCHMARK_DEPS`. Replace with: - "Set `onboarding_deps: ClassVar[BenchmarkDeps] = BenchmarkDeps(...)` on - the class body." -- **UPDATE** the Code map table: add a `BenchmarkDeps` row pointing at - `ergon_core/api/benchmark_deps.py`. - -## Alternatives considered - -- **Validation test only** — a unit test that cross-checks the two registries. - Rejected: does not remove the duplication. A contributor still has to update - two files; the test only catches forgotten updates after the fact. -- **Status quo** — keep the dict and rely on code review. Rejected: already - regressed once (`swebench-verified`, 2026-04-17). Review is not load-bearing. -- **Entry-points based registration** — have each benchmark's package expose - its deps via a Python entry point. Rejected for now: we do not need external - package contribution (see `docs/architecture/06_builtins.md#extension-points`); - entry points add complexity without buying anything at current scale. - -## Relationship to `2026-04-18-template-spec-public-api.md` - -The template-spec RFC (`docs/rfcs/active/2026-04-18-template-spec-public-api.md`) -adds a second required `ClassVar` to `Benchmark`: `template_spec: -ClassVar[TemplateSpec | NoSetupSentinel]`. The two RFCs are **orthogonal** — -they each add a different class-level attribute, touch different consumers -(`ergon onboard` here, `ergon benchmark setup` there), and have no shared -logic. However they **cannot land in arbitrary order** without coordination: - -- If this RFC lands first: `Benchmark` gains `__init_subclass__` checking for - `onboarding_deps`. The template-spec RFC must extend `__init_subclass__` to - also check `template_spec`, or add its own independent check. -- If the template-spec RFC lands first: this RFC must similarly not overwrite - the template-spec RFC's `__init_subclass__` hook. The safest approach is for - `__init_subclass__` to check both attributes in one pass. - -**Recommendation:** land this RFC first; when the template-spec RFC ships, -extend `__init_subclass__` in `benchmark.py` to validate both fields. Do not -implement two independent `__init_subclass__` overrides — only the last one -in MRO would execute. Coordinate in the second RFC's implementation PR. - -## Open questions - -- Should `BenchmarkDeps` become frozen (Pydantic `frozen=True`)? Yes, this - RFC proposes `frozen=True`. It is a class-level `ClassVar` and mutation - would be surprising. No caller mutates it today. The field types change from - `list[str]` to `tuple[str, ...]` to satisfy the frozen constraint. -- Should we also move `SANDBOX_MANAGERS` and `SANDBOX_TEMPLATES` onto the ABC - at the same time for consistency? No — those registries are consumed - differently (by the Inngest runtime and `ergon benchmark setup`). The - template-spec RFC (`docs/rfcs/active/2026-04-18-template-spec-public-api.md`) - already covers the `SANDBOX_TEMPLATES` angle. Keep this RFC focused. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`: - - Update `docs/architecture/06_builtins.md#invariants` to drop the - `BENCHMARK_DEPS` parallel-dict invariant and replace with the - `onboarding_deps` class-attribute invariant. - - Update `docs/architecture/06_builtins.md` section 3, section 6 - (anti-patterns), and the Code map table. - - Update `docs/architecture/01_public_api.md` to include `BenchmarkDeps` - and the `onboarding_deps` field on the public `Benchmark` API surface. - - Coordinate with the template-spec RFC author on `__init_subclass__` - ordering if that RFC is still active. - - Link the implementation plan in `docs/superpowers/plans/`. diff --git a/docs/rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md b/docs/rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md deleted file mode 100644 index 5c7cfb81a..000000000 --- a/docs/rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md +++ /dev/null @@ -1,735 +0,0 @@ ---- -status: active -opened: 2026-04-22 -author: deepflow-research -architecture_refs: - - docs/architecture/01_public_api.md#core-abstractions - - docs/architecture/06_builtins.md#core-abstractions - - docs/architecture/cross_cutting/artifacts.md -supersedes: [] -superseded_by: null ---- - -# RFC: Worker tool-interface cleanup + artifact→evaluator routing - -## Problem - -Three tangled problems have accumulated in PR #27 (`feature/real-llm-harness-infra`): - -### 1. The `BenchmarkAdapter` ABC is the wrong abstraction - -Commit `1ad4aac` (this branch) introduced a `BenchmarkAdapter(ABC)` at -`ergon_builtins/ergon_builtins/workers/baselines/adapters/base.py` with four -hooks — `build_tools`, `on_run_start`, `on_run_end`, `transform_output` — and -concrete `MiniF2FAdapter` / `SWEBenchAdapter` subclasses. The stated goal was -to remove per-benchmark `Worker` subclasses and make `ReActWorker` generic. - -The hook surface is too wide: - -- `on_run_start` and `build_tools` both let an adapter run setup code once - per `execute()`. Which one "owns" sandbox creation / setup scripts? The - SWE-Bench adapter puts the sandbox handle construction in `on_run_start` - and the toolkit construction in `build_tools` — two hooks doing the same - per-task prep because both are available. -- `transform_output` exists solely because the runtime drops - `WorkerOutput.artifacts` on the durable path (see problem 2): adapters - route artifacts back through `WorkerOutput.output` as a workaround. This - is not a benchmark concern; it is a hole in the runtime's output - serialization. -- `on_run_end` is a catch-all teardown hook. It is useful but is the third - way to "run something per task" alongside `on_run_start` and `build_tools`. - -For an LLM reading the ABC, four hooks read as "pick whichever one seems -right." The result is the exact duplication pattern in -`adapters/swebench.py` (`on_run_start` opens sandbox + runs setup scripts; -`build_tools` builds the toolkit; `on_run_end` captures patch; -`transform_output` routes it). Three of those four should not be -adapter concerns at all — they belong to the sandbox manager or the -criterion. - -The worker's construction contract should be: **a list of tools and a -prompt**. Nothing more. Per-task setup belongs to the sandbox manager -(declarative, at sandbox boot). Artifact capture belongs to the criterion -(which already has the runtime surface it needs). - -### 2. `WorkerOutput.artifacts` is dead on the durable path - -Concrete drop point: -`ergon_core/core/runtime/inngest/worker_execute.py:144-147` — - -```python -return WorkerExecuteResult( - success=True, - output_text=output.output, # ← only .output survives -) -``` - -Everything in `output.artifacts` is discarded at this Inngest-step boundary. -Downstream: - -- `ergon_core/core/runtime/inngest/execute_task.py:182` — passes only - `output_text` to `finalize_success(FinalizeTaskExecutionCommand(...))`. -- `ergon_core/core/runtime/services/task_execution_service.py:270` — writes - `output_text` into `RunTaskExecution.output_text`. **No `artifacts` - column exists** on `RunTaskExecution` - (`ergon_core/telemetry/models.py:96-151`). -- `ergon_core/core/runtime/inngest_executor.py:90-92` — artificially - reconstructs `WorkerOutput(output=task_context.agent_reasoning, - artifacts={})` for evaluator dispatch. `artifacts` is always `{}` by - construction at that point. - -### 3. `output_text` is a vague field name - -`WorkerExecuteResult.output_text`, `FinalizeTaskExecutionCommand.output_text`, -and `RunTaskExecution.output_text` all carry the same thing: **the final -assistant-text message the agent emitted before the run terminated** — -the last `assistant_text` context event, verbatim. Not stdout, not a -summary, not a report. - -"Output" is ambiguous in a codebase where "output" also refers to -sandbox command output (`CommandResult.stdout`), sandbox files under -`/workspace/final_output/`, and `WorkerOutput.output`. A reader can't -tell from the field name which thing it is. - -So today the MiniF2F and SWE-Bench criteria read `worker.artifacts["proof"]` -/ `worker.artifacts["patch"]` — and the only reason those reads don't -return empty is that both adapters route the artifact into -`WorkerOutput.output` via `transform_output` as a workaround. The -`artifacts` dict on the return value is a red herring; the durable path -doesn't carry it. - -The fix is not to add a `run_task_executions.artifacts_json` column. The -fix is to use the channels that already exist for file-shaped outputs: - -- Workers write files into the sandbox under `/workspace/final_output/`. -- `SandboxResourcePublisher.sync()` auto-scans that directory before - teardown and publishes every file as a content-addressed blob with a - `run_resources` row. -- Criteria read what they need from the live sandbox - (`CriterionRuntime.run_command`) or from the published blobs - (`CriterionRuntime.read_resource` / `list_resources`), both of which - already exist per RFC `2026-04-17-criterion-runtime-di-container.md`. - -One helper is missing: a materializing "give me all files for this task" -convenience. - -## Proposal - -Four simultaneous cleanups, landed on PR #27. - -### 1. Worker interface — tools + prompt, nothing else - -Delete `ergon_builtins/workers/baselines/adapters/` entirely. `ReActWorker` -takes tools directly via a shared type alias: - -```python -# ergon_core/ergon_core/api/types.py — NEW FILE -"""Shared type aliases for the public API surface.""" - -from typing import Any - -type Tool = Any # slopcop: ignore[no-typing-any] -"""Framework-agnostic tool carrier. - -Intentionally unconstrained so workers can integrate with any agent -framework. ``ReActWorker`` passes these through to pydantic-ai's -``Agent(tools=...)``; nothing in our code enforces a structural protocol. -If we pin to pydantic-ai, tighten this. -""" -``` - -```python -# ergon_core/ergon_core/api/worker.py — base class tightens too -class Worker(ABC): - def __init__( - self, - *, - name: str, - model: str | None, # required, no default - metadata: Mapping[str, Any] | None = None, - ) -> None: ... -``` - -```python -# ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -from ergon_core.api import Tool - -class ReActWorker(Worker): - def __init__( - self, - *, - name: str, - model: str | None, - tools: list[Tool], - system_prompt: str | None, - max_iterations: int, - ) -> None: ... -``` - -**All kwargs on both classes are required — no defaults.** Rationale: - -- `model: str | None` — the union still carries meaning (`None` routes - through the platform default resolver), but the `= None` default is - dropped. Factories must declare the model they want, or explicitly - pass `None` to opt into the resolver's fallback. -- `tools: list[Tool]` — no `| None` and no default. `None` and `[]` - would be semantically identical, so the type loses the union; a - tools-less ReAct loop is a degenerate shape and the caller should - pass `[]` at the construction site so it's visible. -- `system_prompt: str | None` — `None` means "no instructions" - (pydantic-ai receives `instructions=None`). The union stays; the - default is dropped so the factory declares intent. Empty string is - *not* a synonym for `None` — `""` would mean "instructions are - literally the empty string," which is a legitimate (if weird) - caller choice. -- `max_iterations: int` — no default. `10` was a silent guess carried - over from the adapter migration; SWE-Bench wants ~50, MiniF2F is - comfortable at ~10, future benchmarks will have their own budgets. - Sharing a default means every new benchmark silently inherits `10` - until a regression shows up. Force the factory to pick. - -The base `Worker` class follows the same rule: `model` drops its -`= None` default. The anti-pattern ("nullable-with-default on worker -`__init__`") isn't about abstract vs. concrete — it's about hiding -sizing decisions, which applies everywhere in the class hierarchy. -Concrete `Worker` subclasses across the tree (`StubWorker`, -`SmokeTestWorker`, `ManagerResearcherWorker`, etc.) and their test -fixtures will all need to start passing `model=` explicitly. This is -the ripple cost we accept for a clean contract. - -No `adapter` kwarg. No `on_run_start` / `on_run_end` / `transform_output`. -The worker's `execute()` just runs the ReAct loop over `self.tools` with -`self.system_prompt` for up to `self.max_iterations` nodes. It yields -`GenerationTurn`s. `get_output()` returns a `WorkerOutput` with the final -assistant text as `.output` and nothing in `.artifacts`. - -Hoisting `Tool = Any` to `ergon_core.api.types` means call sites read as -"a list of tools" instead of "a list of anything" and the slopcop -`no-typing-any` ignore lives at exactly one definition site. - -Per-benchmark registration becomes a factory closure that builds the -toolkit inline and passes the concrete tool list. Because all -`ReActWorker` kwargs are required, every sizing decision (model, -iteration budget, prompt) lives visibly in the factory. - -The factory is invoked at `ergon_core/core/runtime/inngest/worker_execute.py:60`, -after the sandbox-setup step has run — so `task_id` and `sandbox_id` -are both in scope. The factory call signature grows those two kwargs -so benchmark factories can build toolkits bound to the live sandbox: - -```python -# ergon_core/core/runtime/inngest/worker_execute.py — updated call -worker = worker_cls( - name=payload.assigned_worker_slug, - model=payload.model_target, - task_id=payload.task_id, - sandbox_id=payload.sandbox_id, -) -``` - -```python -# ergon_builtins/registry_core.py -def _swebench_react( - *, - name: str, - model: str | None, - task_id: UUID, - sandbox_id: str, -) -> ReActWorker: - sandbox = SWEBenchSandboxManager().get_sandbox(task_id) - toolkit = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - return ReActWorker( - name=name, - model=model, - tools=list(toolkit.get_tools()), - system_prompt=SWEBENCH_SYSTEM_PROMPT, - max_iterations=50, - ) -``` - -Plain classes that don't need sandbox access (`StubWorker`, -`TrainingStubWorker`, etc.) are promoted to thin factory closures that -accept-and-ignore the extra kwargs, or the call site passes them -defensively — either shape is fine; plan writer decides. - -**The bare `WORKERS["react-v1"] = ReActWorker` registry entry is -removed.** Under the new contract, `ReActWorker` can no longer be -instantiated with just `(name=, model=)` — there is no meaningful -registration for a "generic" ReAct worker because every real use -binds a concrete toolkit + prompt + iteration budget. Callers that -depended on the raw entry must migrate to a benchmark-specific -factory. - -### 2. Setup scripts — move into `_install_dependencies`, fetch payload via queries - -`BaseSandboxManager` already has a per-task hook that runs during -`create()`: the abstract `_install_dependencies(sandbox, task_id)` -method. Every benchmark manager already implements it (MiniF2F and -SWE-Bench as no-ops, GDPEval as pip install, ResearchRubrics as -workspace bootstrap). The reason SWE-Bench's hook is a no-op today is -that `task_id` alone is insufficient — the per-instance setup scripts -need `repo`, `base_commit`, `version`, `environment_setup_commit`, all -of which live in `ExperimentDefinitionTask.task_payload`. - -**The data layer should expose that lookup, not the sandbox manager.** - -Add one method to the existing `TaskExecutionsQueries` singleton at -`ergon_core/core/persistence/queries.py`: - -```python -class TaskExecutionsQueries(BaseQueries[RunTaskExecution]): - # ... existing methods unchanged ... - - def get_task_payload(self, task_execution_id: UUID) -> dict[str, Any] | None: - """Return the immutable task_payload dict for a task execution. - - Joins ``run_task_executions`` → ``experiment_definition_tasks``. - Returns ``None`` if the execution row doesn't exist or has no - ``definition_task_id`` (run-scoped tasks that aren't tied to a - definition). - """ - with get_session() as session: - stmt = ( - select(ExperimentDefinitionTask.task_payload) - .join( - RunTaskExecution, - RunTaskExecution.definition_task_id == ExperimentDefinitionTask.id, - ) - .where(RunTaskExecution.id == task_execution_id) - ) - return session.exec(stmt).first() -``` - -`SWEBenchSandboxManager._install_dependencies` uses it: - -```python -# ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py - -from ergon_core.core.persistence.queries import queries - - -class SWEBenchSandboxManager(BaseSandboxManager): - async def _install_dependencies( - self, - sandbox: AsyncSandbox, - task_id: UUID, - ) -> None: - payload = queries.task_executions.get_task_payload(task_id) - if payload is None: - raise SandboxSetupError( - f"No task_payload for task_id={task_id}; " - "prepare step must commit before sandbox-setup dispatches." - ) - row = _payload_to_swebench_row(payload) - harness = make_test_spec(row) - for label, script in ( - ("setup_env", harness.setup_env_script), - ("install_repo", harness.install_repo_script), - ): - r = await sandbox.commands.run( - f"bash -c {shlex.quote(script)}", - timeout=1800, - ) - if r.exit_code != 0: - raise SandboxSetupError( - f"swebench {label} failed: {(r.stdout or '')[-1000:]}" - ) -``` - -What this gets us: - -- **`BaseSandboxManager.create()` signature unchanged.** No ripple on - MiniF2F / GDPEval / ResearchRubrics. -- **DB access lives in the data layer** where it belongs — next to the - sibling `get_by_task` / `get_latest_by_task` methods already on - `TaskExecutionsQueries`. -- **Only benchmarks that need payload pay the DB round-trip.** MiniF2F's - no-op hook stays a no-op. -- **No Inngest event-payload changes.** `SandboxSetupRequest` unchanged. - -Once this lands, the workers and criteria can assume "sandbox exists ⇒ -repo is cloned, deps are installed" for SWE-Bench. MiniF2F has no -equivalent per-task setup — the Lean container is ready at boot and its -`_install_dependencies` stays a no-op. - -### 3. Artifact→evaluator routing — pull from sandbox or RunResources - -Workers stop producing "artifacts" in the `WorkerOutput.artifacts` sense. -File-shaped outputs leave the worker one of two ways: - -**A. Written to `/workspace/final_output/` in the sandbox.** -Already auto-published by `SandboxResourcePublisher.sync()` as blob-backed -`RunResource` rows before sandbox teardown. -Criterion reads via `context.runtime.read_resource(name)`. - -**B. Left in the sandbox filesystem at a known path.** -Criterion runs commands directly via `context.runtime.run_command(...)` to -compute or extract what it needs. - -Concrete application on this PR: - -- **MiniF2F proof**: the `lean_write_file` tool writes to - `/workspace/final_output/final_solution.lean` directly. Auto-published. - Criterion does - `proof = (await context.runtime.read_resource("final_solution.lean")).decode()`. - -- **SWE-Bench patch**: the criterion computes the patch itself by running - `cd /workspace/repo && git add -A && git diff HEAD` via - `context.runtime.run_command(...)`. The patch never rides on - `WorkerOutput`. The adapter's `_extract_patch` disappears. - -#### New `CriterionRuntime` method: `get_all_files_for_task` - -```python -# ergon_core/api/criterion_runtime.py — addition - -async def get_all_files_for_task(self) -> dict[str, bytes]: ... -``` - -```python -# ergon_core/core/runtime/evaluation/criterion_runtime.py — addition - -async def get_all_files_for_task(self) -> dict[str, bytes]: - """Return {name: bytes} for every run_resource produced by this task. - - Scoped to the ``(run_id, task_execution_id)`` the runtime was - constructed with. Not size-capped — callers that expect large - resources should use ``list_resources()`` + ``read_resource()`` for - selective reads instead. - """ - with get_session() as session: - stmt = ( - select(RunResource) - .where(RunResource.run_id == self._run_id) - .where(RunResource.task_execution_id == self._task_id) - .order_by(RunResource.created_at.desc()) # type: ignore[arg-type] - ) - rows = list(session.exec(stmt).all()) - - # Keep the latest row per name; older revisions are skipped. - seen: set[str] = set() - out: dict[str, bytes] = {} - for row in rows: - if row.name in seen: - continue - seen.add(row.name) - out[row.name] = Path(row.file_path).read_bytes() - return out -``` - -Runtime is already task-scoped (the accepted RFC adds `task_id` to -`DefaultCriterionRuntime.__init__`); the helper takes no arguments — -passing a `task_id` in would invite criteria to read across tasks. - -### 4. Rename `output_text` → `final_assistant_message` - -Propagate the rename through the three layers that carry this value: - -| Before | After | File | -|---|---|---| -| `WorkerExecuteResult.output_text` | `WorkerExecuteResult.final_assistant_message` | `ergon_core/core/runtime/services/inngest_function_results.py` | -| `FinalizeTaskExecutionCommand.output_text` | `FinalizeTaskExecutionCommand.final_assistant_message` | `ergon_core/core/runtime/services/task_execution_service.py` | -| `RunTaskExecution.output_text` (DB column) | `RunTaskExecution.final_assistant_message` (DB column) | `ergon_core/core/persistence/telemetry/models.py` + Alembic migration | - -The name matches the `assistant_text` context event type that produces -this value — same vocabulary as the persistence layer. It also removes -the collision with `CommandResult.stdout` (which is also "output" in -casual English) and with files under `/workspace/final_output/`. - -The DB column rename requires an Alembic revision (simple rename — no -data transform). Downstream readers in the dashboard API and the CLI -need updating in the same PR; grep pass will catch them. - -#### `WorkerOutput.artifacts` — deprecated in place - -The field stays on the model (it is on the public `Worker.execute()` -return contract and removing it would ripple into every Worker subclass -and test in the repo). It is marked deprecated with a docstring that says: - -```python -class WorkerOutput(BaseModel): - ... - artifacts: dict[str, Any] = Field( # noqa — deprecated - default_factory=dict, - description=( - "DEPRECATED. This field is NOT carried across the durable " - "worker→evaluator boundary (dropped at " - "inngest/worker_execute.py). Do not use for files or data " - "the criterion needs to read. Files → write to " - "/workspace/final_output/ (auto-published as RunResources). " - "Computed artifacts → have the criterion run commands in " - "the sandbox via CriterionRuntime.run_command." - ), - ) -``` - -Removal of the field is a follow-up once all built-in workers stop -writing to it. - -## Invariants affected - -### `docs/architecture/01_public_api.md#core-abstractions` - -The `Worker` section should state clearly: - -> The `Worker` base class and every concrete subclass declare their -> construction contract with **required keyword-only kwargs and no -> nullable-with-default fallbacks**. `Worker` takes `name: str`, -> `model: str | None` (required, no default); `ReActWorker` adds -> `tools: list[Tool]`, `system_prompt: str | None`, `max_iterations: int` -> — also required. A default value on worker `__init__` is an -> anti-pattern: it hides sizing decisions (iteration budget, model -> choice, system prompt) that should live visibly in the registry -> factory for each benchmark. Workers MUST NOT own per-task environment -> setup — setup belongs to the sandbox manager (see -> `BaseSandboxManager.create`). Workers MUST NOT return files or blobs -> through `WorkerOutput.artifacts` — the runtime serialization layer -> does not carry that field. Files → write to `/workspace/final_output/`. - -### `docs/architecture/06_builtins.md` - -Remove the "ReAct adapter composition" section added in commit -`a9819fe`. Replace with a "ReAct toolkit composition" section describing -the toolkit-as-list-of-tools pattern (no adapter ABC, no hooks). - -Strengthen the anti-patterns list: - -> - **Worker subclasses for per-benchmark glue.** Benchmark-specific -> wiring is a factory-closure concern (registry), not a class hierarchy. -> The worker `__init__` contract is `tools: list[Tool]` + prompt only. -> - **Per-task setup inside workers.** Setup scripts (clone, install deps, -> environment bootstrap) belong to `BaseSandboxManager.create()` — -> sandbox lifecycle, not worker lifecycle. -> - **Nullable-with-default kwargs on concrete Worker `__init__`.** -> `tools: list[Tool] | None = None`, `max_iterations: int = 10`, etc. -> hide sizing decisions in a shared default and mask per-benchmark -> intent. Concrete workers declare their required construction -> contract; factories pass every kwarg explicitly. - -### `docs/architecture/cross_cutting/artifacts.md` - -Add an "Invariants" item: - -> - `WorkerOutput.artifacts` is a non-durable field. It is dropped at the -> Inngest `worker_execute` step boundary and is not a channel to -> criteria. File-shaped artifacts are published via -> `SandboxResourcePublisher.sync()` from `/workspace/final_output/`; -> criteria read them via `CriterionRuntime.read_resource(name)` or -> `get_all_files_for_task()`. Computed artifacts (e.g. `git diff`) are -> produced by the criterion itself via -> `CriterionRuntime.run_command(...)`. - -Also remove any "use `WorkerOutput.artifacts` for evaluator-visible data" -guidance if present. - -### `docs/architecture/01_public_api.md#criterionruntime` - -Add `get_all_files_for_task` to the method list. Update the surface-area -constraint note — the Protocol is now sandbox lifecycle (7) + resource -I/O (3 — adds `get_all_files_for_task`) + DB read (1) + event emission -(1) = 12 methods. - -### `docs/architecture/04_sandbox_lifecycle.md` (per-task setup) - -The invariant becomes: - -> For benchmarks that require per-task environment setup (clone a -> specific commit, install version-pinned deps, apply a harness spec), -> that work runs inside `BaseSandboxManager._install_dependencies( -> sandbox, task_id)` — not inside the worker's `execute()`, not inside -> a separate `on_run_start` hook, and not inside the criterion. Managers -> that need per-task data (payload, instance-id metadata) read it from -> the data layer via `queries.task_executions.get_task_payload(task_id)`; -> `SandboxSetupRequest` carries only `task_id`, not the full payload. - -### Naming invariant: `final_assistant_message` - -The field that carries the agent's final assistant-text message is named -`final_assistant_message` end-to-end — from the Inngest -`WorkerExecuteResult` through `FinalizeTaskExecutionCommand` to the -`RunTaskExecution.final_assistant_message` column. Future work that -touches this field MUST NOT reintroduce `output_text` as a synonym, and -the rename MUST propagate into dashboard readers and any CLI surfaces in -the same PR that renames the column. - -## Migration - -All changes land in PR #27 (`feature/real-llm-harness-infra`). The PR -will not merge until this RFC is accepted. - -### Files to delete - -- `ergon_builtins/ergon_builtins/workers/baselines/adapters/__init__.py` -- `ergon_builtins/ergon_builtins/workers/baselines/adapters/base.py` -- `ergon_builtins/ergon_builtins/workers/baselines/adapters/minif2f.py` -- `ergon_builtins/ergon_builtins/workers/baselines/adapters/swebench.py` -- Any `tests/**/test_*adapter*.py` that target the deleted module. - -### Files to add - -| File | Purpose | -|---|---| -| `ergon_core/ergon_core/api/types.py` | New module holding the `Tool = Any` alias (and any future public type aliases). Re-exported from `ergon_core.api.__init__`. | - -### Files to modify - -| File | Change | -|---|---| -| `ergon_core/ergon_core/api/worker.py` | Drop `model: str \| None = None` default on the base `Worker.__init__`. `model` becomes required (union stays). | -| `ergon_builtins/workers/baselines/react_worker.py` | Remove `adapter` kwarg and the three hook call sites in `execute()`/`get_output()`. Tighten construction contract: `tools: list[Tool]`, `system_prompt: str \| None`, `max_iterations: int`, `model: str \| None` — all **required, no defaults**. Drop the internal `tools or []` / `max_iterations = max_iterations or 10` fallback branches. | -| `ergon_builtins/registry_core.py` | `_minif2f_react` / `_swebench_react` factories build toolkits inline and pass concrete `tools=` into `ReActWorker`. System prompts and `max_iterations` move here as plain constants. Factory call signature grows `task_id: UUID` and `sandbox_id: str` kwargs. **Remove** bare `"react-v1": ReActWorker` entry from `WORKERS`. Non-benchmark workers (`StubWorker`, `TrainingStubWorker`, `SmokeTestWorker`, `ManagerResearcherWorker`, etc.) either accept-and-ignore the new kwargs or get thin factory closures. | -| `ergon_core/core/runtime/inngest/worker_execute.py` | `worker_cls(...)` call at line ~60 passes `task_id=payload.task_id` and `sandbox_id=payload.sandbox_id` in addition to `name` / `model`. | -| `ergon_builtins/workers/baselines/stub_worker.py`, `training_stub_worker.py`, `smoke_test_worker.py`, `manager_researcher_worker.py`, `research_rubrics/stub_worker.py`, `stubs/canonical_smoke_worker.py` | Callers stop relying on `model` default. Either add explicit `model=` kwargs at construction sites, or tighten these subclasses to also require `model`. Plan writer picks the strategy. | -| Test fixtures that construct `Worker` subclasses without `model=` | Pass `model=None` (or a concrete value) explicitly. Grep for `StubWorker(name=`, `SmokeTestWorker(name=`, etc. Ripple is mechanical. | -| `ergon_core/core/persistence/queries.py` | Add `TaskExecutionsQueries.get_task_payload(task_execution_id)` — one new method, joins `run_task_executions → experiment_definition_tasks`, returns the payload dict. | -| `ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py` | `_install_dependencies` stops being a no-op: reads `task_payload` via `queries.task_executions.get_task_payload`, builds the harness spec, runs `setup_env_script` + `install_repo_script`. | -| `ergon_builtins/benchmarks/swebench_verified/criterion.py` | `evaluate()` runs `git add -A && git diff HEAD` via `context.runtime.run_command` to get the patch. No read of `worker.artifacts["patch"]`. | -| `ergon_builtins/benchmarks/minif2f/criterion.py` (if it currently reads `worker.artifacts`) | Read proof via `context.runtime.read_resource("final_solution.lean")`. | -| `ergon_builtins/benchmarks/minif2f/toolkit.py` | `lean_write_file` writes to `/workspace/final_output/` so the publisher auto-captures it. | -| `ergon_core/api/results.py` | Docstring on `WorkerOutput.artifacts` marks it deprecated with explanation. | -| `ergon_core/api/criterion_runtime.py` | Add `get_all_files_for_task` to Protocol. | -| `ergon_core/core/runtime/evaluation/criterion_runtime.py` | Implement `get_all_files_for_task` on `DefaultCriterionRuntime`. | -| `ergon_core/core/runtime/services/inngest_function_results.py` | Rename `WorkerExecuteResult.output_text` → `final_assistant_message`. | -| `ergon_core/core/runtime/services/task_execution_service.py` | Rename `FinalizeTaskExecutionCommand.output_text` → `final_assistant_message`. Update write site. | -| `ergon_core/core/persistence/telemetry/models.py` | Rename `RunTaskExecution.output_text` column → `final_assistant_message`. | -| `ergon_core/alembic/versions/<new>.py` | Simple column rename; no data transform. | -| Dashboard API + CLI readers of `output_text` | Grep and rename. | -| `docs/architecture/01_public_api.md` | Update Worker contract + CriterionRuntime surface; document `Tool` alias. | -| `docs/architecture/06_builtins.md` | Replace adapter section with toolkit section; add anti-patterns entries. | -| `docs/architecture/cross_cutting/artifacts.md` | Add non-durability invariant for `WorkerOutput.artifacts`. | - -### Tests to add / rewrite - -- `tests/unit/test_react_worker.py` — construct `ReActWorker(tools=[…])` - with fake tools; assert it runs loop and yields turns. No adapter - fixtures. -- `tests/unit/test_swebench_sandbox_manager.py` — assert `create()` runs - setup scripts, raises on non-zero exit. -- `tests/unit/test_swebench_criterion.py` — supply a mock - `CriterionRuntime.run_command` that returns a canned `git diff HEAD` - output; assert the criterion grades it correctly without touching - `worker.artifacts`. -- `tests/unit/test_criterion_runtime_get_all_files.py` — unit test the - new helper: task-scoped filter, dedup by name keeping newest, returns - materialized bytes. -- Delete any existing `test_*adapter*` tests. - -### Data migrations - -None. No DB schema changes. `RunResource` already has the -`task_execution_id` column used by `get_all_files_for_task`. - -## Alternatives considered - -- **Add a `run_task_executions.artifacts_json` column and serialize - `WorkerOutput.artifacts` there.** Rejected. This would preserve a field - that is the wrong shape for the thing it tries to do: durable artifacts - are files (potentially large, content-addressed, dedup-friendly) and - we already have a table (`run_resources`) + a publisher - (`SandboxResourcePublisher.sync()`) tuned for exactly that. Adding an - `artifacts_json` column would duplicate half of `run_resources` - inline on the task-execution row, with no size limit, no dedup, no - blob storage. - -- **Keep the adapter ABC but trim it to `build_tools` only.** Rejected. - The reason to have the ABC is the hooks; with only `build_tools` left, - it degenerates into "pass a `list[Tool]`." Do that directly. - -- **Teach the runtime to copy `WorkerOutput.artifacts` through the - boundary.** Rejected for the same reason as the column: wrong channel. - Files are not cheap to copy inline through Inngest event payloads; - `run_resources` blob storage is the right place. - -- **`get_all_files_for_task(size_cap=N)`.** Discussed on the design - thread; rejected (user preference). `list_resources()` + selective - `read_resource()` remain available for callers with specific size - concerns. - -- **Delete `WorkerOutput.artifacts` in this PR.** Rejected — it is on - the public Worker return contract. Ripples into every subclass + - test. Do it as a follow-up once built-ins stop using it. - -- **Add a `setup_spec: SWEBenchSetupSpec` kwarg to - `BaseSandboxManager.create()`.** Rejected. The shape of that spec - differs per benchmark (SWE-Bench wants `install_repo_script` + - `setup_env_script`; GDPEval wants `pip install <pkgs>`; MiniF2F wants - nothing), so the kwarg would either have to be a `dict[str, Any]` - catch-all or a discriminated union the base class knows about. Both - versions leak benchmark-specific knowledge into `BaseSandboxManager`, - which defeats the point. The `_install_dependencies` hook already - solves this — each manager knows how to fetch its own per-task data. - -- **Pass `BenchmarkTask` (or `task_payload`) through the Inngest - `SandboxSetupRequest` event payload.** Rejected. The sandbox-setup step - already has `task_id`; broadcasting the full payload inside the event - duplicates data that already lives in Postgres and encourages workers / - managers to read from the event blob rather than the canonical - `experiment_definition_tasks` row. The data-layer lookup via - `queries.task_executions.get_task_payload` keeps the event payload - narrow and puts the read next to the sibling per-task query methods. - -- **Put the payload JOIN helper on `BaseSandboxManager` itself (e.g. as - `_load_task_payload`).** Rejected. The JOIN is a data-layer concern and - belongs in `TaskExecutionsQueries`, next to `get_by_task` / - `get_latest_by_task`. Sandbox managers should consume the data layer, - not re-implement it. This also keeps the method testable in isolation - via the existing queries-layer test fixtures. - -- **Keep the current nullable-with-default `ReActWorker.__init__` shape - (`tools = None`, `system_prompt = None`, `max_iterations = 10`).** - Rejected. Defaults on a concrete worker `__init__` hide sizing - decisions — a new benchmark silently inherits `max_iterations=10` - until someone notices runs are cutting short, and factory closures - read as "pass the few things I care about" rather than "declare my - full sizing contract." Making all kwargs required pushes every - decision up to the registry (where it belongs) and lets `ty` flag - "forgot to pick a value" at the type level. - -## Open questions - -All three resolved during plan execution on branch -`feature/real-llm-harness-infra` (PR #27). The rest had been absorbed -into the proposal above (MiniF2F proof filename is -`final_solution.lean`; `get_task_payload` takes `session: Session` -mirroring its siblings; the base `Worker` class also drops its -`model` default). - -1. **Non-benchmark-specific workers' kwarg shape.** **Resolved — - option (b).** Every bare worker class - (`StubWorker`, `TrainingStubWorker`, `SmokeTestWorker`, - `ManagerResearcherWorker`, `StubResearchRubricsWorker`, - `CanonicalSmokeWorker`, `ResearchRubricsManagerWorker`, - `ResearchRubricsResearcherWorker`) is wrapped in the `_plain(cls)` - factory closure in `ergon_builtins/registry_core.py` and re-used - from `registry_data.py`. The closure accepts the registry's - uniform `(name, model, task_id, sandbox_id)` signature and - forwards only `(name, model)` to the underlying `__init__`. - Subclasses never learn about `task_id` / `sandbox_id`. - -2. **`ensure_sandbox()` idempotence check.** **Resolved.** Regression - test landed at - `tests/unit/sandbox/test_ensure_sandbox_idempotence.py` — patches - `AsyncSandbox`, calls `ensure_sandbox()` three times, and asserts - `_install_dependencies` runs exactly once. - -3. **Do we need a `read_resource_text` convenience?** **Resolved — - no.** `CriterionRuntime.read_resource` returns `bytes`; criteria - decode at the call site (e.g. - `raw.decode("utf-8", errors="replace")` in - `ProofVerificationCriterion._extract_proof`). The Protocol stays - at 12 methods with the new `get_all_files_for_task` addition. - -## On acceptance - -- [ ] Move `docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md` - → `docs/rfcs/accepted/`. -- [ ] Update `docs/architecture/01_public_api.md` — Worker contract + - CriterionRuntime surface. -- [ ] Update `docs/architecture/06_builtins.md` — remove adapter section, - add anti-patterns. -- [ ] Update `docs/architecture/cross_cutting/artifacts.md` — - non-durability invariant for `WorkerOutput.artifacts`. -- [ ] Link the implementation plan in - `docs/superpowers/plans/` when written. -- [ ] After PR #27 merges, track the follow-up to fully delete - `WorkerOutput.artifacts` once no in-tree worker writes to it. diff --git a/docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md b/docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md deleted file mode 100644 index a207dbe57..000000000 --- a/docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md +++ /dev/null @@ -1,921 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/02_runtime_lifecycle.md#anti-patterns, docs/architecture/cross_cutting/sandbox_lifecycle.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Wire `cleanup_cancelled_task_fn.release-sandbox` to actually close the E2B sandbox - -## Problem - -### Current state - -`cleanup_cancelled_task_fn` at -`ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py:20-61` -is documented as executing two durable steps — `update-db-rows` and -`release-sandbox`. Only `update-db-rows` is implemented. The function body -ends at line 60 with no `release-sandbox` step, and -`TaskCleanupService.cleanup` at -`ergon_core/ergon_core/core/runtime/services/task_cleanup_service.py:53-54` -hard-codes `sandbox_released = False` with a `slopcop: ignore[no-todo-comment]` -suppressor. - -**Consequence (cost leak):** `ergon run cancel <run_id>` — backed by -`ergon_core/ergon_core/core/runtime/services/run_service.py:107-140` — marks -the `RunRecord` CANCELLED and sends `run/cancelled` + `run/cleanup`. The -`RUN_CANCEL` Inngest matcher kills in-flight `task-execute` functions. -`cleanup_cancelled_task_fn` fires on each `task/cancelled` event but does not -call `BaseSandboxManager.terminate_by_sandbox_id`. The E2B sandbox stays alive -until its creation-time `timeout` expires. For SWE-Bench tasks, that timeout -is ~30 minutes by default (up to ~70 minutes after -`docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md` lands). -A user who cancels two minutes in pays the full timeout. - -**Consequence (missing payload fields):** `TaskCancelledEvent` at -`ergon_core/ergon_core/core/runtime/events/task_events.py:105-123` carries -`run_id`, `definition_id`, `node_id`, `execution_id`, and `cause`. It does -NOT carry `sandbox_id` or `benchmark_slug`. The cleanup function therefore -cannot know which sandbox to close or which `BaseSandboxManager` subclass owns -it. Every emission site that constructs a `TaskCancelledEvent` — -`subtask_cancellation_service.py:98-104`, -`task_management_service.py:214-220`, `task_management_service.py:555-563`, -`propagate_execution.py:73-83`, `propagate_execution.py:165-175` — omits both -fields because the model has no slots for them. - -**Consequence (no manager lookup):** The slug-to-manager map `SANDBOX_MANAGERS` -at `ergon_builtins/ergon_builtins/registry_core.py:84-88` maps benchmark_slug -to a `type[BaseSandboxManager]` (not an instance). The cleanup function would -need to resolve the class and call the static method -`BaseSandboxManager.terminate_by_sandbox_id(sandbox_id)`. -`terminate_by_sandbox_id` is a `@staticmethod` at -`ergon_core/ergon_core/core/sandbox/manager.py:472-490` that calls -`AsyncSandbox.kill(sandbox_id=..., api_key=...)` directly via E2B, so no -instance is needed. However, `cleanup_cancelled_task_fn` currently has no -import path to `SANDBOX_MANAGERS`. - -**Consequence (missing column):** `RunTaskExecution` at -`ergon_core/ergon_core/core/persistence/telemetry/models.py:96-148` has no -`sandbox_id` column. `sandbox_id` is tracked at the Inngest-event level (on -`TaskCompletedEvent`, `TaskFailedEvent`, and child payloads) but not persisted -on the execution row. The cleanup function receives only `execution_id`; it -cannot look up `sandbox_id` from the DB without a new column. - -### Architecture doc cross-references - -- `docs/architecture/02_runtime_lifecycle.md §4 Known limits` — "Cancellation - does not release sandboxes." offender at line 117. -- `docs/architecture/cross_cutting/sandbox_lifecycle.md §5 Failure modes` — - "Cancellation … `release-sandbox` step is currently a STUB" offender at - line 61. -- `docs/architecture/cross_cutting/sandbox_lifecycle.md §8 Anti-patterns` — - "Leaking sandboxes on cancellation. Current offender: …" at line 85. - ---- - -## Proposal - -### Option A: Extend the event, add a column, add the step (chosen) - -Three coordinated changes land in two PRs: - -**PR 1 — Schema + event extension (safe to deploy independently):** - -1. Add `sandbox_id: str | None = None` and `benchmark_slug: str | None = None` - to `TaskCancelledEvent`. Backward-compatible; consumers that do not read - them are unaffected; Inngest event queues tolerate extra fields via - `model_config = {"extra": "allow"}` already set on the model. -2. Add `sandbox_id: str | None` (nullable, no default) column to - `RunTaskExecution` via an additive Alembic migration. No backfill required - — historical rows are `NULL`, which the cleanup step already handles as - "no sandbox to release." -3. Update `TaskExecutionService.prepare` (or its callers) to write - `sandbox_id` onto the execution row after `sandbox-setup` completes. In - practice this is in `execute_task_fn` after the `sandbox-setup` step - returns, before dispatching `worker-execute`. -4. Update every `TaskCancelledEvent` emission site to look up - `sandbox_id` from `RunTaskExecution` (via `execution_id`) and - `benchmark_slug` from the definition row (via `run_id`) and include both - fields. Sites where `execution_id` is `None` (tasks that never ran) leave - both new fields `None`. - -**PR 2 — Cleanup step (depends on PR 1):** - -5. Add a `release-sandbox` durable step to `cleanup_cancelled_task_fn` that - calls `BaseSandboxManager.terminate_by_sandbox_id(payload.sandbox_id)` when - both `payload.sandbox_id` and `payload.benchmark_slug` are non-`None`. - Falls through (no-op, no raise) when either is absent. Uses - `SANDBOX_MANAGERS` from `ergon_builtins.registry` to verify the slug is - known; if unknown, logs a warning and returns. Safe to add to an already- - deployed function because the step is a no-op if PR 1 fields are absent. -6. Remove the `# slopcop: ignore[no-todo-comment]` suppressor from - `task_cleanup_service.py:53` once the step is wired (the service no longer - needs to own sandbox release). - -### Rejected options - -See [Alternatives considered](#alternatives-considered). - ---- - -## Architecture overview - -### Before (current) - -``` -task/cancelled - │ - ├─ cancel_orphan_subtasks_fn (recurse children) - ├─ execute_task_fn via TASK_CANCEL matcher (drop/kill) - └─ cleanup_cancelled_task_fn - └─ step: update-db-rows - TaskCleanupService.cleanup(execution_id) - → mark RunTaskExecution.status = CANCELLED - → sandbox_released = False ← STUB, sandbox leaks -``` - -### After (this RFC) - -``` -task/cancelled {sandbox_id, benchmark_slug now populated} - │ - ├─ cancel_orphan_subtasks_fn (unchanged) - ├─ execute_task_fn via TASK_CANCEL matcher (unchanged) - └─ cleanup_cancelled_task_fn - ├─ step: update-db-rows - │ TaskCleanupService.cleanup(execution_id) - │ → mark RunTaskExecution.status = CANCELLED - └─ step: release-sandbox - if sandbox_id is None or benchmark_slug is None: return - BaseSandboxManager.terminate_by_sandbox_id(sandbox_id) - → AsyncSandbox.kill(sandbox_id, api_key) - → idempotent; logs if already gone -``` - -### Data flow: how `sandbox_id` reaches the event - -``` -execute_task_fn - ├─ step: sandbox-setup → SandboxReadyResult.sandbox_id - ├─ (new) update RunTaskExecution.sandbox_id = sandbox_result.sandbox_id - └─ ... (rest unchanged) - -cancellation paths that emit TaskCancelledEvent: - ├─ subtask_cancellation_service.cancel_orphans - │ → _latest_execution_id(session, node_id) - │ → (new) _lookup_sandbox(session, execution_id) → sandbox_id - │ → (new) _lookup_benchmark_slug(session, run_id) → benchmark_slug - ├─ task_management_service.cancel_task (manager-initiated) - │ (same lookups) - ├─ propagate_execution (dep_invalidated, execution_id=None) - │ → sandbox_id=None, benchmark_slug=None (task never ran) - └─ task_management_service._cancel_downstream (downstream_invalidation) - (same as cancel_task path) -``` - ---- - -## Type / interface definitions - -### Extended `TaskCancelledEvent` - -```python -# ergon_core/ergon_core/core/runtime/events/task_events.py - -class TaskCancelledEvent(InngestEventContract): - """Emitted whenever a node transitions from non-terminal into CANCELLED. - - Consumers: - - cancel_orphan_subtasks_fn (recurse cascade to descendants) - - cleanup_cancelled_task_fn (release sandbox, mark execution row) - - execute_task_fn (via TASK_CANCEL matcher — drops queued / terminates running) - - dashboard_emitter - """ - - name: ClassVar[str] = "task/cancelled" - - run_id: UUID - definition_id: UUID - node_id: UUID - execution_id: UUID | None - cause: CancelCause - sandbox_id: str | None = None # NEW — E2B sandbox_id string; None if task never ran - benchmark_slug: str | None = None # NEW — benchmark type slug; None if task never ran - - model_config = {"frozen": True, "extra": "allow"} -``` - -### New `RunTaskExecution` column - -```python -# ergon_core/ergon_core/core/persistence/telemetry/models.py -# Add to RunTaskExecution (after existing fields, before final_assistant_message): - - sandbox_id: str | None = Field( - default=None, - index=False, - ) - """E2B sandbox_id string written by execute_task_fn after sandbox-setup. - - NULL for tasks that ran before this column was added (pre-migration), - and for tasks whose sandbox was skipped (SANDBOX_SKIPPED sentinel). - The cleanup function treats NULL as 'no sandbox to release'. - """ -``` - -### Alembic migration - -```python -# ergon_core/migrations/versions/<hash>_add_sandbox_id_to_run_task_executions.py - -"""add sandbox_id to run_task_executions - -Revision ID: <auto> -Revises: b5b36e45e5e6 -Create Date: 2026-04-21 -""" - -from alembic import op -import sqlalchemy as sa - -revision = "<auto>" -down_revision = "b5b36e45e5e6" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.add_column( - "run_task_executions", - sa.Column("sandbox_id", sa.String(), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("run_task_executions", "sandbox_id") -``` - ---- - -## Full implementations - -### Modified: `cleanup_cancelled_task_fn` - -```python -# ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py - -"""Inngest function: clean up resources for a cancelled task. - -Two durable steps: -1. update-db-rows — mark execution CANCELLED (idempotent) -2. release-sandbox — close the E2B sandbox if sandbox_id is present -""" - -import logging - -import inngest - -from ergon_builtins.registry import SANDBOX_MANAGERS -from ergon_core.core.sandbox.manager import BaseSandboxManager -from ergon_core.core.runtime.events.task_events import TaskCancelledEvent -from ergon_core.core.runtime.inngest_client import RUN_CANCEL, inngest_client -from ergon_core.core.runtime.services.task_cleanup_dto import CleanupResult -from ergon_core.core.runtime.services.task_cleanup_service import TaskCleanupService - -logger = logging.getLogger(__name__) - - -@inngest_client.create_function( - fn_id="cleanup-cancelled-task", - trigger=inngest.TriggerEvent(event="task/cancelled"), - cancel=RUN_CANCEL, - retries=3, -) -async def cleanup_cancelled_task_fn(ctx: inngest.Context) -> dict: - """Clean up a single cancelled task's resources.""" - payload = TaskCancelledEvent.model_validate(ctx.event.data) - logger.info( - "cleanup-cancelled node_id=%s execution_id=%s cause=%s sandbox_id=%s", - payload.node_id, - payload.execution_id, - payload.cause, - payload.sandbox_id, - ) - - if payload.execution_id is None: - return CleanupResult( - run_id=payload.run_id, - node_id=payload.node_id, - execution_id=None, - sandbox_released=False, - execution_row_updated=False, - ).model_dump(mode="json") - - svc = TaskCleanupService() - - def _update_db_rows() -> dict: - from ergon_core.core.persistence.shared.db import get_session - - with get_session() as session: - result = svc.cleanup( - session, - run_id=payload.run_id, - node_id=payload.node_id, - execution_id=payload.execution_id, - ) - return result.model_dump(mode="json") - - db_result_raw = await ctx.step.run("update-db-rows", _update_db_rows) - - async def _release_sandbox() -> dict: - if payload.sandbox_id is None or payload.benchmark_slug is None: - logger.info( - "release-sandbox skipped: no sandbox_id or benchmark_slug " - "for node_id=%s", - payload.node_id, - ) - return {"sandbox_released": False, "reason": "no_payload"} - - mgr_cls = SANDBOX_MANAGERS.get(payload.benchmark_slug) - if mgr_cls is None: - logger.warning( - "release-sandbox: no manager for benchmark_slug=%s node_id=%s", - payload.benchmark_slug, - payload.node_id, - ) - return {"sandbox_released": False, "reason": "unknown_slug"} - - released = await BaseSandboxManager.terminate_by_sandbox_id(payload.sandbox_id) - logger.info( - "release-sandbox sandbox_id=%s benchmark_slug=%s released=%s", - payload.sandbox_id, - payload.benchmark_slug, - released, - ) - return {"sandbox_released": released, "reason": "terminated"} - - sandbox_result = await ctx.step.run("release-sandbox", _release_sandbox) - - # Merge results for the function return value. - db_result = CleanupResult.model_validate(db_result_raw) - return CleanupResult( - run_id=db_result.run_id, - node_id=db_result.node_id, - execution_id=db_result.execution_id, - sandbox_released=sandbox_result.get("sandbox_released", False), - execution_row_updated=db_result.execution_row_updated, - ).model_dump(mode="json") -``` - -### Modified: `TaskCleanupService` - -Remove the `slopcop: ignore[no-todo-comment]` suppressor. Sandbox release is -now owned by the Inngest step, not by this service. The service remains -responsible only for the DB row. - -```diff -# ergon_core/ergon_core/core/runtime/services/task_cleanup_service.py - -- # slopcop: ignore[no-todo-comment] — sandbox teardown, wire when sandbox management exists -- sandbox_released = False -+ sandbox_released = False # Sandbox release is handled by the release-sandbox Inngest step. -``` - -### Modified: `execute_task_fn` — persist `sandbox_id` on execution row - -```diff -# ergon_core/ergon_core/core/runtime/inngest/execute_task.py - - task_sandbox_id = sandbox_result.sandbox_id - -+ def _persist_sandbox_id() -> None: -+ from ergon_core.core.persistence.shared.db import get_session -+ from ergon_core.core.persistence.telemetry.models import RunTaskExecution -+ -+ with get_session() as session: -+ exe = session.get(RunTaskExecution, prepared.execution_id) -+ if exe is not None and exe.sandbox_id is None: -+ exe.sandbox_id = task_sandbox_id -+ session.add(exe) -+ session.commit() -+ -+ await ctx.step.run("persist-sandbox-id", _persist_sandbox_id) -+ - if not prepared.worker_type: -``` - -**Note:** `SANDBOX_SKIPPED` is the sentinel string `"skipped"` defined at -`ergon_core/ergon_core/core/runtime/events/task_events.py:11`. The step writes -it literally to the column; the cleanup function treats it the same as `None` -because `SANDBOX_MANAGERS.get("skipped")` returns `None` (not a registered -slug), so the step skips and returns `{"sandbox_released": False}`. - -### Modified: `TaskCancelledEvent` emission sites - -All four sites that currently emit `TaskCancelledEvent` must be updated. Three -of them have access to a DB session and an `execution_id`; one emits with -`execution_id=None` (dependency-invalidation path where the task never ran). - -**Helper functions (add to each file, or extract to a shared `_cancel_helpers.py`):** - -```python -# Shared helpers — inline in each service or extract to -# ergon_core/ergon_core/core/runtime/services/_cancel_helpers.py - -from uuid import UUID -from sqlmodel import Session, select - - -def _lookup_sandbox_id(session: Session, execution_id: UUID | None) -> str | None: - """Return RunTaskExecution.sandbox_id for the given execution, or None.""" - if execution_id is None: - return None - # reason: deferred to avoid circular import at module level - from ergon_core.core.persistence.telemetry.models import RunTaskExecution - - result = session.exec( - select(RunTaskExecution.sandbox_id).where(RunTaskExecution.id == execution_id) - ).first() - return result # type: ignore[return-value] - - -def _lookup_benchmark_slug(session: Session, run_id: UUID) -> str | None: - """Return the benchmark_type (slug) for the run's experiment definition, or None.""" - # reason: deferred to avoid circular import at module level - from ergon_core.core.persistence.telemetry.models import RunRecord - from ergon_core.core.persistence.definitions.models import ExperimentDefinition - - run = session.get(RunRecord, run_id) - if run is None: - return None - defn = session.get(ExperimentDefinition, run.experiment_definition_id) - if defn is None: - return None - return defn.benchmark_type -``` - -**Site 1: `subtask_cancellation_service.py:98-104`** — BFS cascade. - -```diff - events = [ - TaskCancelledEvent( - run_id=run_id, - definition_id=definition_id, - node_id=nid, - execution_id=_latest_execution_id(session, nid), - cause=cause, -+ sandbox_id=_lookup_sandbox_id(session, _latest_execution_id(session, nid)), -+ benchmark_slug=_lookup_benchmark_slug(session, run_id), - ) - for nid in transitioned - ] -``` - -**Site 2: `task_management_service.py:214-220`** — manager `cancel_task`. - -```diff - event = TaskCancelledEvent( - run_id=command.run_id, - definition_id=definition_id, - node_id=command.node_id, - execution_id=execution_id, - cause="manager_decision", -+ sandbox_id=_lookup_sandbox_id(session, execution_id), -+ benchmark_slug=_lookup_benchmark_slug(session, command.run_id), - ) -``` - -**Site 3: `task_management_service.py:555-563`** — downstream invalidation. - -```diff - event = TaskCancelledEvent( - run_id=run_id, - definition_id=definition_id, - node_id=node_id, - execution_id=execution_id, - cause="downstream_invalidation", -+ sandbox_id=_lookup_sandbox_id(session, execution_id), -+ benchmark_slug=_lookup_benchmark_slug(session, run_id), - ) -``` - -**Sites 4 + 5: `propagate_execution.py:73-83` and `165-175`** — dependency -invalidation. These paths emit with `execution_id=None` (the task was never -dispatched, so no sandbox was ever created). `sandbox_id` and `benchmark_slug` -remain `None`; no change needed beyond the model accepting `None`. - ---- - -## Package structure - -No new packages. The `_cancel_helpers` module is optional; the two functions -can alternatively be inlined in each service. If extracted: - -``` -ergon_core/ergon_core/core/runtime/services/ - _cancel_helpers.py ADD — _lookup_sandbox_id, _lookup_benchmark_slug -``` - ---- - -## Implementation order - -| Step | Phase | What | Files touched | -|---|---|---|---| -| **1** | PR 1 | Add `sandbox_id: str \| None = None` and `benchmark_slug: str \| None = None` to `TaskCancelledEvent` | MODIFY `runtime/events/task_events.py` | -| **2** | PR 1 | Add `sandbox_id: str \| None` column to `RunTaskExecution` SQLModel | MODIFY `persistence/telemetry/models.py` | -| **3** | PR 1 | Write Alembic migration adding the nullable column | ADD `migrations/versions/<hash>_add_sandbox_id_to_run_task_executions.py` | -| **4** | PR 1 | Add `_lookup_sandbox_id` + `_lookup_benchmark_slug` helpers (inline or extracted) | ADD `runtime/services/_cancel_helpers.py` or inline | -| **5** | PR 1 | Update `SubtaskCancellationService.cancel_orphans` to populate new fields | MODIFY `runtime/services/subtask_cancellation_service.py` | -| **6** | PR 1 | Update `TaskManagementService.cancel_task` + `_cancel_downstream` to populate new fields | MODIFY `runtime/services/task_management_service.py` | -| **7** | PR 1 | Add `persist-sandbox-id` step in `execute_task_fn` after `sandbox-setup` | MODIFY `runtime/inngest/execute_task.py` | -| **8** | PR 1 | Unit tests for new event fields; state tests for helper lookups | ADD `tests/state/test_cancel_event_sandbox_fields.py` | -| **9** | PR 2 | Add `release-sandbox` step to `cleanup_cancelled_task_fn`; import `SANDBOX_MANAGERS` | MODIFY `runtime/inngest/cleanup_cancelled_task.py` | -| **10** | PR 2 | Remove `slopcop: ignore[no-todo-comment]` from `TaskCleanupService` | MODIFY `runtime/services/task_cleanup_service.py` | -| **11** | PR 2 | Unit tests for step: sandbox released, sandbox skipped, unknown slug | MODIFY `tests/state/test_task_cleanup_service.py` | - -**PR 1** (Steps 1–8): event extension + column + emission site updates. Safe to -deploy ahead of PR 2 because `cleanup_cancelled_task_fn` ignores unknown extra -fields. PR 1 itself is backward-compatible: existing queued events without -`sandbox_id` will continue to work (field defaults to `None`). - -**PR 2** (Steps 9–11): the actual cleanup step. Depends on PR 1 merged and -deployed because it reads `payload.sandbox_id` and `payload.benchmark_slug`. - ---- - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `ergon_core/ergon_core/migrations/versions/<hash>_add_sandbox_id_to_run_task_executions.py` | Additive Alembic migration: nullable `sandbox_id` column on `run_task_executions` | -| `ergon_core/ergon_core/core/runtime/services/_cancel_helpers.py` | `_lookup_sandbox_id`, `_lookup_benchmark_slug` — shared DB helpers for emission sites | -| `ergon/tests/state/test_cancel_event_sandbox_fields.py` | Unit + state tests for new event fields and helper lookup functions | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/runtime/events/task_events.py` | Add `sandbox_id: str \| None = None` and `benchmark_slug: str \| None = None` to `TaskCancelledEvent` | -| `ergon_core/ergon_core/core/persistence/telemetry/models.py` | Add `sandbox_id: str \| None = Field(default=None)` to `RunTaskExecution` | -| `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` | Add `persist-sandbox-id` durable step after `sandbox-setup` returns | -| `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py` | Populate `sandbox_id` + `benchmark_slug` in `cancel_orphans` event construction | -| `ergon_core/ergon_core/core/runtime/services/task_management_service.py` | Populate `sandbox_id` + `benchmark_slug` in `cancel_task` + `_cancel_downstream` | -| `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` | Add `release-sandbox` durable step; import `SANDBOX_MANAGERS` + `BaseSandboxManager` | -| `ergon_core/ergon_core/core/runtime/services/task_cleanup_service.py` | Remove `slopcop: ignore[no-todo-comment]` suppressor; update comment | -| `ergon/tests/state/test_task_cleanup_service.py` | Add tests asserting `sandbox_released=True` when step fires | - ---- - -## Testing approach - -### State tests (fast-tier, no E2B, no Inngest) - -```python -# ergon/tests/state/test_cancel_event_sandbox_fields.py - -"""Tests for sandbox_id + benchmark_slug population on TaskCancelledEvent.""" - -from __future__ import annotations - -from uuid import uuid4 - -import pytest -from sqlmodel import Session - -from ergon_core.core.persistence.shared.enums import TaskExecutionStatus -from ergon_core.core.persistence.telemetry.models import RunTaskExecution -from ergon_core.core.runtime.services._cancel_helpers import ( - _lookup_benchmark_slug, - _lookup_sandbox_id, -) - - -def _seed_execution( - session: Session, - *, - run_id, - node_id, - sandbox_id: str | None = "sbx-abc123", - status=TaskExecutionStatus.RUNNING, -) -> RunTaskExecution: - exe = RunTaskExecution( - run_id=run_id, - node_id=node_id, - status=status, - sandbox_id=sandbox_id, - ) - session.add(exe) - session.flush() - return exe - - -class TestLookupSandboxId: - def test_returns_sandbox_id_for_known_execution(self, session: Session) -> None: - run_id = uuid4() - node_id = uuid4() - exe = _seed_execution(session, run_id=run_id, node_id=node_id, sandbox_id="sbx-xyz") - - result = _lookup_sandbox_id(session, exe.id) - - assert result == "sbx-xyz" - - def test_returns_none_for_none_execution_id(self, session: Session) -> None: - result = _lookup_sandbox_id(session, None) - assert result is None - - def test_returns_none_for_missing_execution(self, session: Session) -> None: - result = _lookup_sandbox_id(session, uuid4()) - assert result is None - - def test_returns_none_for_null_sandbox_id_column(self, session: Session) -> None: - run_id = uuid4() - node_id = uuid4() - exe = _seed_execution(session, run_id=run_id, node_id=node_id, sandbox_id=None) - - result = _lookup_sandbox_id(session, exe.id) - - assert result is None - - -class TestTaskCancelledEventFields: - """Verify emission sites populate the new fields.""" - - def test_subtask_cancellation_service_populates_sandbox_id( - self, session: Session - ) -> None: - """cancel_orphans should include sandbox_id on transitioned nodes.""" - from ergon_core.core.persistence.graph.models import RunGraphNode - from ergon_core.core.persistence.graph.status_conventions import RUNNING - from ergon_core.core.runtime.services.subtask_cancellation_service import ( - SubtaskCancellationService, - ) - # Seed a parent and a child node with a running execution - run_id = uuid4() - definition_id = uuid4() - parent_id = uuid4() - child_id = uuid4() - - parent = RunGraphNode( - id=parent_id, - run_id=run_id, - task_key="parent", - instance_key="inst", - status=RUNNING, - ) - child = RunGraphNode( - id=child_id, - run_id=run_id, - task_key="child", - instance_key="inst", - status=RUNNING, - parent_node_id=parent_id, - ) - session.add_all([parent, child]) - session.flush() - - exe = _seed_execution( - session, run_id=run_id, node_id=child_id, sandbox_id="sbx-child" - ) - session.flush() - - svc = SubtaskCancellationService() - result = svc.cancel_orphans( - session, - run_id=run_id, - definition_id=definition_id, - parent_node_id=parent_id, - cause="parent_terminal", - ) - - assert len(result.events_to_emit) == 1 - event = result.events_to_emit[0] - assert event.sandbox_id == "sbx-child" - assert event.execution_id == exe.id -``` - -### Unit tests — `cleanup_cancelled_task_fn` release-sandbox step - -```python -# Add to: ergon/tests/state/test_task_cleanup_service.py - -from unittest.mock import AsyncMock, patch - - -class TestReleaseSandboxStep: - """Verify the release-sandbox logic (extracted to testable form).""" - - async def test_releases_sandbox_when_fields_present(self) -> None: - """terminate_by_sandbox_id called exactly once for valid payload.""" - with patch( - "ergon_core.core.sandbox.manager.BaseSandboxManager" - ".terminate_by_sandbox_id", - new_callable=AsyncMock, - return_value=True, - ) as mock_terminate: - from ergon_builtins.registry import SANDBOX_MANAGERS - from ergon_core.core.sandbox.manager import BaseSandboxManager - - # Any known slug from SANDBOX_MANAGERS - slug = next(iter(SANDBOX_MANAGERS)) - sandbox_id = "sbx-test-abc" - - released = await BaseSandboxManager.terminate_by_sandbox_id(sandbox_id) - - mock_terminate.assert_called_once_with(sandbox_id) - assert released is True - - async def test_no_release_when_sandbox_id_none(self) -> None: - """Step is a no-op when sandbox_id is None.""" - # Simulate the guard at the top of _release_sandbox - sandbox_id = None - benchmark_slug = "swebench-verified" - - # Guard: neither field triggers a terminate call - assert sandbox_id is None # no call should be made - - async def test_no_release_when_unknown_slug(self) -> None: - """Step logs warning and returns False for unknown benchmark_slug.""" - from ergon_builtins.registry import SANDBOX_MANAGERS - - unknown_slug = "not-a-real-benchmark" - assert unknown_slug not in SANDBOX_MANAGERS - - # SANDBOX_MANAGERS.get returns None → no terminate call - mgr_cls = SANDBOX_MANAGERS.get(unknown_slug) - assert mgr_cls is None -``` - -### Integration / contract - -- After this RFC lands, cancel a live `swebench-verified` run mid-execution - and assert `terminate_by_sandbox_id` is called within 30 seconds. -- Assert `CleanupResult.sandbox_released = True` appears in the Inngest - function's output for the cancelled execution. -- Assert the `RunTaskExecution` row for the cancelled execution has a non-null - `sandbox_id` (written by the new `persist-sandbox-id` step). - ---- - -## Trace / observability impact - -### Span attributes - -No new span added. The existing `"cleanup.cancelled"` span (if emitted by a -future cleanup-specific span factory) should include: - -```python -{ - "sandbox_id": payload.sandbox_id or "none", - "benchmark_slug": payload.benchmark_slug or "none", - "sandbox_released": sandbox_result.get("sandbox_released", False), - "release_reason": sandbox_result.get("reason", "unknown"), -} -``` - -`cleanup_cancelled_task_fn` does not currently emit a `CompletedSpan`. No -tracing change is required; the log lines added in step 9 are sufficient for -observability until a dedicated span factory is added. - -### Log lines added - -- `cleanup-cancelled node_id=... execution_id=... cause=... sandbox_id=...` - (INFO, already present; `sandbox_id` added to existing format string) -- `release-sandbox skipped: no sandbox_id or benchmark_slug for node_id=...` - (INFO, new) -- `release-sandbox: no manager for benchmark_slug=... node_id=...` (WARNING, new) -- `release-sandbox sandbox_id=... benchmark_slug=... released=...` (INFO, new) - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| `terminate_by_sandbox_id` raises on E2B API error | Inngest retries the `release-sandbox` step (retries=3). Idempotent on repeated call (E2B `kill` on an already-dead sandbox returns not-found, `terminate_by_sandbox_id` handles this at line 487-489). | No change needed; existing error handling in `BaseSandboxManager.terminate_by_sandbox_id` is already broad-except with a warning log. | -| `persist-sandbox-id` step adds latency to `execute_task_fn` hot path | Adds ~1 DB write per task execution (single `UPDATE` with a session context). | Negligible; `execute_task_fn` already does multiple DB writes in `prepare-execution` and `finalize_success`. No performance risk. | -| Race: `cleanup_cancelled_task_fn` fires before `persist-sandbox-id` writes the column | `RunTaskExecution.sandbox_id` is NULL; cleanup event has `sandbox_id=None`; step is a no-op; sandbox leaks until E2B timeout. | Acceptable fallback; this path only affects tasks cancelled between sandbox creation and `persist-sandbox-id` completing. Window is ~100ms. For correctness at millisecond boundaries, the event carries `sandbox_id` from the emission site lookup — not from the column — so this race only matters if the cancellation fires before the task execution writes to the column AND the emitter also fails to look it up. Probability very low in practice. | -| `_lookup_benchmark_slug` adds a DB read at every `TaskCancelledEvent` emission site | Adds 1 `RunRecord` read + 1 `ExperimentDefinition` read per cancelled node. | Both rows are small and indexed by PK. At cancellation-time concurrency these reads are negligible. If it becomes a bottleneck, pass `benchmark_type` down from the calling context (already available in `execute_task_fn` via `PreparedTaskExecution.benchmark_type`). | -| `RunTaskExecution.sandbox_id` column NULL for historical rows | Pre-migration executions have `sandbox_id=NULL`. Cleanup step is a no-op for these. | Correct behavior; the sandbox for a historical cancelled task has already leaked or expired. No backfill required. | -| `SANDBOX_SKIPPED` sentinel stored literally in column | `cleanup_cancelled_task_fn` calls `SANDBOX_MANAGERS.get("skipped")` → `None` → no-op. Sandbox was never created so nothing to release. | Correct behavior. No special-case needed. | -| Deploy ordering: PR 2 deployed before PR 1 | `payload.sandbox_id` and `payload.benchmark_slug` are always `None` (old events lack the field). Step is a no-op. No regression. | PRs are designed to be safe in either order. PR 1 should land first for correctness but it is not required for safety. | - ---- - -## Invariants affected - -### `docs/architecture/02_runtime_lifecycle.md §4 Known limits` - -Remove the bullet at line 117: -> "Cancellation does not release sandboxes. `cleanup-cancelled-task` updates -> the execution row but its `release-sandbox` step is a stub …" - -Replace with (in `§4 Invariants`, not `§4.1 Known limits`): -> "Cancellation releases the sandbox. `cleanup_cancelled_task_fn` calls -> `BaseSandboxManager.terminate_by_sandbox_id` in its `release-sandbox` step -> when `sandbox_id` is present on the `TaskCancelledEvent`. Tasks that were -> cancelled before a sandbox was created (dep_invalidated with no execution) -> emit the event with `sandbox_id=None`; the step is a safe no-op in that -> case." - -### `docs/architecture/cross_cutting/sandbox_lifecycle.md §5 Failure modes` - -Update the "Cancellation" failure mode at line 61 — change "is currently a -STUB" to "calls `BaseSandboxManager.terminate_by_sandbox_id`." - -### `docs/architecture/cross_cutting/sandbox_lifecycle.md §4 Invariants` - -Add (or update) invariant 4: -> "`close(sandbox_id)` / `terminate_by_sandbox_id(sandbox_id)` is idempotent -> and safe to call from any cancellation path. Calling it twice is a no-op on -> the second call. The `cleanup_cancelled_task_fn.release-sandbox` step is the -> primary cancellation path; `run-cleanup` via -> `BaseSandboxManager.terminate_by_sandbox_id` is the backstop." - -### `docs/architecture/cross_cutting/sandbox_lifecycle.md §8 Anti-patterns` - -Remove the "current offender" annotation from the "Leaking sandboxes on -cancellation" bullet. - ---- - -## Alternatives considered - -- **Look up `sandbox_id` on-the-fly from the DB inside `cleanup_cancelled_task_fn`.** - Rejected: adds a DB read inside an Inngest step that already has the info in - its event payload; slower and creates a dependency on the execution row still - existing. Also interacts badly with `update-db-rows` running before - `release-sandbox` — we would be reading a row we just mutated. - -- **Centralize sandbox cleanup at run-level `finalize_failure` / `finalize_cancelled`.** - Rejected: per-task cleanup is correct — some runs partially cancel (one - failed subtask, rest still running), and waiting for run finalization would - leak sandboxes for hours. Run-level cleanup is a backstop, not the primary - path. - -- **Let E2B platform timeouts handle it.** - Rejected: explicit cleanup is correct; timeouts are a safety net, not a - plan. Relying on timeouts also masks bugs where we thought we cancelled but - did not. - -- **Put the release in `update-db-rows` as a single step.** - Rejected: violates the Inngest step-per-side-effect convention and muddies - retry semantics (a DB failure would force sandbox-close retry too). - -- **`__init_subclass__` auto-registration on `BaseSandboxManager`.** - Considered for populating a `SANDBOX_MANAGER_REGISTRY`. Rejected: auto- - registration hides the side effect of importing a module and breaks the - explicit registration model already in `registry_core.py`. Using `SANDBOX_MANAGERS` - from `ergon_builtins.registry` (already populated) is simpler and consistent - with `sandbox_setup_fn`. - -- **Add a `SANDBOX_MANAGER_REGISTRY` singleton separate from `SANDBOX_MANAGERS`.** - Rejected: `SANDBOX_MANAGERS` already exists and maps benchmark slug to - manager class. `BaseSandboxManager.terminate_by_sandbox_id` is a `@staticmethod` - — it does not need an instance. Using the existing dict avoids introducing a - parallel registry. - ---- - -## Open questions - -- **Does the `sandbox_id` column race matter?** See Risks table. The practical - window is ~100ms between sandbox creation and the `persist-sandbox-id` step - completing. For tasks cancelled in that window, the fallback is the E2B - platform timeout. Acceptable for now; if it proves problematic, pass - `sandbox_id` to the cancellation event directly from `execute_task_fn`'s - exception handler. - -- **`retries=3` on `cleanup_cancelled_task_fn` — is that right?** Sandbox - close is idempotent; 3 retries are safe. If `release-sandbox` is flaky, - retries save us; if not, extra retries are harmless. Keep as-is. - -- **What if `benchmark_slug` is set but the sandbox was already closed by - `check_evaluators`?** `terminate_by_sandbox_id` calls `AsyncSandbox.kill`; - E2B returns not-found for an already-closed sandbox; the method logs at INFO - and returns `False`. `CleanupResult.sandbox_released = False` in that case. - Correct behavior — the sandbox is gone regardless. - -- **Should we store `benchmark_slug` on `RunTaskExecution` instead of looking - it up via `RunRecord → ExperimentDefinition`?** Possible optimization if the - lookup proves slow. Not needed now. - ---- - -## On acceptance - -- Update `docs/architecture/02_runtime_lifecycle.md §4.1 Known limits` — - remove the "release-sandbox stub" offender. -- Update `docs/architecture/cross_cutting/sandbox_lifecycle.md §4 Invariants` - and `§5 Failure modes` and `§8 Anti-patterns` as described above. -- Move this file to `docs/rfcs/accepted/`. -- If a separate bug file exists at - `docs/bugs/open/2026-04-17-cleanup-cancelled-task-release-sandbox-stub.md`, - move it to `docs/bugs/fixed/` with `fixed_pr` set. (No such file exists in - `docs/bugs/open/` as of 2026-04-21; skip this step.) -- Link the implementation plan at - `docs/superpowers/plans/2026-04-21-cleanup-cancelled-task-release-sandbox.md`. diff --git a/docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md b/docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md deleted file mode 100644 index e54a8f9ed..000000000 --- a/docs/rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md +++ /dev/null @@ -1,1193 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/03_providers.md#invariants, docs/architecture/cross_cutting/sandbox_lifecycle.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Sandbox timeout must cover task + criteria; expose `BaseSandboxManager.reconnect` - -## Problem - -### Current state - -`BaseSandboxManager.create()` at -`ergon_core/ergon_core/core/sandbox/manager.py:226` accepts a single -`timeout_minutes: int = 30` parameter. Every call site passes a literal or -relies on the default: - -- `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py:103` — - `timeout_minutes=30` (literal) -- `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py:56` — - `timeout_minutes=30` (literal) -- `tests/swebench_verified/test_sandbox_manager.py:129` — `timeout_minutes=5` -- `tests/minif2f/test_sandbox_manager.py:121` — `timeout_minutes=5` - -After the task finishes, `check_and_run_evaluators` -(`ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py:42`) fans out -criteria sequentially. Each criterion runs inside the **same** sandbox via -in-process reconnect through `BaseSandboxManager.get_sandbox(task_id)` at -`manager.py:394`, which reads the singleton's class-level `_sandboxes` dict. -The sandbox's timeout counter started at `Manager.create()` and has been -ticking ever since. If the task runtime + at least one agentic criterion's -wall-clock > `timeout_minutes`, E2B kills the sandbox mid-criterion. The next -`sandbox.commands.run(...)` raises. The criterion marks the task -`evaluation-failed`; the trajectory is silently dropped from RL training (see -`docs/architecture/08_rl_loop.md`). - -This is a data-loss bug class: the worker produced correct output; the system -failed to evaluate it. - -### Missing `reconnect` - -There is no `BaseSandboxManager.reconnect(sandbox_id: str)` method today. This -is documented as a known limit at `docs/architecture/03_providers.md:143`: - -> **No `reconnect()` method.** Cross-process criteria must spawn their own -> sandbox (see `ergon_builtins/benchmarks/swebench_verified/criterion.py:66`). - -The SWE-bench criterion works around this at -`ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py:66-78` -by calling `_spawn_eval_sandbox(run_id)` which constructs a fresh -`SWEBenchSandboxManager()` (line 72) and calls `manager.create(...)` (line 74) -for a brand-new sandbox on every criterion invocation. Consequences: - -1. Every SWE-bench eval pays the full sandbox cold-start cost (clone + install). -2. The criterion grades the agent patch against a **clean** environment, not - the worker's actual on-disk state — breaking the "criterion sees what the - worker produced" invariant. -3. The pattern is contagious: it is being used as a template by future - criterion authors (tracked as P2 bug - `docs/bugs/open/2026-04-18-swebench-criterion-spawns-sandbox.md`). -4. `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` will enforce - the "criteria go through `CriterionRuntime`" invariant; without a - `reconnect` method on the manager, `CriterionRuntime.ensure_sandbox()` at - `criterion_runtime.py:53` cannot correctly reconnect cross-process criteria - — it calls `manager.create(...)` unconditionally when `get_sandbox()` returns - `None`, provisioning a fresh sandbox instead. - -The `cross_cutting/sandbox_lifecycle.md` invariant 3 states: - -> Criteria MUST reconnect via the manager, never by constructing `AsyncSandbox` -> directly. ... The reconnect path through the manager is the only correct way -> to attach to a live sandbox. - -That invariant cannot be enforced while `reconnect` is absent. - ---- - -## Proposal - -### Option chosen: split `timeout_minutes` into two typed parameters; add `reconnect` - -Three coordinated changes, shipped as two PRs (see Implementation order): - -**Change 1 — Split `create()` timeout.** -Change `BaseSandboxManager.create()`'s signature from: - -```python -async def create( - self, - sandbox_key: UUID, - run_id: UUID, - timeout_minutes: int = 30, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, -) -> str: -``` - -to: - -```python -async def create( - self, - sandbox_key: UUID, - run_id: UUID, - task_timeout_minutes: int = 30, - max_criterion_timeout_minutes: int = 10, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, -) -> str: -``` - -The E2B sandbox is provisioned with -`timeout = task_timeout_minutes + max_criterion_timeout_minutes` minutes. -Callers that previously passed `timeout_minutes=N` must be migrated to -`task_timeout_minutes=N` (default `max_criterion_timeout_minutes=10` adds -headroom automatically). No caller today passes a custom `timeout_minutes` to -`create()` other than tests; the migration is mechanical. - -`DefaultSandboxManager.create()` at `manager.py:503` overrides the base method; -it must be updated to the same signature, forwarding both new params to -`super().create(...)`. - -**Change 2 — Add `BaseSandboxManager.reconnect(sandbox_id)`.** -Add a concrete method: - -```python -async def reconnect(self, sandbox_id: str) -> "AsyncSandbox": - """Attach to a running sandbox by its E2B sandbox_id. - - Returns the AsyncSandbox handle. Raises SandboxExpiredError if the - sandbox is not found or has already timed out. Idempotent: calling - reconnect twice for the same sandbox_id is safe — both calls return - equivalent handles. - """ -``` - -Implementation calls `AsyncSandbox.connect(sandbox_id=sandbox_id, -api_key=settings.e2b_api_key)`. On E2B "not found" / "expired" error, raises -`SandboxExpiredError` (Change 3). This is the single blessed cross-process -reconnect path; `CriterionRuntime.ensure_sandbox()` will call it once RFC -`2026-04-17-criterion-runtime-di-container` lands. - -**Change 3 — Define `SandboxExpiredError`.** -New exception class at -`ergon_core/ergon_core/core/sandbox/errors.py`. Subclasses the base -`Exception` (not `ErgonNonRetriableError` — sandbox expiry is not a -definition-level error; it is a transient infrastructure condition). Criteria -that catch it should surface a `"sandbox-expired"` evaluation outcome rather -than a generic failure. - -**Change 4 — Update `sandbox_setup_fn` call site.** -`ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py:103`: -- Change `timeout_minutes=30` to `task_timeout_minutes=30` (no semantic change - for now; the new default `max_criterion_timeout_minutes=10` adds 10 min - automatically). - -**Change 5 — Update `DefaultCriterionRuntime.ensure_sandbox` call site.** -`ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py:56`: -- Same mechanical rename. - ---- - -## Architecture overview - -### Before (timeout bug path) - -``` -sandbox_setup_fn - └─ BaseSandboxManager.create(task_id, timeout_minutes=30) - └─ E2B provisions sandbox with timeout=30m - -Worker.execute() runs ... wall-clock 25m elapsed ... - -task/completed fires - └─ check_and_run_evaluators - └─ evaluate_task_run - └─ SWEBenchTestCriterion.evaluate() - └─ _spawn_eval_sandbox() ← FRESH sandbox (WRONG) - └─ SWEBenchSandboxManager.create(...) - └─ sandbox.commands.run(...) ← may hit timeout at 30m - └─ E2B kills sandbox ← SILENT DATA LOSS -``` - -### After (this RFC) - -``` -sandbox_setup_fn - └─ BaseSandboxManager.create(task_id, - task_timeout_minutes=30, - max_criterion_timeout_minutes=10) - └─ E2B provisions sandbox with timeout=40m - -Worker.execute() runs ... wall-clock 25m elapsed ... - -task/completed fires - └─ check_and_run_evaluators - └─ evaluate_task_run - └─ CriterionRuntime.ensure_sandbox() ← via RFC criterion-runtime-di-container - └─ BaseSandboxManager.reconnect(sandbox_id) - └─ AsyncSandbox.connect(sandbox_id) - └─ raises SandboxExpiredError → criterion returns "sandbox-expired" score - └─ sandbox.commands.run(...) ← 10m of headroom remaining - └─ _terminate_sandbox(sandbox_id) ← unchanged teardown path -``` - -### Data flow: `timeout_minutes` parameter through the stack - -``` -SandboxSetupRequest (payload) - └─ sandbox_setup_fn (_create_sandbox) - └─ BaseSandboxManager.create( - task_timeout_minutes=30, ← from payload / default - max_criterion_timeout_minutes=10) ← default; overridable per-subclass - │ - ├─ timeout_seconds = (task_timeout_minutes + max_criterion_timeout_minutes) * 60 - └─ AsyncSandbox.create(timeout=timeout_seconds) - └─ E2B sandbox: will live for 40 minutes from create -``` - ---- - -## Type / interface definitions - -```python -# ergon_core/ergon_core/core/sandbox/errors.py - -"""Sandbox-specific exception types.""" - - -class SandboxError(Exception): - """Base for sandbox infrastructure errors.""" - - -class SandboxExpiredError(SandboxError): - """Raised when a sandbox is not reachable because it has timed out or - been terminated. - - Callers (criteria, CriterionRuntime) should catch this and surface a - ``"sandbox-expired"`` evaluation outcome rather than a generic error. - The underlying task output is not lost — the sandbox's state was already - published to the blob store by the worker's resource publisher before the - sandbox timed out. - """ - - def __init__(self, sandbox_id: str, detail: str = "") -> None: - self.sandbox_id = sandbox_id - msg = f"Sandbox {sandbox_id!r} is expired or not found" - if detail: - msg = f"{msg}: {detail}" - super().__init__(msg) -``` - ---- - -## Full implementations - -### `errors.py` (new file) - -```python -# ergon_core/ergon_core/core/sandbox/errors.py - -"""Sandbox-specific exception types.""" - - -class SandboxError(Exception): - """Base for sandbox infrastructure errors.""" - - -class SandboxExpiredError(SandboxError): - """Raised when a sandbox is not reachable because it has timed out or - been terminated. - - Callers (criteria, CriterionRuntime) should catch this and surface a - ``"sandbox-expired"`` evaluation outcome rather than a generic error. - The underlying task output is not lost — the sandbox's state was already - published to the blob store by the worker's resource publisher before the - sandbox timed out. - """ - - def __init__(self, sandbox_id: str, detail: str = "") -> None: - self.sandbox_id = sandbox_id - msg = f"Sandbox {sandbox_id!r} is expired or not found" - if detail: - msg = f"{msg}: {detail}" - super().__init__(msg) -``` - -### `reconnect` method (added to `BaseSandboxManager`) - -```python -# Added to: ergon_core/ergon_core/core/sandbox/manager.py -# Location: after get_sandbox() at line 394, before get_sandbox_path() - -async def reconnect(self, sandbox_id: str) -> "AsyncSandbox": - """Attach to a running sandbox by its E2B sandbox_id. - - Returns an AsyncSandbox handle connected to the running sandbox. - Raises SandboxExpiredError if the sandbox is not found or has already - timed out. Idempotent: calling reconnect twice for the same sandbox_id - is safe — both calls invoke AsyncSandbox.connect() which returns an - equivalent handle. - - Use this for cross-process criterion reconnect. In-process criteria - should prefer get_sandbox(task_id) (reads shared class state). - This method does NOT register the sandbox in class-level state; - callers should not assume it shows up in _sandboxes. - """ - from ergon_core.core.sandbox.errors import SandboxExpiredError - - if AsyncSandbox is None: - raise RuntimeError( - "e2b_code_interpreter is not installed. " - "Install it with: pip install e2b-code-interpreter" - ) - try: - sandbox = await AsyncSandbox.connect( - sandbox_id=sandbox_id, - api_key=settings.e2b_api_key, - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - err = str(exc).lower() - if "not found" in err or "404" in err or "expired" in err or "timeout" in err: - raise SandboxExpiredError(sandbox_id, detail=str(exc)) from exc - raise - return sandbox -``` - -### Updated `create()` signature — `BaseSandboxManager` - -```python -# ergon_core/ergon_core/core/sandbox/manager.py -# Replace lines 226-295 (existing create method) - -async def create( - self, - sandbox_key: UUID, - run_id: UUID, - task_timeout_minutes: int = 30, - max_criterion_timeout_minutes: int = 10, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, -) -> str: - """Create a new E2B sandbox, set up directories, install deps. - - The sandbox is provisioned with a timeout of - ``task_timeout_minutes + max_criterion_timeout_minutes`` to ensure - criteria running after the task has finished still have a live sandbox - to reconnect to. See RFC 2026-04-17-sandbox-lifetime-covers-criteria. - """ - if AsyncSandbox is None: - raise RuntimeError( - "e2b_code_interpreter is not installed. " - "Install it with: pip install e2b-code-interpreter" - ) - - display_task_id = display_task_id or sandbox_key - lock = self._creation_locks.setdefault(sandbox_key, asyncio.Lock()) - async with lock: - if sandbox_key in self._sandboxes: - return self._sandboxes[sandbox_key].sandbox_id - - if not settings.e2b_api_key: - raise ValueError( - "E2B_API_KEY is not set. " - "Please set E2B_API_KEY in your .env file or environment variables." - ) - - try: - # Provision for task + criterion headroom. - total_timeout_minutes = task_timeout_minutes + max_criterion_timeout_minutes - timeout_seconds = total_timeout_minutes * 60 - create_kwargs: dict[str, str | int] = { - "api_key": settings.e2b_api_key, - "timeout": timeout_seconds, - } - if envs: - create_kwargs["envs"] = envs - if self.template: - create_kwargs["template"] = self.template - sandbox = await AsyncSandbox.create(**create_kwargs) - except Exception as e: # slopcop: ignore[no-broad-except] - raise RuntimeError( - f"Failed to create sandbox for sandbox_key={sandbox_key}: {e}" - ) from e - - if not sandbox: - raise RuntimeError("Sandbox object is None after creation") - - self._sandboxes[sandbox_key] = sandbox - self._ensure_registries(sandbox_key) - self._run_ids[sandbox_key] = run_id - self._display_task_ids[sandbox_key] = display_task_id - - await self._event_sink.sandbox_created( - run_id=run_id, - task_id=display_task_id, - sandbox_id=sandbox.sandbox_id, - timeout_minutes=total_timeout_minutes, - ) - await self._emit_wal_entry( - sandbox_key, - command="sandbox.created", - stdout=( - f"sandbox_id={sandbox.sandbox_id}\n" - f"task_timeout={task_timeout_minutes}m\n" - f"max_criterion_timeout={max_criterion_timeout_minutes}m\n" - f"total_timeout={total_timeout_minutes}m" - ), - exit_code=0, - duration_ms=0, - ) - - await self._create_directory_structure(sandbox, sandbox_key) - await self._install_dependencies(sandbox, display_task_id) - await self._verify_setup(sandbox, display_task_id) - - return sandbox.sandbox_id -``` - -### Updated `DefaultSandboxManager.create()` override - -```python -# ergon_core/ergon_core/core/sandbox/manager.py -# Replace lines 503-526 (existing DefaultSandboxManager.create override) - -async def create( - self, - sandbox_key: UUID, - run_id: UUID, - task_timeout_minutes: int = 30, - max_criterion_timeout_minutes: int = 10, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, -) -> str: - if not settings.e2b_api_key: - # Deferred: avoid a circular import between providers and runtime events. - from ergon_core.core.runtime.events.task_events import SANDBOX_SKIPPED - - logger.info( - "E2B_API_KEY not set — skipping sandbox creation for task %s (stub mode)", - sandbox_key, - ) - return SANDBOX_SKIPPED - return await super().create( - sandbox_key, - run_id=run_id, - task_timeout_minutes=task_timeout_minutes, - max_criterion_timeout_minutes=max_criterion_timeout_minutes, - envs=envs, - display_task_id=display_task_id, - ) -``` - -### Updated `__init__.py` (sandbox package) - -```python -# ergon_core/ergon_core/core/sandbox/__init__.py -# Add SandboxExpiredError, SandboxError to exports - -"""Sandbox management: provisioning, file I/O, lifecycle.""" - -from ergon_core.core.sandbox.errors import ( - SandboxError, - SandboxExpiredError, -) -from ergon_core.core.sandbox.event_sink import ( - DashboardEmitterSandboxEventSink, - NoopSandboxEventSink, - SandboxEventSink, -) -from ergon_core.core.sandbox.manager import ( - BaseSandboxManager, - DefaultSandboxManager, - DownloadedFile, - DownloadedFiles, -) - -__all__ = [ - "BaseSandboxManager", - "DashboardEmitterSandboxEventSink", - "DefaultSandboxManager", - "DownloadedFile", - "DownloadedFiles", - "NoopSandboxEventSink", - "SandboxError", - "SandboxEventSink", - "SandboxExpiredError", -] -``` - ---- - -## Exact diffs for modified files - -### `ergon_core/ergon_core/core/sandbox/manager.py` - -```diff -@@ -226,13 +226,16 @@ class BaseSandboxManager(ABC): - async def create( - self, - sandbox_key: UUID, - run_id: UUID, -- timeout_minutes: int = 30, -+ task_timeout_minutes: int = 30, -+ max_criterion_timeout_minutes: int = 10, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, - ) -> str: -- """Create a new E2B sandbox, set up directories, install deps.""" -+ """Create a new E2B sandbox, set up directories, install deps. -+ -+ The sandbox is provisioned with a timeout of -+ ``task_timeout_minutes + max_criterion_timeout_minutes`` to ensure -+ criteria running after the task has finished still have a live sandbox -+ to reconnect to. See RFC 2026-04-17-sandbox-lifetime-covers-criteria. -+ """ - if AsyncSandbox is None: - raise RuntimeError(...) - -@@ -250,7 +253,8 @@ class BaseSandboxManager(ABC): - try: -- timeout_seconds = timeout_minutes * 60 -+ total_timeout_minutes = task_timeout_minutes + max_criterion_timeout_minutes -+ timeout_seconds = total_timeout_minutes * 60 - create_kwargs: dict[str, str | int] = { - "api_key": settings.e2b_api_key, - "timeout": timeout_seconds, - } - -@@ -278,8 +283,12 @@ class BaseSandboxManager(ABC): - await self._event_sink.sandbox_created( - run_id=run_id, - task_id=display_task_id, - sandbox_id=sandbox.sandbox_id, -- timeout_minutes=timeout_minutes, -+ timeout_minutes=total_timeout_minutes, - ) - await self._emit_wal_entry( - sandbox_key, - command="sandbox.created", -- stdout=f"sandbox_id={sandbox.sandbox_id}\ntimeout={timeout_minutes}m", -+ stdout=( -+ f"sandbox_id={sandbox.sandbox_id}\n" -+ f"task_timeout={task_timeout_minutes}m\n" -+ f"max_criterion_timeout={max_criterion_timeout_minutes}m\n" -+ f"total_timeout={total_timeout_minutes}m" -+ ), - exit_code=0, - duration_ms=0, - ) - -+ async def reconnect(self, sandbox_id: str) -> "AsyncSandbox": -+ """Attach to a running sandbox by its E2B sandbox_id. -+ -+ Returns an AsyncSandbox handle. Raises SandboxExpiredError if the -+ sandbox is not found or has already timed out. Idempotent. -+ Does NOT register in class-level _sandboxes state. -+ """ -+ from ergon_core.core.sandbox.errors import SandboxExpiredError -+ -+ if AsyncSandbox is None: -+ raise RuntimeError( -+ "e2b_code_interpreter is not installed. " -+ "Install it with: pip install e2b-code-interpreter" -+ ) -+ try: -+ sandbox = await AsyncSandbox.connect( -+ sandbox_id=sandbox_id, -+ api_key=settings.e2b_api_key, -+ ) -+ except Exception as exc: # slopcop: ignore[no-broad-except] -+ err = str(exc).lower() -+ if "not found" in err or "404" in err or "expired" in err or "timeout" in err: -+ raise SandboxExpiredError(sandbox_id, detail=str(exc)) from exc -+ raise -+ return sandbox - - @@ -503,10 +543,12 @@ class DefaultSandboxManager(BaseSandboxManager): - async def create( - self, - sandbox_key: UUID, - run_id: UUID, -- timeout_minutes: int = 30, -+ task_timeout_minutes: int = 30, -+ max_criterion_timeout_minutes: int = 10, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, - ) -> str: - if not settings.e2b_api_key: - ... - return SANDBOX_SKIPPED - return await super().create( - sandbox_key, - run_id=run_id, -- timeout_minutes=timeout_minutes, -+ task_timeout_minutes=task_timeout_minutes, -+ max_criterion_timeout_minutes=max_criterion_timeout_minutes, - envs=envs, - display_task_id=display_task_id, - ) -``` - -### `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` - -```diff -@@ -103,7 +103,8 @@ async def _create_sandbox(...) -> SandboxReadyResult: - sandbox_id = await sandbox_manager.create( - task_id, - run_id=run_id, -- timeout_minutes=30, -+ task_timeout_minutes=30, -+ # max_criterion_timeout_minutes uses default (10) - envs=envs, - display_task_id=task_id, - ) -``` - -### `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` - -```diff -@@ -53,8 +53,9 @@ class DefaultCriterionRuntime: - async def ensure_sandbox(self) -> None: - sandbox = self.sandbox_manager.get_sandbox(self.context.run_id) - if sandbox is None: - await self.sandbox_manager.create( - self.context.run_id, - run_id=self.context.run_id, -- timeout_minutes=30, -+ task_timeout_minutes=30, -+ # max_criterion_timeout_minutes uses default (10) - ) - self._owns_sandbox = True - return -- await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=30) -+ await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=40) -``` - -Note: `reset_timeout` call changes from 30 to 40 to match the new provisioned total. The signature of `reset_timeout` at `manager.py:407` is unchanged (still accepts `timeout_minutes`). - -### `ergon_core/ergon_core/core/sandbox/__init__.py` - -```diff -@@ -1,6 +1,11 @@ - """Sandbox management: provisioning, file I/O, lifecycle.""" - -+from ergon_core.core.sandbox.errors import ( -+ SandboxError, -+ SandboxExpiredError, -+) - from ergon_core.core.sandbox.event_sink import ( - DashboardEmitterSandboxEventSink, - NoopSandboxEventSink, - SandboxEventSink, - ) - from ergon_core.core.sandbox.manager import ( - BaseSandboxManager, - DefaultSandboxManager, - DownloadedFile, - DownloadedFiles, - ) - - __all__ = [ - "BaseSandboxManager", - "DashboardEmitterSandboxEventSink", - "DefaultSandboxManager", - "DownloadedFile", - "DownloadedFiles", - "NoopSandboxEventSink", -+ "SandboxError", - "SandboxEventSink", -+ "SandboxExpiredError", - ] -``` - ---- - -## Package structure - -New file, no new package. The errors module sits alongside the existing sandbox -package files: - -``` -ergon_core/ergon_core/core/sandbox/ -├── __init__.py MODIFY (add SandboxError, SandboxExpiredError exports) -├── errors.py ADD (SandboxError, SandboxExpiredError) -├── event_sink.py no change -├── instrumentation.py no change -├── manager.py MODIFY (create signature, reconnect method) -├── research_rubrics_manager.py no change -├── resource_publisher.py no change -└── utils.py no change -``` - ---- - -## Implementation order - -| Step | Phase | What | Files touched | -|------|-------|------|---------------| -| 1 | PR 1 | Create `errors.py` with `SandboxError` and `SandboxExpiredError` | ADD `ergon_core/ergon_core/core/sandbox/errors.py` | -| 2 | PR 1 | Add `errors` imports to sandbox `__init__.py` | MODIFY `ergon_core/ergon_core/core/sandbox/__init__.py` | -| 3 | PR 1 | Update `BaseSandboxManager.create()` signature: `timeout_minutes` → `task_timeout_minutes + max_criterion_timeout_minutes`; update WAL entry log | MODIFY `ergon_core/ergon_core/core/sandbox/manager.py` | -| 4 | PR 1 | Update `DefaultSandboxManager.create()` override with same signature change | MODIFY `ergon_core/ergon_core/core/sandbox/manager.py` | -| 5 | PR 1 | Migrate `sandbox_setup.py` call site: `timeout_minutes=30` → `task_timeout_minutes=30` | MODIFY `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` | -| 6 | PR 1 | Migrate `criterion_runtime.py` call sites: same rename; `reset_timeout` 30 → 40 | MODIFY `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | -| 7 | PR 1 | Migrate test call sites: `timeout_minutes=5` → `task_timeout_minutes=5` in `tests/swebench_verified/test_sandbox_manager.py` and `tests/minif2f/test_sandbox_manager.py` | MODIFY 2 test files | -| 8 | PR 1 | Unit tests: `create()` passes correct total timeout to E2B; `task_timeout + max_criterion_timeout` arithmetic | ADD `tests/unit/test_sandbox_timeout.py` | -| 9 | PR 2 | Add `BaseSandboxManager.reconnect(sandbox_id)` method | MODIFY `ergon_core/ergon_core/core/sandbox/manager.py` | -| 10 | PR 2 | Unit tests for `reconnect`: successful connect, E2B-not-found raises `SandboxExpiredError`, non-expired E2B error re-raises | ADD to `tests/unit/test_sandbox_reconnect.py` | -| 11 | PR 2 | Canary e2e test: deliberately-slow criterion (sleep > task_timeout) still finds sandbox reachable | ADD `tests/e2e/test_sandbox_criterion_timeout_canary.py` | -| 12 | PR 2 | (Deferred — depends on `2026-04-17-criterion-runtime-di-container`) Migrate `DefaultCriterionRuntime.ensure_sandbox()` to use `reconnect` when `get_sandbox` returns `None`, handling `SandboxExpiredError` | MODIFY `criterion_runtime.py` | - -Steps 1–8 land as PR 1 ("sandbox-lifetime/split-timeout"). Steps 9–11 land as PR 2 -("sandbox-lifetime/reconnect"). Step 12 is gated on the DI container RFC. - ---- - -## File map - -### ADD - -| File | Purpose | -|------|---------| -| `ergon_core/ergon_core/core/sandbox/errors.py` | `SandboxError` base class; `SandboxExpiredError` raised by `reconnect()` on expired sandbox | -| `tests/unit/test_sandbox_timeout.py` | Unit tests: `create()` arithmetic, `task_timeout + max_criterion_timeout` passed to E2B | -| `tests/unit/test_sandbox_reconnect.py` | Unit tests: `reconnect()` success, not-found raises `SandboxExpiredError`, other errors re-raise | -| `tests/e2e/test_sandbox_criterion_timeout_canary.py` | E2e canary: slow criterion still reaches sandbox when timeout is correctly provisioned | - -### MODIFY - -| File | Changes | -|------|---------| -| `ergon_core/ergon_core/core/sandbox/manager.py` | Split `timeout_minutes` into `task_timeout_minutes + max_criterion_timeout_minutes` in `BaseSandboxManager.create()` and `DefaultSandboxManager.create()`; add `reconnect()` method | -| `ergon_core/ergon_core/core/sandbox/__init__.py` | Export `SandboxError`, `SandboxExpiredError` | -| `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` | Rename `timeout_minutes=30` → `task_timeout_minutes=30` at line 106 | -| `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | Rename `timeout_minutes=30` → `task_timeout_minutes=30` at line 59; `reset_timeout(..., timeout_minutes=30)` → `timeout_minutes=40` at line 63 | -| `tests/swebench_verified/test_sandbox_manager.py` | Rename `timeout_minutes=5` → `task_timeout_minutes=5`; update assertion `call_kwargs["timeout"] == 5 * 60` → `== (5 + 10) * 60` | -| `tests/minif2f/test_sandbox_manager.py` | Same rename and timeout assertion update | - ---- - -## Testing approach - -### Unit tests — `tests/unit/test_sandbox_timeout.py` - -```python -# tests/unit/test_sandbox_timeout.py - -"""Unit tests: BaseSandboxManager.create() provisions task + criterion timeout.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 - -import pytest - -from ergon_core.core.sandbox.manager import BaseSandboxManager, DefaultSandboxManager - - -@pytest.fixture(autouse=True) -def _reset_singleton() -> None: - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - BaseSandboxManager._creation_locks = {} - BaseSandboxManager._run_ids = {} - BaseSandboxManager._display_task_ids = {} - BaseSandboxManager._file_registries = {} - BaseSandboxManager._created_files_registry = {} - yield - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - - -class _MinimalManager(BaseSandboxManager): - """Concrete manager with no-op hooks for unit testing.""" - - async def _install_dependencies(self, sandbox, task_id): # type: ignore[override] - pass - - async def _create_directory_structure(self, sandbox, sandbox_key): # type: ignore[override] - pass - - -@pytest.mark.asyncio -async def test_create_passes_total_timeout_to_e2b(monkeypatch: pytest.MonkeyPatch) -> None: - """E2B AsyncSandbox.create receives task + criterion timeout in seconds.""" - fake_sandbox = MagicMock() - fake_sandbox.sandbox_id = "sbx-test" - fake_create = AsyncMock(return_value=fake_sandbox) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(create=fake_create), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - await mgr.create( - sandbox_key=uuid4(), - run_id=uuid4(), - task_timeout_minutes=30, - max_criterion_timeout_minutes=10, - ) - - call_kwargs = fake_create.await_args.kwargs - assert call_kwargs["timeout"] == 40 * 60, "Should be (30 + 10) * 60 = 2400s" - - -@pytest.mark.asyncio -async def test_create_default_max_criterion_timeout(monkeypatch: pytest.MonkeyPatch) -> None: - """Default max_criterion_timeout_minutes=10 is applied when not supplied.""" - fake_sandbox = MagicMock() - fake_sandbox.sandbox_id = "sbx-default" - fake_create = AsyncMock(return_value=fake_sandbox) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(create=fake_create), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - await mgr.create(sandbox_key=uuid4(), run_id=uuid4(), task_timeout_minutes=20) - - call_kwargs = fake_create.await_args.kwargs - assert call_kwargs["timeout"] == 30 * 60, "Should be (20 + 10) * 60 = 1800s" - - -@pytest.mark.asyncio -async def test_create_zero_criterion_timeout(monkeypatch: pytest.MonkeyPatch) -> None: - """Caller can pass max_criterion_timeout_minutes=0 to opt out of headroom.""" - fake_sandbox = MagicMock() - fake_sandbox.sandbox_id = "sbx-zero" - fake_create = AsyncMock(return_value=fake_sandbox) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(create=fake_create), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - await mgr.create( - sandbox_key=uuid4(), - run_id=uuid4(), - task_timeout_minutes=15, - max_criterion_timeout_minutes=0, - ) - - call_kwargs = fake_create.await_args.kwargs - assert call_kwargs["timeout"] == 15 * 60, "Zero criterion timeout: total = task only" -``` - -### Unit tests — `tests/unit/test_sandbox_reconnect.py` - -```python -# tests/unit/test_sandbox_reconnect.py - -"""Unit tests: BaseSandboxManager.reconnect().""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 - -import pytest - -from ergon_core.core.sandbox.errors import SandboxExpiredError -from ergon_core.core.sandbox.manager import BaseSandboxManager - - -@pytest.fixture(autouse=True) -def _reset_singleton() -> None: - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - yield - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - - -class _MinimalManager(BaseSandboxManager): - async def _install_dependencies(self, sandbox, task_id): # type: ignore[override] - pass - - async def _create_directory_structure(self, sandbox, sandbox_key): # type: ignore[override] - pass - - -@pytest.mark.asyncio -async def test_reconnect_returns_sandbox_on_success(monkeypatch: pytest.MonkeyPatch) -> None: - """reconnect() returns the AsyncSandbox handle on successful connect.""" - fake_sandbox = MagicMock() - fake_connect = AsyncMock(return_value=fake_sandbox) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(connect=fake_connect), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - result = await mgr.reconnect("sbx-live-001") - - assert result is fake_sandbox - fake_connect.assert_awaited_once_with(sandbox_id="sbx-live-001", api_key="test-key") - - -@pytest.mark.asyncio -async def test_reconnect_raises_sandbox_expired_on_not_found( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """reconnect() raises SandboxExpiredError when E2B returns 'not found'.""" - fake_connect = AsyncMock(side_effect=Exception("sandbox not found (404)")) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(connect=fake_connect), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - with pytest.raises(SandboxExpiredError) as exc_info: - await mgr.reconnect("sbx-expired-001") - - assert exc_info.value.sandbox_id == "sbx-expired-001" - assert "sbx-expired-001" in str(exc_info.value) - - -@pytest.mark.asyncio -async def test_reconnect_reraises_non_expiry_errors(monkeypatch: pytest.MonkeyPatch) -> None: - """reconnect() re-raises unexpected E2B errors unchanged.""" - fake_connect = AsyncMock(side_effect=ConnectionError("network blip")) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.AsyncSandbox", - MagicMock(connect=fake_connect), - ) - monkeypatch.setattr( - "ergon_core.core.sandbox.manager.settings.e2b_api_key", - "test-key", - ) - - mgr = _MinimalManager() - with pytest.raises(ConnectionError, match="network blip"): - await mgr.reconnect("sbx-network-error") -``` - -### E2e canary — `tests/e2e/test_sandbox_criterion_timeout_canary.py` - -This test confirms the invariant end-to-end. It requires a live E2B key and is -gated behind the `e2e` marker. - -```python -# tests/e2e/test_sandbox_criterion_timeout_canary.py - -"""Canary: a sandbox provisioned with task+criterion timeout survives a slow criterion. - -If this test starts failing, we have regressed on the invariant that -sandbox_timeout >= task_timeout + max_criterion_timeout. - -Requires E2B_API_KEY. Only runs in CI on feature/* branches (e2e suite). -""" - -import asyncio -import pytest -from uuid import uuid4 - -from ergon_core.core.sandbox.manager import DefaultSandboxManager, BaseSandboxManager - - -@pytest.fixture(autouse=True) -def _reset_singleton(): - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - BaseSandboxManager._creation_locks = {} - BaseSandboxManager._run_ids = {} - BaseSandboxManager._display_task_ids = {} - BaseSandboxManager._file_registries = {} - BaseSandboxManager._created_files_registry = {} - yield - BaseSandboxManager._instance = None - BaseSandboxManager._sandboxes = {} - - -@pytest.mark.e2e -@pytest.mark.asyncio -async def test_sandbox_survives_slow_criterion() -> None: - """Provision sandbox with task_timeout=1m, max_criterion_timeout=2m. - Sleep for 90s (longer than task_timeout alone), then reconnect. - Confirm sandbox is still reachable. - """ - mgr = DefaultSandboxManager() - task_id = uuid4() - run_id = uuid4() - - sandbox_id = await mgr.create( - sandbox_key=task_id, - run_id=run_id, - task_timeout_minutes=1, - max_criterion_timeout_minutes=2, - ) - - # Simulate task completion + slow criterion: wait 90s (> 1-minute task timeout) - await asyncio.sleep(90) - - # Criterion reconnects via manager.reconnect — sandbox must still be live. - sandbox = await mgr.reconnect(sandbox_id) - result = await sandbox.commands.run("echo 'sandbox alive'", timeout=10) - assert result.exit_code == 0, "Sandbox must be reachable after criterion headroom" - assert "sandbox alive" in (result.stdout or "") - - await BaseSandboxManager.terminate_by_sandbox_id(sandbox_id) -``` - ---- - -## Trace / observability impact - -### Updated WAL entry log - -`BaseSandboxManager.create()` currently emits a `sandbox.created` WAL entry -with `stdout=f"sandbox_id=...\ntimeout={timeout_minutes}m"`. After this RFC, -it emits: - -``` -sandbox_id=sbx-abc123 -task_timeout=30m -max_criterion_timeout=10m -total_timeout=40m -``` - -This is a log content change only; no schema migration or DB change. Operators -inspecting sandbox WAL events in the dashboard will now see the split values. - -### `sandbox_created` event sink - -`SandboxEventSink.sandbox_created` receives `timeout_minutes: int` -(`event_sink.py:15`). After this RFC, the value passed is -`total_timeout_minutes` (= `task_timeout + max_criterion_timeout`). The -contract is unchanged (the field is "how many minutes this sandbox was -provisioned for"); the value now correctly reflects actual provisioned lifetime -rather than the mistaken task-only value. - -### Span attributes - -`sandbox_setup_fn` emits a `sandbox.setup` span with `timeout_minutes` absent -today (it emits `sandbox_id`, `benchmark_type`, `input_resource_count`). No -new span attributes are required. If observability of the split is needed, add -`task_timeout_minutes` and `max_criterion_timeout_minutes` to span attributes -in a follow-up. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|------|--------|------------| -| `AsyncSandbox.connect()` API not available in installed `e2b_code_interpreter` version | `reconnect()` raises `AttributeError` at runtime | Check against current E2B SDK before merge; the method exists in `e2b-code-interpreter>=0.7`. Pin if needed. | -| Callers passing positional `timeout_minutes` break at rename | `TypeError` at call site | Grep confirms no positional callers; all call sites use `timeout_minutes=N` kwargs. Migration is mechanical. | -| `max_criterion_timeout_minutes=10` default is insufficient for long agentic criteria | Sandbox expires; criterion returns `"sandbox-expired"` | Default is conservative; benchmarks with longer criteria override at their `create()` call site. Calibrate via p99 criterion wall-clock. | -| `reconnect()` does not update `_sandboxes` class dict | `get_sandbox(task_id)` returns `None` after reconnect | Documented: `reconnect()` is for cross-process use; in-process criteria continue to use `get_sandbox(task_id)`. Cross-process callers hold the returned handle directly. | -| Test call sites forget to update `timeout_minutes=5` → `task_timeout_minutes=5` | Tests break with `TypeError` | Grep for `timeout_minutes=` before merge; all affected files listed in the file map. | -| `SandboxExpiredError` swallowed by `CriterionRuntime.ensure_sandbox` before RFC `2026-04-17-criterion-runtime-di-container` lands | Criterion gets `None` sandbox and raises a generic `RuntimeError` | Acceptable interim state; `reconnect` is not wired into `ensure_sandbox` until the DI container RFC. Until then, `ensure_sandbox` provisions a fresh sandbox on `None`, which is the current behavior. | -| `DefaultSandboxManager` stub mode (`SANDBOX_SKIPPED`) short-circuits before reaching the new timeout logic | No effect; stub mode returns early before any E2B call | No change to behavior; `SANDBOX_SKIPPED` path is unaffected. | - ---- - -## Invariants affected - -### `docs/architecture/03_providers.md#invariants` - -**Invariant 5** currently reads: - -> **Sandbox lives across evaluator fan-out.** Teardown runs at the end of -> `check_evaluators`, not at task completion, not in `finalize_success`. -> Enforced by the evaluator harness, not by the manager itself. - -Tighten to: - -> **Sandbox lives across evaluator fan-out.** Teardown runs at the end of -> `check_evaluators`, not at task completion, not in `finalize_success`. -> Enforced by the evaluator harness. **The sandbox timeout on create MUST be at -> least `task_timeout + max_criterion_timeout`; `BaseSandboxManager.create()` -> enforces this by accepting both parameters and passing their sum to E2B.** - -**Known limits section** — remove: - -> **No `reconnect()` method.** Cross-process criteria must spawn their own -> sandbox (see `ergon_builtins/benchmarks/swebench_verified/criterion.py:66`). - -Replace with: - -> **`reconnect()` does not update class-level `_sandboxes` state.** It is -> intended for cross-process criterion use. In-process criteria should use -> `get_sandbox(task_id)` which reads shared class state. The reconnect path -> for cross-process criteria is gated on RFC -> `2026-04-17-criterion-runtime-di-container` landing. - -### `docs/architecture/cross_cutting/sandbox_lifecycle.md` - -**Invariant 2** currently reads: - -> **Sandbox timeout on creation MUST be at least `task_timeout + max_criterion_timeout`.** -> ... Pending enforcement in RFC 2026-04-17-sandbox-lifetime-covers-criteria. -> Today this is a convention. - -Update to: - -> **Sandbox timeout on creation MUST be at least `task_timeout + -> max_criterion_timeout`.** Enforced by `BaseSandboxManager.create()` which -> accepts `task_timeout_minutes` and `max_criterion_timeout_minutes` -> separately and passes their sum to E2B. The invariant is machine-checked -> by `tests/unit/test_sandbox_timeout.py`. - -**Invariant 3** currently reads: - -> **Criteria MUST reconnect via the manager, never by constructing `AsyncSandbox` -> directly.** ... The reconnect path through the manager is the only correct way -> to attach to a live sandbox. - -Update anti-pattern list (Section 8) to add: - -> **Calling `BaseSandboxManager.reconnect(sandbox_id)` from in-process -> criteria.** In-process criteria should use `get_sandbox(task_id)`; `reconnect` -> is for cross-process use and does not register the sandbox in class-level state. - ---- - -## Alternatives considered - -- **Leave timeout at `task_timeout`; have criteria pause and re-create the - sandbox if expired.** Rejected: complicated, loses in-sandbox state (file - contents, mounted volumes, process state), forces criteria to handle a new - failure mode. -- **Use a separate "evaluation sandbox" spawned by criteria.** Rejected: system - owner explicitly does not want criteria spawning sandboxes (see - `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`). Also - doubles sandbox cost per task. -- **Globally configure a generous timeout (e.g. `task_timeout * 3`).** Rejected: - opaque; per-benchmark overrides are easier to tune. Multiplicative defaults - also scale poorly for very short or very long tasks. -- **Keep a single `timeout` arg but document that callers should pre-add - criterion headroom.** Rejected: the policy leaks out of the manager; every - caller implements it slightly differently. -- **Add `reconnect` return value to class-level `_sandboxes` state.** Rejected: - would require passing `task_id` to `reconnect`, which is not always known - cross-process. The design explicitly keeps cross-process use (reconnect) and - in-process use (get_sandbox) separate. - ---- - -## Open questions - -- What is the right default for `max_criterion_timeout`? 10 minutes is a guess; - needs empirical calibration against existing benchmarks. Proposed calibration: - log p99 criterion wall-clock across a week of runs, set default to p99 * 1.5, - round up to the nearest minute. -- Should `reconnect()` retry once on transient E2B platform errors (e.g. 502s), - or raise immediately? Recommend raise; callers can retry at their level. - Retries in the manager hide backpressure signals. -- Is there a use case where a benchmark needs `max_criterion_timeout_minutes=0` - (no criteria run in-sandbox)? If yes, `create()` should skip the addition and - pass `timeout=task_timeout`. Today all benchmarks run at least one in-sandbox - criterion, but this is worth supporting for future criterion-free benchmarks. - The current implementation already supports `max_criterion_timeout_minutes=0` - correctly (total = task timeout only). -- How does this interact with sandbox-pool reuse (if the codebase grows one)? - A pooled sandbox has its own lifetime independent of a specific task; out of - scope for this RFC, flagged for later. - ---- - -## On acceptance - -- Update `docs/architecture/03_providers.md#invariants` — tighten sandbox-lifetime - invariant 5 and remove the "no reconnect" known limit per the text above. -- Update `docs/architecture/cross_cutting/sandbox_lifecycle.md` invariants 2 and 3, - and the anti-patterns section, per the text above. -- Link the implementation plan at `docs/superpowers/plans/2026-04-17-sandbox-lifetime-covers-criteria.md`. -- Move this file to `docs/rfcs/accepted/`. diff --git a/docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md b/docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md deleted file mode 100644 index 67d3ca844..000000000 --- a/docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md +++ /dev/null @@ -1,1005 +0,0 @@ ---- -status: active -opened: 2026-04-17 -author: deepflow-research -architecture_refs: [docs/architecture/02_runtime_lifecycle.md#invariants, docs/architecture/cross_cutting/error_propagation.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Align static-sibling failure propagation with fractal-OS semantics (stay PENDING) - -## Problem - -`on_task_completed_or_failed` in -`ergon_core/ergon_core/core/runtime/execution/propagation.py:438-586` -treats static workflow siblings and managed subtasks differently on upstream -failure: - -- **Managed subtasks** (`parent_node_id is not None`, lines 505-513): the edge - is INVALIDATED but the target node is left PENDING. The manager observes the - failure and decides whether to retry, cancel, or re-plan. Matches fractal-OS - semantics. -- **Static workflow siblings** (`parent_node_id is None`, lines 515-527): the - target is written CANCELLED via `graph_repo.update_node_status` with - `only_if_not_terminal=True`, then appended to the `invalidated` return list. - Diverges from fractal-OS semantics. - -The diverging gate is at lines 505-513: - -```python -if not is_success: - if candidate_node.parent_node_id is not None: - continue # manager-owned: stay PENDING - - graph_repo.update_node_status( # static: auto-cancel - session, run_id=run_id, node_id=candidate_id, - new_status=CANCELLED, - meta=MutationMeta(actor="system:propagation", - reason=f"dependency {node_id} {terminal_status}"), - only_if_not_terminal=True, - ) - invalidated.append(candidate_id) - continue -``` - -The `propagate_task_failure_fn` Inngest function in -`ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py:137-200` -calls `TaskPropagationService.propagate_failure`, which calls -`on_task_completed_or_failed` with `terminal_status=FAILED` and then emits a -`TaskCancelledEvent` (cause `"dep_invalidated"`) for every node in the -`invalidated_targets` list. This causes `cancel_orphan_subtasks_fn` to fire, -which calls `SubtaskCancellationService.cancel_orphans` on the now-CANCELLED -static node — cancelling any dynamic children that static node happened to -have. The chain is load-bearing for the current semantics. - -Secondary consequence: `is_workflow_complete_v2` at lines 594-601: - -```python -def is_workflow_complete_v2(session: Session, run_id: UUID) -> bool: - statuses = list( - session.exec(select(RunGraphNode.status).where(RunGraphNode.run_id == run_id)).all() - ) - if not statuses: - return True - return all(s in TERMINAL_STATUSES for s in statuses) and not any(s == FAILED for s in statuses) -``` - -This returns `True` only when **every node** is in `{COMPLETED, FAILED, -CANCELLED}` and no node is FAILED. Under the new semantics, a failed node -leaves its static downstream siblings PENDING. `is_workflow_complete_v2` then -never returns `True` for those runs. The workflow hangs: `task-propagate` will -not emit `workflow/completed` or `workflow/failed` because neither -`is_workflow_complete_v2` nor `is_workflow_failed_v2` (line 604) becomes True -while unblocked-PENDING nodes exist. - -`is_workflow_failed_v2` (lines 604-609) is fine as-is — it returns `True` -whenever any node is FAILED. After the semantic flip, a FAILED node still -triggers `is_workflow_failed_v2=True`, so `propagate_task_failure_fn` will -detect `WorkflowTerminalState.FAILED` correctly once `is_workflow_complete_v2` -no longer fires first on PENDING-blocked runs. - -Existing test `tests/state/test_dep_failure_cascade.py:43-86` -(`TestStaticNodeAutoCancel`) encodes the current auto-cancel semantic and will -fail after the change. `tests/state/test_dep_failure_cascade.py:219-280` -(`TestDynamicSubtaskNoAutoCancel::test_mixed_static_and_dynamic_targets`) -also asserts `c_row.status == "cancelled"` for the static node; this must be -inverted. - -System owner steer (2026-04-17): the intended model is uniform — "if B depends -on A and A fails, B stays PENDING forever unless an adaptive planner unblocks -it." Auto-cancelling static siblings removes optionality and bakes in an -assumption that no planner will ever exist above the static DAG. - ---- - -## Proposal - -Four coordinated changes, across two PRs: - -1. **Rewrite `is_workflow_complete_v2`** with a blocked-chain rule: the - workflow is complete when every non-terminal node is blocked by at least one - INVALIDATED incoming edge and none of its incoming edges are in a - satisfiable (PENDING) state. A node that is PENDING with all PENDING - incoming edges is not blocked; it is waiting. The new rule must not confuse - the two. - -2. **Remove the static auto-cancel branch** in `on_task_completed_or_failed`: - delete the `graph_repo.update_node_status(CANCELLED)` write and the - `invalidated.append` for static nodes. Both static and managed nodes stay - PENDING; both edges become INVALIDATED uniformly. The `invalidated` return - list becomes empty on the failure path for all node types. - -3. **Stop emitting `TaskCancelledEvent` for invalidated static targets** in - `propagate_task_failure_fn`: after step 2, `propagation.invalidated_targets` - will be empty for static nodes, so the event fan-out loop already does the - right thing. No code change needed in `propagate_execution.py` beyond - removing an assertion or comment. - -4. **Rewrite the affected tests** in `tests/state/test_dep_failure_cascade.py` - and `tests/state/test_propagation_graph_native.py` against the new semantic; - add new tests for the `is_workflow_complete_v2` blocked-chain rule. - -PR order: - -- **PR 1** — Rewrite `is_workflow_complete_v2`, add tests for the new rule - (run them against the old propagation semantic where static nodes are still - auto-cancelled — the new completion check should still work because - CANCELLED ∈ TERMINAL_STATUSES and blocked-PENDING does not yet exist). -- **PR 2** — Remove the auto-cancel branch from `on_task_completed_or_failed`; - rewrite `test_dep_failure_cascade.py`; rewrite any other tests that assert - the old auto-cancel outcome. - -No data migration. This is a forward-only behavior change. In-flight runs at -deploy time continue under whichever state their nodes have already reached. -Existing CANCELLED nodes are not touched. - ---- - -## Architecture overview - -### Before (current) - -``` -task/failed fires propagate_task_failure_fn - │ - ├── TaskPropagationService.propagate_failure - │ ├── update_node_status(FAILED) ← terminal node - │ └── on_task_completed_or_failed(FAILED) - │ ├── edges → INVALIDATED - │ ├── managed targets → stay PENDING (manager adapts) - │ └── static targets → CANCELLED ← divergence - │ - ├── emit TaskCancelledEvent(cause="dep_invalidated") - │ for each invalidated_target (static nodes only) - │ └── cancel_orphan_subtasks_fn - │ └── SubtaskCancellationService.cancel_orphans - │ (BFS children of the now-CANCELLED static node) - │ - └── is_workflow_failed_v2 → True - emit workflow/failed -``` - -### After (proposed) - -``` -task/failed fires propagate_task_failure_fn - │ - ├── TaskPropagationService.propagate_failure - │ ├── update_node_status(FAILED) ← terminal node - │ └── on_task_completed_or_failed(FAILED) - │ ├── edges → INVALIDATED - │ ├── managed targets → stay PENDING (manager adapts) - │ └── static targets → stay PENDING ← uniform - │ - ├── invalidated_targets == [] - │ no TaskCancelledEvent emitted for dep-blocked siblings - │ - └── is_workflow_failed_v2 → True (FAILED node present) - is_workflow_complete_v2 (new rule): - any non-terminal node blocked by ≥1 INVALIDATED edge - and zero PENDING incoming edges → counts as blocked - when ALL non-terminal nodes are blocked → True - emit workflow/failed -``` - -### `is_workflow_complete_v2` rule change - -| Condition | Old result | New result | -|---|---|---| -| All nodes COMPLETED | True | True | -| All nodes terminal (COMPLETED + CANCELLED), no FAILED | True | True | -| Any node FAILED | False (is_workflow_failed_v2 handles) | False (unchanged) | -| FAILED node + static PENDING siblings (new case) | hang (never True) | True — siblings are blocked-by-failed | -| PENDING node with PENDING incoming edges | False | False (not blocked, still waiting) | - ---- - -## Type / interface definitions - -No new public types. Two functions change signatures (same signature, different -body): - -```python -# ergon_core/ergon_core/core/runtime/execution/propagation.py - -def is_workflow_complete_v2(session: Session, run_id: UUID) -> bool: - """Workflow is complete when no non-terminal node can make progress. - - A node can make progress only if at least one of its incoming edges is - PENDING (i.e. the upstream might yet succeed). A node that has only - INVALIDATED incoming edges is blocked and will never advance on its own. - - Rules: - - Zero nodes → True (empty run completes immediately). - - Any FAILED node → False (is_workflow_failed_v2 handles termination). - - Any non-terminal node with ≥1 PENDING incoming edge → False. - - Any non-terminal node with 0 PENDING incoming edges but ≥1 INVALIDATED - incoming edge is "blocked-by-failed" and does NOT block completion. - - All nodes terminal or blocked-by-failed → True. - """ -``` - -```python -def on_task_completed_or_failed( - session: Session, - run_id: UUID, - node_id: UUID, - terminal_status: str, - *, - graph_repo: WorkflowGraphRepository, -) -> tuple[list[UUID], list[UUID]]: - """Handle a node reaching COMPLETED, FAILED, or CANCELLED. - - Returns (newly_ready_node_ids, invalidated_target_node_ids). - - - COMPLETED: outgoing edges → SATISFIED; targets with all deps satisfied - → PENDING (ready). - - FAILED / CANCELLED: outgoing edges → INVALIDATED. Targets stay PENDING - regardless of parent_node_id. invalidated_target_node_ids is always [] - on the failure path (edges are invalidated; nodes are not touched). - - The asymmetry between COMPLETED re-activation (managed subtasks only) and - failure propagation (uniform PENDING) is intentional and documented in - docs/architecture/cross_cutting/error_propagation.md. - """ -``` - ---- - -## Full implementations - -### Change 1 — `is_workflow_complete_v2` (new body) - -```python -# ergon_core/ergon_core/core/runtime/execution/propagation.py - -def is_workflow_complete_v2(session: Session, run_id: UUID) -> bool: - """Workflow is complete when no non-terminal node can make progress. - - See RFC docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md - for the full rule description and rationale. - """ - nodes = list( - session.exec(select(RunGraphNode).where(RunGraphNode.run_id == run_id)).all() - ) - if not nodes: - return True - - node_by_id = {n.id: n for n in nodes} - - # Any FAILED node means this is a failed workflow, not a completed one. - if any(n.status == FAILED for n in nodes): - return False - - non_terminal = [n for n in nodes if n.status not in TERMINAL_STATUSES] - if not non_terminal: - # All nodes are terminal and none FAILED → completed. - return True - - # For each non-terminal node, check whether it has a PENDING incoming edge. - # If it does, it can still make progress → workflow is not complete. - # If all its incoming edges are INVALIDATED → it is blocked-by-failed and - # does not prevent completion. - all_edges = list( - session.exec(select(RunGraphEdge).where(RunGraphEdge.run_id == run_id)).all() - ) - incoming_by_node: dict[UUID, list[RunGraphEdge]] = {} - for edge in all_edges: - incoming_by_node.setdefault(edge.target_node_id, []).append(edge) - - for node in non_terminal: - incoming = incoming_by_node.get(node.id, []) - if not incoming: - # Root node that is non-terminal and has no edges: - # it is legitimately waiting (e.g. initial PENDING before first - # task/ready is dispatched) or running. Not blocked. - return False - has_pending_edge = any(e.status == EDGE_PENDING for e in incoming) - if has_pending_edge: - # At least one upstream is still reachable → not blocked. - return False - # All incoming edges are INVALIDATED (or SATISFIED, which would - # only occur if a source completed but the node hasn't been marked - # ready yet — transient, not a hang). Treat as blocked. - - # Every non-terminal node is blocked by invalidated deps. - return True -``` - -### Change 2 — `on_task_completed_or_failed` failure branch (removal) - -Exact unified diff against `propagation.py:505-527`: - -```diff -- if not is_success: -- # Dynamic subtasks (parent_node_id set) are manager-owned. -- # The manager polls via list_subtasks and decides whether to -- # retry, cancel, or re-plan — so we leave the target PENDING -- # and only invalidate the edge. Static workflow nodes -- # (parent_node_id is None) have no adaptive supervisor, so -- # auto-cancel is the correct behaviour. -- if candidate_node.parent_node_id is not None: -- continue -- -- graph_repo.update_node_status( -- session, -- run_id=run_id, -- node_id=candidate_id, -- new_status=CANCELLED, -- meta=MutationMeta( -- actor="system:propagation", -- reason=f"dependency {node_id} {terminal_status}", -- ), -- only_if_not_terminal=True, -- ) -- invalidated.append(candidate_id) -- continue -+ if not is_success: -+ # Both managed subtasks (parent_node_id set) and static workflow -+ # nodes (parent_node_id is None) stay PENDING. Failure is a -+ # signal for an adaptive planner, not a reason to auto-cancel. -+ # The edge is already INVALIDATED above. is_workflow_complete_v2 -+ # uses the blocked-by-failed rule to detect when the run is done. -+ # See RFC docs/rfcs/active/2026-04-17-static-sibling-failure-semantics.md -+ continue -``` - -After this change the `invalidated` list is always `[]` on the failure path. -The `invalidated_targets` returned by `on_task_completed_or_failed` for a -FAILED source is an empty list. `propagate_task_failure_fn` already iterates -`propagation.invalidated_targets` to emit `TaskCancelledEvent`; an empty list -means no events are emitted, which is correct. - -### Change 2 full replacement — `on_task_completed_or_failed` failure block - -Full function body after edit (lines 438-586 become): - -```python -def on_task_completed_or_failed( - session: Session, - run_id: UUID, - node_id: UUID, - terminal_status: str, - *, - graph_repo: WorkflowGraphRepository, -) -> tuple[list[UUID], list[UUID]]: - """Handle a node reaching COMPLETED, FAILED, or CANCELLED. - - Returns (newly_ready_node_ids, invalidated_target_node_ids). - - - COMPLETED: outgoing edges → SATISFIED; targets with all deps satisfied - → PENDING (ready). - - FAILED / CANCELLED: outgoing edges → INVALIDATED. All downstream - candidates stay PENDING regardless of parent_node_id. - invalidated_target_node_ids is always [] on the failure path. - - Precondition: node_id is already in terminal_status before calling. - The node's own status is NOT written here — only edge statuses and - downstream candidate statuses are updated. - """ - is_success = terminal_status == TaskExecutionStatus.COMPLETED - - outgoing = list( - session.exec( - select(RunGraphEdge).where( - RunGraphEdge.run_id == run_id, - RunGraphEdge.source_node_id == node_id, - ) - ).all() - ) - - edge_status = EDGE_SATISFIED if is_success else EDGE_INVALIDATED - for edge in outgoing: - graph_repo.update_edge_status( - session, - run_id=run_id, - edge_id=edge.id, - new_status=edge_status, - meta=_PROPAGATION_META, - ) - - candidate_node_ids = {e.target_node_id for e in outgoing} - - newly_ready: list[UUID] = [] - invalidated: list[UUID] = [] - - for candidate_id in candidate_node_ids: - candidate_node = session.get(RunGraphNode, candidate_id) - if candidate_node is None: - continue - if candidate_node.status in TERMINAL_STATUSES and candidate_node.status != CANCELLED: - continue - if candidate_node.status == CANCELLED and not is_success: - continue - - if not is_success: - # Both managed subtasks and static workflow nodes stay PENDING. - # Failure is a signal for an adaptive planner; is_workflow_complete_v2 - # uses the blocked-by-failed rule to detect run termination. - continue - - # Source completed — check if this candidate can become READY. - # - # Eligibility: - # - PENDING (first activation): normal case. - # - CANCELLED managed subtask (parent_node_id is not None): - # re-activation after the manager or an upstream restart - # invalidated it. - # - CANCELLED static workflow node (parent_node_id is None): - # NOT re-activated — no supervisor to adapt. - # - # Everything else (COMPLETED, FAILED, RUNNING) is skipped. - status = candidate_node.status - is_managed_subtask = candidate_node.parent_node_id is not None - is_pending = status == TaskExecutionStatus.PENDING - is_reactivatable_cancelled = status == CANCELLED and is_managed_subtask - - if not (is_pending or is_reactivatable_cancelled): - continue - - incoming = list( - session.exec( - select(RunGraphEdge).where( - RunGraphEdge.run_id == run_id, - RunGraphEdge.target_node_id == candidate_id, - ) - ).all() - ) - - source_nodes = [session.get(RunGraphNode, e.source_node_id) for e in incoming] - if all(n is not None and n.status == TaskExecutionStatus.COMPLETED for n in source_nodes): - reason = ( - f"all dependencies satisfied after {node_id}" - if is_pending - else f"re-activating cancelled subtask after {node_id}" - ) - graph_repo.update_node_status( - session, - run_id=run_id, - node_id=candidate_id, - new_status=TaskExecutionStatus.PENDING, - meta=MutationMeta( - actor="system:propagation", - reason=reason, - ), - only_if_not_terminal=False, - ) - newly_ready.append(candidate_id) - - session.commit() - return newly_ready, invalidated -``` - ---- - -## Exact diffs for modified files - -### `ergon_core/ergon_core/core/runtime/execution/propagation.py` - -**Hunk 1** — Replace `is_workflow_complete_v2` body (lines 594-601): - -```diff --def is_workflow_complete_v2(session: Session, run_id: UUID) -> bool: -- """Every node terminal; zero FAILED. CANCELLED is neutral.""" -- statuses = list( -- session.exec(select(RunGraphNode.status).where(RunGraphNode.run_id == run_id)).all() -- ) -- if not statuses: -- return True -- return all(s in TERMINAL_STATUSES for s in statuses) and not any(s == FAILED for s in statuses) -+def is_workflow_complete_v2(session: Session, run_id: UUID) -> bool: -+ """Workflow is complete when no non-terminal node can make progress. -+ -+ A node is blocked-by-failed when all its incoming edges are INVALIDATED. -+ Blocked-by-failed nodes do not prevent completion. -+ Any FAILED node → False (is_workflow_failed_v2 handles termination). -+ """ -+ nodes = list( -+ session.exec(select(RunGraphNode).where(RunGraphNode.run_id == run_id)).all() -+ ) -+ if not nodes: -+ return True -+ if any(n.status == FAILED for n in nodes): -+ return False -+ non_terminal = [n for n in nodes if n.status not in TERMINAL_STATUSES] -+ if not non_terminal: -+ return True -+ all_edges = list( -+ session.exec(select(RunGraphEdge).where(RunGraphEdge.run_id == run_id)).all() -+ ) -+ incoming_by_node: dict[UUID, list[RunGraphEdge]] = {} -+ for edge in all_edges: -+ incoming_by_node.setdefault(edge.target_node_id, []).append(edge) -+ for node in non_terminal: -+ incoming = incoming_by_node.get(node.id, []) -+ if not incoming: -+ return False -+ if any(e.status == EDGE_PENDING for e in incoming): -+ return False -+ return True -``` - -**Hunk 2** — Remove static auto-cancel branch (lines 505-527): - -```diff - if not is_success: -- # Dynamic subtasks (parent_node_id set) are manager-owned. -- # The manager polls via list_subtasks and decides whether to -- # retry, cancel, or re-plan — so we leave the target PENDING -- # and only invalidate the edge. Static workflow nodes -- # (parent_node_id is None) have no adaptive supervisor, so -- # auto-cancel is the correct behaviour. -- if candidate_node.parent_node_id is not None: -- continue -- -- graph_repo.update_node_status( -- session, -- run_id=run_id, -- node_id=candidate_id, -- new_status=CANCELLED, -- meta=MutationMeta( -- actor="system:propagation", -- reason=f"dependency {node_id} {terminal_status}", -- ), -- only_if_not_terminal=True, -- ) -- invalidated.append(candidate_id) -- continue -+ # Both managed subtasks and static workflow nodes stay PENDING. -+ # is_workflow_complete_v2 uses the blocked-by-failed edge rule -+ # to detect run termination without requiring auto-cancel. -+ continue -``` - -No other files in the production path require changes for this RFC. The -`propagate_task_failure_fn` (`propagate_execution.py:137-200`) already iterates -`propagation.invalidated_targets`; an empty list means no `TaskCancelledEvent` -is emitted, which is the correct new behavior. - ---- - -## Package structure - -No new packages. Changes are confined to one source file and three test files. - ---- - -## Implementation order - -| Step | PR | What | Files touched | -|---|---|---|---| -| 1 | PR 1 | Add `is_blocked_by_failed` helper predicate (private, used by `is_workflow_complete_v2`) | `propagation.py` | -| 2 | PR 1 | Rewrite `is_workflow_complete_v2` with the blocked-chain rule | `propagation.py` | -| 3 | PR 1 | Add `TestBlockedChainCompletion` class to `test_propagation_graph_native.py` | `tests/state/test_propagation_graph_native.py` | -| 4 | PR 1 | Verify existing `TestIsWorkflowCompleteV2` tests still pass (CANCELLED-as-terminal and RUNNING cases unchanged) | no changes | -| 5 | PR 2 | Remove static auto-cancel branch from `on_task_completed_or_failed` | `propagation.py` | -| 6 | PR 2 | Rewrite `TestStaticNodeAutoCancel` → `TestStaticNodeStaysPending` in `test_dep_failure_cascade.py` | `tests/state/test_dep_failure_cascade.py` | -| 7 | PR 2 | Invert `test_mixed_static_and_dynamic_targets` assertion for static node (PENDING not CANCELLED) | `tests/state/test_dep_failure_cascade.py` | -| 8 | PR 2 | Add `TestWorkflowCompleteOnBlockedChain` integration test: FAILED node + PENDING-blocked siblings → workflow terminates via `propagate_failure` path | `tests/state/test_dep_failure_cascade.py` | -| 9 | PR 2 | Audit remaining tests for implicit dependency on auto-cancel (see Testing section) | `tests/state/` | - -Steps 1–4 land as **PR 1** (safe: new completion logic is tested before the -semantic flip). Steps 5–9 land as **PR 2** (semantic flip, test rewrites). - ---- - -## File map - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/runtime/execution/propagation.py` | (1) Replace `is_workflow_complete_v2` body with blocked-chain rule. (2) Remove static auto-cancel branch from `on_task_completed_or_failed` (lines 505-527). | -| `tests/state/test_dep_failure_cascade.py` | Rewrite `TestStaticNodeAutoCancel` → `TestStaticNodeStaysPending`; invert `test_mixed_static_and_dynamic_targets` assertion for static node; add `TestWorkflowCompleteOnBlockedChain`. | -| `tests/state/test_propagation_graph_native.py` | Add `TestBlockedChainCompletion` class for new `is_workflow_complete_v2` rule. | - -### ADD - -None. All changes are in-place rewrites. - ---- - -## Testing approach - -### Unit — `TestBlockedChainCompletion` (PR 1, `test_propagation_graph_native.py`) - -```python -class TestBlockedChainCompletion: - """is_workflow_complete_v2 with the blocked-by-failed rule.""" - - def test_pending_node_with_invalidated_edges_does_not_block(self, session: Session): - """A fails. B is PENDING with only an INVALIDATED A->B edge. - Workflow should be complete (blocked-by-failed).""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED) - b = _add_node(repo, session, run_id, "B", status=TaskExecutionStatus.PENDING) - repo.add_edge( - session, run_id, - source_node_id=a.id, target_node_id=b.id, - status="invalidated", meta=META, - ) - session.flush() - - assert is_workflow_complete_v2(session, run_id) is False - # is_workflow_failed_v2 returns True; completion check must be False - # so the caller emits workflow/failed rather than workflow/completed. - - def test_failed_node_alone_blocks_complete(self, session: Session): - """A FAILED node alone → is_workflow_complete_v2 is False.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED) - session.flush() - - assert is_workflow_complete_v2(session, run_id) is False - - def test_all_completed_no_failed_returns_true(self, session: Session): - """Original happy path unchanged.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.COMPLETED) - _add_node(repo, session, run_id, "B", status=TaskExecutionStatus.COMPLETED) - session.flush() - - assert is_workflow_complete_v2(session, run_id) is True - - def test_pending_node_with_pending_edge_not_complete(self, session: Session): - """B has a PENDING incoming edge (upstream still running) → not complete.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.RUNNING) - b = _add_node(repo, session, run_id, "B", status=TaskExecutionStatus.PENDING) - repo.add_edge( - session, run_id, - source_node_id=a.id, target_node_id=b.id, - status="pending", meta=META, - ) - session.flush() - - assert is_workflow_complete_v2(session, run_id) is False - - def test_chain_failure_all_blocked_pending(self, session: Session): - """A fails. B PENDING (A->B invalidated). C PENDING (B->C pending). - C still has a PENDING edge so run is not complete.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED) - b = _add_node(repo, session, run_id, "B", status=TaskExecutionStatus.PENDING) - c = _add_node(repo, session, run_id, "C", status=TaskExecutionStatus.PENDING) - repo.add_edge( - session, run_id, - source_node_id=a.id, target_node_id=b.id, - status="invalidated", meta=META, - ) - repo.add_edge( - session, run_id, - source_node_id=b.id, target_node_id=c.id, - status="pending", meta=META, - ) - session.flush() - - # C has a PENDING edge from B → not fully blocked → not complete - assert is_workflow_complete_v2(session, run_id) is False -``` - -**Note on the first test:** `is_workflow_complete_v2` returns `False` because -`FAILED ∈ nodes`. The caller (`TaskPropagationService.propagate_failure`) checks -`is_workflow_failed_v2` first (line 143); that returns `True`, and -`WorkflowTerminalState.FAILED` is set. `is_workflow_complete_v2` is not the -primary finalization path on failure. The test documents the invariant that the -two checks do not conflict. - -### Unit — `TestStaticNodeStaysPending` (PR 2, rewrite of `TestStaticNodeAutoCancel`) - -```python -class TestStaticNodeStaysPending: - """Static workflow nodes stay PENDING when a dependency fails.""" - - def test_failure_leaves_static_downstream_pending(self, session: Session): - """A -> B, A -> C (all static). A fails. B and C stay PENDING.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED) - b = _add_node(repo, session, run_id, "B") - c = _add_node(repo, session, run_id, "C") - - repo.add_edge(session, run_id, source_node_id=a.id, target_node_id=b.id, - status="pending", meta=META) - repo.add_edge(session, run_id, source_node_id=a.id, target_node_id=c.id, - status="pending", meta=META) - session.flush() - - _ready, invalidated = on_task_completed_or_failed( - session, run_id, a.id, TaskExecutionStatus.FAILED, graph_repo=repo, - ) - - assert invalidated == [] # no auto-cancel - - b_row = session.get(RunGraphNode, b.id) - c_row = session.get(RunGraphNode, c.id) - assert b_row is not None and b_row.status == TaskExecutionStatus.PENDING - assert c_row is not None and c_row.status == TaskExecutionStatus.PENDING - - def test_edges_are_still_invalidated(self, session: Session): - """Edge state is INVALIDATED even though the target node stays PENDING.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED) - b = _add_node(repo, session, run_id, "B") - edge = repo.add_edge(session, run_id, source_node_id=a.id, target_node_id=b.id, - status="pending", meta=META) - session.flush() - - on_task_completed_or_failed( - session, run_id, a.id, TaskExecutionStatus.FAILED, graph_repo=repo, - ) - - from ergon_core.core.persistence.graph.models import RunGraphEdge - edge_row = session.get(RunGraphEdge, edge.id) - assert edge_row is not None - assert edge_row.status == "invalidated" - - def test_managed_subtask_regression(self, session: Session): - """Managed subtask behavior is unchanged: stays PENDING, not in invalidated.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - manager = _add_node(repo, session, run_id, "manager", - status=TaskExecutionStatus.RUNNING) - a = _add_node(repo, session, run_id, "A", status=TaskExecutionStatus.FAILED, - parent_node_id=manager.id, level=1) - b = _add_node(repo, session, run_id, "B", parent_node_id=manager.id, level=1) - repo.add_edge(session, run_id, source_node_id=a.id, target_node_id=b.id, - status="pending", meta=META) - session.flush() - - _ready, invalidated = on_task_completed_or_failed( - session, run_id, a.id, TaskExecutionStatus.FAILED, graph_repo=repo, - ) - - assert b.id not in invalidated - b_row = session.get(RunGraphNode, b.id) - assert b_row is not None and b_row.status == TaskExecutionStatus.PENDING - - -class TestWorkflowCompleteOnBlockedChain: - """After the semantic flip, propagate_failure terminates runs via - is_workflow_failed_v2, not is_workflow_complete_v2.""" - - def test_propagate_failure_detects_failed_state(self, session: Session): - """TaskPropagationService.propagate_failure on A with static B downstream - returns WorkflowTerminalState.FAILED (not NONE).""" - from ergon_core.core.runtime.services.task_propagation_service import ( - TaskPropagationService, - ) - from ergon_core.core.runtime.services.orchestration_dto import ( - PropagateTaskCompletionCommand, - WorkflowTerminalState, - ) - - # Build graph via repo (no definition tables) - repo = WorkflowGraphRepository() - run_id = uuid4() - def_id = uuid4() - - a_node = repo.add_node(session, run_id, task_key="A", instance_key="i0", - description="A", status=TaskExecutionStatus.RUNNING, meta=META) - b_node = repo.add_node(session, run_id, task_key="B", instance_key="i0", - description="B", status=TaskExecutionStatus.PENDING, meta=META) - repo.add_edge(session, run_id, source_node_id=a_node.id, target_node_id=b_node.id, - status="pending", meta=META) - session.commit() - - svc = TaskPropagationService() - result = svc.propagate_failure( - PropagateTaskCompletionCommand( - run_id=run_id, - definition_id=def_id, - task_id=a_node.definition_task_id or uuid4(), - execution_id=uuid4(), - node_id=a_node.id, - ) - ) - - assert result.workflow_terminal_state == WorkflowTerminalState.FAILED - assert result.invalidated_targets == [] -``` - -### Regression guard - -All existing tests in `tests/state/test_propagation_reactivation.py` must pass -unchanged — they test the CANCELLED managed-subtask re-activation path, which -is untouched. `test_static_cancelled_does_not_reactivate` (line 176) remains -correct: a CANCELLED static node (reached by some prior path, not by this -RFC's change) stays CANCELLED on the success path. - ---- - -## Trace / observability impact - -### Spans - -`task.propagate` span in `propagate_task_failure_fn` (emitted from -`propagate_execution.py:119-133`) already carries: - -```python -"newly_ready_tasks": len(propagation.ready_tasks), # always 0 on failure path -"workflow_terminal": str(propagation.workflow_terminal_state), -``` - -After the change, `workflow_terminal` will be `"failed"` where it previously -may have been `"failed"` (unchanged for runs where the failed node itself was -the last to trigger finalization). No attribute schema change. - -### Logs - -`propagate_task_failure_fn` logs: -``` -task-failure-propagate run_id=... task_id=... error=... -``` -No change. - -`on_task_completed_or_failed` has no structured logging of its own; the -auto-cancel branch being removed produces no loss of log signal. - -### Dashboard events - -`propagate_task_failure_fn` currently emits `TaskCancelledEvent(cause="dep_invalidated")` -for each entry in `invalidated_targets`. Under the new semantics, that list is -empty for static nodes; no `TaskCancelledEvent` fires for dep-blocked siblings. -The dashboard will show those nodes as PENDING rather than CANCELLED. - -The dashboard badge for "blocked by failed dep" versus "cancelled" is a -follow-up (noted in Open Questions). This RFC does not add a new node status — -PENDING remains the node status. The edge INVALIDATED state is readable via the -mutations log if dashboard wants to distinguish. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| `is_workflow_complete_v2` edge-query cost | PR 1 adds a full `RunGraphEdge` scan per finalization check. Propagation runs once per task terminal; edge count ≤ task count. Acceptable at experiment scale. | Monitor `task.propagate` span duration. If P99 degrades, add `GROUP BY target_node_id` aggregate query. | -| Root PENDING node (no incoming edges) with failing chain elsewhere in DAG | `is_workflow_complete_v2` returns `False` (root has no edges → treated as not blocked). Correct: root is still runnable. | Covered by `test_pending_node_with_pending_edge_not_complete`. | -| In-flight runs at deploy: nodes already CANCELLED by old path | CANCELLED is absorbing. Existing CANCELLED nodes stay CANCELLED. New runs get the new semantic. No hybrid run possible mid-task. | Acceptable; document in migration note. | -| Tests that assert `TaskCancelledEvent` is emitted for dep-blocked static nodes | `test_propagation_graph_native.py` and `test_dep_failure_cascade.py` have explicit assertions on `invalidated` list contents | PR 2 rewrites those assertions. Grep: `dep_invalidated` in `tests/` before merging PR 2. | -| `cancel_orphans_on_failed_fn` fires but `SubtaskCancellationService.cancel_orphans` finds nothing to cancel | No-op; `cancel_orphans` uses BFS on `parent_node_id` — static nodes with `parent_node_id=None` are never BFS roots. | Verified: `cancel_orphan_subtasks.py:98-107` passes `parent_node_id=payload.node_id` to the service; static siblings are not children of the failing node. No behavior change needed. | -| `propagate_task_failure_fn` still emits `TaskCancelledEvent` via the `for inv_node_id in propagation.invalidated_targets` loop | After Change 2, `invalidated_targets == []` so the loop body never executes. | Zero code change needed in `propagate_execution.py`. Confirmed by inspection of lines 164-176. | -| Interaction with `SubtaskCancellationService` assumption | Today `cancel_orphans_on_failed_fn` fires because `TaskFailedEvent` triggers it. It walks `parent_node_id` children of the FAILED node, not the PENDING siblings. The siblings' subtrees are unaffected. | Confirmed: `cancel_orphan_subtasks.py:98-107` passes the FAILED node's `node_id` as `parent_node_id`. Static siblings (`parent_node_id=None`) are not children of the failed node. No change to `cancel_orphans_on_*_fn` needed. | - ---- - -## Invariants affected - -### `docs/architecture/02_runtime_lifecycle.md#invariants` - -**Section 4, Known limits, bullet 1 (line 116):** Remove the bullet entirely on -merge — it describes the divergence this RFC closes. - -**Section 4, Invariants:** Add new invariant after existing invariant 2: - -> **Failure propagation is uniform across static and managed nodes.** When a -> node reaches FAILED or CANCELLED, all outgoing edges become INVALIDATED and -> all downstream targets stay PENDING — regardless of `parent_node_id`. The -> only asymmetry is re-activation on the success path: managed subtasks -> (`parent_node_id is not None`) can re-activate to PENDING when all deps -> re-satisfy; static workflow nodes cannot. See -> `ergon_core/core/runtime/execution/propagation.py::on_task_completed_or_failed` -> for the enforcement point. - -**Section 5, Extension points:** Add a bullet: - -> **Blocking-chain completion seam.** `is_workflow_complete_v2` is the single -> check for "is the workflow done without a FAILED node". Future features -> (retry policies, planners) that want to unblock PENDING-blocked nodes do so -> by satisfying or replacing INVALIDATED edges — not by patching the completion -> check. The completion check must never be taught about domain-specific states; -> it must stay a pure graph query. - -### `docs/architecture/cross_cutting/error_propagation.md` - -**"Current behavior" section:** Move items 1-5 to "Before 2026-04-17". After -merge, "Current behavior" becomes: - -1. Task COMPLETED → dependents with all deps satisfied → PENDING. Correct. -2. Task FAILED or CANCELLED, any downstream target (managed or static) → stays - PENDING, edge → INVALIDATED. Uniform fractal-OS semantics. -3. Parent terminal → `cancel_orphans_on_*_fn` cancels the full subtask - subtree. Correct. -4. CANCELLED managed subtask with re-satisfied deps → re-activates to PENDING. - Static workflow CANCELLED nodes do NOT re-activate. - -**"Intended behavior" section:** Delete (it is now current behavior). - -**Control flow diagram:** Update the `static sibling target` line from -`AUTO-CANCELLED (CURRENT)` to `STAY PENDING (uniform)`. - -**Invariants section:** Add: -> - Failure propagation is uniform: all downstream candidates stay PENDING -> on a failed/cancelled source, regardless of `parent_node_id`. - -**Anti-patterns section:** Update: -> - **Assuming "failed → children cancel" today.** No longer true for ANY -> target — neither static nor managed. Check edge status (`INVALIDATED`) to -> determine whether a node's dependency chain has failed. - ---- - -## Alternatives considered - -- **Keep static auto-cancel; add a config flag per-benchmark.** Rejected: two - semantics in parallel makes every PR reviewer's life harder; the system owner - wants uniform. -- **Auto-cancel static siblings but also emit a signal to a future planner.** - Rejected: no planner exists today; speculative coupling. Adding a signal - channel we do not consume ages into dead code. -- **Leave `is_workflow_complete_v2` as-is and rely on run-level timeouts.** - Rejected: timeouts are a crutch; blocked-chain detection is a correctness - invariant. Timeouts also make it impossible to distinguish "hung" from - "legitimately slow." -- **Move the gate to a strategy object on the run.** Rejected: same problem as - the config flag, with extra indirection. -- **Add a new node status `BLOCKED`.** Rejected: `TaskExecutionStatus` is - frozen (architecture doc §3.2). Adding a status requires coordinated changes - across propagation, evaluator gating, finalization, and dashboard. The blocked - state is fully encodable as PENDING + all-INVALIDATED incoming edges; no new - status needed. - ---- - -## Open questions - -- **Transitive closure definition.** "Blocked by a failed-dependency chain" is - defined as: the node has ≥1 INVALIDATED incoming edge and 0 PENDING incoming - edges. This is a single-hop check, not transitive. A deeper chain (A fails, - B stays PENDING with INVALIDATED edge from A, C stays PENDING with PENDING - edge from B) is NOT blocked because C still has a PENDING edge. This is - correct: B may yet be unblocked by a planner, which would satisfy B's dep and - allow C to activate. Transitive closure is not needed and would be wrong. - -- **Dashboard badge for PENDING-blocked.** A node that is PENDING with only - INVALIDATED incoming edges looks identical to a legitimately waiting PENDING - node in the current dashboard. Operators need a way to distinguish. Specific - event name and badge design TBD; recommended as a follow-up PR after this - RFC lands. The INVALIDATED edge status is available via the mutations log - today. - -- **Dashboard event for "workflow finalized due to blocked chains".** The - `WorkflowFailedEvent` covers this case (run ends in FAILED state). A separate - event may be useful for UX. TBD. - -- **Mixed static + managed siblings test.** Covered in - `TestStaticNodeStaysPending::test_managed_subtask_regression` above and by - the existing `TestDynamicSubtaskNoAutoCancel::test_mixed_static_and_dynamic_targets` - after the assertion inversion in PR 2. - -- **Interaction with `SubtaskCancellationService`.** Confirmed no handler - assumes `cancel_orphans` fires for dep-failed static siblings (see Risks - table). The three `cancel_orphans_on_*_fn` functions pass the terminal - node's own `node_id` as `parent_node_id`; static siblings are not children. - ---- - -## On acceptance - -- Update `docs/architecture/02_runtime_lifecycle.md#invariants` — add the - "failure propagation is UNIFORM" invariant; remove the Known Limits bullet - for static auto-cancel. -- Update `docs/architecture/02_runtime_lifecycle.md#extension-points` — add - blocking-chain completion seam bullet. -- Update `docs/architecture/cross_cutting/error_propagation.md` — merge - intended-behavior into current-behavior; update control flow diagram; add - invariant; update anti-pattern. -- Link the implementation plan at - `docs/superpowers/plans/2026-04-17-static-sibling-pending.md`. -- Move this file to `docs/rfcs/accepted/`. diff --git a/docs/rfcs/active/2026-04-18-dashboard-paginated-runs-api.md b/docs/rfcs/active/2026-04-18-dashboard-paginated-runs-api.md deleted file mode 100644 index c19bac03e..000000000 --- a/docs/rfcs/active/2026-04-18-dashboard-paginated-runs-api.md +++ /dev/null @@ -1,1163 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: agent -architecture_refs: [docs/architecture/05_dashboard.md, docs/architecture/04_persistence.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Paginated REST Runs API; Store Holds Only Subscribed Runs - -## 1. Problem - -Three defects compound in the current runs index path. - -**1a. Runs older than 50 are inaccessible.** -`DashboardStore.pruneOldRuns` (`ergon-dashboard/src/lib/state/store.ts:409`) evicts the oldest -completed/failed runs when `runs.size > config.maxRunsToKeep` (`lib/config.ts:29`, default 50). -There is no REST endpoint that reads from Postgres for the index view: the only paginated data -surface is `GET /api/runs/{run_id}` (per-run detail) and `GET /cohorts/{cohort_id}` (cohort-scoped -list). A researcher who runs more than 50 benchmarks in one session loses index visibility of older -runs. - -**1b. Store conflates cache and index-of-record.** -`doc/architecture/05_dashboard.md §4` states: "DashboardStore is a cache. The durable source of -truth is Postgres." But the Socket.io `request:runs` handler -(`ergon-dashboard/src/lib/socket/server.ts:79`) serves the all-runs index directly from -`store.getAllRuns()` — a process-local, pruned, cache — not from Postgres. This violates the stated -invariant: if the process restarts or the 50-run window fills, the client receives a stale or -truncated index. - -**1c. HA is structurally impossible while the store is the index-of-record.** -`docs/bugs/open/2026-04-17-dashboard-process-local-state.md` documents that `global.__dashboardStore` -and `global.__socketIO` are process-local singletons. Two Next.js replicas each hold independent -50-run windows. Even if the store is made read-through for detail pages, the index page cannot be -consistent without a shared store or a direct Postgres read. This RFC resolves the index-of-record -half of that bug (the live-event fan-out half remains; see §13). - -**Exact code locations:** - -| File | Line | What is wrong | -|---|---|---| -| `ergon-dashboard/src/lib/state/store.ts` | 409–423 | `pruneOldRuns` evicts completed/failed beyond cap | -| `ergon-dashboard/src/lib/state/store.ts` | 53–55 | `getAllRuns()` is the only index source | -| `ergon-dashboard/src/lib/socket/server.ts` | 79–93 | `request:runs` handler reads `store.getAllRuns()` | -| `ergon-dashboard/src/lib/config.ts` | 29 | `maxRunsToKeep: 50` default cap | -| `ergon-dashboard/src/inngest/functions/index.ts` | 92 | `store.pruneOldRuns()` call in `onWorkflowStarted` | - ---- - -## 2. Proposal - -### 2.1 Design decision: REST not Socket.io - -Three options were evaluated (see §15). REST is chosen because: -- Simple cursor pagination is trivial over HTTP. -- Responses are cache-friendly (HTTP cache-control, CDN, stale-while-revalidate). -- Decouples the index view from the live stream; Socket.io retains its event-delivery role. -- Consistent with the existing `/api/runs/{id}` pattern already used by `useRunState`. - -### 2.2 Cursor design: opaque base64 - -Cursor shape: base64-encoded `<started_at_iso>:<run_id_uuid>`. This is: -- Stable under inserts: inserting a new run between pages does not shift existing cursors. -- Unambiguous when `started_at` has ties: the UUID component breaks ties deterministically. -- Opaque to the client: the client treats it as a black box, enabling server-side format migration. - -### 2.3 Query shape - -Order: `(started_at DESC, id DESC)`. For a forward-only index this is the natural recency order. -Cursor decode: split the base64 string, parse ISO and UUID, then apply the keyset condition: - -```sql -WHERE (started_at, id) < (:cursor_started_at, :cursor_id) -``` - -### 2.4 Response schema - -``` -GET /api/runs?cursor=<opaque>&limit=<n<=100> -→ { runs: RunIndexRow[], next_cursor: string | null } -``` - -`RunIndexRow` is a lightweight summary row — not a full `RunSnapshotDto`. Fields: `id`, `name`, -`status`, `experiment_id`, `started_at`, `completed_at`, `duration_seconds`, `final_score`, -`error`. This matches the subset the all-runs list page needs to render. - -### 2.5 Store narrowing - -`DashboardStore` is narrowed to hold only currently-subscribed runs: -- On first `subscribe(runId)` (Socket.io `subscribe` event): if the run is not in the store, - load it via `build_run_snapshot` (existing path: `request:run` handler, unchanged). -- On last `unsubscribe(runId)` (when socket count for a run room drops to 0): evict the run from - the store. -- `pruneOldRuns` is deleted. `onWorkflowStarted` no longer calls it. -- `store.getAllRuns()` becomes internal-only for live-event fan-out; it is never the index source. - -### 2.6 What does NOT change - -- `GET /api/runs/{run_id}` — per-run detail REST endpoint (`ergon_core/core/api/runs.py:517`). - Unchanged. -- `WorkflowGraphRepository`, `RunRecord`, all persistence models. No Alembic revision required. -- Socket.io `subscribe`, `unsubscribe`, `request:run`, `sync:run` — unchanged. -- `useRunState` hook — unchanged; it already uses `fetch('/api/runs/${runId}')` for the REST - snapshot. -- Inngest event handlers other than the `pruneOldRuns` call site. - ---- - -## 3. Architecture Overview - -### Before - -``` -Browser "all-runs" view - | - | socket.emit("request:runs") - v -Socket.io server.ts:79 - | - | store.getAllRuns() ← pruned, process-local, up to 50 runs - v -socket.emit("sync:runs", runs) -``` - -### After - -``` -Browser "all-runs" view - | - | fetch("/api/runs?cursor=...&limit=50") - v -Next.js route: ergon-dashboard/src/app/api/runs/route.ts (NEW) - | - | fetchErgonApi("/runs?cursor=...&limit=50") - v -FastAPI: ergon_core/core/api/runs.py (ADD endpoint) - | - | SELECT id, name, status, ... FROM runs - | WHERE (started_at, id) < (:cursor_ts, :cursor_id) - | ORDER BY started_at DESC, id DESC LIMIT :limit - v -Postgres RunRecord table -``` - -``` -Live delta stream (unchanged): -Python runtime --> dashboard/* Inngest events --> DashboardStore + Socket.io room - (only for subscribed runs) -``` - -### Store subscription lifecycle (narrowed) - -``` -subscribe(runId) arrives - → if run not in store: socket.emit("request:run") triggers existing load path - → room.count += 1 - -unsubscribe(runId) arrives - → room.count -= 1 - → if room.count == 0: store.runs.delete(runId) - -disconnect - → for each room socket was in: decrement count; if 0, evict -``` - ---- - -## 4. Type and Interface Definitions - -### 4.1 Python: `RunIndexRowDto` (new Pydantic model) - -```python -# ergon_core/ergon_core/core/api/schemas.py (ADD after RunSnapshotDto) - -class RunIndexRowDto(CamelModel): - """Lightweight run summary for the paginated runs index.""" - - id: str - name: str - status: str - experiment_id: str - started_at: datetime | None = None - completed_at: datetime | None = None - duration_seconds: float | None = None - final_score: float | None = None - error: str | None = None - - -class RunsPageDto(CamelModel): - """Paginated runs index page.""" - - runs: list[RunIndexRowDto] = Field(default_factory=list) - next_cursor: str | None = None -``` - -### 4.2 Python: cursor helpers - -```python -# ergon_core/ergon_core/core/api/runs.py (ADD near top of file) - -import base64 -from datetime import datetime -from uuid import UUID - - -def _encode_cursor(started_at: datetime, run_id: UUID) -> str: - """Encode a keyset cursor as opaque base64. - - Format before encoding: "<started_at_iso>:<run_id_uuid>" - """ - raw = f"{started_at.isoformat()}:{run_id}" - return base64.urlsafe_b64encode(raw.encode()).decode() - - -def _decode_cursor(cursor: str) -> tuple[datetime, UUID]: - """Decode an opaque base64 cursor. Raises ValueError on malformed input.""" - try: - raw = base64.urlsafe_b64decode(cursor.encode()).decode() - ts_str, uuid_str = raw.split(":", 1) - return datetime.fromisoformat(ts_str), UUID(uuid_str) - except Exception as exc: - raise ValueError(f"Invalid cursor: {cursor!r}") from exc -``` - -### 4.3 TypeScript: `RunIndexRow` and `RunsPage` - -```typescript -// ergon-dashboard/src/lib/types.ts (ADD alongside existing WorkflowRunState etc.) - -export interface RunIndexRow { - id: string; - name: string; - status: string; // RunLifecycleStatus - experimentId: string; - startedAt: string | null; - completedAt: string | null; - durationSeconds: number | null; - finalScore: number | null; - error: string | null; -} - -export interface RunsPage { - runs: RunIndexRow[]; - nextCursor: string | null; -} -``` - -### 4.4 TypeScript: `useRunsPage` hook - -```typescript -// ergon-dashboard/src/hooks/useRunsPage.ts (NEW FILE) - -"use client"; - -import { useState, useEffect, useCallback } from "react"; -import type { RunIndexRow, RunsPage } from "@/lib/types"; - -interface UseRunsPageResult { - runs: RunIndexRow[]; - isLoading: boolean; - error: string | null; - hasMore: boolean; - loadMore: () => void; -} - -export function useRunsPage(limit: number = 50): UseRunsPageResult { - const [runs, setRuns] = useState<RunIndexRow[]>([]); - const [cursor, setCursor] = useState<string | null>(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState<string | null>(null); - const [hasMore, setHasMore] = useState(false); - const [pendingLoad, setPendingLoad] = useState(true); - - const fetchPage = useCallback( - async (pageCursor: string | null, append: boolean) => { - setIsLoading(true); - try { - const params = new URLSearchParams({ limit: String(limit) }); - if (pageCursor) params.set("cursor", pageCursor); - const response = await fetch(`/api/runs?${params.toString()}`, { - cache: "no-store", - }); - if (!response.ok) { - throw new Error(`Failed to load runs (${response.status})`); - } - const page = (await response.json()) as RunsPage; - setRuns((prev) => (append ? [...prev, ...page.runs] : page.runs)); - setCursor(page.nextCursor); - setHasMore(page.nextCursor !== null); - setError(null); - } catch (err) { - setError(err instanceof Error ? err.message : "Failed to load runs"); - } finally { - setIsLoading(false); - } - }, - [limit], - ); - - useEffect(() => { - if (pendingLoad) { - setPendingLoad(false); - void fetchPage(null, false); - } - }, [pendingLoad, fetchPage]); - - const loadMore = useCallback(() => { - if (!isLoading && hasMore && cursor !== null) { - void fetchPage(cursor, true); - } - }, [isLoading, hasMore, cursor, fetchPage]); - - return { runs, isLoading, error, hasMore, loadMore }; -} -``` - ---- - -## 5. Full Implementations - -### 5.1 FastAPI: `GET /runs` paginated index endpoint - -Add to `ergon_core/ergon_core/core/api/runs.py` after the existing `get_run` endpoint: - -```python -# ergon_core/ergon_core/core/api/runs.py (ADD after line 524) - -import base64 - - -def _encode_cursor(started_at: datetime, run_id: UUID) -> str: - raw = f"{started_at.isoformat()}:{run_id}" - return base64.urlsafe_b64encode(raw.encode()).decode() - - -def _decode_cursor(cursor: str) -> tuple[datetime, UUID]: - try: - raw = base64.urlsafe_b64decode(cursor.encode()).decode() - ts_str, uuid_str = raw.split(":", 1) - return datetime.fromisoformat(ts_str), UUID(uuid_str) - except Exception as exc: - raise ValueError(f"Invalid cursor: {cursor!r}") from exc - - -@router.get("", response_model=RunsPageDto) -def list_runs( - cursor: str | None = None, - limit: int = 50, -) -> RunsPageDto: - """Paginated runs index. Reads directly from Postgres. - - Query params: - cursor: Opaque base64 keyset cursor from a prior response's ``next_cursor``. - Absent on the first page. - limit: Page size, clamped to [1, 100]. Default 50. - - Returns: - ``runs``: List of ``RunIndexRowDto`` ordered by (started_at DESC, id DESC). - ``next_cursor``: Opaque cursor to retrieve the next page, or null if exhausted. - """ - limit = max(1, min(limit, 100)) - - cursor_started_at: datetime | None = None - cursor_run_id: UUID | None = None - if cursor is not None: - try: - cursor_started_at, cursor_run_id = _decode_cursor(cursor) - except ValueError: - raise HTTPException(status_code=400, detail="Invalid cursor") - - with get_session() as session: - # Fetch limit+1 to determine hasMore - stmt = ( - select(RunRecord) - .order_by(RunRecord.started_at.desc(), RunRecord.id.desc()) - .limit(limit + 1) - ) - if cursor_started_at is not None and cursor_run_id is not None: - # Keyset: rows strictly before the cursor position - stmt = stmt.where( - (RunRecord.started_at < cursor_started_at) - | ( - (RunRecord.started_at == cursor_started_at) - & (RunRecord.id < cursor_run_id) - ) - ) - rows = list(session.exec(stmt).all()) - - has_more = len(rows) > limit - page_rows = rows[:limit] - - # Resolve names from definition metadata - with get_session() as session: - def_ids = [r.experiment_definition_id for r in page_rows] - if def_ids: - defs_stmt = select(ExperimentDefinition).where( - ExperimentDefinition.id.in_(def_ids) # type: ignore[union-attr] - ) - defs = {d.id: d for d in session.exec(defs_stmt).all()} - else: - defs = {} - - # Aggregate scores per run - run_ids = [r.id for r in page_rows] - if run_ids: - evals_stmt = select(RunTaskEvaluation).where( - RunTaskEvaluation.run_id.in_(run_ids) # type: ignore[union-attr] - ) - evals_by_run: dict[UUID, list[float]] = {} - for ev in session.exec(evals_stmt).all(): - if ev.score is not None: - evals_by_run.setdefault(ev.run_id, []).append(ev.score) - else: - evals_by_run = {} - - run_rows: list[RunIndexRowDto] = [] - for run in page_rows: - defn = defs.get(run.experiment_definition_id) - if defn is not None: - meta = defn.parsed_metadata() - name = str(meta.get("name", defn.benchmark_type)) - else: - name = str(run.experiment_definition_id) - - duration_seconds: float | None = None - if run.started_at and run.completed_at: - duration_seconds = (run.completed_at - run.started_at).total_seconds() - - scores = evals_by_run.get(run.id, []) - final_score = (sum(scores) / len(scores)) if scores else None - - run_rows.append( - RunIndexRowDto( - id=str(run.id), - name=name, - status=run.status, - experiment_id=str(run.experiment_definition_id), - started_at=run.started_at, - completed_at=run.completed_at, - duration_seconds=duration_seconds, - final_score=final_score, - error=run.error_message, - ) - ) - - next_cursor: str | None = None - if has_more and page_rows: - last = page_rows[-1] - ref_time = last.started_at or last.created_at - next_cursor = _encode_cursor(ref_time, last.id) - - return RunsPageDto(runs=run_rows, next_cursor=next_cursor) -``` - -### 5.2 Next.js: `GET /api/runs` route handler - -```typescript -// ergon-dashboard/src/app/api/runs/route.ts (NEW FILE) - -import { NextRequest, NextResponse } from "next/server"; -import { fetchErgonApi } from "@/lib/serverApi"; - -export async function GET(req: NextRequest): Promise<NextResponse> { - const cursor = req.nextUrl.searchParams.get("cursor"); - const limitRaw = req.nextUrl.searchParams.get("limit"); - const limit = Math.min( - 100, - Math.max(1, parseInt(limitRaw ?? "50", 10) || 50), - ); - - const params = new URLSearchParams({ limit: String(limit) }); - if (cursor) params.set("cursor", cursor); - - try { - const response = await fetchErgonApi(`/runs?${params.toString()}`); - const body = await response.json(); - return NextResponse.json(body, { status: response.status }); - } catch (error) { - return NextResponse.json( - { - detail: "Ergon API is unavailable while loading runs.", - error: - error instanceof Error ? error.message : "Unknown backend fetch failure", - }, - { status: 503 }, - ); - } -} -``` - -### 5.3 `DashboardStore`: delete `pruneOldRuns`, add `evictRun` - -The relevant mutation to `ergon-dashboard/src/lib/state/store.ts`: - -Delete lines 408–423 (the `pruneOldRuns` method) entirely. - -Add the following method after `seedRun`: - -```typescript - /** - * Evict a run from the store when the last subscriber disconnects. - * Called by the Socket.io server when room membership drops to zero. - */ - evictRun(runId: string): void { - this.runs.delete(runId); - this.pendingSandboxCommands.delete(runId); - } -``` - -### 5.4 Socket.io server: delete `request:runs` handler, add eviction logic - -In `ergon-dashboard/src/lib/socket/server.ts`: - -**Delete** lines 79–93 (the `request:runs` / `sync:runs` handler block). - -**Add** room-membership tracking and eviction in the `connection` handler, after the existing -`unsubscribe` handler: - -```typescript - // Evict run from store when last subscriber leaves - socket.on("unsubscribe", (runId: string) => { - const room = `run:${runId}`; - socket.leave(room); - console.log(`[Socket.io] ${socket.id} unsubscribed from ${room}`); - - // Count remaining sockets in the room (excluding this one, which just left) - const roomSockets = io.sockets.adapter.rooms.get(room); - const remaining = roomSockets ? roomSockets.size : 0; - if (remaining === 0) { - store.evictRun(runId); - console.log(`[Socket.io] No subscribers left for ${room} — evicted from store`); - } - }); -``` - -Note: the existing `socket.on("unsubscribe", ...)` handler at lines 120–125 is replaced by this -version. The `subscribe` handler at lines 113–118 is unchanged. - -Also add eviction on disconnect to handle ungraceful disconnects: - -```typescript - socket.on("disconnect", (reason) => { - console.log(`[Socket.io] Client disconnected: ${socket.id} (${reason})`); - - // For each run room this socket was in, check if it was the last subscriber - for (const [roomName] of socket.rooms) { - if (roomName.startsWith("run:")) { - const runId = roomName.slice(4); - const roomSockets = io.sockets.adapter.rooms.get(roomName); - const remaining = roomSockets ? roomSockets.size : 0; - if (remaining === 0) { - store.evictRun(runId); - console.log( - `[Socket.io] No subscribers left for ${roomName} after disconnect — evicted`, - ); - } - } - } - }); -``` - -### 5.5 Inngest `onWorkflowStarted`: delete `pruneOldRuns` call - -In `ergon-dashboard/src/inngest/functions/index.ts`, line 92: delete the line `store.pruneOldRuns();`. - ---- - -## 6. Exact Diffs for Modified Files - -### 6.1 `ergon_core/ergon_core/core/api/schemas.py` - -```diff -+class RunIndexRowDto(CamelModel): -+ """Lightweight run summary for the paginated runs index.""" -+ -+ id: str -+ name: str -+ status: str -+ experiment_id: str -+ started_at: datetime | None = None -+ completed_at: datetime | None = None -+ duration_seconds: float | None = None -+ final_score: float | None = None -+ error: str | None = None -+ -+ -+class RunsPageDto(CamelModel): -+ """Paginated runs index page.""" -+ -+ runs: list[RunIndexRowDto] = Field(default_factory=list) -+ next_cursor: str | None = None -``` - -### 6.2 `ergon_core/ergon_core/core/api/runs.py` - -```diff -+import base64 - from collections import defaultdict - from datetime import datetime - ... -+from ergon_core.core.api.schemas import ( -+ RunIndexRowDto, -+ RunsPageDto, - RunCommunicationMessageDto, - ... - ) -+ -+ -+def _encode_cursor(started_at: datetime, run_id: UUID) -> str: -+ raw = f"{started_at.isoformat()}:{run_id}" -+ return base64.urlsafe_b64encode(raw.encode()).decode() -+ -+ -+def _decode_cursor(cursor: str) -> tuple[datetime, UUID]: -+ try: -+ raw = base64.urlsafe_b64decode(cursor.encode()).decode() -+ ts_str, uuid_str = raw.split(":", 1) -+ return datetime.fromisoformat(ts_str), UUID(uuid_str) -+ except Exception as exc: -+ raise ValueError(f"Invalid cursor: {cursor!r}") from exc -+ - ... -+@router.get("", response_model=RunsPageDto) -+def list_runs(cursor: str | None = None, limit: int = 50) -> RunsPageDto: -+ ... # full implementation in §5.1 -``` - -### 6.3 `ergon-dashboard/src/lib/state/store.ts` - -```diff -- pruneOldRuns(keepCount: number = config.maxRunsToKeep): void { -- const runs = this.getAllRuns(); -- if (runs.length <= keepCount) return; -- -- // Sort by startedAt descending, keep the newest -- const sorted = runs.sort((a, b) => b.startedAt.localeCompare(a.startedAt)); -- const toRemove = sorted.slice(keepCount); -- -- for (const run of toRemove) { -- // Only remove completed/failed runs -- if (run.status === "completed" || run.status === "failed") { -- this.runs.delete(run.id); -- } -- } -- } -+ evictRun(runId: string): void { -+ this.runs.delete(runId); -+ this.pendingSandboxCommands.delete(runId); -+ } -``` - -The `import { config } from "../config";` at line 14 may be removed if `maxRunsToKeep` is the only -use of `config` in this file. Verify before removing. - -### 6.4 `ergon-dashboard/src/lib/socket/server.ts` - -```diff -- // Send current runs to newly connected client -- socket.on("request:runs", () => { -- console.log(`[Socket.io] Client ${socket.id} requested runs sync`); -- const runs = store.getAllRuns(); -- console.log(`[Socket.io] Sending ${runs.length} runs to client`); -- socket.emit("sync:runs", runs.map(r => ({ -- runId: r.id, -- name: r.name, -- status: r.status, -- startedAt: r.startedAt, -- completedAt: r.completedAt, -- durationSeconds: r.durationSeconds, -- finalScore: r.finalScore, -- error: r.error, -- }))); -- }); -+ // (request:runs removed; index view uses REST GET /api/runs) - - ... - -- // Client unsubscribes from a run's updates -- socket.on("unsubscribe", (runId: string) => { -- const room = `run:${runId}`; -- socket.leave(room); -- console.log(`[Socket.io] ${socket.id} unsubscribed from ${room}`); -- }); -+ // Client unsubscribes; evict from store if last subscriber -+ socket.on("unsubscribe", (runId: string) => { -+ const room = `run:${runId}`; -+ socket.leave(room); -+ console.log(`[Socket.io] ${socket.id} unsubscribed from ${room}`); -+ const roomSockets = io.sockets.adapter.rooms.get(room); -+ const remaining = roomSockets ? roomSockets.size : 0; -+ if (remaining === 0) { -+ store.evictRun(runId); -+ console.log(`[Socket.io] No subscribers for ${room} — evicted`); -+ } -+ }); - -- socket.on("disconnect", (reason) => { -- console.log(`[Socket.io] Client disconnected: ${socket.id} (${reason})`); -- }); -+ socket.on("disconnect", (reason) => { -+ console.log(`[Socket.io] Client disconnected: ${socket.id} (${reason})`); -+ for (const [roomName] of socket.rooms) { -+ if (roomName.startsWith("run:")) { -+ const runId = roomName.slice(4); -+ const roomSockets = io.sockets.adapter.rooms.get(roomName); -+ const remaining = roomSockets ? roomSockets.size : 0; -+ if (remaining === 0) { -+ store.evictRun(runId); -+ } -+ } -+ } -+ }); -``` - -### 6.5 `ergon-dashboard/src/inngest/functions/index.ts` - -```diff - broadcastRunStarted(run_id, workflow_name); - console.log("[Dashboard] broadcastRunStarted completed"); - -- // Prune old runs to prevent memory growth -- store.pruneOldRuns(); -- - return { success: true }; -``` - ---- - -## 7. Package Structure - -No new Python packages. The two new DTOs go into the existing `schemas.py`. The cursor helpers and -the new route handler go into the existing `runs.py`. - -No new TypeScript packages. New files are: -- `ergon-dashboard/src/app/api/runs/route.ts` — new file in the existing `app/api/` tree. -- `ergon-dashboard/src/hooks/useRunsPage.ts` — new file in the existing `hooks/` tree. - -The types `RunIndexRow` and `RunsPage` are added to the existing types index. If -`ergon-dashboard/src/lib/types/index.ts` re-exports from sub-files, add the new types to the -appropriate sub-file and re-export. - ---- - -## 8. Implementation Order - -Phased into two PRs. - -### PR 1 — Backend: paginated endpoint (no frontend change) - -| Step | What | Files touched | -|---|---|---| -| 1 | Add `RunIndexRowDto` and `RunsPageDto` to `schemas.py` | MODIFY `ergon_core/ergon_core/core/api/schemas.py` | -| 2 | Add `_encode_cursor`, `_decode_cursor` helpers to `runs.py` | MODIFY `ergon_core/ergon_core/core/api/runs.py` | -| 3 | Add `list_runs` (`GET /runs`) endpoint to `runs.py` | MODIFY `ergon_core/ergon_core/core/api/runs.py` | -| 4 | Unit tests: cursor encode/decode round-trip; empty DB; one page; two pages with next_cursor; cursor boundary correctness | ADD `ergon/tests/state/test_paginated_runs_api.py` | -| 5 | Integration test: create 5 RunRecords, call `list_runs` with limit=3, verify next_cursor, fetch second page, verify no overlap | ADD to `ergon/tests/state/test_paginated_runs_api.py` | - -### PR 2 — Frontend: REST index + store narrowing - -| Step | What | Files touched | -|---|---|---| -| 6 | Add `RunIndexRow`, `RunsPage` types to TS types file | MODIFY `ergon-dashboard/src/lib/types.ts` | -| 7 | Add Next.js `GET /api/runs` proxy route | ADD `ergon-dashboard/src/app/api/runs/route.ts` | -| 8 | Add `useRunsPage` hook | ADD `ergon-dashboard/src/hooks/useRunsPage.ts` | -| 9 | Delete `pruneOldRuns` from `DashboardStore`, add `evictRun` | MODIFY `ergon-dashboard/src/lib/state/store.ts` | -| 10 | Delete `request:runs` handler from `server.ts`; update `unsubscribe` and `disconnect` handlers to call `evictRun` | MODIFY `ergon-dashboard/src/lib/socket/server.ts` | -| 11 | Delete `store.pruneOldRuns()` call from `onWorkflowStarted` | MODIFY `ergon-dashboard/src/inngest/functions/index.ts` | -| 12 | Wire `useRunsPage` into the runs index page (or the component that currently renders `sync:runs` data) | MODIFY the relevant page/component | -| 13 | Remove `ClientToServerEvents["request:runs"]` (`lib/types.ts:417`) and `ServerToClientEvents["sync:runs"]` (`lib/types.ts:401`); delete `RunListEntry`, `SyncRunsSchema`, `parseSyncRuns` from `lib/contracts/events.ts` | MODIFY `ergon-dashboard/src/lib/types.ts`, MODIFY `ergon-dashboard/src/lib/contracts/events.ts` | -| 14 | Frontend hook tests: mock `fetch`, verify page-1 load; loadMore appends; error state | ADD `ergon-dashboard/src/__tests__/useRunsPage.test.ts` | - -**Step 12 dependency note:** The current home page (`ergon-dashboard/src/app/page.tsx:1`) renders -`<CohortListView />`, not a standalone runs index. If the all-runs index view is a component that -currently calls `socket.emit("request:runs")`, that component must be identified and updated in PR 2. -A grep for `request:runs` in the `src/` tree will locate all call sites. If no component currently -uses `request:runs` (the handler exists in server.ts but has no verified client call site beyond -test harness), step 12 is a new feature — create a new `RunsIndexView` component that calls -`useRunsPage`. - ---- - -## 9. File Map - -### ADD - -| File | Purpose | -|---|---| -| `ergon-dashboard/src/app/api/runs/route.ts` | Next.js GET /api/runs proxy to ergon_core | -| `ergon-dashboard/src/hooks/useRunsPage.ts` | React hook for paginated runs index | -| `ergon/tests/state/test_paginated_runs_api.py` | Unit + integration tests for `list_runs` endpoint | -| `ergon-dashboard/src/__tests__/useRunsPage.test.ts` | Frontend hook tests | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/api/schemas.py` | Add `RunIndexRowDto`, `RunsPageDto` | -| `ergon_core/ergon_core/core/api/runs.py` | Add `_encode_cursor`, `_decode_cursor`, `list_runs` endpoint; add `base64` import | -| `ergon-dashboard/src/lib/state/store.ts` | Delete `pruneOldRuns`; add `evictRun`; remove `config` import if unused | -| `ergon-dashboard/src/lib/socket/server.ts` | Delete `request:runs` handler; update `unsubscribe` to evict on last subscriber; update `disconnect` to evict on last subscriber | -| `ergon-dashboard/src/inngest/functions/index.ts` | Delete `store.pruneOldRuns()` call in `onWorkflowStarted` | -| `ergon-dashboard/src/lib/types.ts` | Add `RunIndexRow`, `RunsPage`; remove `ClientToServerEvents["request:runs"]` (line 417) and `ServerToClientEvents["sync:runs"]` (line 401) entries; remove `RunListEntry` import (line 33) | -| `ergon-dashboard/src/lib/contracts/events.ts` | Delete `RunListEntrySchema` (line 201), `SyncRunsSchema` (line 212), `RunListEntry` type (line 273), `parseSyncRuns` (line 355) | -| `docs/architecture/05_dashboard.md` | Update §2 (retention) and §4 (invariants) on acceptance | - ---- - -## 10. Testing Approach - -### 10.1 Backend unit tests — `test_paginated_runs_api.py` - -```python -# ergon/tests/state/test_paginated_runs_api.py - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from uuid import uuid4 - -import pytest -from fastapi.testclient import TestClient - -from ergon_core.core.api.app import app -from ergon_core.core.api.runs import _decode_cursor, _encode_cursor -from ergon_core.core.persistence.shared.db import ensure_db, get_session -from ergon_core.core.persistence.shared.enums import RunStatus -from ergon_core.core.persistence.telemetry.models import RunRecord - - -class TestCursorHelpers: - def test_round_trip(self) -> None: - ts = datetime(2026, 4, 18, 12, 0, 0, tzinfo=UTC) - run_id = uuid4() - cursor = _encode_cursor(ts, run_id) - decoded_ts, decoded_id = _decode_cursor(cursor) - assert decoded_ts == ts - assert decoded_id == run_id - - def test_malformed_cursor_raises(self) -> None: - with pytest.raises(ValueError, match="Invalid cursor"): - _decode_cursor("not-a-valid-cursor") - - def test_opaque_looks_like_base64(self) -> None: - cursor = _encode_cursor(datetime(2026, 1, 1, tzinfo=UTC), uuid4()) - # Should not contain raw ISO or UUID separators - assert ":" not in cursor - assert "-" not in cursor or True # urlsafe base64 may contain "-" - - -class TestListRunsEndpoint: - @pytest.fixture(autouse=True) - def db(self) -> None: - ensure_db() - - def test_empty_db_returns_empty_page(self) -> None: - client = TestClient(app) - resp = client.get("/runs") - assert resp.status_code == 200 - body = resp.json() - assert body["runs"] == [] - assert body["nextCursor"] is None - - def test_returns_runs_ordered_by_started_at_desc(self) -> None: - base_time = datetime(2026, 4, 18, 10, 0, 0, tzinfo=UTC) - run_ids = [] - with get_session() as session: - for i in range(3): - defn_id = uuid4() - run = RunRecord( - id=uuid4(), - experiment_definition_id=defn_id, - status=RunStatus.COMPLETED, - started_at=base_time + timedelta(hours=i), - created_at=base_time + timedelta(hours=i), - ) - session.add(run) - run_ids.append(run.id) - session.commit() - - client = TestClient(app) - resp = client.get("/runs?limit=10") - assert resp.status_code == 200 - returned_ids = [r["id"] for r in resp.json()["runs"]] - # Most recent first - assert returned_ids[0] == str(run_ids[2]) - assert returned_ids[1] == str(run_ids[1]) - assert returned_ids[2] == str(run_ids[0]) - - def test_cursor_pagination_no_overlap(self) -> None: - base_time = datetime(2026, 4, 18, 12, 0, 0, tzinfo=UTC) - with get_session() as session: - for i in range(5): - defn_id = uuid4() - run = RunRecord( - id=uuid4(), - experiment_definition_id=defn_id, - status=RunStatus.COMPLETED, - started_at=base_time + timedelta(minutes=i), - created_at=base_time + timedelta(minutes=i), - ) - session.add(run) - session.commit() - - client = TestClient(app) - page1 = client.get("/runs?limit=3").json() - assert len(page1["runs"]) == 3 - assert page1["nextCursor"] is not None - - page2 = client.get(f"/runs?limit=3&cursor={page1['nextCursor']}").json() - assert len(page2["runs"]) == 2 - assert page2["nextCursor"] is None - - ids1 = {r["id"] for r in page1["runs"]} - ids2 = {r["id"] for r in page2["runs"]} - assert ids1.isdisjoint(ids2), "Pages must not overlap" - - def test_limit_clamped_to_100(self) -> None: - client = TestClient(app) - resp = client.get("/runs?limit=500") - # Should succeed (not 400) and return at most 100 - assert resp.status_code == 200 - - def test_invalid_cursor_returns_400(self) -> None: - client = TestClient(app) - resp = client.get("/runs?cursor=not-valid-base64!!!") - assert resp.status_code == 400 -``` - -### 10.2 Frontend hook tests — `useRunsPage.test.ts` - -```typescript -// ergon-dashboard/src/__tests__/useRunsPage.test.ts - -import { renderHook, act, waitFor } from "@testing-library/react"; -import { useRunsPage } from "@/hooks/useRunsPage"; - -const mockRun = { - id: "run-1", - name: "Test Run", - status: "completed", - experimentId: "exp-1", - startedAt: "2026-04-18T10:00:00Z", - completedAt: "2026-04-18T10:05:00Z", - durationSeconds: 300, - finalScore: 0.9, - error: null, -}; - -describe("useRunsPage", () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it("loads first page on mount", async () => { - global.fetch = jest.fn().mockResolvedValueOnce({ - ok: true, - json: async () => ({ runs: [mockRun], nextCursor: null }), - }); - - const { result } = renderHook(() => useRunsPage(10)); - - await waitFor(() => !result.current.isLoading); - - expect(result.current.runs).toHaveLength(1); - expect(result.current.runs[0].id).toBe("run-1"); - expect(result.current.hasMore).toBe(false); - }); - - it("loadMore appends next page", async () => { - global.fetch = jest - .fn() - .mockResolvedValueOnce({ - ok: true, - json: async () => ({ runs: [mockRun], nextCursor: "abc123" }), - }) - .mockResolvedValueOnce({ - ok: true, - json: async () => ({ - runs: [{ ...mockRun, id: "run-2" }], - nextCursor: null, - }), - }); - - const { result } = renderHook(() => useRunsPage(1)); - await waitFor(() => !result.current.isLoading); - expect(result.current.hasMore).toBe(true); - - act(() => result.current.loadMore()); - await waitFor(() => !result.current.isLoading); - - expect(result.current.runs).toHaveLength(2); - expect(result.current.hasMore).toBe(false); - }); - - it("sets error state on fetch failure", async () => { - global.fetch = jest.fn().mockResolvedValueOnce({ - ok: false, - status: 503, - json: async () => ({}), - }); - - const { result } = renderHook(() => useRunsPage(10)); - await waitFor(() => !result.current.isLoading); - - expect(result.current.error).toMatch(/503/); - expect(result.current.runs).toHaveLength(0); - }); -}); -``` - -### 10.3 API contract test - -The existing codegen at `ergon-dashboard/src/generated/rest/contracts` is generated from the FastAPI -OpenAPI schema. After adding `RunIndexRowDto` and `RunsPageDto`, regenerate the contracts and verify -the Zod schemas include `RunIndexRowDto` and `RunsPageDto` with the expected fields (id, name, -status, experimentId, startedAt, completedAt, durationSeconds, finalScore, error, nextCursor). - -If the codegen is run via a script, add a CI step to fail if the generated file is stale (diff -non-empty after generation). - -### 10.4 Store eviction test - -Add to an existing or new store unit test: - -```typescript -// ergon-dashboard/src/__tests__/store.test.ts - -import { store } from "@/lib/state/store"; - -describe("DashboardStore.evictRun", () => { - afterEach(() => store.reset()); - - it("removes the run and its pending sandbox commands", () => { - store.initializeRun("r1", "exp-1", "Test", { id: "t1", name: "task", description: "", - children: [], depends_on: [], is_leaf: true, assigned_to: null }, - new Date().toISOString(), 1, 1); - expect(store.getRun("r1")).toBeDefined(); - store.evictRun("r1"); - expect(store.getRun("r1")).toBeUndefined(); - }); -}); -``` - ---- - -## 11. Trace / Observability Impact - -**No new spans.** The `list_runs` endpoint is a synchronous FastAPI GET; it runs within the scope -of any existing HTTP middleware tracing (e.g., OpenTelemetry FastAPI instrumentation if added in -future). - -**Logging.** The `evictRun` call sites in `server.ts` log at `console.log` level (consistent with -existing Socket.io log lines), e.g. -`[Socket.io] No subscribers for run:<id> — evicted`. - -**Metrics (optional, out of scope v1).** A future `store.size()` gauge metric could be exported to -observe live subscription counts. Not required for this RFC. - -**No changes to the `CompletedSpan` attribute schema** (that lives in `worker_execute.py`). - ---- - -## 12. Risks and Mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| `(started_at, id)` keyset misses runs where `started_at IS NULL` | Runs with `started_at = null` fall outside cursor ordering and are skipped | `list_runs` falls back to `created_at` when constructing the cursor (see §5.1); ordering clause uses `started_at.desc().nullslast()` to push nulls to the end. Mark as known limitation: null-started runs appear last and are not re-orderable. | -| FastAPI `GET /runs` route conflicts with existing `GET /runs/{run_id}` | 404 or routing ambiguity if FastAPI matches `/runs` before `/runs/{run_id}` | `@router.get("")` (empty path, relative to prefix `/runs`) is distinct from `@router.get("/{run_id}")`. FastAPI resolves static paths before parameterized ones. Verified: existing routes use `/{run_id}`. No conflict. | -| Store eviction races: evict fires before a tab's `request:run` completes | Tab sends `request:run`, then is evicted before `sync:run` response arrives; tab shows empty state | `evictRun` only fires when room membership is zero. If a tab emits `subscribe` before `request:run`, it is in the room and will not be evicted. If a tab subscribes, gets evicted (no-one left in room), then reconnects — `useRunState` re-emits `request:run` and the load path reconstructs from Postgres. No data loss. | -| Removing `pruneOldRuns` causes unbounded memory for long-lived processes with many subscribers | OOM under very high run count if clients never unsubscribe | Narrowed store holds only subscribed runs. A process handling 100 concurrent subscribers with large run states is bounded by 100 runs in memory, not 50. Monitoring store size (§11 metric) is the safety valve. | -| `sync:runs` removal breaks existing clients that call `request:runs` | Legacy clients (e.g., old browser tabs) emit `request:runs` and receive no response | Server silently ignores unknown events. Client receives no `sync:runs` and sits in loading state. Acceptable for a dev tool; document in the PR description. | -| codegen drift: `RunIndexRowDto` added to FastAPI but generated contracts stale | Frontend type errors or silent mismatches | Follow §10.3 contract test procedure; regenerate `src/generated/rest/contracts` in PR 2. | - ---- - -## 13. Invariants Affected - -**`docs/architecture/05_dashboard.md §2` (DashboardStore retention)** -Currently: "oldest runs beyond a cap are pruned on new-run arrival. The cap is configurable via env; -the default lives in `ergon-dashboard/src/lib/config.ts:29`." -After this RFC: the store holds only currently-subscribed runs. No count cap, no time-based pruning. -The `maxRunsToKeep` config key becomes dead; it may be removed from `config.ts`. - -**`docs/architecture/05_dashboard.md §4` invariant ("store is a cache")** -Currently stated but violated in practice by `request:runs`. After this RFC: strictly enforced. -The all-runs index view reads only from Postgres via `GET /api/runs`. The store is never queried for -index data. - -**`docs/architecture/05_dashboard.md §4` invariant ("browser clients subscribe to specific run -rooms; they do NOT receive events for runs they have not subscribed to")** -Unchanged and strengthened: the store now also does not hold runs that have no subscribers. - -**`docs/architecture/04_persistence.md §3` (Dashboard rehydration reads)** -Unchanged: "Dashboard rehydration reads a full run snapshot composed from the mutable tables -directly." This RFC adds a second read path (the index) that also reads directly from mutable -tables (`runs`). Both paths are consistent with the invariant. - -**`docs/bugs/open/2026-04-17-dashboard-process-local-state.md`** -This RFC resolves the index-of-record half: the all-runs view no longer depends on the -process-local store, so two replicas show the same index. The residual bug — live-event fan-out -diverges across replicas for subscribed runs — is unchanged and remains open. - ---- - -## 14. Alternatives Considered - -- **Larger cap (100, 500, 1000 runs in store).** Rejected: does not fix HA or cold-start of - historical runs; pushes the cliff further out. Postgres can always return historical data; the - store should not pretend to be a database. - -- **Redis-backed shared store.** Rejected for now: heavier than needed for a research tool when - Postgres is already authoritative for the index. The live-event HA problem (fan-out) is a - separate concern addressed by the existing bug report. Using Redis to fix the index problem would - add operational complexity without benefit over a direct Postgres read. - -- **Server-sent events (SSE) for the index stream.** Rejected: SSE adds complexity for a view that - is not latency-sensitive at the page level. The index is a historical list; a "pull to refresh" - UX is sufficient. Socket.io `run:started` / `run:completed` events already notify clients when - the list changes; the client can re-fetch `/api/runs` on those events. - -- **Offset pagination instead of cursor.** Rejected: offset pagination is unstable under concurrent - inserts (a new run shifts all subsequent pages). The keyset cursor provides stable, efficient - pagination even under high write rates. The cost is slightly more complex cursor serialization, - which is encapsulated in `_encode_cursor` / `_decode_cursor`. - -- **Serve the paginated index from the Socket.io handshake.** Rejected: REST is simpler, - cache-friendly, easy to paginate, and decouples the index view from the live stream. This is - the same reasoning as the original RFC proposal. - ---- - -## 15. Open Questions - -- **Cursor shape for `started_at IS NULL`.** Currently the cursor falls back to `created_at`. If - `started_at` is always set at run creation time in practice, this is moot. Verify in production - data before shipping. - -- **Filtering (status, cohort, provider).** Out of scope for v1. The query param surface can be - extended (`?status=completed`, `?cohort_id=<uuid>`) in a follow-up RFC without breaking the - cursor contract, provided the filter is included in the cursor encoding. - -- **Index page location.** The current home page renders `<CohortListView />` — there is no - standalone all-runs index page in the current app. This RFC provides the hook (`useRunsPage`) and - the API route. The UI component and routing are left to the implementer; the RFC does not prescribe - a page path. - ---- - -## 16. On Acceptance - -When this RFC moves from `active/` to `accepted/`: - -- Update `docs/architecture/05_dashboard.md` §2 (DashboardStore retention: from cap-based to - subscription-based) and §4 (invariants: add "the all-runs index view reads only from Postgres via - REST; it never reads from DashboardStore"). -- Update `docs/architecture/05_dashboard.md` §7 (Follow-ups): mark the paginated-runs-api RFC as - accepted; update the process-local-state bug entry to reflect that the index-of-record half is - resolved. -- Close or rescope `docs/bugs/open/2026-04-17-dashboard-process-local-state.md` to "subscribed-run - live-event fan-out diverges across replicas", which is the residual scope. -- Link the implementation plan in `docs/superpowers/plans/` once created. -- Regenerate `ergon-dashboard/src/generated/rest/contracts` to include the new schemas. diff --git a/docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md b/docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md deleted file mode 100644 index e0452646d..000000000 --- a/docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md +++ /dev/null @@ -1,747 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: deepflow-research -architecture_refs: - - docs/architecture/03_providers.md#sandbox-managers -supersedes: [] -superseded_by: null ---- - -# RFC: Sandbox manager — collapse `sandbox_key` / `display_task_id` to a single `task_id` - -## Relationship to the process-state RFC - -`docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md` is a broader -reform that removes the singleton, moves class-level state to instances, and -adds cross-process reconnect. This RFC is scoped narrowly to the naming debt in -`create()`'s parameter list. The two RFCs are independent — either can land -first — but this RFC's changes are purely additive cleanup that simplifies the -surface the process-state RFC will later restructure. Landing this one first -reduces the diff size for that RFC. - ---- - -## Problem - -`BaseSandboxManager.create()` -(`ergon_core/ergon_core/core/sandbox/manager.py:226-233`) takes three -conceptual task-keys as positional/keyword arguments: - -```python -async def create( - self, - sandbox_key: UUID, - run_id: UUID, - timeout_minutes: int = 30, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, -) -> str: -``` - -Every production call site passes `sandbox_key == display_task_id`, either -explicitly or via the default-collapse at `manager.py:241`: - -```python -display_task_id = display_task_id or sandbox_key -``` - -The three call sites: - -1. **`sandbox_setup.py:103-108`** — passes positional `task_id` as `sandbox_key` - and then `display_task_id=task_id` explicitly (both the same UUID). -2. **`evaluation/criterion_runtime.py:56-60`** — calls - `manager.create(self.context.run_id, run_id=self.context.run_id, ...)` - omitting `display_task_id`; the default-to-`sandbox_key` branch fires. -3. **`benchmarks/swebench_verified/criterion.py:74`** — calls - `manager.create(sandbox_key=sandbox_key, run_id=run_id)` omitting - `display_task_id`; same default fires. - -The `_display_task_ids: dict[UUID, UUID]` class-level dict at `manager.py:70` -is written only in `create()` at line 275 and read only by -`_get_display_task_id()` at `manager.py:96-97`, which is only called from -`_emit_wal_entry()` at `manager.py:124` and `terminate()` at `manager.py:444`. - -The indirection was introduced to support a hypothetical "subtask reuses parent -sandbox, emits events under parent's task_id" case. That case is unrealized -anywhere in the codebase. The split actively hurts readability: `create()`'s -first parameter is named `sandbox_key` while every other method in the class -names the same concept `task_id` (`get_sandbox(task_id)`, `upload_inputs(task_id, -...)`, `terminate(task_id, ...)`). - -This inconsistency is explicitly called out as debt in -`docs/architecture/03_providers.md:42` ("The `sandbox_key` / `task_id` / -`display_task_id` triplet is **debt**") and in the Known Limits section at -`03_providers.md:142`. - -Additionally, the class-level `_display_task_ids` dict is one of six dicts -whose unbounded growth is tracked as a latent issue in -`docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`. Deleting -it is a small but real reduction in that surface. - ---- - -## Proposal - -### Option chosen: collapse to `task_id`, delete `display_task_id` (YAGNI) - -Rename `sandbox_key` → `task_id` throughout `create()` and its internals. -Delete the `display_task_id` parameter and the backing `_display_task_ids` -dict. If a future use case genuinely needs "emit under a different task id," -implement it at the caller with an explicit `parent_task_id` parameter at that -time. - -Concrete steps: - -1. `BaseSandboxManager.create()` (`manager.py:226-295`): rename first param - `sandbox_key` → `task_id`; remove `display_task_id` param; delete line 241 - (`display_task_id = display_task_id or sandbox_key`); write - `self._display_task_ids[task_id] = task_id` is eliminated (just remove). -2. Delete `_display_task_ids: dict[UUID, UUID] = {}` class attribute - (`manager.py:70`). -3. Delete `_get_display_task_id()` method (`manager.py:96-97`). -4. Simplify `_emit_wal_entry()` (`manager.py:99-131`): rename `sandbox_key` - param to `task_id`; replace `self._get_display_task_id(sandbox_key)` at - line 124 with bare `task_id`. -5. Simplify `terminate()` (`manager.py:429-469`): delete - `display_task_id = self._get_display_task_id(task_id)` at line 444; delete - `self._display_task_ids.pop(task_id, None)` at lines 440 and 454; replace - `display_task_id` with `task_id` in `sandbox_closed()` and - `_emit_wal_entry()` calls at lines 457 and 468. -6. `DefaultSandboxManager.create()` (`manager.py:503-526`): rename - `sandbox_key` → `task_id`; remove `display_task_id` param; update - `super().create()` call accordingly. -7. Update production call sites: - - `sandbox_setup.py:103-108`: drop `display_task_id=task_id` kwarg (already - redundant; positional arg is `task_id`). - - `criterion_runtime.py:56-60`: no rename needed (already passes - `self.context.run_id` positionally; just drop the `display_task_id` kwarg - if present — it is absent today). - - `swebench_verified/criterion.py:74`: rename `sandbox_key=sandbox_key` → - `task_id=sandbox_key` (the local variable name `sandbox_key` can remain or - be renamed to `task_id` at the caller's discretion). -8. Update tests that reset `BaseSandboxManager._display_task_ids = {}`: - - `tests/minif2f/test_sandbox_manager.py:30` - - `tests/swebench_verified/test_sandbox_manager.py:31` - - Remove those lines (the attribute no longer exists). -9. Update tests that call `create(sandbox_key=...)`: - - `tests/minif2f/test_sandbox_manager.py:121,172,206` - - `tests/swebench_verified/test_sandbox_manager.py:129,180` - - `tests/minif2f/test_verification_integration.py:79` - - Rename `sandbox_key=` → `task_id=` at each call. - ---- - -## Architecture overview - -### Before - -``` -create(sandbox_key, run_id, timeout_minutes, envs, display_task_id) - │ - ├─ display_task_id = display_task_id or sandbox_key ← always collapses - ├─ self._sandboxes[sandbox_key] = sandbox - ├─ self._run_ids[sandbox_key] = run_id - ├─ self._display_task_ids[sandbox_key] = display_task_id ← always == sandbox_key - │ - ├─ _emit_wal_entry(sandbox_key, ...) - │ task_id = self._get_display_task_id(sandbox_key) ← reads _display_task_ids - │ - └─ terminate(task_id) - display_task_id = self._get_display_task_id(task_id) ← reads again - self._display_task_ids.pop(task_id, None) ← deleted twice - sandbox_closed(task_id=display_task_id, ...) -``` - -### After - -``` -create(task_id, run_id, timeout_minutes, envs) - │ - ├─ self._sandboxes[task_id] = sandbox - ├─ self._run_ids[task_id] = run_id - │ ← _display_task_ids gone - ├─ _emit_wal_entry(task_id, ...) - │ task_id used directly ← _get_display_task_id gone - │ - └─ terminate(task_id) - sandbox_closed(task_id=task_id, ...) ← direct -``` - -No behavioral change. The only observable difference is that the `task_id` -argument to `sandbox_closed()` and `sandbox_command()` was previously -`_display_task_ids.get(sandbox_key, sandbox_key)` — i.e. `sandbox_key` in all -production cases — which is exactly what `task_id` is after the rename. - ---- - -## Full implementation - -### Modified file: `ergon_core/ergon_core/core/sandbox/manager.py` - -#### 1. Remove `_display_task_ids` class attribute - -```diff -- _display_task_ids: dict[UUID, UUID] = {} - _creation_locks: dict[UUID, asyncio.Lock] = {} -``` - -Location: `manager.py:70`. The line is between `_run_ids` and `_creation_locks`. - -#### 2. Delete `_get_display_task_id()` - -```diff -- def _get_display_task_id(self, sandbox_key: UUID) -> UUID: -- return self._display_task_ids.get(sandbox_key, sandbox_key) -- - async def _emit_wal_entry( # slopcop: ignore[max-function-params] -``` - -Location: `manager.py:96-98`. - -#### 3. Rename `sandbox_key` → `task_id` in `_emit_wal_entry()` and remove lookup - -```diff - async def _emit_wal_entry( # slopcop: ignore[max-function-params] - self, -- sandbox_key: UUID, -+ task_id: UUID, - command: str, - stdout: str | None = None, - stderr: str | None = None, - exit_code: int | None = 0, - started_at: float | None = None, - duration_ms: int | None = None, - sandbox_id: str | None = None, - task_id: UUID | None = None, - ) -> None: -- raw_sandbox = self._sandboxes.get(sandbox_key) -+ raw_sandbox = self._sandboxes.get(task_id) - resolved_sandbox_id = sandbox_id or (raw_sandbox.sandbox_id if raw_sandbox else None) - if resolved_sandbox_id is None: - return - - resolved_duration_ms = duration_ms - if resolved_duration_ms is None and started_at is not None: - resolved_duration_ms = int((time.time() - started_at) * 1000) - - max_len = settings.otel_stdout_stderr_max_length -- resolved_run_id = self._run_ids.get(sandbox_key, sandbox_key) -+ resolved_run_id = self._run_ids.get(task_id, task_id) - await self._event_sink.sandbox_command( - run_id=resolved_run_id, -- task_id=task_id or self._get_display_task_id(sandbox_key), -+ task_id=task_id, - sandbox_id=resolved_sandbox_id, -``` - -**Note:** The `task_id` parameter in the current signature (`manager.py:109`) is -an optional override for the WAL `task_id` field. After this change, that param -is still present as an explicit override but is never used by the existing -callers (all pass `task_id=None` or omit it). It can be removed as a follow-up -cleanup but is kept here to avoid broadening the diff. - -Wait — the current signature has a naming collision: the method parameter -`sandbox_key: UUID` at position 2 and the optional param `task_id: UUID | None` -at position 9. After renaming `sandbox_key` → `task_id` at position 2, the -optional override at position 9 must be renamed to avoid collision. Rename it -to `override_task_id`: - -```diff - async def _emit_wal_entry( # slopcop: ignore[max-function-params] - self, -- sandbox_key: UUID, -+ task_id: UUID, - command: str, - stdout: str | None = None, - stderr: str | None = None, - exit_code: int | None = 0, - started_at: float | None = None, - duration_ms: int | None = None, - sandbox_id: str | None = None, -- task_id: UUID | None = None, -+ override_task_id: UUID | None = None, - ) -> None: -- raw_sandbox = self._sandboxes.get(sandbox_key) -+ raw_sandbox = self._sandboxes.get(task_id) - resolved_sandbox_id = sandbox_id or (raw_sandbox.sandbox_id if raw_sandbox else None) - if resolved_sandbox_id is None: - return - - resolved_duration_ms = duration_ms - if resolved_duration_ms is None and started_at is not None: - resolved_duration_ms = int((time.time() - started_at) * 1000) - - max_len = settings.otel_stdout_stderr_max_length -- resolved_run_id = self._run_ids.get(sandbox_key, sandbox_key) -+ resolved_run_id = self._run_ids.get(task_id, task_id) - await self._event_sink.sandbox_command( - run_id=resolved_run_id, -- task_id=task_id or self._get_display_task_id(sandbox_key), -+ task_id=override_task_id or task_id, - sandbox_id=resolved_sandbox_id, -``` - -#### 4. Rename `sandbox_key` → `task_id` in `BaseSandboxManager.create()`, remove `display_task_id` - -```diff - async def create( - self, -- sandbox_key: UUID, -+ task_id: UUID, - run_id: UUID, - timeout_minutes: int = 30, - envs: dict[str, str] | None = None, -- display_task_id: UUID | None = None, - ) -> str: - """Create a new E2B sandbox, set up directories, install deps.""" - if AsyncSandbox is None: - raise RuntimeError( - "e2b_code_interpreter is not installed. " - "Install it with: pip install e2b-code-interpreter" - ) - -- display_task_id = display_task_id or sandbox_key -- lock = self._creation_locks.setdefault(sandbox_key, asyncio.Lock()) -+ lock = self._creation_locks.setdefault(task_id, asyncio.Lock()) - async with lock: -- if sandbox_key in self._sandboxes: -- return self._sandboxes[sandbox_key].sandbox_id -+ if task_id in self._sandboxes: -+ return self._sandboxes[task_id].sandbox_id - - if not settings.e2b_api_key: - raise ValueError( - "E2B_API_KEY is not set. " - "Please set E2B_API_KEY in your .env file or environment variables." - ) - - try: - timeout_seconds = timeout_minutes * 60 - create_kwargs: dict[str, str | int] = { - "api_key": settings.e2b_api_key, - "timeout": timeout_seconds, - } - if envs: - create_kwargs["envs"] = envs - if self.template: - create_kwargs["template"] = self.template - sandbox = await AsyncSandbox.create(**create_kwargs) - except Exception as e: # slopcop: ignore[no-broad-except] - raise RuntimeError( -- f"Failed to create sandbox for sandbox_key={sandbox_key}: {e}" -+ f"Failed to create sandbox for task_id={task_id}: {e}" - ) from e - - if not sandbox: - raise RuntimeError("Sandbox object is None after creation") - -- self._sandboxes[sandbox_key] = sandbox -- self._ensure_registries(sandbox_key) -- self._run_ids[sandbox_key] = run_id -- self._display_task_ids[sandbox_key] = display_task_id -+ self._sandboxes[task_id] = sandbox -+ self._ensure_registries(task_id) -+ self._run_ids[task_id] = run_id - - await self._event_sink.sandbox_created( - run_id=run_id, -- task_id=display_task_id, -+ task_id=task_id, - sandbox_id=sandbox.sandbox_id, - timeout_minutes=timeout_minutes, - ) - await self._emit_wal_entry( -- sandbox_key, -+ task_id, - command="sandbox.created", - stdout=f"sandbox_id={sandbox.sandbox_id}\ntimeout={timeout_minutes}m", - exit_code=0, - duration_ms=0, - ) - -- await self._create_directory_structure(sandbox, sandbox_key) -- await self._install_dependencies(sandbox, display_task_id) -- await self._verify_setup(sandbox, display_task_id) -+ await self._create_directory_structure(sandbox, task_id) -+ await self._install_dependencies(sandbox, task_id) -+ await self._verify_setup(sandbox, task_id) - - return sandbox.sandbox_id -``` - -#### 5. Simplify `terminate()`: remove `_display_task_ids` pops and rename - -```diff - async def terminate(self, task_id: UUID, reason: str = "completed") -> None: - """Terminate sandbox by task_id key and clean up all registries.""" - sandbox = self._sandboxes.pop(task_id, None) - if sandbox is None: - logger.warning( - "Sandbox not found for task_id=%s. Already terminated or never created.", - task_id, - ) - self._file_registries.pop(task_id, None) - self._created_files_registry.pop(task_id, None) - self._run_ids.pop(task_id, None) -- self._display_task_ids.pop(task_id, None) - return - - sandbox_id = sandbox.sandbox_id -- display_task_id = self._get_display_task_id(task_id) - try: - await sandbox.kill() - except Exception as e: # slopcop: ignore[no-broad-except] - logger.warning("Error killing sandbox for task_id=%s: %s", task_id, e) - reason = "error" - finally: - self._file_registries.pop(task_id, None) - self._created_files_registry.pop(task_id, None) - self._run_ids.pop(task_id, None) -- self._display_task_ids.pop(task_id, None) - - await self._event_sink.sandbox_closed( -- task_id=display_task_id, -+ task_id=task_id, - sandbox_id=sandbox_id, - reason=reason, - ) - await self._emit_wal_entry( - task_id, - command=f"sandbox.closed: {reason}", - stdout=f"sandbox_id={sandbox_id}", - exit_code=0, - duration_ms=0, - sandbox_id=sandbox_id, -- task_id=display_task_id, -+ override_task_id=task_id, - ) -``` - -#### 6. Update `DefaultSandboxManager.create()` - -```diff - async def create( - self, -- sandbox_key: UUID, -+ task_id: UUID, - run_id: UUID, - timeout_minutes: int = 30, - envs: dict[str, str] | None = None, -- display_task_id: UUID | None = None, - ) -> str: - if not settings.e2b_api_key: - from ergon_core.core.runtime.events.task_events import SANDBOX_SKIPPED - - logger.info( - "E2B_API_KEY not set — skipping sandbox creation for task %s (stub mode)", -- sandbox_key, -+ task_id, - ) - return SANDBOX_SKIPPED - return await super().create( -- sandbox_key, -+ task_id, - run_id=run_id, - timeout_minutes=timeout_minutes, - envs=envs, -- display_task_id=display_task_id, - ) -``` - ---- - -### Modified file: `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` - -Drop the redundant `display_task_id=task_id` kwarg (`sandbox_setup.py:108`): - -```diff - sandbox_id = await sandbox_manager.create( - task_id, - run_id=run_id, - timeout_minutes=30, - envs=envs, -- display_task_id=task_id, - ) -``` - -The first positional argument is already `task_id`; no other change needed. - ---- - -### Modified file: `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` - -`ensure_sandbox()` at `criterion_runtime.py:53-60` already passes -`self.context.run_id` positionally with no `display_task_id` kwarg. No -parameter rename is needed at the call site. No change required. - ---- - -### Modified file: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` - -Rename the keyword argument at `criterion.py:74`: - -```diff -- await manager.create(sandbox_key=sandbox_key, run_id=run_id) -+ await manager.create(task_id=sandbox_key, run_id=run_id) -``` - -The local variable `sandbox_key` retains its name (it is a freshly-generated -`uuid4()` at line 73 used only for this call and the subsequent `get_sandbox` -call at line 75). - ---- - -### Modified files: tests - -**`tests/minif2f/test_sandbox_manager.py`** - -```diff -- BaseSandboxManager._display_task_ids = {} -``` -Remove line 30 from the `_reset_sandbox_singleton` fixture. Also remove it from -the yield-teardown block if present. - -Rename keyword arg at three call sites (lines 121, 172, 206): -```diff -- sandbox_id = await mgr.create(sandbox_key=uuid4(), run_id=uuid4(), timeout_minutes=5) -+ sandbox_id = await mgr.create(task_id=uuid4(), run_id=uuid4(), timeout_minutes=5) - -- await mgr.create(sandbox_key=uuid4(), run_id=uuid4()) # line 172 -+ await mgr.create(task_id=uuid4(), run_id=uuid4()) - -- await mgr.create(sandbox_key=uuid4(), run_id=uuid4()) # line 206 -+ await mgr.create(task_id=uuid4(), run_id=uuid4()) -``` - -**`tests/swebench_verified/test_sandbox_manager.py`** - -```diff -- BaseSandboxManager._display_task_ids = {} -``` -Remove line 31 from the fixture. - -Rename keyword arg at two call sites (lines 129, 180): -```diff -- sandbox_id = await mgr.create(sandbox_key=uuid4(), run_id=uuid4(), timeout_minutes=5) -+ sandbox_id = await mgr.create(task_id=uuid4(), run_id=uuid4(), timeout_minutes=5) - -- await mgr.create(sandbox_key=uuid4(), run_id=uuid4()) # line 180 -+ await mgr.create(task_id=uuid4(), run_id=uuid4()) -``` - -**`tests/minif2f/test_verification_integration.py`** - -```diff - await sandbox_manager.create( -- sandbox_key=run_id, -+ task_id=run_id, - run_id=run_id, - timeout_minutes=10, - ) -``` - ---- - -## Package structure - -No new packages. No `__init__.py` changes. This RFC modifies existing files only. - ---- - -## Implementation order - -| Step | Phase | What | Files touched | -|---|---|---|---| -| 1 | PR 1 | Delete `_display_task_ids` class attr and `_get_display_task_id()` method; rename `sandbox_key` → `task_id` and remove `display_task_id` param in `BaseSandboxManager.create()`; rename `override_task_id` in `_emit_wal_entry()`; simplify `terminate()` | MODIFY `manager.py` | -| 2 | PR 1 | Update `DefaultSandboxManager.create()` to match | MODIFY `manager.py` | -| 3 | PR 1 | Drop `display_task_id=task_id` in `sandbox_setup.py` | MODIFY `sandbox_setup.py` | -| 4 | PR 1 | Rename `sandbox_key=` → `task_id=` in SWEBench criterion | MODIFY `swebench_verified/criterion.py` | -| 5 | PR 1 | Update all tests: remove `_display_task_ids` resets; rename `sandbox_key=` → `task_id=` at call sites | MODIFY `test_sandbox_manager.py` (×2), `test_verification_integration.py` | -| 6 | PR 1 | Grep-verify: zero remaining occurrences of `sandbox_key`, `display_task_id`, `_display_task_ids`, `_get_display_task_id` anywhere under `ergon/` (docs and tests included) | No file changes | - -All steps fit a single PR. No staged rollout needed — the rename is mechanical -and all call sites are in-repo. - ---- - -## File map - -### ADD - -None. - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/core/sandbox/manager.py` | Delete `_display_task_ids` attr (line 70); delete `_get_display_task_id()` (lines 96-97); rename `sandbox_key`→`task_id` + remove `display_task_id` in `BaseSandboxManager.create()` (lines 226-295); rename `sandbox_key`→`task_id` + rename `task_id`→`override_task_id` in `_emit_wal_entry()` (lines 99-131); simplify `terminate()` (lines 429-469); rename + remove `display_task_id` in `DefaultSandboxManager.create()` (lines 503-526) | -| `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` | Drop `display_task_id=task_id` kwarg at line 108 | -| `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` | Rename `sandbox_key=` → `task_id=` at line 74 | -| `tests/minif2f/test_sandbox_manager.py` | Remove `BaseSandboxManager._display_task_ids = {}` at line 30; rename `sandbox_key=` → `task_id=` at lines 121, 172, 206 | -| `tests/swebench_verified/test_sandbox_manager.py` | Remove `BaseSandboxManager._display_task_ids = {}` at line 31; rename `sandbox_key=` → `task_id=` at lines 129, 180 | -| `tests/minif2f/test_verification_integration.py` | Rename `sandbox_key=run_id` → `task_id=run_id` at line 79 | - ---- - -## Testing approach - -### Unit — verifying the rename does not break existing behavior - -The existing test suites in `tests/minif2f/test_sandbox_manager.py` and -`tests/swebench_verified/test_sandbox_manager.py` already exercise -`create()`/`terminate()` paths against a mocked `AsyncSandbox`. After the -keyword-arg renames, all existing tests must pass unchanged (aside from the -fixture cleanup and call-site renames listed above). - -Representative new assertions to add to `test_sandbox_manager.py`: - -```python -# tests/minif2f/test_sandbox_manager.py - -async def test_create_uses_task_id_as_event_task_id( - fake_create: AsyncMock, - recording_sink: RecordingSandboxEventSink, -) -> None: - """After the rename, sandbox_created fires with task_id equal to the - task_id arg — no display_task_id indirection.""" - mgr = MiniF2FSandboxManager() - task_id = uuid4() - run_id = uuid4() - await mgr.create(task_id=task_id, run_id=run_id) - - assert len(recording_sink.created_events) == 1 - event = recording_sink.created_events[0] - assert event["task_id"] == task_id - assert event["run_id"] == run_id - - -async def test_terminate_fires_closed_with_same_task_id( - fake_create: AsyncMock, - recording_sink: RecordingSandboxEventSink, -) -> None: - """terminate() emits sandbox_closed with the same task_id passed to create().""" - mgr = MiniF2FSandboxManager() - task_id = uuid4() - await mgr.create(task_id=task_id, run_id=uuid4()) - await mgr.terminate(task_id) - - assert len(recording_sink.closed_events) == 1 - assert recording_sink.closed_events[0]["task_id"] == task_id - - -def test_display_task_ids_attr_absent() -> None: - """Class-level _display_task_ids must not exist after this RFC.""" - assert not hasattr(BaseSandboxManager, "_display_task_ids") - - -def test_get_display_task_id_method_absent() -> None: - """_get_display_task_id must not exist after this RFC.""" - assert not hasattr(BaseSandboxManager, "_get_display_task_id") -``` - -### Integration — no new integration tests required - -The three production call sites are each covered by the existing test suites -once the keyword-arg renames are applied. No new integration test surface is -introduced because no behavior changes. - -### Regression grep check - -Before merging, run: - -```bash -grep -rn "sandbox_key\|display_task_id\|_display_task_ids\|_get_display_task_id" \ - ergon_core ergon_builtins ergon_cli ergon_infra tests -``` - -Expected: zero matches. Any remaining hit is a missed call site. - ---- - -## Trace / observability impact - -`_emit_wal_entry()` emits `task_id` to `sandbox_command()` on the event sink. -Before this RFC, it passed `self._get_display_task_id(sandbox_key)` which -resolved to `sandbox_key` in every production case. After this RFC it passes -`task_id` directly. The value is identical in all current production cases, -so no span or event payload changes. - -The `override_task_id` optional parameter in `_emit_wal_entry()` is only used -by `terminate()`'s own `_emit_wal_entry()` call (passing the explicit `task_id` -again). That is a no-op equivalent and exists for forward compatibility if a -future caller genuinely needs to override the emitted task id. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| External `BaseSandboxManager` subclass overriding `create(sandbox_key, ...)` | `TypeError` on mismatch after rename | No external subclasses exist in-repo. Architecture doc Section 5.2 says subclasses are owned per-benchmark inside `ergon_builtins/`. The change is intra-repo only. | -| `_emit_wal_entry`'s `task_id` / `override_task_id` rename breaks a test that passes it as a keyword | Test-only `TypeError` | Grep confirms only `terminate()` calls `_emit_wal_entry()` with the positional `task_id` — and that positional slot is now the first `task_id` param, not the old optional one. No callers use the optional by name today. | -| The swebench criterion local variable `sandbox_key` (line 73) confuses future readers | Minor readability cost | Acceptable: the local var is used for two lines only. A follow-up can rename it. Does not affect behavior. | -| Missing call site — a test helper passes `sandbox_key=` to a manager subclass we haven't grepped | Silent wrong kwarg → `TypeError` at test time | The post-PR grep check (`grep -rn sandbox_key ...`) is mandatory before merge. CI will also catch any `TypeError` at test time. | -| `DefaultSandboxManager.create()` log line still says "task %s" after rename | No behavioral impact | Line already says "task %s" (not "sandbox_key %s"); the variable passed changes from `sandbox_key` to `task_id` — correct after rename. | - ---- - -## Invariants affected - -**`docs/architecture/03_providers.md` — updates required on acceptance:** - -- **Section 2.2, line 42** ("The `sandbox_key` / `task_id` / `display_task_id` triplet is **debt**"): replace with "Collapsed to `task_id` in - `docs/rfcs/accepted/2026-04-18-sandbox-manager-key-cleanup.md`." -- **Section 2.2, control-flow block** (`03_providers.md:93`): - `create(sandbox_key=task_id, ...)` → `create(task_id, ...)`. -- **Section 4, invariant 4** (`03_providers.md:131`): "Enforced by `create` - accepting `sandbox_key`" → "Enforced by `create` accepting `task_id`." -- **Section 4.1 Known Limits** (`03_providers.md:142`): drop the - "Key-triplet debt" bullet. -- **Section 6 Anti-patterns**: no change needed. - -The `_display_task_ids` class-level dict is one of the six class-level mutable -dicts flagged in `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` -and in the process-state RFC. Removing it reduces the unbounded-growth surface -without requiring the broader singleton reform. The bug report's scope statement -should be updated to reflect that `_display_task_ids` is gone; the remaining -five dicts are still addressed by the process-state RFC. - ---- - -## Alternatives considered - -- **Rename `sandbox_key` → `task_id` but keep `display_task_id` for future - flexibility.** Rejected: YAGNI. If the subtask-sharing case appears, solve it - when it is real — with an explicit `parent_task_id` at the caller. -- **Rename everywhere but keep `_display_task_ids` around as dead structure.** - Rejected: a dict that is always `{k: k}` is noise and contributes to the - unbounded-growth surface. -- **Keep `sandbox_key` as-is and just alias `task_id` as an alternate param.** - Rejected: two names for the same concept in the same class hierarchy is the - exact problem being solved; adding a third alias makes it worse. - ---- - -## Open questions - -- Interaction with `docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md`: - light coupling only — that RFC will restructure the class-level dicts this RFC - cleans up. The rename is purely additive from that RFC's perspective; it will - see `task_id` everywhere and fewer dicts to move. - ---- - -## On acceptance - -- Move this RFC to `accepted/`. -- Update `docs/architecture/03_providers.md` as described in "Invariants - affected" above: simplify the `create()` signature documentation; remove the - key-triplet debt bullet; update the control-flow block. -- Update `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`: note - that `_display_task_ids` has been removed; the remaining class-level dicts are - still addressed by the process-state RFC. -- Grep for remaining uses of the terms `sandbox_key`, `display_task_id`, - `_display_task_ids`, `_get_display_task_id` in `docs/` and `tests/` and update - to `task_id`. -- No Alembic migration needed — all changed state is in-memory. diff --git a/docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md b/docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md deleted file mode 100644 index 0d82db052..000000000 --- a/docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md +++ /dev/null @@ -1,927 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: deepflow-research -architecture_refs: - - docs/architecture/03_providers.md#sandbox-managers - - docs/architecture/03_providers.md#invariants -supersedes: [] -superseded_by: null ---- - -# RFC: Sandbox manager — drop singleton, move state to instances, add cross-process reconnect - -## 1. Problem - -`BaseSandboxManager` -(`ergon_core/ergon_core/core/sandbox/manager.py`) is wired as a -singleton-per-subclass via `__new__` at `manager.py:78-81`: - -```python -def __new__(cls, *args: object, **kwargs: object): - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance -``` - -Six class-level mutable dicts are declared at `manager.py:65-71`: - -```python -_instance: "BaseSandboxManager | None" = None -_sandboxes: dict[UUID, "AsyncSandbox"] = {} -_file_registries: dict[UUID, dict[str, str]] = {} -_created_files_registry: dict[UUID, set[str]] = {} -_run_ids: dict[UUID, UUID] = {} -_display_task_ids: dict[UUID, UUID] = {} -_creation_locks: dict[UUID, asyncio.Lock] = {} -``` - -These dicts are **shared across every instance of a subclass** for the -process lifetime. Entries are cleared only by `terminate()` at -`manager.py:429-469`. Three concrete problems follow. - -### 1.1 Invisible in-process coupling - -Workers reconnect to their sandbox in `execute()` by re-instantiating the -subclass and calling `get_sandbox(task_id)` at `manager.py:394-396`: - -```python -def get_sandbox(self, task_id: UUID) -> "AsyncSandbox | None": - return self._sandboxes.get(task_id) -``` - -Example: `MiniF2FReActWorker.execute()` at -`ergon_builtins/ergon_builtins/workers/baselines/minif2f_react_worker.py:111-113`: - -```python -manager = MiniF2FSandboxManager() -sandbox = manager.get_sandbox(context.task_id) -``` - -Nothing in the signature says "worker must run in same process as the -`sandbox_setup_fn` that created it." The coupling is invisible: if the -Inngest worker runs in a different process replica the call silently returns -`None` and the task fails with a confusing error. - -The same pattern appears in: - -- `ergon_builtins/ergon_builtins/workers/baselines/swebench_worker.py:123` -- `ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py:74` -- `ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py:78` -- `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py:72` - -`ResearchRubricsSandboxManager` (in -`ergon_builtins/ergon_builtins/benchmarks/researchrubrics/sandbox_manager.py`) also -calls `self._sandboxes[task_id]` directly at `research_rubrics_manager.py:105` -in `publisher_for()`, relying on the class-level dict. - -`DefaultCriterionRuntime.ensure_sandbox()` at -`ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py:53-63` -calls `self.sandbox_manager.get_sandbox(self.context.run_id)` — it would -return `None` across processes even though the sandbox row exists in the DB. - -### 1.2 Latent `__init__` stomp race - -`__init__` at `manager.py:85-87` is last-write-wins: - -```python -def __init__(self, event_sink: SandboxEventSink | None = None): - if event_sink is not None: - self._event_sink = event_sink -``` - -Because `__new__` always returns the cached instance, every call to -`MiniF2FSandboxManager(event_sink=sink)` re-runs `__init__` on the **same** -object. A second construction with a different `event_sink` silently replaces -the first. This is the root cause documented in -`docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md`. No -production site hits it today: all 5 construction sites omit `event_sink=`. -The latent hazard is real and blocks any future DI injection of per-test sinks. - -Tests work around this at -`tests/minif2f/test_sandbox_manager.py:23-35` by manually resetting all six -class-level dicts in an `autouse` fixture — a sign that the design is leaking -its internals. - -### 1.3 Unbounded memory growth - -The class-level dicts grow with every task. Entries are evicted only by -`terminate()`. Any task that crashes before reaching `terminate()` leaks its -entry for the process lifetime. This is noted in `manager.py:63-64` and in -`docs/architecture/03_providers.md` Section 4.1: - -> **Class-dict unbounded growth.** Manager class-level state is cleared only -> inside `terminate()`. Any task that never reaches `terminate()` leaks its -> entries for the process lifetime. - -For long-lived research training processes this is a genuine memory hazard. - -### 1.4 Impossible multi-process scale-out - -`docs/architecture/03_providers.md` Section 2.2 states: - -> A subclass's class-level state is the only source of truth for in-process -> reconnect. Applies only within a single Python process; cross-process actors -> must use `terminate_by_sandbox_id` or provision their own sandbox. - -There is no `reconnect(sandbox_id)` method. In-process criteria reconnect via -`get_sandbox(task_id)`. Cross-process criteria (SWE-Bench) provision fresh -sandboxes at `swebench_verified/criterion.py:66-78`. If the backend is ever -split into multi-replica Inngest workers or async worker pools, the reconnect -path silently returns `None` and evaluation breaks. - -### 1.5 Scope boundary with key-cleanup RFC - -`docs/rfcs/active/2026-04-18-sandbox-manager-key-cleanup.md` collapses the -`sandbox_key` / `display_task_id` triplet in `create()`. That RFC is -**orthogonal**: either order of landing is correct. This RFC does not change -the `create()` signature; key-cleanup can proceed independently. Implementation -order: prefer key-cleanup first (smaller scope, unblocks readable diffs), then -process-state. - ---- - -## 2. Proposal - -Stop relying on process-global state in the sandbox manager. Four concrete -changes, implemented in three staged PRs. - -### Option A — Instance-state migration (chosen) - -1. **Move state from class-level to instance-level.** `_sandboxes`, `_run_ids`, - `_file_registries`, `_created_files_registry`, `_display_task_ids`, and - `_creation_locks` become instance attributes initialized in `__init__`. No - class-level mutable dicts. -2. **Drop the singleton.** Remove `__new__` (`manager.py:78-81`) and - `_instance` (`manager.py:65`). The construction site in - `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py:57` - (`sandbox_manager = manager_cls()`) already creates one manager per - function invocation; that behavior becomes correct rather than accidentally - correct. -3. **Add `reconnect(sandbox_id: str) -> AsyncSandbox`.** Uses the E2B API - (`AsyncSandbox.connect`) to rehydrate a sandbox by its persisted id. Criteria - in a different process can reconnect given only the `sandbox_id` column on - the execution row. -4. **Update criterion path.** `DefaultCriterionRuntime.ensure_sandbox()` at - `criterion_runtime.py:53-63` stops calling `manager.get_sandbox(task_id)` on - a shared dict and instead calls `reconnect(context.sandbox_id)` when the - manager's in-memory dict has no entry. - -### Options considered and rejected - -See Section 13. - ---- - -## 3. Architecture overview - -### 3.1 Before (current state) - -``` -sandbox_setup_fn (process A) - │ - ├─ manager_cls = SANDBOX_MANAGERS["minif2f"] # MiniF2FSandboxManager - ├─ sandbox_manager = manager_cls() # returns SAME instance via __new__ - │ MiniF2FSandboxManager._instance cached - │ MiniF2FSandboxManager._sandboxes[task_id] = <AsyncSandbox> - │ - └─ returns sandbox_id="sbx-abc123" (persisted to DB) - -worker_execute_fn (process A — must be SAME process) - │ - ├─ manager = MiniF2FSandboxManager() # hits __new__, gets cached instance - └─ sandbox = manager.get_sandbox(context.task_id) # reads class-level _sandboxes dict - # returns None if different process -``` - -### 3.2 After (this RFC) - -``` -sandbox_setup_fn (process A) - │ - ├─ manager_cls = SANDBOX_MANAGERS["minif2f"] - ├─ sandbox_manager = manager_cls() # new fresh instance — no singleton - │ instance._sandboxes[task_id] = <AsyncSandbox> - │ - └─ returns sandbox_id="sbx-abc123" (persisted to DB) - -worker_execute_fn (any process) - │ - ├─ manager = manager_cls() # new fresh instance - └─ sandbox = await manager.reconnect("sbx-abc123") # AsyncSandbox.connect via E2B API - # works cross-process - -DefaultCriterionRuntime.ensure_sandbox() (any process) - │ - ├─ sandbox = self.sandbox_manager.get_sandbox(task_id) # in-memory: None in new proc - └─ if None: sandbox = await manager.reconnect(context.sandbox_id) -``` - -### 3.3 Data flow change - -| Field | Before | After | -|---|---|---| -| `_instance` | `ClassVar` | removed | -| `_sandboxes` | `ClassVar[dict]` — shared | `instance dict` — per-construction | -| `_run_ids` | `ClassVar[dict]` | `instance dict` | -| `_file_registries` | `ClassVar[dict]` | `instance dict` | -| `_created_files_registry` | `ClassVar[dict]` | `instance dict` | -| `_display_task_ids` | `ClassVar[dict]` | `instance dict` | -| `_creation_locks` | `ClassVar[dict]` | `instance dict` | -| `reconnect(sandbox_id)` | does not exist | `async def reconnect(str) -> AsyncSandbox` | - ---- - -## 4. Type / interface definitions - -### 4.1 Updated `BaseSandboxManager.__init__` - -```python -# ergon_core/ergon_core/core/sandbox/manager.py - -class BaseSandboxManager(ABC): - """Abstract base class for E2B sandbox lifecycle management. - - One instance per sandbox_setup_fn invocation. Instance-level dicts - replace the former class-level shared state; the singleton-per-subclass - pattern (__new__) is removed. - - Cross-process reconnect: call reconnect(sandbox_id) with the sandbox_id - persisted on the TaskExecution row to rehydrate an AsyncSandbox without - access to the in-process _sandboxes dict. - """ - - template: str | None = None # ClassVar stays; it is read-only config - - def __init__(self, event_sink: SandboxEventSink | None = None) -> None: - # Instance-level state — no class-level shared dicts. - self._sandboxes: dict[UUID, AsyncSandbox] = {} - self._file_registries: dict[UUID, dict[str, str]] = {} - self._created_files_registry: dict[UUID, set[str]] = {} - self._run_ids: dict[UUID, UUID] = {} - self._display_task_ids: dict[UUID, UUID] = {} - self._creation_locks: dict[UUID, asyncio.Lock] = {} - self._event_sink: SandboxEventSink = event_sink or NoopSandboxEventSink() -``` - -### 4.2 `reconnect` method signature - -```python -# ergon_core/ergon_core/core/sandbox/manager.py - - async def reconnect(self, sandbox_id: str) -> "AsyncSandbox": - """Rehydrate a running sandbox by its E2B sandbox_id. - - Uses the E2B API directly — does not require the sandbox to have - been created by this instance or in this process. Returns a live - AsyncSandbox handle. - - Does NOT cache the result on self._sandboxes — callers are expected - to hold the reference for their scope. Revisit if E2B rate limits - become a problem (see Open Questions). - - Raises RuntimeError if e2b_code_interpreter is not installed or - if the sandbox is no longer alive. - """ - if AsyncSandbox is None: - raise RuntimeError( - "e2b_code_interpreter is not installed; cannot reconnect to sandbox." - ) - return await AsyncSandbox.connect(sandbox_id, api_key=settings.e2b_api_key) -``` - -### 4.3 `DefaultCriterionRuntime.ensure_sandbox` updated signature - -```python -# ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py - - async def ensure_sandbox(self) -> None: - """Ensure a live sandbox is available for this criterion's context. - - In-process: reads self.sandbox_manager._sandboxes via get_sandbox(). - Cross-process: reconnects via reconnect(context.sandbox_id) if - get_sandbox returns None and context.sandbox_id is set. - Creates a fresh sandbox only if both paths yield nothing. - """ - sandbox = self.sandbox_manager.get_sandbox(self.context.run_id) - if sandbox is None and self.context.sandbox_id: - # Cross-process case: sandbox was created in a different process. - await self.sandbox_manager.reconnect(self.context.sandbox_id) - return - if sandbox is None: - await self.sandbox_manager.create( - self.context.run_id, - run_id=self.context.run_id, - timeout_minutes=30, - ) - self._owns_sandbox = True - return - await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=30) -``` - ---- - -## 5. Full implementations - -### 5.1 `BaseSandboxManager` — full diff region - -The diff below covers `manager.py:50-90` (the class header and `__init__`): - -```diff - class BaseSandboxManager(ABC): -- """Abstract base class for E2B sandbox management.""" -+ """Abstract base class for E2B sandbox lifecycle management. -+ -+ One instance per sandbox_setup_fn invocation. Instance-level dicts -+ replace the former class-level shared state; the singleton pattern is -+ removed. Use reconnect(sandbox_id) for cross-process sandbox access. -+ """ - - # Optional name or ID of a pre-built E2B template. ClassVar — read-only. - template: str | None = None - -- # Class-level state: shared across every instance of this subclass. -- # Entries are only evicted by terminate(). Tasks that crash without -- # terminate() leak their entries forever. -- # docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md. -- _instance: "BaseSandboxManager | None" = None -- _sandboxes: dict[UUID, "AsyncSandbox"] = {} -- _file_registries: dict[UUID, dict[str, str]] = {} -- _created_files_registry: dict[UUID, set[str]] = {} -- _run_ids: dict[UUID, UUID] = {} -- _display_task_ids: dict[UUID, UUID] = {} -- _creation_locks: dict[UUID, asyncio.Lock] = {} -- _event_sink: SandboxEventSink -- -- # TODO(2026-04-18): Singleton-per-subclass with class-level state dicts is load-bearing -- # for same-process criterion reconnect via `get_sandbox(task_id)`. This forecloses -- # multi-process scale-out and creates a latent stomp risk on `_event_sink` in __init__. -- # Broader reform tracked in docs/rfcs/active/2026-04-18-sandbox-manager-process-state.md. -- def __new__(cls, *args: object, **kwargs: object): -- if cls._instance is None: -- cls._instance = super().__new__(cls) -- return cls._instance -- -- _event_sink: SandboxEventSink = NoopSandboxEventSink() -- -- def __init__(self, event_sink: SandboxEventSink | None = None): -- if event_sink is not None: -- self._event_sink = event_sink -+ def __init__(self, event_sink: SandboxEventSink | None = None) -> None: -+ # Instance-level state — no class-level shared dicts. -+ self._sandboxes: dict[UUID, "AsyncSandbox"] = {} -+ self._file_registries: dict[UUID, dict[str, str]] = {} -+ self._created_files_registry: dict[UUID, set[str]] = {} -+ self._run_ids: dict[UUID, UUID] = {} -+ self._display_task_ids: dict[UUID, UUID] = {} -+ self._creation_locks: dict[UUID, asyncio.Lock] = {} -+ self._event_sink: SandboxEventSink = event_sink or NoopSandboxEventSink() -``` - -### 5.2 `reconnect` — new method added after `get_sandbox` - -Add after `manager.py:396` (after `get_sandbox`): - -```diff -+ async def reconnect(self, sandbox_id: str) -> "AsyncSandbox": -+ """Rehydrate a running sandbox by its E2B sandbox_id. -+ -+ Uses the E2B API directly — works cross-process. Does NOT cache the -+ result on self._sandboxes; callers hold the reference. -+ """ -+ if AsyncSandbox is None: -+ raise RuntimeError( -+ "e2b_code_interpreter is not installed; cannot reconnect to sandbox." -+ ) -+ return await AsyncSandbox.connect(sandbox_id, api_key=settings.e2b_api_key) -+ - def get_sandbox_path(self, task_id: UUID, local_path: str) -> str | None: -``` - -### 5.3 `DefaultCriterionRuntime.ensure_sandbox` — updated - -Diff against `criterion_runtime.py:53-63`: - -```diff -- async def ensure_sandbox(self) -> None: -- sandbox = self.sandbox_manager.get_sandbox(self.context.run_id) -- if sandbox is None: -- await self.sandbox_manager.create( -- self.context.run_id, -- run_id=self.context.run_id, -- timeout_minutes=30, -- ) -- self._owns_sandbox = True -- return -- await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=30) -+ async def ensure_sandbox(self) -> None: -+ sandbox = self.sandbox_manager.get_sandbox(self.context.run_id) -+ if sandbox is None and self.context.sandbox_id: -+ # Cross-process path: reconnect to the sandbox created in sandbox_setup_fn. -+ await self.sandbox_manager.reconnect(self.context.sandbox_id) -+ return -+ if sandbox is None: -+ await self.sandbox_manager.create( -+ self.context.run_id, -+ run_id=self.context.run_id, -+ timeout_minutes=30, -+ ) -+ self._owns_sandbox = True -+ return -+ await self.sandbox_manager.reset_timeout(self.context.run_id, timeout_minutes=30) -``` - -**Note:** `CriterionContext.sandbox_id` must be a `str | None` field. If it -does not exist today, it must be added to -`ergon_core/ergon_core/core/runtime/evaluation/evaluation_schemas.py` as part -of Stage 3. Check this file before landing Stage 3. - -### 5.4 `ResearchRubricsSandboxManager.publisher_for` — no change needed - -`publisher_for` at `research_rubrics_manager.py:93-110` calls -`self._sandboxes[task_id]`. Once `_sandboxes` is an instance dict, this is -correct: the caller (a worker's `execute()`) holds the manager instance that -was used to create the sandbox, so `_sandboxes[task_id]` is populated. No -code change needed here; the test fixture cleanup (`_reset_sandbox_singleton`) -is deleted since there is no singleton to reset. - -### 5.5 `sandbox_setup_fn` construction site — no change needed - -`sandbox_setup.py:57`: - -```python -sandbox_manager = manager_cls() -``` - -This already constructs a fresh instance per invocation. After removing -`__new__`, the call is identical but now returns a genuinely fresh object -rather than the cached singleton. No code change. - -### 5.6 Worker construction sites — no change needed - -`minif2f_react_worker.py:111`, `swebench_worker.py:123`, -`researcher_worker.py:74`, `stub_worker.py:78`, `criterion.py:72` all do -`ManagerClass()`. After this RFC, those calls produce fresh instances. If a -worker calls `get_sandbox(task_id)` on a fresh instance, the dict will be -empty (reconnect path needed). Stage 3 addresses the criterion path -(`criterion_runtime.py`). For workers that call `get_sandbox` in `execute()`: -the sandbox was created by `sandbox_setup_fn` in a **different** Inngest step, -so the worker's freshly constructed manager has no entry. Workers must either: - -- Receive the manager instance via DI (prerequisite: - `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`), or -- Call `reconnect(context.sandbox_id)` directly. - -**Stage 3 must update these worker sites.** Until then, keep the singleton -(Stage 1 only) to avoid a regression. Stage 2 (drop singleton) gates on all -worker sites being updated. - ---- - -## 6. Implementation order - -Phased into three PRs to avoid a flag day. - -### Phase 1 — Extract to instance-level (keep singleton) - -**PR 1: "sandbox-manager: instance-state migration (singleton preserved)"** - -| Step | What | Files touched | -|---|---|---| -| 1 | Move the six class-level dicts into `__init__` as instance attrs; set `_event_sink` cleanly in `__init__` | `manager.py` | -| 2 | Keep `__new__` in place (singleton preserved) | `manager.py` | -| 3 | Remove the vestigial `_event_sink: SandboxEventSink = NoopSandboxEventSink()` class-level assignment at `manager.py:83` | `manager.py` | -| 4 | Add test: construct same subclass twice; assert both dicts are the same object (singleton still) and are initialized | `tests/state/test_sandbox_manager_instance_state.py` (ADD) | -| 5 | Add test: assert `BaseSandboxManager` has no class-level mutable dicts (inspects `__dict__` and class `__dict__`) — fails if any reappear | `tests/state/test_sandbox_manager_instance_state.py` | -| 6 | Delete the `_reset_sandbox_singleton` autouse fixture from `tests/minif2f/test_sandbox_manager.py` and `tests/swebench_verified/test_sandbox_manager.py` — replace with normal per-test construction | `tests/minif2f/test_sandbox_manager.py`, `tests/swebench_verified/test_sandbox_manager.py` | - -Behavior unchanged. Stage 1 is a pure refactor. - -### Phase 2 — Remove singleton - -**PR 2: "sandbox-manager: remove __new__ singleton" (depends on PR 1)** - -| Step | What | Files touched | -|---|---|---| -| 7 | Delete `__new__` from `BaseSandboxManager` | `manager.py` | -| 8 | Delete `_instance: ClassVar` from `BaseSandboxManager` | `manager.py` | -| 9 | Update docstrings in architecture doc (Section 2.2 `03_providers.md`) — remove singleton-per-subclass language | `docs/architecture/03_providers.md` | -| 10 | Add `reconnect(sandbox_id: str) -> AsyncSandbox` to `BaseSandboxManager` | `manager.py` | -| 11 | Add unit test: `reconnect` calls `AsyncSandbox.connect` with correct `sandbox_id` and `api_key`; mock `AsyncSandbox` | `tests/state/test_sandbox_manager_instance_state.py` | -| 12 | Add multi-run isolation test: two manager instances with overlapping `task_id` values do not share sandbox state | `tests/state/test_sandbox_manager_instance_state.py` | - -### Phase 3 — Update criterion path - -**PR 3: "sandbox-manager: criterion reconnect via sandbox_id" (depends on PR 2)** - -| Step | What | Files touched | -|---|---|---| -| 13 | Add `sandbox_id: str | None = None` to `CriterionContext` if absent | `ergon_core/ergon_core/core/runtime/evaluation/evaluation_schemas.py` | -| 14 | Update `DefaultCriterionRuntime.ensure_sandbox()` to call `reconnect` when `get_sandbox` returns None and `context.sandbox_id` is set | `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | -| 15 | Update worker sites that call `get_sandbox` in `execute()`: replace with `reconnect(context.sandbox_id)` or receive manager via DI | `minif2f_react_worker.py`, `researcher_worker.py`, `stub_worker.py` (MODIFY) | -| 16 | Update `swebench_verified/criterion.py:72-78` (`_spawn_eval_sandbox`) to construct a standalone fresh manager (already does this — verify no class-level state bleed) | `swebench_verified/criterion.py` | -| 17 | Update `docs/architecture/03_providers.md` Section 2.2 — replace "in-process reconnect via class state" language with cross-process reconnect contract | `docs/architecture/03_providers.md` | -| 18 | Close `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` — move to `fixed/` | `docs/bugs/` | -| 19 | Integration test: criterion running in a simulated second-process context reconnects via `sandbox_id` and executes a command | `tests/state/test_sandbox_manager_instance_state.py` | - ---- - -## 7. File map - -### ADD - -| File | Purpose | -|---|---| -| `ergon/tests/state/test_sandbox_manager_instance_state.py` | Unit + integration tests: instance isolation, singleton removal, reconnect, multi-run isolation | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon/ergon_core/ergon_core/core/sandbox/manager.py` | Stage 1: move six dicts to `__init__`, fix `_event_sink` init; Stage 2: remove `__new__` + `_instance`, add `reconnect()` | -| `ergon/ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` | Stage 3: update `ensure_sandbox()` to use `reconnect()` on cross-process miss | -| `ergon/ergon_core/ergon_core/core/runtime/evaluation/evaluation_schemas.py` | Stage 3: add `sandbox_id: str \| None = None` to `CriterionContext` if absent | -| `ergon/ergon_builtins/ergon_builtins/workers/baselines/minif2f_react_worker.py` | Stage 3: replace `manager.get_sandbox(context.task_id)` with `reconnect` or DI | -| `ergon/ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` | Stage 3: same | -| `ergon/ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py` | Stage 3: same | -| `ergon/tests/minif2f/test_sandbox_manager.py` | Stage 1: remove `_reset_sandbox_singleton` autouse fixture | -| `ergon/tests/swebench_verified/test_sandbox_manager.py` | Stage 1: same | -| `ergon/docs/architecture/03_providers.md` | Stage 2+3: remove singleton-per-subclass language; update Section 2.2, 3, 4, 5.2, 6 | - ---- - -## 8. Testing approach - -### 8.1 Unit tests — `test_sandbox_manager_instance_state.py` - -```python -# ergon/tests/state/test_sandbox_manager_instance_state.py - -"""Tests for sandbox manager instance-state migration (RFC 2026-04-18-process-state).""" - -from __future__ import annotations - -import asyncio -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import pytest - -from ergon_core.core.sandbox.manager import BaseSandboxManager - - -class _MinimalManager(BaseSandboxManager): - """Concrete subclass for testing — no deps.""" - - async def _install_dependencies(self, sandbox, task_id): # noqa: ANN001, ARG002 - pass - - -class TestInstanceIsolation: - """After Stage 1: two instances of the same subclass must not share state.""" - - def test_two_instances_have_independent_sandboxes_dicts(self) -> None: - m1 = _MinimalManager() - m2 = _MinimalManager() - # After Stage 2 (singleton removed), these are distinct objects. - # After Stage 1 (singleton preserved), same object — skip in Stage 1 run. - assert m1._sandboxes is not m2._sandboxes or m1 is m2 # noqa: S101 - - def test_sandbox_added_to_one_not_visible_in_other(self) -> None: - m1 = _MinimalManager() - m2 = _MinimalManager() - if m1 is m2: - pytest.skip("singleton still present — Stage 1 only") - fake_sb = MagicMock() - task_id = uuid4() - m1._sandboxes[task_id] = fake_sb - assert m2.get_sandbox(task_id) is None - - def test_no_class_level_mutable_dicts(self) -> None: - """BaseSandboxManager class dict must not contain mutable dicts.""" - class_dict = vars(BaseSandboxManager) - mutable_class_attrs = { - k: v - for k, v in class_dict.items() - if isinstance(v, dict) and k.startswith("_") and k != "__dict__" - } - assert mutable_class_attrs == {}, ( - f"Found class-level mutable dicts: {list(mutable_class_attrs.keys())}" - ) - - def test_event_sink_initialized_in_init(self) -> None: - from ergon_core.core.sandbox.event_sink import NoopSandboxEventSink - m = _MinimalManager() - assert isinstance(m._event_sink, NoopSandboxEventSink) - - def test_custom_event_sink_set_without_stomp(self) -> None: - from ergon_core.core.sandbox.event_sink import NoopSandboxEventSink - sink_a = NoopSandboxEventSink() - sink_b = NoopSandboxEventSink() - m1 = _MinimalManager(event_sink=sink_a) - m2 = _MinimalManager(event_sink=sink_b) - if m1 is m2: - pytest.skip("singleton still present — stomp still possible in Stage 1") - assert m1._event_sink is sink_a - assert m2._event_sink is sink_b - - -class TestMultiRunIsolation: - """Cross-run isolation: two manager instances with overlapping task_ids.""" - - def test_overlapping_task_ids_independent(self) -> None: - m1 = _MinimalManager() - m2 = _MinimalManager() - if m1 is m2: - pytest.skip("singleton still present") - task_id = uuid4() # same UUID on both instances - fake_sb1 = MagicMock() - fake_sb2 = MagicMock() - m1._sandboxes[task_id] = fake_sb1 - m2._sandboxes[task_id] = fake_sb2 - assert m1.get_sandbox(task_id) is fake_sb1 - assert m2.get_sandbox(task_id) is fake_sb2 - - -class TestReconnect: - """Stage 2: reconnect() calls AsyncSandbox.connect with correct args.""" - - @pytest.mark.asyncio - async def test_reconnect_calls_connect(self, monkeypatch: pytest.MonkeyPatch) -> None: - from ergon_core.core.sandbox import manager as mgr_module - - fake_sandbox = MagicMock() - fake_connect = AsyncMock(return_value=fake_sandbox) - fake_async_sandbox = MagicMock() - fake_async_sandbox.connect = fake_connect - monkeypatch.setattr(mgr_module, "AsyncSandbox", fake_async_sandbox) - monkeypatch.setattr(mgr_module.settings, "e2b_api_key", "test-key") - - m = _MinimalManager() - result = await m.reconnect("sbx-xyz-999") - - fake_connect.assert_awaited_once_with("sbx-xyz-999", api_key="test-key") - assert result is fake_sandbox - - @pytest.mark.asyncio - async def test_reconnect_raises_when_e2b_not_installed( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from ergon_core.core.sandbox import manager as mgr_module - - monkeypatch.setattr(mgr_module, "AsyncSandbox", None) - - m = _MinimalManager() - with pytest.raises(RuntimeError, match="e2b_code_interpreter is not installed"): - await m.reconnect("sbx-xyz-999") - - @pytest.mark.asyncio - async def test_reconnect_does_not_cache_result( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """reconnect() must not populate self._sandboxes (stateless by design).""" - from ergon_core.core.sandbox import manager as mgr_module - - fake_sandbox = MagicMock() - fake_connect = AsyncMock(return_value=fake_sandbox) - monkeypatch.setattr(mgr_module, "AsyncSandbox", MagicMock(connect=fake_connect)) - monkeypatch.setattr(mgr_module.settings, "e2b_api_key", "key") - - m = _MinimalManager() - await m.reconnect("sbx-001") - assert len(m._sandboxes) == 0 - - -class TestEnsureSandboxCrossProcess: - """Stage 3: DefaultCriterionRuntime.ensure_sandbox() uses reconnect.""" - - @pytest.mark.asyncio - async def test_ensure_sandbox_reconnects_when_get_sandbox_returns_none( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from ergon_core.core.runtime.evaluation.criterion_runtime import ( - DefaultCriterionRuntime, - ) - from ergon_core.core.runtime.evaluation.evaluation_schemas import CriterionContext - - fake_sandbox = MagicMock() - fake_reconnect = AsyncMock(return_value=fake_sandbox) - fake_manager = MagicMock() - fake_manager.get_sandbox = MagicMock(return_value=None) - fake_manager.reconnect = fake_reconnect - - ctx = CriterionContext( - run_id=uuid4(), - task_id=uuid4(), - sandbox_id="sbx-from-db", - ) - runtime = DefaultCriterionRuntime(context=ctx, sandbox_manager=fake_manager) - await runtime.ensure_sandbox() - - fake_reconnect.assert_awaited_once_with("sbx-from-db") - assert not runtime._owns_sandbox -``` - -### 8.2 Regression tests - -Existing tests in `tests/minif2f/test_sandbox_manager.py` and -`tests/swebench_verified/test_sandbox_manager.py` must continue to pass. The -`_reset_sandbox_singleton` autouse fixtures are deleted in Stage 1; tests -that previously needed them now work naturally because each test constructs -a fresh instance. - -After Stage 2, add a regression test that `BaseSandboxManager` has no `__new__` -override (inspect `BaseSandboxManager.__new__ is object.__new__`). - -### 8.3 Multi-run isolation test (critical) - -The most important behavioral test: two concurrent runs must not share sandbox -entries. Covered by `TestMultiRunIsolation.test_overlapping_task_ids_independent` -above. This test skips under Stage 1 and passes under Stage 2. - ---- - -## 9. Trace / observability impact - -### 9.1 Spans - -No span schema change. The `sandbox.setup` span emitted at `sandbox_setup.py:76-90` -already records `sandbox_id` as an attribute. After this RFC, that `sandbox_id` -is the durable reconnect key. - -### 9.2 Logs - -Add one `logger.debug` to `reconnect()`: - -```python -logger.debug( - "sandbox.reconnect sandbox_id=%s", - sandbox_id, -) -``` - -This distinguishes reconnect calls from create calls in log output during -Stage 3 rollout, when both paths coexist. - -### 9.3 Metrics - -No new metrics. The `sandbox.setup` span attributes are sufficient for -observing the lifecycle transition. If reconnect latency becomes a concern, -add a `sandbox.reconnect` span mirroring `sandbox.setup`. - ---- - -## 10. Risks and mitigations - -| Risk | Likelihood | Impact | Mitigation | -|---|---|---|---| -| Worker `execute()` calls `get_sandbox()` on fresh instance — returns None | High (Stage 2) | Task fails at first `sandbox.commands.run` call | Stage 2 gates on worker sites updated to use `reconnect` or DI. Stage 1 keeps singleton — no regression. | -| `AsyncSandbox.connect` fails if sandbox expired | Medium | Criterion or worker fails with `RuntimeError` | E2B 30-minute idle timeout is the canonical safety net. `reconnect` raises `RuntimeError`; callers should surface it with the `sandbox_id` for diagnosis. | -| Stage 2 removes singleton before all callers updated | Medium | Silent `None` return from `get_sandbox` | PR sequence enforced: PR 2 must list updated callers in PR description. CI test `test_no_class_level_mutable_dicts` catches regression. | -| `CriterionContext` missing `sandbox_id` field | Low (field may already exist) | Stage 3 `ensure_sandbox` cannot pass `sandbox_id` to `reconnect` | Verify `evaluation_schemas.py` before Stage 3 PR. Add field if absent. | -| Subclass `__init__` forgets `super().__init__()` | Low | Instance dicts not initialized; `AttributeError` on first `create()` | Add `test_no_class_level_mutable_dicts` which instantiates all registered subclasses and checks `_sandboxes` is an instance attribute. | -| Test suite assumes singleton reset via `_reset_sandbox_singleton` fixture | Low | Flaky tests after Stage 1 fixture deletion | Delete fixture in Stage 1 PR; confirm all tests pass before merging. | - ---- - -## 11. Invariants affected - -### 11.1 `docs/architecture/03_providers.md` — invariants to rewrite - -**Section 2.2 (Sandbox managers) — changes:** - -- Remove: "singleton per subclass" — `BaseSandboxManager` is both abstract and a - **singleton per subclass**. -- Remove: "This works only because all actors run inside the same Python process." -- Add: "One instance per `sandbox_setup_fn` invocation. Instance-level dicts - replace class-level shared state. No `__new__` caching." -- Add: "`reconnect(sandbox_id)` provides cross-process access to a running sandbox - given only the `sandbox_id` string persisted on the `TaskExecution` row." -- Remove footnote: "event_sink stomp" (Section 2.2 last paragraph). - -**Section 3 (Control flow diagram) — changes:** - -Replace: - -``` -+-> ManagerClass() (singleton; returns cached instance) -| ManagerClass().create(sandbox_key=task_id, run_id=run_id, ...) -``` - -With: - -``` -+-> sandbox_manager = ManagerClass() (fresh instance per setup invocation) -| sandbox_manager.create(task_id=task_id, run_id=run_id, ...) -``` - -And replace: - -``` -criteria reconnect via ManagerClass().get_sandbox(task_id) - (works because singleton + shared class state; - cross-process criteria spawn a fresh sandbox instead) -``` - -With: - -``` -criteria reconnect via manager.reconnect(sandbox_id) - (E2B API call; works cross-process) -``` - -**Section 4 (Invariants) — changes:** - -- Invariant 3 currently: "Singleton managers hold authoritative sandbox state." - Replace with: "**Manager instances own per-invocation sandbox state.** No - class-level mutable dicts. Cross-process reconnect uses `reconnect(sandbox_id)` - against the E2B API." -- Section 4.1 "Known limits" — remove "Constructor `event_sink` stomp" and - "Class-dict unbounded growth" bullets. - Add: "**Memory:** each `BaseSandboxManager` instance holds refs to its sandboxes - for its lifetime. After `terminate()`, all refs are released." - -**Section 5.2 (Add a new sandbox manager) — changes:** - -Step 4: replace "Treat the class as a singleton — re-instantiation is how callers -acquire the cached instance" with "Construct one instance per run in -`sandbox_setup_fn`; pass it through context or DI to workers and criteria." - -Step 5: replace "Do NOT pass `event_sink=` at construction today; the stomp -described in Section 2.2 makes that unsafe" with "Pass `event_sink=` at -construction safely; the stomp is eliminated." - -**Section 6 (Anti-patterns) — changes:** - -- Remove: "Constructing an `AsyncSandbox` directly from worker or criterion code." - Replace: keep the anti-pattern but add "use `reconnect(sandbox_id)` to acquire - a cross-process handle; the manager still owns template pinning and teardown." -- Remove: "Passing `event_sink=` at manager construction." (stomp no longer exists) - ---- - -## 12. Alternatives considered - -### Keep the singleton, wrap class-level dicts in a bounded LRU - -Rejected. Hides the coupling problem behind an eviction policy and still cannot -scale out of one process. Evicting an entry under load would silently break a -running task. The root cause — shared mutable class state — is not addressed. - -### Leave as-is and document the single-process assumption forever - -Rejected. The assumption is load-bearing for every rollout path; codifying it -forecloses future scale-out. Ergon is research-grade today but should not -hard-lock itself in with a design that is actively wrong under the most likely -scaling path (multi-replica Inngest). - -### Move reconnect into criterion runtime only, keep manager as singleton - -Rejected. Partial fix: the unbounded-growth and invisible-coupling problems -remain for every non-criterion code path. The `_reset_sandbox_singleton` test -fixture would still be needed. - -### Thread manager instance through `WorkerContext` - -A reasonable approach for Stage 3. Workers receive `WorkerContext` in -`execute()` and could receive a `sandbox_manager` field on it. Deferred: this -couples `WorkerContext` (an `ergon_core` public API type) to -`BaseSandboxManager`, which is also in `ergon_core` but has E2B as an optional -dep. The DI path in -`docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` is the right -vehicle for this — Stage 3 should coordinate with that RFC. - ---- - -## 13. Open questions - -- Should `reconnect` cache the rehydrated `AsyncSandbox` on the instance, or - return a fresh handle each call? Lean stateless for cross-process safety; - revisit if E2B rate limits bite. -- Do long-running RL training loops want a manager pool keyed by run_id, or - is per-run construction cheap enough? Defer until measured. -- Stage 3 coordination with - `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md`: that RFC - exposes `get_sandbox()` through `CriterionRuntime`. After this RFC, - `get_sandbox` becomes an in-process-only convenience; the primary - cross-process entry point is `reconnect`. The DI container RFC should - expose `reconnect` or accept a pre-connected `AsyncSandbox` handle. - ---- - -## 14. On acceptance - -- Move this RFC to `accepted/`. -- Update `docs/architecture/03_providers.md` per Section 11 above — remove - singleton-per-subclass paragraphs (`03_providers.md` Section 2.2, line 32 - and line 43–44), Section 3 diagram, Section 4 invariant 3, Section 4.1 - known-limits bullets, Section 5.2 step 4–5, Section 6 anti-patterns. -- Close `docs/bugs/open/2026-04-18-sandbox-manager-shared-state-race.md` — per - the new invariant, instances cannot race on shared class-level dicts. Move - to `docs/bugs/fixed/`. -- Address TODO at `manager.py:74-77` referencing this RFC; the comment block - is deleted along with `__new__`. -- Link the implementation plan under `docs/superpowers/plans/`. -- Coordinate Stage 3 timing with - `docs/rfcs/active/2026-04-17-criterion-runtime-di-container.md` — both touch - `criterion_runtime.py`. diff --git a/docs/rfcs/active/2026-04-18-template-spec-public-api.md b/docs/rfcs/active/2026-04-18-template-spec-public-api.md deleted file mode 100644 index 16a9ea745..000000000 --- a/docs/rfcs/active/2026-04-18-template-spec-public-api.md +++ /dev/null @@ -1,1049 +0,0 @@ ---- -status: active -opened: 2026-04-18 -author: architecture-qa -architecture_refs: [docs/architecture/06_builtins.md#extension-points, docs/architecture/01_public_api.md] -supersedes: [] -superseded_by: null ---- - -# RFC: `TemplateSpec` as a public API on the `Benchmark` ABC - -## Relationship to sibling RFC - -`docs/rfcs/active/2026-04-18-onboarding-deps-on-benchmark-abc.md` moves -`BENCHMARK_DEPS` onto `Benchmark` as `onboarding_deps: ClassVar[BenchmarkDeps]` -using the same pattern this RFC uses for `template_spec`. Both RFCs should land -in the same PR or in consecutive PRs targeting the same migration checklist: if -`onboarding-deps` lands first, the nine benchmark migrations it requires are the -same nine benchmarks that need `template_spec` here. Duplicate that sweep once, -not twice. The shared migration step is explicit in the Implementation order -(§9) below. Neither RFC supersedes the other; they are additive class-level -attributes on the same ABC. - ---- - -## Problem - -Sandbox-template setup is implicit today. Every benchmark that needs a sandbox -handles setup a different way, and the variation is bespoke and undiscoverable: - -1. **`minif2f`** — `MiniF2FSandboxManager.__init__` calls - `sandbox/utils.py:resolve_template()` which reads - `~/.ergon/sandbox_templates.json["minif2f"]["template_id"]` and falls back to - the mutable template name `ergon-minif2f-v1` hardcoded in - `ergon_builtins/benchmarks/minif2f/sandbox/utils.py:13`. - `ergon_builtins/benchmarks/minif2f/sandbox/e2b.toml.template` declares - `dockerfile = "Dockerfile"` and `cpu_count = 2, memory_mb = 8192`. - `MiniF2FSandboxManager._install_dependencies` is a no-op (line 78–79) because - the template has elan + Lean 4 + mathlib4 baked in. - -2. **`swebench-verified`** — `SWEBenchSandboxManager.__init__` calls - `sandbox/utils.py:resolve_template()` with `DEFAULT_TEMPLATE_NAME = - "ergon-swebench-v1"` and `REGISTRY_SLUG = "swebench-verified"` in - `ergon_builtins/benchmarks/swebench_verified/sandbox/utils.py:13–15`. - `_install_dependencies` is a no-op (line 59–62). Template declared in - `sandbox/e2b.toml.template`: `cpu_count = 4, memory_mb = 8192`. - -3. **`gdpeval`** — No pre-built template. `GDPEvalSandboxManager._install_dependencies` - (in `ergon_builtins/benchmarks/gdpeval/sandbox.py:28–41`) installs - `pdfplumber PyPDF2 reportlab pytesseract` at sandbox-prep time via - `pip install`. The `sandbox_utils.py` file provides download helpers - (unrelated to template setup). There is no `e2b.toml.template` and no - entry in `SANDBOX_TEMPLATES` for `gdpeval`. - -4. **`smoke-test`** — `SmokeTestBenchmark` in - `ergon_builtins/benchmarks/smoke_test/benchmark.py` has no sandbox - at all. No `SandboxManager` entry in `SANDBOX_MANAGERS` or `SANDBOX_TEMPLATES` - for `smoke-test`. - -5. **`delegation-smoke`** — `DelegationSmokeBenchmark` in - `ergon_builtins/benchmarks/delegation_smoke/benchmark.py` has no sandbox. - No entry in `SANDBOX_MANAGERS` or `SANDBOX_TEMPLATES`. - -6. **`researchrubrics-smoke`** — `ResearchRubricsSmokeTestBenchmark` in - `ergon_builtins/benchmarks/researchrubrics/smoke.py`. No sandbox. No - entry in `SANDBOX_MANAGERS` or `SANDBOX_TEMPLATES`. - -7. **`researchrubrics`** — `ResearchRubricsBenchmark` in - `ergon_builtins/benchmarks/researchrubrics/benchmark.py`. No sandbox. - No entry in `SANDBOX_MANAGERS` or `SANDBOX_TEMPLATES`. - -8. **`researchrubrics-ablated`** — Registered in `registry_data.py:24` as - `ResearchRubricsBenchmark` (same class as `researchrubrics`, different slug). - No sandbox. No entry in `SANDBOX_MANAGERS` or `SANDBOX_TEMPLATES`. - -9. **`researchrubrics-vanilla`** — `ResearchRubricsVanillaBenchmark` in - `ergon_builtins/benchmarks/researchrubrics/vanilla.py` subclasses - `ResearchRubricsBenchmark`. No sandbox. No entry in `SANDBOX_MANAGERS` or - `SANDBOX_TEMPLATES`. - -The `ergon benchmark setup <slug>` path in -`ergon_cli/ergon_cli/commands/benchmark.py:60–175` dispatches by looking up -`SANDBOX_TEMPLATES[slug]` (the dict in `registry_core.py:90–93`). That dict -is hardcoded to `minif2f` and `swebench-verified`. Any new benchmark that needs -a template must remember to edit `SANDBOX_TEMPLATES`; there is no public API -that declares setup requirements. A contributor who forgets has no signal from -the system — the benchmark silently fails at first run with an opaque sandbox -error. - -There is no declared shape for "this benchmark has no template" vs. "this -benchmark ships a pre-built template ID" vs. "this benchmark installs deps at -sandbox startup". The architecture doc -(`docs/architecture/06_builtins.md:124–126`) already calls this out as an open -problem under "Template setup". - ---- - -## Proposal - -Introduce `TemplateSpec` as a public Pydantic model in `ergon_core/api/` and -require every concrete `Benchmark` subclass to declare a `template_spec` class -variable. The `NoSetup` sentinel forces the author to make an explicit -declaration even when no setup is needed. - -### `TemplateSpec` model - -```python -# ergon_core/ergon_core/api/template_spec.py - -from __future__ import annotations - -from pathlib import Path - -from pydantic import BaseModel - - -class TemplateSpec(BaseModel, frozen=True): - """Declarative description of a benchmark's sandbox-template setup. - - A Benchmark subclass sets ``template_spec`` to either a ``TemplateSpec`` - describing how its sandbox is prepared, or to the ``NoSetup`` sentinel to - declare intentionally that no template setup is required. - - Fields are intentionally additive: a benchmark may set ``e2b_template_id`` - (pre-built), ``build_recipe_path`` (buildable), ``runtime_install`` - (installed at sandbox-prep time), or any combination. - """ - - e2b_template_id: str | None = None - """Pre-built E2B template name or pinned template_id. - - When set, ``ergon benchmark setup <slug>`` verifies the template exists in - the user's E2B account and prints rebuild instructions if absent. - - Example: ``"ergon-minif2f-v1"`` - """ - - build_recipe_path: Path | None = None - """Path to the Dockerfile or setup directory that ``ergon benchmark setup`` - uses to build the E2B template. - - Typically ``Path(__file__).parent / "sandbox"`` pointing to the - per-benchmark ``sandbox/`` folder (which must contain a ``Dockerfile`` and - an ``e2b.toml.template``). - - When set alongside ``e2b_template_id``, the setup command can rebuild the - template from this recipe. - """ - - runtime_install: tuple[str, ...] = () - """pip package specifiers installed at sandbox-prep time. - - Strings are passed verbatim to ``pip install``; extras markers such as - ``"foo[bar]==1.2.3"`` are supported. - - When non-empty and ``build_recipe_path`` is None, ``ergon benchmark setup`` - prints a note that setup is deferred to sandbox prep (no build step is - needed). - """ -``` - -### `NoSetup` sentinel - -```python -# ergon_core/ergon_core/api/template_spec.py (continued) - -from typing import TypeAlias - - -class _NoSetupType: - """Singleton sentinel: this benchmark has no template setup requirements. - - Use the pre-constructed ``NoSetup`` instance, not this class directly. - """ - - _instance: _NoSetupType | None = None - - def __new__(cls) -> _NoSetupType: - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def __repr__(self) -> str: - return "NoSetup" - - -NoSetup: _NoSetupType = _NoSetupType() -NoSetupSentinel: TypeAlias = _NoSetupType -``` - -### `Benchmark` ABC change - -`template_spec` is declared without a default. A concrete subclass that omits -it will raise `AttributeError` on attribute access — the same early failure -mode used by unimplemented abstract methods. No `= None` default. The system -cannot notice a `None` default; it can notice an `AttributeError`. - -```python -# ergon_core/ergon_core/api/benchmark.py (modified) - -from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence -from typing import Any, ClassVar - -from ergon_core.api.dependencies import check_packages -from ergon_core.api.errors import DependencyError -from ergon_core.api.task_types import BenchmarkTask -from ergon_core.api.template_spec import NoSetupSentinel, TemplateSpec - - -class Benchmark(ABC): - """Base class for all benchmarks. - - Subclasses must set ``type_slug``, ``template_spec``, and implement - ``build_instances``. - """ - - type_slug: ClassVar[str] - template_spec: ClassVar[TemplateSpec | NoSetupSentinel] - required_packages: ClassVar[list[str]] = [] - install_hint: ClassVar[str] = "" - - def __init__( - self, - *, - name: str | None = None, - description: str | None = None, - metadata: Mapping[str, Any] | None = None, # slopcop: ignore[no-typing-any] - ) -> None: - self.name = name or self.__class__.__name__ - self.description = description or "" - self.metadata: dict[str, Any] = dict(metadata or {}) # slopcop: ignore[no-typing-any] - - @abstractmethod - def build_instances(self) -> Mapping[str, Sequence[BenchmarkTask]]: - """Materialize benchmark instances. - - Returns a mapping of instance_key -> tasks for that instance. - """ - ... - - def evaluator_requirements(self) -> Sequence[str]: - """Declare evaluator slot names required by this benchmark.""" - return ("default",) - - def validate(self) -> None: - """Check that runtime dependencies are available.""" - errors = check_packages( - self.required_packages, - f"Benchmark '{self.type_slug}'", - ) - if errors: - parts = [*errors] - if self.install_hint: - parts.append(f"Install with: {self.install_hint}") - raise DependencyError("\n".join(parts)) -``` - -### `ergon_core/api/__init__.py` additions - -```python -# ergon_core/ergon_core/api/__init__.py (additions only — full file not shown) - -from ergon_core.api.template_spec import NoSetup, NoSetupSentinel, TemplateSpec - -__all__ = [ - # ... existing exports unchanged ... - "NoSetup", - "NoSetupSentinel", - "TemplateSpec", -] -``` - -### `ergon benchmark setup <slug>` dispatch (rewritten) - -The CLI no longer looks up `SANDBOX_TEMPLATES`. It reads the benchmark class's -`template_spec` attribute and dispatches accordingly. - -```python -# ergon_cli/ergon_cli/commands/benchmark.py setup_benchmark() — full replacement - -def setup_benchmark(args: Namespace) -> int: - """Build and register the E2B sandbox template for *args.slug*. - - Dispatches off ``Benchmark.template_spec`` instead of the hardcoded - ``SANDBOX_TEMPLATES`` dict. No explicit dispatch table is needed; the spec - carries all information. - """ - # reason: deferred to avoid pulling heavy ergon_builtins deps at CLI startup - from ergon_builtins.registry import BENCHMARKS - - from ergon_core.api.template_spec import NoSetup, TemplateSpec - - slug: str = args.slug - force: bool = args.force - - # 1. Look up benchmark class - if slug not in BENCHMARKS: - available = ", ".join(sorted(BENCHMARKS)) or "(none)" - return _fail(f"Error: unknown benchmark slug '{slug}'.\nAvailable slugs: {available}") - - benchmark_cls = BENCHMARKS[slug] - - # 2. Read template_spec - spec = benchmark_cls.template_spec - - # 3. NoSetup sentinel — nothing to do - if isinstance(spec, NoSetup.__class__): - print(f"Benchmark '{slug}' declares NoSetup: no template build required.") - return 0 - - if not isinstance(spec, TemplateSpec): - return _fail( - f"Error: '{slug}'.template_spec is neither TemplateSpec nor NoSetup. " - f"Got: {spec!r}" - ) - - # 4. runtime_install only — deferred to sandbox prep; no build step needed - if spec.runtime_install and spec.build_recipe_path is None and spec.e2b_template_id is None: - pkgs = ", ".join(spec.runtime_install) - print( - f"Benchmark '{slug}' installs packages at sandbox-prep time ({pkgs}). " - "No template build required." - ) - return 0 - - # 5. E2B template required - if not settings.e2b_api_key: - return _fail( - "Error: E2B_API_KEY is not set.\n" - "Export your E2B API key before running this command:\n" - " export E2B_API_KEY=<your-key>\n" - "Get a key at https://e2b.dev/dashboard" - ) - - # 6. No build recipe — verify-only path - if spec.build_recipe_path is None: - print( - f"Benchmark '{slug}' references E2B template '{spec.e2b_template_id}' " - "but declares no build_recipe_path. Cannot rebuild automatically.\n" - f"Ensure template '{spec.e2b_template_id}' exists in your E2B account." - ) - return 0 - - template_dir = spec.build_recipe_path - - # 7. Load e2b.toml.template from the recipe directory - template_spec_path = template_dir / "e2b.toml.template" - if not template_spec_path.exists(): - return _fail(f"Error: template spec not found at {template_spec_path}") - - with open(template_spec_path, "rb") as f: - toml_spec = tomllib.load(f) - - template_name = toml_spec.get("template_name") or spec.e2b_template_id - if not template_name: - return _fail( - f"Error: no template_name in {template_spec_path} and " - "no e2b_template_id on TemplateSpec." - ) - - # 8. Idempotency check - config = _config_dir() - registry_path = config / "sandbox_templates.json" - - existing_templates: dict[str, object] = {} - if registry_path.exists(): - with open(registry_path) as f: - existing_templates = json.load(f) - - if not force and slug in existing_templates: - tid = existing_templates[slug].get("template_id", "unknown") # type: ignore[union-attr] - print(f"Template already built: {tid}. Use --force to rebuild.") - return 0 - - # 9. Build via E2B SDK - cpu_count = int(toml_spec.get("cpu_count", 2)) - memory_mb = int(toml_spec.get("memory_mb", 8192)) - start_cmd = toml_spec.get("start_cmd", "/bin/bash") - dockerfile_name = toml_spec.get("dockerfile", "Dockerfile") - dockerfile_path = template_dir / dockerfile_name - - if not dockerfile_path.exists(): - return _fail(f"Error: Dockerfile not found at {dockerfile_path}") - - dockerfile_content = dockerfile_path.read_text() - - print(f"Building E2B template '{template_name}' from {template_dir} ...") - print(f" cpu_count={cpu_count}, memory_mb={memory_mb}") - - def _on_build_logs(log: object) -> None: - print(f" [build] {log}", flush=True) - - template_def = ( - Template(file_context_path=str(template_dir)) - .from_dockerfile(dockerfile_content) - .set_start_cmd(start_cmd=start_cmd, ready_cmd="echo ready") - ) - - t0 = time.monotonic() - try: - build_info = Template.build( - template_def, - name=template_name, - cpu_count=cpu_count, - memory_mb=memory_mb, - on_build_logs=_on_build_logs, - ) - except Exception as exc: # noqa: BLE001 # slopcop: ignore[no-broad-except] - return _fail(f"Error: E2B SDK Template.build() failed: {exc}") - - build_time = round(time.monotonic() - t0, 1) - template_id = build_info.template_id - - # 10. Persist - config.mkdir(parents=True, exist_ok=True) - existing_templates[slug] = { - "template_id": template_id, - "template_name": template_name, - "build_id": build_info.build_id, - "built_at": datetime.now(timezone.utc).isoformat(), - } - with open(registry_path, "w") as f: - json.dump(existing_templates, f, indent=2) - - print(f"\nSuccess! Template ID: {template_id} (build {build_info.build_id}, {build_time}s)") - print(f"Now run: `ergon benchmark run {slug} --worker <worker> --model <model> --limit 1`") - return 0 -``` - ---- - -## Architecture overview - -### Before - -``` -ergon benchmark setup <slug> - │ - ├─ look up SANDBOX_TEMPLATES[slug] ← hardcoded dict in registry_core.py:90–93 - │ (only "minif2f" and "swebench-verified" present) - │ - ├─ read e2b.toml.template from that path - └─ build E2B template via SDK -``` - -Unknown slugs fail with "Error: unknown benchmark slug". Slugs without a -`SANDBOX_TEMPLATES` entry (gdpeval, smoke-test, researchrubrics*) also fail -even though some have no template requirement and some install deps at runtime. - -``` -SandboxManager.__init__ - └─ calls per-benchmark resolve_template() in sandbox/utils.py - └─ reads ~/.ergon/sandbox_templates.json[slug] - or falls back to hardcoded DEFAULT_TEMPLATE_NAME string -``` - -Two per-benchmark `sandbox/utils.py` files duplicate the same JSON-read logic -(minif2f:18–36, swebench_verified:18–36). - -### After - -``` -ergon benchmark setup <slug> - │ - ├─ BENCHMARKS[slug].template_spec ← ClassVar on Benchmark subclass - │ - ├─ isinstance(spec, _NoSetupType) → "nothing to do", exit 0 - │ - ├─ TemplateSpec with runtime_install only - │ and no build_recipe_path → "deferred to sandbox prep", exit 0 - │ - ├─ TemplateSpec with build_recipe_path - │ (and optional e2b_template_id) → build E2B template from recipe - │ persist to ~/.ergon/sandbox_templates.json - │ - └─ TemplateSpec with e2b_template_id only → verify template exists; print - rebuild instructions if absent -``` - -The `SandboxManager` side is unchanged in this RFC: `sandbox/utils.py` files -continue to work as-is. The `resolve_template()` helpers could be folded into a -shared utility in a follow-up once `template_spec` stabilises. - ---- - -## Type / interface definitions - -### Full `TemplateSpec` model and `_NoSetupType` sentinel - -```python -# ergon_core/ergon_core/api/template_spec.py - -from __future__ import annotations - -from pathlib import Path -from typing import TypeAlias - -from pydantic import BaseModel - - -class TemplateSpec(BaseModel, frozen=True): - """Declarative description of a benchmark's sandbox-template setup. - - A Benchmark subclass sets ``template_spec`` to either a ``TemplateSpec`` - describing how its sandbox is prepared, or to the ``NoSetup`` sentinel to - declare intentionally that no template setup is required. - - Fields are intentionally additive: a benchmark may set ``e2b_template_id`` - (pre-built), ``build_recipe_path`` (buildable), ``runtime_install`` - (installed at sandbox-prep time), or any combination. - """ - - e2b_template_id: str | None = None - """Pre-built E2B template name or pinned template_id. - - When set, ``ergon benchmark setup <slug>`` verifies the template exists in - the user's E2B account and prints rebuild instructions if absent. - """ - - build_recipe_path: Path | None = None - """Path to the Dockerfile or setup directory that ``ergon benchmark setup`` - uses to build the E2B template. - - Typically ``Path(__file__).parent / "sandbox"`` pointing to the - per-benchmark ``sandbox/`` folder containing a ``Dockerfile`` and an - ``e2b.toml.template``. - """ - - runtime_install: tuple[str, ...] = () - """pip package specifiers installed at sandbox-prep time. - - Strings are passed verbatim to ``pip install``; extras markers such as - ``"foo[bar]==1.2.3"`` are supported. When non-empty and - ``build_recipe_path`` is None, no template build step is required. - """ - - -class _NoSetupType: - """Singleton sentinel: this benchmark has no template setup requirements. - - Use the pre-constructed ``NoSetup`` instance, not this class directly. - """ - - _instance: _NoSetupType | None = None - - def __new__(cls) -> _NoSetupType: - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def __repr__(self) -> str: - return "NoSetup" - - -NoSetup: _NoSetupType = _NoSetupType() -NoSetupSentinel: TypeAlias = _NoSetupType -``` - ---- - -## Full implementations - -### New file: `ergon_core/ergon_core/api/template_spec.py` - -Full content shown in "Type / interface definitions" above. - -### Modified file: `ergon_core/ergon_core/api/benchmark.py` - -Full content shown in "Proposal" above. The only substantive change is adding -`template_spec: ClassVar[TemplateSpec | NoSetupSentinel]` with no default, and -importing the new types. - -### Modified file: `ergon_core/ergon_core/api/__init__.py` - -Add three new exports: - -```python -from ergon_core.api.template_spec import NoSetup, NoSetupSentinel, TemplateSpec -``` - -Add `"NoSetup"`, `"NoSetupSentinel"`, `"TemplateSpec"` to `__all__`. - ---- - -## Exact diffs for all nine benchmarks - -Each benchmark adds exactly one `ClassVar` declaration. - -### 1. `smoke-test` — `ergon_builtins/benchmarks/smoke_test/benchmark.py` - -```diff -+from ergon_core.api.template_spec import NoSetup, NoSetupSentinel -+ - class SmokeTestBenchmark(Benchmark): - type_slug: ClassVar[str] = "smoke-test" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup -``` - -### 2. `delegation-smoke` — `ergon_builtins/benchmarks/delegation_smoke/benchmark.py` - -```diff -+from ergon_core.api.template_spec import NoSetup, NoSetupSentinel -+ - class DelegationSmokeBenchmark(Benchmark): - type_slug: ClassVar[str] = "delegation-smoke" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup -``` - -### 3. `researchrubrics-smoke` — `ergon_builtins/benchmarks/researchrubrics/smoke.py` - -```diff -+from ergon_core.api.template_spec import NoSetup, NoSetupSentinel -+ - class ResearchRubricsSmokeTestBenchmark(Benchmark): - type_slug: ClassVar[str] = "researchrubrics-smoke" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup -``` - -### 4. `minif2f` — `ergon_builtins/benchmarks/minif2f/benchmark.py` - -```diff -+from pathlib import Path -+from ergon_core.api.template_spec import NoSetupSentinel, TemplateSpec -+ - class MiniF2FBenchmark(Benchmark): - type_slug: ClassVar[str] = "minif2f" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = TemplateSpec( -+ e2b_template_id="ergon-minif2f-v1", -+ build_recipe_path=Path(__file__).parent / "sandbox", -+ ) -``` - -Template name matches `DEFAULT_TEMPLATE_NAME` in `sandbox/utils.py:13` and -`template_name` in `sandbox/e2b.toml.template:9`. - -### 5. `swebench-verified` — `ergon_builtins/benchmarks/swebench_verified/benchmark.py` - -```diff -+from pathlib import Path -+from ergon_core.api.template_spec import NoSetupSentinel, TemplateSpec -+ - class SweBenchVerifiedBenchmark(Benchmark): - type_slug: ClassVar[str] = "swebench-verified" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = TemplateSpec( -+ e2b_template_id="ergon-swebench-v1", -+ build_recipe_path=Path(__file__).parent / "sandbox", -+ ) -``` - -Template name matches `DEFAULT_TEMPLATE_NAME` in `sandbox/utils.py:13` and -`template_name` in `sandbox/e2b.toml.template:9`. - -### 6. `gdpeval` — `ergon_builtins/benchmarks/gdpeval/benchmark.py` - -```diff -+from ergon_core.api.template_spec import NoSetupSentinel, TemplateSpec -+ - class GDPEvalBenchmark(Benchmark): - type_slug: ClassVar[str] = "gdpeval" - required_packages: ClassVar[list[str]] = ["pandas", "huggingface_hub"] - install_hint: ClassVar[str] = "pip install 'ergon-builtins[data]'" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = TemplateSpec( -+ runtime_install=( -+ "pdfplumber", -+ "PyPDF2", -+ "reportlab", -+ "pytesseract", -+ ), -+ ) -``` - -Package names match `_GDP_PACKAGES` in -`ergon_builtins/benchmarks/gdpeval/sandbox.py:19`. - -### 7. `researchrubrics` — `ergon_builtins/benchmarks/researchrubrics/benchmark.py` - -```diff -+from ergon_core.api.template_spec import NoSetup, NoSetupSentinel -+ - class ResearchRubricsBenchmark(Benchmark): - type_slug: ClassVar[str] = "researchrubrics" - required_packages: ClassVar[list[str]] = ["datasets", "huggingface_hub"] - install_hint: ClassVar[str] = "pip install 'ergon-builtins[data]'" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup -``` - -`researchrubrics` has no sandbox; all evaluation is pure Python. - -### 8. `researchrubrics-ablated` — `registry_data.py` (class alias) - -`researchrubrics-ablated` is registered in `registry_data.py:24` as -`ResearchRubricsBenchmark` directly (same class, different slug). Because the -slug is only a registry key and does not have its own class, it inherits -`template_spec = NoSetup` from `ResearchRubricsBenchmark`. No per-file change -needed — the diff is the `ResearchRubricsBenchmark` change in item 7. - -If a dedicated class is added in future, it must re-declare `template_spec`. - -### 9. `researchrubrics-vanilla` — `ergon_builtins/benchmarks/researchrubrics/vanilla.py` - -```diff - class ResearchRubricsVanillaBenchmark(ResearchRubricsBenchmark): - type_slug: ClassVar[str] = "researchrubrics-vanilla" -+ template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup -``` - -`NoSetup` is re-declared explicitly even though it inherits from -`ResearchRubricsBenchmark`, because the contract test (§10) validates per-class -directly, and the slug is distinct. Future readers should not have to chase the -inheritance chain to understand this class's setup story. - ---- - -## Package structure - -### New file: `ergon_core/ergon_core/api/template_spec.py` - -Full content is in "Type / interface definitions". No new package needed; the -file is added to the existing `ergon_core/api/` module. - -### `ergon_core/ergon_core/api/__init__.py` — additions only - -```python -# Additions to existing __all__ list and import block: -from ergon_core.api.template_spec import NoSetup, NoSetupSentinel, TemplateSpec - -# In __all__: -"NoSetup", -"NoSetupSentinel", -"TemplateSpec", -``` - -No new `__init__.py` files required. - ---- - -## Implementation order - -Phases are sized for separate PRs. Steps within a phase can land as a single -commit. - -| Step | Phase | What | Files touched | -|---|---|---|---| -| 1 | PR 1 | Create `ergon_core/ergon_core/api/template_spec.py` with `TemplateSpec`, `_NoSetupType`, `NoSetup`, `NoSetupSentinel` | ADD 1 file | -| 2 | PR 1 | Add `template_spec: ClassVar[TemplateSpec \| NoSetupSentinel]` to `Benchmark` ABC; import new types | MODIFY `ergon_core/ergon_core/api/benchmark.py` | -| 3 | PR 1 | Add `NoSetup`, `NoSetupSentinel`, `TemplateSpec` to `ergon_core/api/__init__.py` | MODIFY `ergon_core/ergon_core/api/__init__.py` | -| 4 | PR 1 | Unit tests for `TemplateSpec` (frozen, valid combos) and `NoSetup` singleton | ADD `tests/state/test_template_spec.py` | -| 5 | PR 2 | Migrate all nine benchmarks: add `template_spec` ClassVar per §7 diffs | MODIFY 8 benchmark files (researchrubrics-ablated is free via inheritance) | -| 6 | PR 2 | Add contract test: every benchmark in `registry_core.BENCHMARKS` and `registry_data.BENCHMARKS` has `template_spec` that is `TemplateSpec` or `NoSetup` | ADD `tests/state/test_benchmark_contract.py` | -| 7 | PR 3 | Rewrite `setup_benchmark()` in `ergon_cli/ergon_cli/commands/benchmark.py` to dispatch off `TemplateSpec`; remove `SANDBOX_TEMPLATES` lookup | MODIFY `ergon_cli/ergon_cli/commands/benchmark.py` | -| 8 | PR 3 | Remove `SANDBOX_TEMPLATES` from `ergon_builtins/ergon_builtins/registry_core.py` | MODIFY `ergon_builtins/ergon_builtins/registry_core.py` | -| 9 | PR 3 | Update CLI tests in `tests/cli/test_benchmark_setup.py` to exercise `TemplateSpec` dispatch and `NoSetup` path | MODIFY `tests/cli/test_benchmark_setup.py` | - -**PR ordering constraint:** PR 2 (benchmark migrations) depends on PR 1 -(`TemplateSpec` in `ergon_core/api/`). PR 3 (CLI rewrite + `SANDBOX_TEMPLATES` -removal) depends on PR 2 (all benchmarks have `template_spec`). Steps 5 and 6 -in PR 2 can be combined with the `onboarding-deps` RFC's parallel sweep of the -same nine benchmarks if both RFCs are accepted together. - ---- - -## File map - -### ADD - -| File | Purpose | -|---|---| -| `ergon_core/ergon_core/api/template_spec.py` | `TemplateSpec` model, `_NoSetupType`, `NoSetup` singleton, `NoSetupSentinel` alias | -| `tests/state/test_template_spec.py` | Unit tests: `TemplateSpec` frozen/field contract, `NoSetup` singleton, `isinstance` checks | -| `tests/state/test_benchmark_contract.py` | Contract test: every registered benchmark has a valid `template_spec` | - -### MODIFY - -| File | Changes | -|---|---| -| `ergon_core/ergon_core/api/benchmark.py` | Add `template_spec: ClassVar[TemplateSpec \| NoSetupSentinel]` with no default; add imports | -| `ergon_core/ergon_core/api/__init__.py` | Add `NoSetup`, `NoSetupSentinel`, `TemplateSpec` to imports and `__all__` | -| `ergon_builtins/ergon_builtins/benchmarks/smoke_test/benchmark.py` | Add `template_spec = NoSetup` | -| `ergon_builtins/ergon_builtins/benchmarks/delegation_smoke/benchmark.py` | Add `template_spec = NoSetup` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/smoke.py` | Add `template_spec = NoSetup` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` | Add `template_spec = NoSetup` | -| `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/vanilla.py` | Add `template_spec = NoSetup` (explicit re-declaration) | -| `ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py` | Add `template_spec = TemplateSpec(e2b_template_id="ergon-minif2f-v1", build_recipe_path=...)` | -| `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py` | Add `template_spec = TemplateSpec(e2b_template_id="ergon-swebench-v1", build_recipe_path=...)` | -| `ergon_builtins/ergon_builtins/benchmarks/gdpeval/benchmark.py` | Add `template_spec = TemplateSpec(runtime_install=(...))` | -| `ergon_cli/ergon_cli/commands/benchmark.py` | Rewrite `setup_benchmark()` to dispatch off `template_spec` | -| `ergon_builtins/ergon_builtins/registry_core.py` | Remove `SANDBOX_TEMPLATES` dict (PR 3, after CLI is migrated) | -| `tests/cli/test_benchmark_setup.py` | Update to exercise `TemplateSpec` dispatch and `NoSetup` path | - -**Note:** `researchrubrics-ablated` has no dedicated class file; it inherits -`template_spec` from `ResearchRubricsBenchmark` via `registry_data.py:24`. - ---- - -## Testing approach - -### Unit tests — `tests/state/test_template_spec.py` - -```python -# tests/state/test_template_spec.py - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from ergon_core.api.template_spec import ( - NoSetup, - NoSetupSentinel, - TemplateSpec, - _NoSetupType, -) - - -class TestTemplateSpec: - def test_frozen_rejects_mutation(self) -> None: - spec = TemplateSpec(e2b_template_id="my-template") - with pytest.raises(Exception): # pydantic ValidationError or AttributeError - spec.e2b_template_id = "other" # type: ignore[misc] - - def test_default_all_none(self) -> None: - spec = TemplateSpec() - assert spec.e2b_template_id is None - assert spec.build_recipe_path is None - assert spec.runtime_install == () - - def test_runtime_install_tuple(self) -> None: - spec = TemplateSpec(runtime_install=("pdfplumber", "PyPDF2==3.0.0")) - assert len(spec.runtime_install) == 2 - assert "pdfplumber" in spec.runtime_install - - def test_build_recipe_path_accepts_path(self) -> None: - p = Path("/some/benchmark/sandbox") - spec = TemplateSpec(e2b_template_id="ergon-minif2f-v1", build_recipe_path=p) - assert spec.build_recipe_path == p - - def test_full_combo(self) -> None: - spec = TemplateSpec( - e2b_template_id="ergon-minif2f-v1", - build_recipe_path=Path("/fake"), - runtime_install=("lean4-extra",), - ) - assert spec.e2b_template_id == "ergon-minif2f-v1" - - -class TestNoSetupSingleton: - def test_singleton_identity(self) -> None: - a = _NoSetupType() - b = _NoSetupType() - assert a is b - assert a is NoSetup - - def test_repr(self) -> None: - assert repr(NoSetup) == "NoSetup" - - def test_isinstance_alias(self) -> None: - assert isinstance(NoSetup, NoSetupSentinel) - - def test_not_template_spec(self) -> None: - assert not isinstance(NoSetup, TemplateSpec) -``` - -### Contract test — `tests/state/test_benchmark_contract.py` - -This is the load-bearing enforcement mechanism: every registered benchmark must -declare `template_spec`. The test exercises both the core registry (always -available) and the data registry (guarded by `[data]` extra). - -```python -# tests/state/test_benchmark_contract.py - -from __future__ import annotations - -import pytest - -from ergon_core.api.template_spec import NoSetup, TemplateSpec, _NoSetupType - - -def _all_benchmark_entries(): - """Yield (slug, benchmark_cls) for all registered benchmarks.""" - from ergon_builtins.registry_core import BENCHMARKS as core_benchmarks - - for slug, cls in core_benchmarks.items(): - yield slug, cls - - try: - from ergon_builtins.registry_data import BENCHMARKS as data_benchmarks - - for slug, cls in data_benchmarks.items(): - yield slug, cls - except ImportError: - pass # ergon-builtins[data] not installed; skip data benchmarks - - -@pytest.mark.parametrize("slug,benchmark_cls", list(_all_benchmark_entries())) -def test_benchmark_has_template_spec(slug: str, benchmark_cls: type) -> None: - """Every registered benchmark must declare template_spec as TemplateSpec or NoSetup.""" - assert hasattr(benchmark_cls, "template_spec"), ( - f"Benchmark '{slug}' ({benchmark_cls.__name__}) has no 'template_spec' ClassVar. " - "Add 'template_spec: ClassVar[TemplateSpec | NoSetupSentinel] = NoSetup' " - "(or a TemplateSpec) to the class." - ) - spec = benchmark_cls.template_spec - assert isinstance(spec, (TemplateSpec, _NoSetupType)), ( - f"Benchmark '{slug}' ({benchmark_cls.__name__}).template_spec is neither " - f"TemplateSpec nor NoSetup. Got: {spec!r}" - ) -``` - -### CLI tests — `tests/cli/test_benchmark_setup.py` additions - -The existing tests in `tests/cli/test_benchmark_setup.py` test the -`SANDBOX_TEMPLATES`-based dispatch. After PR 3 they must be updated. New cases: - -```python -# New tests to add to tests/cli/test_benchmark_setup.py - -def test_nosetup_benchmark_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None: - """A benchmark with NoSetup should print a note and return 0.""" - # smoke-test declares NoSetup after migration - monkeypatch.setenv("E2B_API_KEY", "test-key") - rc = setup_benchmark(_make_args(slug="smoke-test")) - assert rc == 0 - - -def test_runtime_install_only_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None: - """A benchmark with only runtime_install should return 0 without building.""" - monkeypatch.setenv("E2B_API_KEY", "test-key") - rc = setup_benchmark(_make_args(slug="gdpeval")) - assert rc == 0 - - -def test_template_spec_build_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """minif2f has e2b_template_id + build_recipe_path: full build path runs.""" - monkeypatch.setenv("E2B_API_KEY", "test-key") - monkeypatch.setenv("ERGON_CONFIG_DIR", str(tmp_path)) - _patch_sdk(monkeypatch) - rc = setup_benchmark(_make_args(slug="minif2f")) - assert rc == 0 -``` - ---- - -## Trace / observability impact - -No new spans, logs, or metrics are introduced by this RFC. The change is -purely in the class hierarchy and CLI dispatch logic. - -One minor observability improvement is implicit: `ergon benchmark setup <slug>` -will now produce meaningful output for `NoSetup` and `runtime_install`-only -benchmarks instead of an opaque "unknown benchmark slug" error. The log surface -at INFO level is unchanged. - -If future work adds an `ergon benchmark status` command, it can iterate -`BENCHMARKS` and read `template_spec` to report setup state per benchmark. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| A benchmark author forgets to declare `template_spec` | `AttributeError` at `Benchmark.template_spec` access; runtime failure if `ergon benchmark setup` is called | Contract test in `test_benchmark_contract.py` catches this in CI. The `Benchmark` ABC has no default so the error surfaces early. | -| `build_recipe_path` points to a directory that exists in dev but not in a package install | `setup_benchmark` fails with "Dockerfile not found" | `Path(__file__).parent / "sandbox"` is relative to the benchmark file; it is always valid when the package is installed (the sandbox directory is shipped with the package). | -| `researchrubrics-ablated` has no dedicated class; inherits `template_spec` from `ResearchRubricsBenchmark` | If `ResearchRubricsBenchmark.template_spec` changes, `researchrubrics-ablated` silently inherits the change | Documented explicitly in §7 (item 8). The contract test validates the resolved class attribute value, so any change to the base class is caught. | -| `SANDBOX_TEMPLATES` removal (PR 3) breaks code that imports it from `registry_core` | `ImportError` or `KeyError` at import time for any external user of `SANDBOX_TEMPLATES` | `SANDBOX_TEMPLATES` is not in `ergon_core/api/` and not part of the public API surface. Internal uses: only `ergon_cli/commands/benchmark.py:71` imports it. PR 3 removes both the import and the dict in the same change. | -| `NoSetup` is a singleton; `isinstance(spec, NoSetup.__class__)` is fragile | Subtle logic errors if the class check is written incorrectly | Use `isinstance(spec, _NoSetupType)` throughout. `NoSetupSentinel` is a TypeAlias for `_NoSetupType` and works the same way. The RFC, tests, and CLI rewrite all use `isinstance(spec, _NoSetupType)`. | -| `runtime_install` extras markers (e.g. `"foo[bar]"`) passed verbatim to pip | Shell injection if strings are user-supplied | Strings are always literal constants in `ClassVar` declarations, not runtime user input. No shell; pip is called via Python API or subprocess with a list. | - ---- - -## Invariants affected - -- **`docs/architecture/06_builtins.md#invariants` — new invariant added.** - "Every concrete `Benchmark` subclass MUST declare `template_spec` as either a - `TemplateSpec` instance or the `NoSetup` sentinel. The `Benchmark` ABC - provides no default. Omitting this declaration is detected at class attribute - access time." - -- **`docs/architecture/06_builtins.md#invariants` — existing invariant - updated.** "A custom sandbox template implies a matching `ergon benchmark - setup <slug>` code path" becomes: "A benchmark with `TemplateSpec.build_recipe_path` - set implies that `ergon benchmark setup <slug>` will build the template. - Benchmarks declaring `NoSetup` or `TemplateSpec(runtime_install=...)`-only are - self-documenting: no build step is required." - -- **`docs/architecture/06_builtins.md#extension-points` — "Template setup" - bullet updated.** The current prose ("the pattern is implicit") is replaced - by: "Template setup is declared via `template_spec: ClassVar[TemplateSpec | - NoSetupSentinel]` on the `Benchmark` subclass. Use `NoSetup` for benchmarks - with no sandbox, `TemplateSpec(runtime_install=(...))` for packages installed - at sandbox-prep time, and `TemplateSpec(e2b_template_id=..., - build_recipe_path=...)` for pre-built E2B templates." - -- **`docs/architecture/06_builtins.md#follow-ups`** — the "Template setup is - implicit" entry is removed on acceptance. - -- **`docs/architecture/01_public_api.md`** — adds `TemplateSpec`, `NoSetup`, - and `NoSetupSentinel` to the public API surface under "core abstractions". - Adds `template_spec` to the description of `Benchmark`. The `code map` table - gains a row for `TemplateSpec | NoSetup | NoSetupSentinel` pointing to - `ergon_core/api/template_spec.py`. - ---- - -## Alternatives considered - -- **Keep the ad-hoc pattern.** Rejected: contributors forget setup steps and - the system cannot notice. New benchmarks land with inconsistent bootstrap - code. - -- **`Optional[TemplateSpec] = None` with `None` meaning "no setup."** - Rejected per system-owner directive: an implicit opt-out defeats the point. - `NoSetup` forces the author to write the intent down. Easier to grep, easier - to review, harder to miss. - -- **String-enum `"none" | "e2b" | "runtime"`.** Rejected: the structured - model carries the template ID and recipe path directly; a tagged string - would require the author to populate parallel fields. The sentinel-plus-model - approach keeps the declaration site one-line simple. - ---- - -## Open questions - -- Should the sentinel be a typealias plus singleton instance (as above) or a - `Literal["NoSetup"]` string? Either works. Pick the one that reads cleaner - in a `ClassVar` annotation; leaning toward the singleton since it avoids - magic-string comparisons. -- Should `runtime_install` support extras markers (e.g. - `"foo[bar]==1.2.3"`)? Probably yes — just document that the strings are - passed verbatim to `pip install`. The `gdpeval` case uses bare package names - today; no incompatibility. -- Does `build_recipe_path` support directories (a whole `sandbox/` folder)? - Yes — both `minif2f` and `swebench-verified` use the full `sandbox/` - directory as the E2B file context path. The path should point to the - directory; the `Dockerfile` name within it comes from `e2b.toml.template`. - ---- - -## On acceptance - -When this RFC moves from `active/` to `accepted/`: - - Update `docs/architecture/06_builtins.md#extension-points`, - `#invariants`, and `#follow-ups` per §12. - - Update `docs/architecture/01_public_api.md` with the new exports per §12. - - Link the implementation plan in `docs/superpowers/plans/`. diff --git a/docs/rfcs/active/2026-04-21-real-llm-debug-harness.md b/docs/rfcs/active/2026-04-21-real-llm-debug-harness.md deleted file mode 100644 index 89b7924a3..000000000 --- a/docs/rfcs/active/2026-04-21-real-llm-debug-harness.md +++ /dev/null @@ -1,363 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: cm2435+agent -architecture_refs: [docs/architecture/06_builtins.md, docs/architecture/07_testing.md] -supersedes: [] -superseded_by: null ---- - -# RFC: Real-LLM debug harness for end-to-end benchmark validation - -## Problem - -Every existing test tier exercises Ergon against **stubbed or deterministic -workers**. `tests/unit/` tests isolated classes; `tests/integration/` tests the -full runtime lifecycle but only with stub workers that don't call LLMs; -`tests/e2e/` is Playwright-driven but the experiments it triggers use stubs -too. This means three things the product actually ships have **never been -exercised end-to-end**: - -1. The three benchmark sandbox templates (`researchrubrics`, `minif2f`, - `swebench-verified`) being functional enough that a real LLM, given the - right tools, can produce the outputs the criterion expects to find. -2. The generic ReAct worker actually using its injected tools correctly under - a real model's decision-making — i.e., `add_subtask` is called with the - right arguments, `bash` invocations land in the sandbox, `cancel_task` - correctly transitions state. -3. The per-benchmark criteria correctly parsing / grading the outputs a real - LLM would write, rather than the synthetic outputs our stubs emit. - -We know these paths are broken in places because ad-hoc local runs surface -bugs that our tiered tests don't catch (an outdated Lean toolchain in the -minif2f sandbox, a missing environment variable in the swebench sandbox, -researchrubrics tools returning structures the criterion doesn't parse). We -need a first-class tier that turns "run a real experiment locally and poke -it" into a reproducible, assertion-backed workflow — one we can point at a -benchmark, let loose with a real LLM, and use to validate every moving piece. -That same harness becomes a **bug-hunting instrument**: run it in an -autonomous loop, file each discovered bug, fix and re-run, until the -configured benchmarks behave as documented. - -## Proposal - -Land a new pytest tier at `tests/real_llm/` that runs real experiments -against a real LLM (Sonnet 4.6 via OpenRouter), asserts Postgres state after -each run, and asserts dashboard state via Playwright. The tier is gated on -two env vars (`ERGON_REAL_LLM=1`, `OPENROUTER_API_KEY`), is marker-skippable -(`@pytest.mark.real_llm`), and ships with a budget guard that skips remaining -tests when cumulative OpenRouter spend exceeds `ERGON_REAL_LLM_BUDGET_USD` -(default `$5`). - -The worker under test is the **existing generic ReAct worker** configured -per-benchmark via a new `benchmark_toolkit_composer` DI factory. This factory -unions the benchmark-specific toolkit with the `SubtaskLifecycleToolkit` -(and, for research-style envs, `ResearchGraphToolkit`), producing a single -ReAct worker that has every tool each benchmark's agent is documented to -need. **No new tools are added in this RFC** — `add_subtask`, `plan_subtasks`, -`cancel_task`, `refine_task`, `restart_task`, `list_subtasks`, `get_subtask`, -and sandboxed `bash` already exist; the composer just wires them. - -Experiments run via `subprocess.run(["ergon", "benchmark", "run", ...])` so -we exercise the shipped CLI path. The pytest session fixture brings the stack -up (Postgres + Inngest + FastAPI + `pnpm dev:test`) via a new -`docker-compose.real-llm.yml` overlay, unless `--assume-stack-up` is passed -(in which case the developer is expected to have the stack already running -for faster iteration). After subprocess exit, the test polls -`/api/test/read/run/{run_id}/state` from the smoke-shared-infra test harness -(RFC `2026-04-21-e2e-smoke-coverage-rewrite.md`, PR #25) until a terminal -status is reached, asserts DB invariants via `get_session()`, then launches a -headless Playwright spec that navigates the cohort page and run detail page -and asserts the expected nodes render. Screenshots are saved for every run. - -### Component sketch - -``` -tests/real_llm/ -├── __init__.py -├── conftest.py # session fixture, --assume-stack-up flag -├── fixtures/ -│ ├── stack.py # docker-compose up/down + health probe -│ ├── openrouter_budget.py # see below -│ └── playwright_client.py # reuse BackendHarnessClient (from smoke PR) -├── benchmarks/ -│ ├── test_smoke_stub.py # PR 1 canary: stub workers, no LLM cost -│ ├── test_researchrubrics.py # PR 2: 3 random instances -│ ├── test_minif2f.py # PR 2: 3 random instances -│ └── test_swebench.py # PR 2: 3 random instances -└── reporting/ - └── results_writer.py # per-run .results.md + PR body emission - -ergon_builtins/ergon_builtins/tools/benchmark_toolkit_composer.py # NEW -tests/real_llm/openrouter_budget.py # NEW -docker-compose.real-llm.yml # NEW -``` - -### Toolkit composition - -```python -# ergon_builtins/tools/benchmark_toolkit_composer.py - -from ergon_core.api.worker_context import WorkerContext - -def compose_benchmark_toolkit( - *, benchmark_slug: str, ctx: WorkerContext, sandbox: AsyncSandbox -) -> list[Tool]: - """Return the union of tools a generic ReAct worker needs for a benchmark.""" - lifecycle = SubtaskLifecycleToolkit( - run_id=ctx.run_id, - parent_node_id=ctx.node_id, - sandbox_id=ctx.sandbox_id, - ).get_tools() - - match benchmark_slug: - case "researchrubrics": - return [ - *lifecycle, - *ResearchRubricsToolkit(...).build_tools(), - *ResearchGraphToolkit( - run_id=ctx.run_id, - task_execution_id=ctx.execution_id, - ).build_tools(), - ] - case "minif2f": - return [*lifecycle, *MiniF2FToolkit(sandbox=sandbox).get_tools()] - case "swebench-verified": - return [*lifecycle, *SWEBenchToolkit(sandbox=sandbox).get_tools()] - case _: - raise ValueError(f"no toolkit composer for {benchmark_slug!r}") -``` - -Registration: a new CLI worker slug `react-generic` that the harness passes -as `--worker react-generic --toolkit-benchmark <slug>`. The composition layer -in `ergon_cli/composition/__init__.py` wires this into -`ReactWorker(tools=compose_benchmark_toolkit(...))` at experiment build time. - -### OpenRouter budget gate - -```python -# tests/real_llm/openrouter_budget.py - -class OpenRouterBudget: - def __init__(self, limit_usd: float) -> None: - self._limit = limit_usd - self._baseline: float | None = None - - async def snapshot_baseline(self) -> None: - data = await self._get_key_status() - self._baseline = data["usage"] - - async def remaining_usd(self) -> float: - data = await self._get_key_status() - return self._limit - (data["usage"] - (self._baseline or data["usage"])) - - async def _get_key_status(self) -> dict: - async with httpx.AsyncClient() as client: - resp = await client.get( - "https://openrouter.ai/api/v1/auth/key", - headers={"Authorization": f"Bearer {os.environ['OPENROUTER_API_KEY']}"}, - ) - resp.raise_for_status() - return resp.json()["data"] -``` - -The session fixture calls `snapshot_baseline()` once. A per-test fixture -calls `remaining_usd()` before each test; if ≤0, it raises `pytest.skip(...)` -with a clear "budget exceeded" reason. - -### Assertion model - -**Hard gates** (infra bugs — test fails if violated): - -- subprocess exits with return code 0 -- `/api/test/read/run/{id}/state` returns `status` in `{completed, failed}` - within budget -- Postgres query via `get_session()`: - - `RunRecord` row exists - - `RunGraphNode` count ≥ 1 (at least root node) - - If benchmark allows subtasks: `RunGraphMutation` with - `mutation_type="add_subtask"` exists iff the ReAct worker decided to - delegate -- Playwright: - - Cohort index page lists the run with the right status - - Run detail page renders without JS errors and shows ≥ 1 graph node - -**Soft gates** (report only — PR 2 adds one asserted form): - -- Per-benchmark, over the 3 instances: at least one produced - `RunTaskEvaluation.score > 0` on the primary criterion -- Per-run tool-call count, per-run wall clock, per-run cost - -Soft gate values are persisted to `tests/real_llm/.results/<run-id>.json` -and summarised into a `.results.md` that PR 2's "ship" step attaches to the -PR body. - -### CLI surface - -``` -uv run pytest tests/real_llm/ -m real_llm # CI default (stack up) -uv run pytest tests/real_llm/ -m real_llm --assume-stack-up -uv run pytest tests/real_llm/benchmarks/test_minif2f.py -vv -s -``` - -Env: - -- `ERGON_REAL_LLM=1` — activates the tier -- `OPENROUTER_API_KEY` — must be set; tests skip cleanly otherwise -- `ERGON_REAL_LLM_MODEL` — default `openrouter/anthropic/claude-sonnet-4.6` -- `ERGON_REAL_LLM_BUDGET_USD` — default `5.0` -- `ERGON_REAL_LLM_MAX_TURNS` — default `40` -- `ERGON_REAL_LLM_WALL_CLOCK_S` — default `300` - -## Invariants affected - -- **`docs/architecture/07_testing.md`** — adds a new tier (`tests/real_llm/`) - alongside `unit/`, `integration/`, `e2e/`. Update the testing-tier matrix - section to list its opt-in activation, budget gate, and non-CI-default - posture. -- **`docs/architecture/06_builtins.md`** — adds - `benchmark_toolkit_composer` to the builtin tools registry section. -- **Cross-cutting:** no new sandbox-lifecycle, artifact, or error-propagation - invariants introduced. The tier uses the same sandbox lifecycle and test - harness as smoke-shared-infra PR #25; only adds a marker and gated - execution path. - -## Migration - -No existing test fails or changes behaviour. Nothing is deleted. New files -only, plus a small addition to `ergon_cli/composition/__init__.py` to -resolve `--worker react-generic --toolkit-benchmark <slug>` into -`ReactWorker(tools=compose_benchmark_toolkit(...))`. - -No Alembic revision. No data migration. - -## Rollout (4 PRs) - -### PR 1 — Harness infrastructure - -**Branch:** `feature/real-llm-harness-infra` -**Blocks on:** smoke-shared-infra PR #25 merged to `main`. - -Scope: - -- `tests/real_llm/` scaffolding (conftest, fixtures, stack fixture with - `--assume-stack-up`, Playwright fixture, budget fixture) -- `benchmark_toolkit_composer.py` + registration under `react-generic` worker - slug -- `openrouter_budget.py` module + unit tests -- `docker-compose.real-llm.yml` stack overlay -- One canary: `test_smoke_stub.py` exercising the existing `smoke_test` - benchmark with **stub workers** (cost = $0), asserting: - - Subprocess ran, ergon CLI exits 0 - - Postgres has a completed `RunRecord` with `RunGraphNode`s - - Playwright finds the run in the cohort index and run detail -- Budget gate unit tests use a mocked OpenRouter API -- Architecture doc updates to `06_builtins.md` + `07_testing.md` - -Acceptance gate: `pnpm run check:fast` + `uv run pytest tests/unit -v` green -AND `uv run pytest tests/real_llm/ -m real_llm` green **without** -`OPENROUTER_API_KEY` set (i.e. the stub canary runs; real-LLM tests skip -cleanly). - -### Bug-hunt phase (between PR 1 and PR 2) - -Autonomous loop runs `uv run pytest tests/real_llm/ -m real_llm` with -`ERGON_REAL_LLM=1` set and `OPENROUTER_API_KEY` provided. Each discovered -non-LLM bug follows the CLAUDE.md workflow: - -1. File `docs/bugs/open/YYYY-MM-DD-<slug>.md` from `docs/bugs/TEMPLATE.md`. -2. If fix is trivial, open a fix PR that also moves the bug file to - `docs/bugs/fixed/` and sets `fixed_pr` in frontmatter. -3. If fix is non-trivial, promote to RFC, link via `related_rfc`. - -The loop terminates when, for each of the three benchmarks, the hard-gate -assertions all pass on a fresh run. - -### PR 2 — Three-example artifact - -**Branch:** `feature/real-llm-three-examples` - -Scope: - -- `test_researchrubrics.py`, `test_minif2f.py`, `test_swebench.py` — each - parametrized over 3 random benchmark instances (seeded via - `ERGON_REAL_LLM_INSTANCE_SEED`, default `42`). -- Hard-gate assertions on every test (as in the Proposal). -- Soft-gate: each benchmark's test class asserts "at least 1 of 3 instances - scored non-zero on primary criterion." -- Results reporter: writes `.results/YYYY-MM-DD-HHMM-<benchmark>.md` - summarising per-instance score, tool-call count, wall-clock, cost. PR 2's - ship step attaches a combined report to the PR body. -- Each soft gate that fires gets a `docs/bugs/open/` entry + follow-up PR. - -Acceptance gate: all hard gates green on three consecutive runs. Soft gates -document whatever reality is, not an idealised pass rate. - -### PR 3 — Results baseline (optional, deferred) - -If PR 2 surfaces an interesting capability curve, PR 3 can promote the -3-random to a hand-picked tiered set (easy/medium/hard) and include a stored -baseline. Skip unless requested. - -### PR 4 — CI integration (optional, deferred) - -Wire a manually-dispatchable workflow (`.github/workflows/real-llm.yml`) that -runs the tier on label `real-llm` on a PR. Defer until PR 2 stabilises. - -## Alternatives considered - -**Sibling repository.** Rejected. Keeps secrets + flaky runs out of the main -repo but makes Ergon imports awkward, breaks refactor-in-place workflows, -and defeats the "debug in a loop" use case where you want to edit Ergon -source and immediately re-run the harness. - -**Examples-dir instead of pytest.** Rejected. The use case is -assertion-backed debugging, not shipped documentation. Example scripts would -still need to implement the assertion model; building that outside pytest -reinvents most of what pytest gives us (fixtures, parametrize, skip, collect, -reports). - -**In-process runtime, no CLI subprocess.** Rejected. We'd bypass the Inngest -event path and the CLI composition layer — the exact production code paths -we most need to validate. The "real LLM + real CLI + real Inngest + real -Postgres + real dashboard" shape is the whole point. - -**Anthropic API direct, no OpenRouter.** Rejected. OpenRouter gives us -per-model swap (matrix-expansion friendliness) and a single billing/budget -surface. OpenRouter is already Ergon's canonical provider router. - -**Hand-picked tiered tasks in PR 2.** Deferred to PR 3. Hand-picked tasks -are more informative but require upfront curation per benchmark; random -instances prove the harness is useful with less human cost. - -**Per-benchmark specialized ReAct workers.** Rejected. The three -benchmark-specific workers already exist, but using them means the PR 2 -signal is "the specialized workers work," not "the generic ReAct loop + -toolkit composer work." Testing the more general primitive is higher -leverage. - -## Open questions - -None blocking RFC acceptance. Items deferred to PR 2 / PR 3: - -- How the soft-gate "at least 1/3 passes" threshold should evolve as models - improve. For Sonnet 4.6 in 2026-04, 1/3 feels right for minif2f and - swebench; researchrubrics may hit 3/3 and the soft gate becomes - uninteresting. -- Whether the `.results.md` reporter should also upload per-run terminal - recordings (asciinema) as CI artifacts. Nice-to-have; not in PR 2 scope. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`: - -- Update `docs/architecture/07_testing.md` testing-tier matrix with the new - `tests/real_llm/` row, its activation env vars, and its non-CI-default - posture. -- Update `docs/architecture/06_builtins.md` to list - `benchmark_toolkit_composer` alongside existing toolkits. -- Link the implementation plan - (`docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md`) from - here. -- Close `docs/bugs/open/` entries that were resolved during the bug-hunt - phase; move to `docs/bugs/fixed/` and link `fixed_pr`. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md deleted file mode 100644 index 82434669e..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md +++ /dev/null @@ -1,155 +0,0 @@ ---- -status: in-flight engineering doc (decisions accepted; implementation pending) -opened: 2026-05-11 -author: charlie + agent -architecture_refs: - - ../../../architecture/01_public_api.md - - ../../../architecture/cross_cutting/sandbox_lifecycle.md -supersedes: ../2026-05-08-authoring-api-redesign/ -superseded_by: null ---- - -# Authoring API redesign — v2 - -> **Note on the "RFC" path.** This doc lives under `docs/rfcs/active/` for -> path continuity, but it is **not** an open proposal. The decisions below -> are accepted; the implementation is pending. Treat it as engineering -> documentation for an in-flight redesign that will graduate into -> `docs/architecture/` once it ships. - -## Why v2 - -v1 of this RFC ([`../2026-05-08-authoring-api-redesign/`](../2026-05-08-authoring-api-redesign/)) -shipped to a worktree but never merged. An end-to-end audit -([`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md)) -found significant drift between the v1 spec and the v1 implementation: - -- **Public API surface — solid.** `Experiment`, `Benchmark`, `Task`, - `Worker`, `Sandbox`, `Criterion`, `Rubric`, `Evaluator` and the - `_type`-discriminated definition serialization all hold up. Authors - building benchmarks against v1 produce code that v2 will accept - unchanged. -- **Implementation underneath — partially rotted.** Three persistence - tiers where two suffice (`ExperimentRecord` ⊕ `ExperimentDefinition` ⊕ - `RunGraphNode`). A second `_prepare_definition` runtime path that - rebuilds `Sandbox`/`Worker` instances from definition rows — read - exclusively from definitions, never from run-tier copies. A - `_persist_single_sample_workflow_definition` CLI path that writes only - to a stub `saved_specs` table no other code reads. Sandbox provisioned - in `worker_execute` then *not* released after inline criteria; per-run - cleanup masks the leak. A `Worker.from_buffer` constructor with no - callers. A `CriterionExecutor` Protocol with one trivial implementation. - -The audit's conclusion: **the v1 design intent is right; the v1 -implementation accumulated parallel paths and stub abstractions that -need to come out before the design can be claimed as load-bearing.** - -v2 is a smaller, better-specced cut at the same target. The v1 docs in -`../2026-05-08-authoring-api-redesign/` remain as the historical -reference (see the archive banner on v1's README); the v2 docs in this -folder are the active spec the next implementation aims at. - -## Reading order - -| # | File | What it owns | Status vs v1 | -|---|---|---|---| -| 00 | [`00-readme.md`](00-readme.md) | This doc — entry point, doc map, deltas vs v1 | NEW | -| 01 | [`01-api-surface.md`](01-api-surface.md) | The public types authors construct (`Experiment`, `Benchmark`, `Task`, `Worker`, `Sandbox`, `Criterion`, `Rubric`, `Evaluator`); definition-time vs runtime split; `_type` discriminator pattern | Carried forward from v1 (cross-refs updated; phase tags marked historical) | -| 02 | [`02-persistence-layer.md`](02-persistence-layer.md) | Two-tier persistence model (`ExperimentDefinition` ⊕ run-tier `RunRecord`/`RunGraphNode`/`RunGraphEdge`/`RunGraphAnnotation`/`RunGraphMutation`); identifier model; `from_definition` convention; **runtime reads only run-tier tables** | Augmented from v1's 02 — `ExperimentRecord` collapsed into `ExperimentDefinition`; runtime read boundary made explicit | -| 03 | [`03-runtime.md`](03-runtime.md) | What happens when a run starts; sandbox provisioning via `Sandbox`-on-Task; `SandboxLifecycleHub`; `WorkerContext`; **synchronous per-evaluator fanout via `ctx.step.invoke` from `worker_execute`**; cross-job sandbox lifetime bounded by orchestrator's `try/finally` | Augmented from v1's 03 — synchronous-fanout shape made explicit; cross-job sandbox lifetime spelled out; cross-ref to 06 for event contracts | -| 04 | [`04-walkthrough.md`](04-walkthrough.md) | Single source of truth for "what running end-to-end looks like" — author code, what hits the database, what events fire, where each Task lives at each step | Carried forward from v1 unchanged (still canonical) | -| 05 | [`05-cli-authoring-interface.md`](05-cli-authoring-interface.md) | What `ergon define` and `ergon run` actually do; composition-convenience semantics (build an `Experiment` and persist it); no second slug-based path | NEW — fills a gap v1 left open | -| 06 | [`06-inngest-event-contracts.md`](06-inngest-event-contracts.md) | Per-event payload schemas, producers, consumers, fan-out semantics, idempotency keys; **single `worker_execute` job (no separate `evaluate_task_run`)** | NEW — replaces v1/05 §14.B; specs the unified worker_execute | -| 07 | [`07-test-strategy.md`](07-test-strategy.md) | Architecture-guard tests (boundary tests, retired-symbol tests); walkthrough as an integration test; regression net for the v1 audit findings | NEW — closes the "v1 stubs slipped through review" failure mode | -| 08 | [`08-decisions-log.md`](08-decisions-log.md) | Accepted decisions + rejected alternatives + the locked decisions inherited from the v1 audit (collapse, read boundary, dynamic-subtask shape, synchronous-fanout criteria, single sandbox lifetime owner, schema reset, deletions) | Carried forward from v1's 06 + locked-decisions section appended | -| 09 | [`09-implementation-plan.md`](09-implementation-plan.md) + [`09-implementation-plan/`](09-implementation-plan/) | Index plus detailed, docs/superpowers-style per-PR implementation plans | NEW — supersedes v1/05's phased work-order entirely | - -## What changed from v1 — the locked deltas - -The full audit lives in -[`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md); -the architectural decisions it crystallized are reproduced verbatim in -[`08-decisions-log.md`](08-decisions-log.md) "Locked decisions -inherited from v1 audit". The summary form: - -| # | Delta | Owner doc | -|---|---|---| -| Δ.1 | **`ExperimentRecord` collapses into `ExperimentDefinition`.** Authoring metadata fields move directly onto `Experiment`. Persistence is two-tier, not three. | [`02-persistence-layer.md`](02-persistence-layer.md) | -| Δ.2 | **Runtime reads only run-tier tables.** No `_prepare_definition` fall-through that hydrates `Sandbox`/`Worker` from `ExperimentDefinition` rows. After `prepare_run` copies the graph from definition into run tables, all subsequent reads come from run-tier. | [`02-persistence-layer.md`](02-persistence-layer.md) | -| Δ.3 | **Dynamic subtasks are graph-native.** Tasks spawned at runtime live only in `RunGraphNode` with a nullable definition `task_id`. There is no "synthesize a definition row" path. | [`02-persistence-layer.md`](02-persistence-layer.md) | -| Δ.4 | **Synchronous fanout, thin payload contract.** `worker_execute` provisions the sandbox, runs the worker, then `asyncio.gather`s a `ctx.step.invoke(evaluate_task_run, ...)` per evaluator. Each `evaluate_task_run` invocation is a full Inngest function (own retry, own concurrency cap, own observability slug). Its payload is `(run_id, task_id, execution_id, evaluator_index)` only — the `sandbox_id` is read from the persisted `run_task_executions` row and the live sandbox is rebuilt on the eval side via `Sandbox.from_definition(json, sandbox_id=...)`. There is no fire-and-forget eval dispatch and no `check_evaluators` coordinator. | [`03-runtime.md`](03-runtime.md), [`06-inngest-event-contracts.md`](06-inngest-event-contracts.md) | -| Δ.5 | **Cross-job sandbox lifetime, single owner, bounded by parent's try/finally.** `worker_execute` is the sole acquirer and releaser for `(run_id, task_id)`. Release happens after `gather` returns, so the sandbox is alive for every eval invocation and dead the moment they all complete. Eval invocations re-attach to the same external sandbox via `sandbox_id`; the live `_runtime` is reconstructed in the eval worker process, not transferred over the wire. Per-run cleanup is a backstop. | [`03-runtime.md`](03-runtime.md) | -| Δ.6 | **Schema reset, not incremental drops.** Wipe the v1 Alembic migrations; generate one fresh "initial schema" migration that matches v2's two-tier model. No "drop a column" migration chain. | [`02-persistence-layer.md`](02-persistence-layer.md), [`09-implementation-plan.md`](09-implementation-plan.md) | -| Δ.7 | **Deletions.** `CriterionExecutor` / `InngestCriterionExecutor` Protocol pair (one trivial impl; reshaped `evaluate_task_run` calls `criterion.evaluate(...)` directly); `Worker.from_buffer` (no callers); `saved_specs` package (write-only, no readers); `_prepare_definition` runtime path; `definition_task_id` column. **Kept and reshaped (not deleted):** `evaluate_task_run` Inngest function and `EvaluateTaskRunRequest` (or its successor `TaskEvaluateRequest`) — they survive in v2 as the per-evaluator fanout target with a thin id-only payload. | [`09-implementation-plan.md`](09-implementation-plan.md) | -| Δ.8 | **CLI is composition convenience, not a parallel persistence path.** `ergon define <slug>` builds an `Experiment` from a registered factory and calls the same `persist_definition` the public Python API uses. There is no second slug→spec persistence flow. | [`05-cli-authoring-interface.md`](05-cli-authoring-interface.md) | - -## What did **not** change from v1 - -The audit explicitly cleared these — they're carried forward in v2 -unchanged: - -- **Public API shape.** All seven definition-time types - (`Experiment`, `Benchmark`, `Task`, `Worker`, `Sandbox`, `Criterion`, - `Rubric`, `Evaluator`), `WorkerContext` curated methods, - `CriterionContext` as a pure data carrier, the `WeightedCriterion` / - rubric aggregation shape — all unchanged. -- **`_type`-discriminated definition serialization.** `_type`, - `to_definition()`, `from_definition()` round-trip — unchanged. -- **`SandboxLifecycleHub` as the framework-internal coordinator.** - Unchanged. What's new is *who calls release*, not the hub itself. -- **Identifier model.** Single `task_id` minted at definition-time and - carried unchanged through run-tier and into the runtime objects via - `_task_id` PrivateAttr. Unchanged from v1's 02. -- **Walkthrough.** [`04-walkthrough.md`](04-walkthrough.md) was - conceptually correct and stays as the single source of truth for - "what running looks like." - -## How to consume this folder - -For the spec reader (e.g. another agent reimplementing v2): - -1. Read `00-readme.md` (you're here). -2. Read `08-decisions-log.md` "Locked decisions inherited from v1 - audit" before reading any other content doc — those decisions are - the load-bearing constraints that distinguish v2 from v1. -3. Read `01` → `02` → `03` → `04` for the design. -4. Read `05` → `06` for the surface contracts (CLI, events). -5. Read `07` for the test strategy. -6. Read `09` for the implementation order. - -For the human reviewer / workshop participant: - -- Each new doc (`05`, `06`, `07`, `09`) ends with a **"Open questions - for workshop"** section — that's where the targeted feedback should - go. -- The augmented docs (`02`, `03`) flag inserted sections with `[v2: - added]` markers so v1↔v2 diff review is straightforward. - -## Status - -**Status:** Workshop completed (2026-05-11); all open questions -resolved or deferred to follow-up. Implementation pending. - -The "Decisions locked at workshop" section at the bottom of each doc -records the resolutions. For provenance: - -| Doc | Section | What it records | -|---|---|---| -| `02` | §6 — Decisions locked at workshop | `name`/`description` are dedicated columns; `is_dynamic` denormalized; no definition versioning; WAL kept forever for v2 launch. | -| `05` | Decisions locked at workshop | No plugin slug registration; no `--override`; no cohort runs; no `ergon launch <slug>` shortcut. | -| `06` | `task/failed` "Failure semantics — the four-axis lock" + Decisions locked at workshop | Spawn-subtree cascade-FAIL; dependency-dependents stay PENDING; non-descendants continue; `runs.status` strict. Minimal `workflow/completed` payload; framework-only retry policy; annotation-WAL stream chunks. | -| `07` | Decisions locked at workshop | Textual architecture guards; hand-written walkthrough integration test; 8-finding regression net; sequential test execution. v1 integration / e2e suite refresh is a post-v2 follow-up. | -| `08` | "Open questions" (most resolved) | PrivateAttr pattern kept; sandbox IO methods on base; CriterionContext kept; backpressure deferred; instance_key drop deferred. | -| `09` | Decisions locked at workshop | Incremental vertical PR chain; no prod data; SQLite for unit + Postgres for walkthrough integration; walkthrough verification lands last after bridge deletion. | - -Follow-up items deferred from v2 (tracked in -[`08-decisions-log.md` "Future work"](08-decisions-log.md)): - -- Synchronous `spawn_task(..., await_completion=True)`. -- Multiple agents per task / sandbox sharing. -- Cross-task data handoff via `Task.inputs` / `Task.outputs`. -- `GraphSpawnGovernor` for fork-bomb backpressure. -- `Task.instance_key` redundancy cleanup. -- Cohort runs from CLI (`--replicas N`). -- v1 integration / e2e test suite refresh. -- v1 worktree deletion (~4 weeks after v2 ships). diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md deleted file mode 100644 index ccfd145c8..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md +++ /dev/null @@ -1,1219 +0,0 @@ -# 01 — API surface - -> The public types authors construct, the design principles that produced -> them, and what `ergon_builtins` looks like once the smells are removed. -> See [`02-persistence-layer.md`](02-persistence-layer.md) -> for how these objects are persisted and reconstructed, -> [`03-runtime.md`](03-runtime.md) for what happens when one runs, -> and [`04-walkthrough.md`](04-walkthrough.md) for an end-to-end trace. - -> **On phase tags `[P1]–[P4]`.** Inline phase tags in this document -> reference the v1 phase plan and remain only as historical provenance -> markers. The v2 implementation order is **different and smaller** — -> see [`09-implementation-plan.md`](09-implementation-plan.md). Read the -> tags as "this concept landed in v1's Pn batch"; they have no v2 force. - -## Two surfaces, one public package - -`ergon_core.api` exposes the entire framework contract. It splits — by -*usage moment*, not by *abstraction tier* — into two surfaces: - -| Surface | Used at | Owns | Who constructs / consumes | -|---|---|---|---| -| **Definition-time** | Pre-run, in the author's process | `Benchmark`, `Task`, `Sandbox`, `Worker`, `Criterion`, `Rubric`, `Evaluator`, `Experiment` | Author constructs them; framework persists them | -| **Runtime** | Inside `worker.execute()` (rollout container) | `WorkerContext` (curated single-target operations) | Framework hands `WorkerContext` to `execute()`; workers invoke it. Toolkits / CLI that need batch / predicate / materialisation ops import the internal services in `core.application.*` directly — see [`03-runtime.md`](03-runtime.md) "Where the implementation lives" for what to import. | - -Definition-time owns *what an experiment is*; runtime owns *what a worker -can do while one runs*. The split exists because the invariants differ -(definition-time is single-writer + frozen-after-define; runtime is -concurrent + scoped to `(run_id, task_id)` with a stricter -constraint set), but the public surface is one — `ergon_core.api` is the -only import path authors and workers ever use. - -```python -from ergon_core.api import ( - # Definition-time (the nouns): - Benchmark, Task, Sandbox, Worker, Criterion, Rubric, Evaluator, Experiment, - WeightedCriterion, - # Runtime: - WorkerContext, WorkerStreamItem, SpawnedTaskHandle, - # Public exception types: - TaskNotMaterializedError, SandboxNotLiveError, SandboxKindMismatch, - ContainmentViolation, -) -``` - -Flat layout, no `runtime` / `services` / `advanced` submodule. An -earlier design promoted three additional public service classes -(`GraphMutator`, `GraphInspector`, `ResourceInspector`) into -`ergon_core.api` to expose batch / cross-scope / CLI-tier operations -through a layered API; that was rolled back as scope creep for v1 (see -[`08-decisions-log.md`](08-decisions-log.md) "Layered public runtime -API" under Alternatives considered, and -[`09-implementation-plan.md`](09-implementation-plan.md) "What's deferred"). For v1, the -"escape hatch" is direct import from `ergon_core.core.application.*` — -the same path `ergon_builtins.tools.*` and the workflow CLI use today. - -The rest of this doc covers the **definition-time** surface — the seven -types authors construct. The runtime surface is documented end-to-end in -[`03-runtime.md`](03-runtime.md), with example consumers in -[`04-walkthrough.md`](04-walkthrough.md). - -## What `ergon_core.api/` looks like on disk - -The full public surface, file by file, with phase tags ([P1]–[P4]) -matching [`09-implementation-plan.md`](09-implementation-plan.md). Everything authors, -workers, toolkits, criteria, and the workflow CLI ever import via the -public path lives under this tree. Toolkits / CLI that need ops not -covered by `WorkerContext` import directly from -`ergon_core.core.application.*` (no boundary enforcement in v1; see -[`09-implementation-plan.md`](09-implementation-plan.md) "What's deferred"). - -Legend: **`[P1]`–`[P4]`** = phase from [`09-implementation-plan.md`](09-implementation-plan.md); -**ADD / MODIFY / DELETE** = action vs. today's tree. - -``` -ergon_core/ergon_core/api/ -│ -├── __init__.py # MODIFY — re-exports the public surface as a flat namespace -│ # (only one import path: `from ergon_core.api import ...`) -│ # Drops: TaskSpec, ComponentRegistry [P1] -│ # Adds: Sandbox, SandboxRuntime [P2], WeightedCriterion [P4], -│ # SpawnedTaskHandle [P3], -│ # new exception types [P2/P3] -│ -│ ── Definition-time: the nouns authors construct ── -│ -├── benchmark/ -│ ├── __init__.py # MODIFY — re-export new from_definition entry points -│ ├── benchmark.py # MODIFY — Benchmark(ABC) gains from_definition classmethod [P1] -│ ├── task.py # MODIFY — Task(BaseModel): worker: Worker, sandbox: Sandbox, -│ │ # evaluators: tuple[Evaluator, ...] direct object bindings [P1/P3]; -│ │ # _task_id PrivateAttr + task_id property [P3]; -│ │ # from_definition classmethod [P3] -│ └── requirements.py # unchanged — Requirements stays as a definition-time helper -│ -├── sandbox/ # ADD — entirely new subpackage [P2] -│ ├── __init__.py # ADD — re-exports Sandbox, SandboxRuntime -│ ├── sandbox.py # ADD — class Sandbox(BaseModel, ABC); abstract provision()/terminate(); -│ │ # IO proxy methods (run_command, read_file, write_file, list_files); -│ │ # _runtime PrivateAttr; from_definition classmethod -│ └── runtime.py # ADD — class SandboxRuntime(Protocol) — the live backing object's contract -│ -├── worker/ -│ ├── __init__.py # MODIFY — re-export new shapes (incl. SpawnedTaskHandle) -│ ├── worker.py # MODIFY — Worker(BaseModel, ABC): execute(task, *, context, sandbox) [P1]; -│ │ # from_definition classmethod [P1] -│ ├── context.py # MODIFY — WorkerContext gains curated single-target methods [P3]: -│ │ # IDs: run_id, task_id, execution_id, definition_id -│ │ # impls: _task_mgmt, _task_inspect, _resource_repo (PrivateAttr, -│ │ # internal services injected at job start) -│ │ # methods: spawn_task, cancel_task, refine_task, restart_task, -│ │ # subtasks, descendants, get_task, resources -│ └── results.py # MODIFY — adds WorkerStreamItem (discriminated union); keeps WorkerOutput -│ # class SpawnedTaskHandle — return of WorkerContext.spawn_task; -│ # .task_id, .wait() (lives here rather than in a separate types.py -│ # since it's worker-runtime-specific) -│ -├── criterion/ -│ ├── __init__.py # MODIFY — re-export new shapes -│ ├── criterion.py # MODIFY — Criterion(BaseModel, ABC): keeps v1 evaluate(context) signature; -│ │ # sandbox accessed via context.task.sandbox [LOCKED — see § Criterion class signature]; -│ │ # from_definition classmethod [P1] -│ ├── context.py # MODIFY — CriterionContext becomes a pure data carrier [P4: runtime proxies dropped; -│ │ # sandbox proxies live on Sandbox itself] -│ ├── score.py # ADD — ScoreScale -│ ├── evidence.py # ADD — EvidenceMessage / CriterionEvidence -│ └── outcome.py # ADD — CriterionOutcome -│ -├── rubric/ -│ ├── __init__.py # MODIFY — re-export WeightedCriterion -│ ├── rubric.py # MODIFY — Rubric: list[WeightedCriterion]; aggregation knobs [P4] -│ │ # class WeightedCriterion(criterion=, weight=) [P4 — split out from Criterion.weight] -│ ├── evaluator.py # MODIFY — Evaluator(ABC): directly embedded on Task.evaluators; -│ │ # Rubric is one impl; future evaluator kinds plug in here -│ └── results.py # unchanged — RubricResult and friends -│ -├── errors.py # MODIFY — already holds DependencyError, CriterionCheckError; -│ # adds the new exception types: -│ # TaskNotMaterializedError — task.task_id accessed before materialization [P3] -│ # SandboxNotLiveError — IO method called before sandbox.provision() [P2] -│ # SandboxKindMismatch — toolkit/criterion requires_sandbox unsatisfied [P2] -│ # ContainmentViolation — WorkerContext mutation targeting non-descendant [P3] -│ -├── experiment.py # ADD [P1] — canonical home of class Experiment. -│ # Lifted out of ergon_core.core.domain.experiments — the -│ # composition root is part of the authoring contract, not an -│ # internal type that happens to be exposed. core.domain owns the -│ # *services* that act on it (validation, definition writing, -│ # launch); core.domain.experiments.experiment.py is DELETED. -│ # Becomes a Pydantic BaseModel with a @model_validator(mode="after") -│ # that runs the requires_sandbox compatibility check (see -│ # "Foundational change C" below for the class sketch). -│ # Decision in 08-decisions-log.md (resolved — re-export pattern rolled back). -│ -└── registry.py # DELETE [P1] — ComponentRegistry replaced by _type discriminator -``` - -**Not in this tree (deferred):** an earlier draft added `graph.py`, -`resources.py`, and `types.py` to host three new public service classes -(`GraphMutator`, `GraphInspector`, `ResourceInspector`) plus the -intermediate value-objects (`SubtaskSpec`, `MaterializeResult`, -`TaskTreeView`, `NextActionHint`, `ResourceRef`). All rolled back as -scope creep — see [`09-implementation-plan.md`](09-implementation-plan.md) "What's -deferred" and [`08-decisions-log.md`](08-decisions-log.md) "Layered -public runtime API" for the rationale. - -Out of scope for the public surface (intentionally — only the rollout -container ever uses them, never author code or workers): - -- `SandboxLifecycleHub` lives in - `ergon_core/core/infrastructure/sandbox/lifecycle.py` (framework - internals). It tracks live sandboxes process-wide for retry-reconnect - / quota / shutdown — purely the rollout job's concern. -- `RunGraphNodeView` lives next to the graph repository in - `ergon_core/core/application/...` (framework internals). It's the - typed return shape of `graph_repo.node(...)`, and only - `worker_execute.py` ever reads one. -- `TaskManagementService`, `TaskInspectionService`, - `RunResourceRepository`, the Command DTOs (`AddSubtaskCommand`, - etc.), `SubtaskInfo`, `RunResourceView` all stay in - `ergon_core/core/application/*`. They are the **implementation - backend** for `WorkerContext`'s curated methods, *and* the direct - import target for any toolkit / CLI / future consumer that needs ops - outside the curated surface. The boundary is porous (no enforcement - test) — that's deliberate v1 simplification, see - [`09-implementation-plan.md`](09-implementation-plan.md) "What's deferred". - -Two architectural rules this layout encodes: - -1. **One import root for the curated surface.** Anything a benchmark / - worker / criterion author needs is in `ergon_core.api`. Toolkits and - the workflow CLI may also import from `ergon_core.core.application.*` - when they need ops not covered by `WorkerContext`; that import path - is documented as the v1 escape hatch. -2. **Concrete benchmark types live in `ergon_builtins`, not `ergon_core.api`.** - `LeanSandbox`, `PythonSandbox`, `MiniF2FToolkit`, etc. are - *implementations* shipped with the framework but not part of its - public contract — third parties can ship their own concrete - `Sandbox` subclasses without touching `ergon_core` (see - [`03-runtime.md#sandbox-provisioning`](03-runtime.md#sandbox-provisioning)). - -## Public exceptions - -The exception types added by this redesign all live in `errors.py` -alongside the existing `DependencyError` and `CriterionCheckError`. -Pinned constructor signatures so toolkit/test code that constructs -or catches them has one canonical shape: - -```python -class TaskNotMaterializedError(RuntimeError): - """Raised by Task.task_id when accessed before from_definition has run. - Indicates author code held onto a definition-time Task; the framework - path always inflates first.""" - def __init__(self, message: str) -> None: - super().__init__(message) - - -class SandboxNotLiveError(RuntimeError): - """Raised by Sandbox IO methods (run_command, read_file, ...) when - called before provision() has attached _runtime. Author tests that - forget to provision get this, not an AttributeError.""" - def __init__(self, sandbox_kind: str) -> None: - super().__init__( - f"{sandbox_kind} method called before provision(); " - "no live runtime attached." - ) - self.sandbox_kind = sandbox_kind - - -class SandboxKindMismatch(TypeError): - """Raised by Experiment.@model_validator when a task's directly bound - worker (or one of its evaluators/criteria) declares requires_sandbox = X - and the task has a Y sandbox where Y is not a subclass of X. - Construction-time error; misconfigured experiments never reach the - database.""" - def __init__( - self, *, - task_id: UUID, component: str, - required: type[Sandbox], actual: type[Sandbox], - ) -> None: - super().__init__( - f"task {task_id} ({component}) requires " - f"a {required.__name__}, got {actual.__name__}" - ) - self.task_id = task_id - self.component = component - self.required = required - self.actual = actual - - -class ContainmentViolation(RuntimeError): - """Raised by WorkerContext mutation/inspection methods when the target - task_id is not a descendant of the calling worker's task_id. Logic - bug, not a recoverable runtime condition; workers should not catch.""" - def __init__(self, *, target: UUID, ancestor: UUID, run_id: UUID) -> None: - super().__init__( - f"task_id={target} is not a descendant of {ancestor} in run {run_id}" - ) - self.target = target - self.ancestor = ancestor - self.run_id = run_id -``` - -Construction conventions: - -- `*` after `self` for any constructor with more than one argument so - callers always read keyword-by-keyword. Single-string ctors - (`TaskNotMaterializedError`, `SandboxNotLiveError`) take their one - argument positionally. -- Each exception keeps its semantic context as instance attributes - (`exc.component`, `exc.task_id`, etc.) for test assertions and - structured logging — not just baked into the message string. -- All inherit from a sensible stdlib base (`RuntimeError`, `TypeError`, - etc.) rather than the bare `Exception` so generic `except RuntimeError:` - blocks behave reasonably. - -## The five types - -Five user-facing types, no Spec siblings, no central registry: - -| Type | Authored as | Holds (public fields) | Holds (private runtime) | Receives at runtime | -|---|---|---|---|---| -| `Benchmark` | subclass | task generator, payload type | — | (definition-time only) | -| `Task` | instance | slug, description, payload, **worker**, **sandbox**, **evaluators**, deps | `_task_id` (set by runtime, read via `task.task_id` property) | (it's the thing passed) | -| `Sandbox` | subclass + instance (`LeanSandbox(...)`, `PythonSandbox(...)`) | env, timeout, requires_network + subclass-specific config (e.g. `lean_version`) | `_runtime` (set after `provision()`; backs `run_command` / `read_file` / etc.) | (provisioned by framework, passed into `execute` / `evaluate`) | -| `Worker` | instance | name, model, strategy config (prompt, max_iter…) | — (pure config + behavior) | `task`, `context` (sandbox is `task.sandbox`) | -| `Criterion` | instance | slug, description | — (pure config + behavior) | `context: CriterionContext` (carries `task`, `worker_result`; sandbox is `context.task.sandbox`) | -| `Rubric` | instance (or subclass) | criteria + weights + aggregation | — | `task`, criterion outcomes | - -## The unifying pattern: one class per concept, runtime as `PrivateAttr` - -The recurring "Spec / Live / Manager" tripling we have today -(`SandboxSpec` / `Sandbox` / `SandboxManager`, `TaskSpec` / `Task` / runtime -DTOs, `WorkerSpec` / `Worker` / `ComponentRegistry`) is not an essential -property of the domain — it's a workaround for the fact that pydantic models -can't carry runtime state without leaking it into their public schema. - -**`CriterionContext` already solves this in-tree** and we generalize from it. -A `CriterionContext` is one class with two layers: - -- **Public pydantic fields** — `task`, `worker_result`, `sandbox_id`, etc. - Serializable, declarative, what an author looks at. -- **Private runtime backdoor** — `_runtime: CriterionRuntime | None` as a - `PrivateAttr`, set via `with_runtime(...)` by the framework after - construction. Public proxy methods (`run_command`, `read_resource`, - `write_file`, …) dispatch to it. - -```python -# Today, in ergon_core/api/criterion/context.py -class CriterionContext(BaseModel): - task: Task - worker_result: WorkerOutput - ... - _runtime: Annotated[CriterionRuntime | None, SkipValidation] = PrivateAttr(default=None) - - async def run_command(self, command: str, timeout: int = 30): - return await self._require_runtime().run_command(command, timeout) -``` - -The pattern works. The author constructs a `CriterionContext` with public -fields; the runtime attaches `_runtime`; downstream consumers call -`context.run_command(...)` and don't care whether they're in a test that -faked the runtime or in a live container that wired the real one. **There is -no parallel `CriterionContextSpec` and no `CriterionContextManager`** — and -nobody misses them. - -We apply the same shape to `Task` and `Sandbox` to collapse the duplicate -classes: - -```python -class Task(BaseModel, Generic[PayloadT]): - """Single Task type. ID is set by the runtime when materialized.""" - model_config = {"frozen": False} # PrivateAttr requires non-frozen - - task_slug: str - instance_key: str - description: str - worker: Worker # NEW, direct object binding - sandbox: Sandbox # NEW, non-optional - evaluators: tuple[Evaluator, ...] = () # NEW, direct object binding - parent_task_slug: str | None = None - dependency_task_slugs: tuple[str, ...] = () - task_payload: PayloadT = Field(default_factory=EmptyTaskPayload) - - _task_id: UUID | None = PrivateAttr(default=None) - - @property - def task_id(self) -> UUID: - if self._task_id is None: - raise TaskNotMaterializedError( - f"Task {self.task_slug!r} has no task_id; it has not been " - "materialized into a run yet. This is a framework error — " - "worker code should never see a non-materialized Task." - ) - return self._task_id - - # Note: there is no `_materialize` classmethod. The single - # framework-only entry point is `from_definition` — defined in - # 02-persistence-layer.md alongside the other class - # `from_definition` conventions. It does construction + identity - # binding in one call so the framework never holds a - # half-materialized Task. Worker/evaluator objects are already part - # of task_json and round-trip via their own `_type` discriminators. - - -class Sandbox(BaseModel, ABC): - """Base for all sandbox kinds. Subclass to add a new kind of environment. - Capabilities are live once `provision()` has been called and `_runtime` - is attached.""" - model_config = {"frozen": False, "arbitrary_types_allowed": True} - - env: dict[str, str] = Field(default_factory=dict) - timeout_seconds: int | None = None - requires_network: bool = False - - # `[v2: locked]` (Q28). The convention path under which workers and - # criteria expect "task output" files to live. Each Sandbox subclass - # picks its default; benchmarks that need a different path override - # via the field. Replaces the v1 hardcoded `/workspace/final_output/` - # in worker code AND in the resource publisher with one source of - # truth on the sandbox itself. - output_path: str = "/workspace/final_output/" - - _runtime: SandboxRuntime | None = PrivateAttr(default=None) - - # ── Subclass MUST implement ── - @abstractmethod - async def provision(self) -> None: - """Spin up the underlying environment, install deps, attach - ``self._runtime``. After this returns, ``self.is_live`` is True.""" - - # ── Subclass MAY override (default = noop) ── - async def terminate(self) -> None: - """Tear down the underlying environment. Cheap sandboxes have nothing - to clean up; expensive ones override.""" - pass - - # ── Inherited; subclasses get these for free via _runtime proxy ── - @property - def is_live(self) -> bool: - return self._runtime is not None - - @property - def sandbox_id(self) -> str: - return self._require_runtime().sandbox_id - - async def run_command( - self, cmd: str | Sequence[str], *, - timeout: int | None = None, - ) -> CommandResult: - return await self._require_runtime().run_command(cmd, timeout=timeout) - - async def write_file(self, path: str, content: bytes | str) -> None: - if isinstance(content, str): - content = content.encode("utf-8") - await self._require_runtime().write_file(path, content) - - async def read_file(self, path: str) -> bytes: - return await self._require_runtime().read_file(path) - - async def list_files(self, path: str) -> list[str]: - return await self._require_runtime().list_files(path) - - def _require_runtime(self) -> SandboxRuntime: - if self._runtime is None: - raise SandboxNotLiveError( - f"{type(self).__name__} has no runtime attached; " - "provision() has not been called yet. This is a framework " - "error — worker code should never see a non-live Sandbox." - ) - return self._runtime - - -# Concrete kind in ergon_builtins: -class LeanSandbox(Sandbox): - lean_version: str = "4.7.0" - e2b_template: str = "ergon-minif2f-v1" - requires_network: bool = False - - async def provision(self) -> None: - sb = await AsyncSandbox.create(template=self.e2b_template, envs=self.env) - await sb.commands.run(f"elan default {self.lean_version}") - object.__setattr__(self, "_runtime", _E2BSandboxRuntime(sb)) - - async def terminate(self) -> None: - if self._runtime is not None: - await self._runtime.close() -``` - -**Author code constructs the typed subclass. Framework calls `provision()` -to attach the runtime. Worker/criterion code uses the inherited proxy -methods.** Same instance, two phases of life. No duplicate classes. - -The "you constructed it wrong" failure mode is real but bounded — the only -two callsites that materialize a `Task` (one in `worker_execute.py`, one in -the dynamic-subtask path) and the only callsite that calls `provision()` -(the worker_execute job, just before invoking the worker) are framework -code. Author code never constructs these in their materialized form, so -the `_require_runtime()` errors only fire on framework bugs, which is -exactly when you want them. - -For **Worker** and **Criterion**, no PrivateAttr is needed — they're pure -config + a single `execute`/`evaluate` method that takes everything runtime -through parameters. The pattern only applies where the type genuinely *has* -runtime capabilities (a sandbox to talk to, an ID minted by the framework). - -## Subclass for *kind*, field for *config* - -Adjacent rule that governs every other "should this be a subclass or a -field?" question this redesign raises: - -- **Subclass when the *kind of thing* is genuinely different** — different - shape, different operations, different lifecycle. The subclass is the - identity; instances of it differ in config. -- **Field when the *configuration of a kind* varies** — same shape, same - operations, just different data. No new subclass; just a different - instance. - -| Axis | Pattern | Examples | -|---|---|---| -| Worker strategies | subclass | `ReActWorker`, `TreeSearchWorker`, `SingleShotWorker` (each implements `execute()` differently) | -| Per-benchmark workers using ReAct | **field** (`toolkit`) | `ReActWorker(toolkit=MiniF2FToolkit(...))` — *not* `MiniF2FReactWorker(ReActWorker)` | -| Sandbox kinds | subclass | `LeanSandbox`, `PythonSandbox`, `LocalDockerSandbox`, `WasmSandbox` (each implements `provision()` differently) | -| Per-benchmark sandbox config | **field** (on the subclass) | `LeanSandbox(lean_version="4.7.0")`, `LeanSandbox(max_kill_time_seconds=600)` — *not* `MiniF2FLeanSandbox(LeanSandbox)` | -| Criteria | subclass | `LeanFileExists(Criterion)`, `WeightedSum(Criterion)` | -| Per-instance criterion config | **field** | `LeanFileExists(path="/workspace/proof.lean")` | - -The smell with the original `MiniF2FReactWorker(ReActWorker)` was using -*subclassing* for what was actually *configuration* (same loop shape, just -different toolkit). The smell with the original -`Sandbox(template="lean", setup_payload={...})` was the opposite mistake — -using a stringly-typed *config* for what was actually a *kind* difference -(`provision()` for Lean+E2B is genuinely a different operation from -`provision()` for local-Docker-Python). Both fixes apply this same rule; -they just point in opposite directions because the original mistakes did. - -The forcing question for any new "should this be a class or a field?" -decision: **does the subclass need its own behavior** (different -`execute`/`provision`/`evaluate` body), **or just different data**? If -behavior, subclass. If only data, field. - -## What the framework sees (and what it doesn't) - -Once we're using subclasses for kinds, the natural worry: if `LeanSandbox` -and `PythonSandbox` carry different config fields and possibly expose -different methods, how does the framework — which only knows the base -`Sandbox` — actually *invoke* anything that varies? The answer comes in -three parts, and the key principle is stronger than it usually gets stated. - -**1. Subclass-specific config fields are invisible to the framework — -pydantic + the `_type` discriminator do all the work.** - -`LeanSandbox(lean_version="4.7.0", e2b_template="ergon-minif2f-v1")` has -fields no other sandbox carries. The framework never reads them. At -construction time pydantic validates them against `LeanSandbox`'s schema, -not `Sandbox`'s. At persist time `model_dump()` includes -`_type: "ergon_builtins.sandboxes:LeanSandbox"` plus every subclass field. -At read time `Sandbox.from_definition(json)` does: - -```python -SandboxCls = import_component_string(json["_type"]) # → LeanSandbox -return SandboxCls.model_validate(json) # uses LeanSandbox schema -``` - -— pydantic uses the *subclass* schema for validation, so `lean_version` -parses correctly without any framework code mentioning it. The only place -that ever reads `self.lean_version` is `LeanSandbox.provision()`, which -gets dispatched polymorphically when the framework calls -`sandbox.provision()`. - -The framework treats subclass config as opaque data. The "different -kwargs per subclass" problem disappears. - -**2. Subclass-specific *methods* are invisible to the framework — only -consumers that import the subclass call them.** - -> The base interface is the contract the *framework* needs from a Sandbox. -> Subclass-specific methods are the contract between a *consumer* (worker, -> criterion) and a particular sandbox kind. The framework never sees the -> second contract. - -Concrete example: suppose `LeanSandbox` exposes -`compile_lean_file(path: str) -> CompileResult` that no other sandbox has. - -| Caller | What it sees | Calls subclass methods? | -|---|---|---| -| `worker_execute` (framework job) | `Sandbox` (base) | No — only `provision()` / `terminate()` | -| `SandboxLifecycleHub` (framework) | `Sandbox` (base) | No — only `provision()` / `terminate()` | -| `MiniF2FToolkit.build_tools()` (worker-side) | imports `LeanSandbox` directly | Yes — calls `compile_lean_file` | -| `LeanProofValid.evaluate()` (criterion-side) | imports `LeanSandbox` directly | Yes — same | - -`compile_lean_file` lives on `LeanSandbox`; the framework neither -references it nor needs to. The minif2f toolkit (which is already paired -with Lean by the experiment author) imports `LeanSandbox` and calls it -directly. Polymorphism handles the framework-facing interface -(`provision` / `terminate`); subclass-specific affordances bypass -polymorphism entirely. - -The same applies to `Worker` (framework calls only `execute()`; everything -else is the subclass's private implementation) and `Criterion` (framework -calls only `evaluate()`). - -**3. Workers that genuinely require a specific sandbox kind declare it — -once, at the binding.** - -Most workers should be sandbox-agnostic and just use the base interface -(`sandbox.run_command(["lean", "--check", file])` works wherever Lean is -installed; the worker doesn't care which subclass is providing it). When -that's genuinely impossible — the worker needs typed access to subclass -methods or to the structured return shapes they give — the worker -declares its compatibility: - -```python -class MiniF2FToolkit(_Toolkit): - requires_sandbox: ClassVar[type[Sandbox]] = LeanSandbox - - def build_tools(self, sandbox: Sandbox, task: Task) -> list[AgentTool]: - # Defensive isinstance — the Experiment validator (below) catches - # mismatches at construction time, so this branch is dead code in - # well-formed setups. Kept as a paranoia guard for hand-constructed - # workers in tests; the live error is the SandboxKindMismatch the - # validator raises before any rollout starts. - assert isinstance(sandbox, self.requires_sandbox), ( - f"{type(sandbox).__name__} given to {type(self).__name__}" - ) - # type-narrowed: sandbox is LeanSandbox here - return [_compile_tool(sandbox.compile_lean_file), ...] -``` - -The framework checks `requires_sandbox` against `task.sandbox` in -`Experiment`'s `@model_validator(mode="after")` — i.e. at -`Experiment(...)` construction in the author's process, before -`persist_definition` runs. A misconfigured pairing raises -`SandboxKindMismatch` immediately and never reaches the database -or a rollout. The validator walks each task's directly bound -`task.worker` and `task.evaluators`; no experiment-level binding pool -or evaluator-key chase is involved. See -[`08-decisions-log.md`](08-decisions-log.md#alternatives-considered) -"Worker → Sandbox compatibility checking" for why the validator -lives on `Experiment` rather than per-`Task` or in the service -layer. - -### The forcing test for what goes on the base - -> A method belongs on the base interface **if and only if the framework -> code itself calls it.** Anything else lives on the subclass. - -Apply that to the proposed `Sandbox` base: - -| Method | Framework calls? | Verdict | -|---|---|---| -| `provision()` | Yes (`SandboxLifecycleHub.acquire`) | Base — framework contract | -| `terminate()` | Yes (`SandboxLifecycleHub.release`) | Base — framework contract | -| `run_command()`, `write_file()`, `read_file()`, `list_files()` | No — workers and criteria call these | **Convenience surface, not framework contract** | - -This is a real wrinkle worth being honest about. The IO methods are -*useful* — every sandbox-backed worker wants them, and every E2B-backed -sandbox can implement them identically — but they're not part of the -framework contract; they're a shared convenience for sandbox-aware -consumers. Two ways to express that: - -- **Default-implementations on the base** (current proposal): the base - `Sandbox` declares these signatures; concrete kinds that *can* support - them override; consumers get a uniform call site. The lie is small — - every sandbox we'll ship in the foreseeable future implements them all, - so no one trips on `NotImplementedError` in practice. -- **A `_RemoteIOSandbox` intermediate base** (`Sandbox` ← `_RemoteIOSandbox` - ← `LeanSandbox`) that provides the IO methods. Any future sandbox kind - that genuinely can't (e.g. a native-process `LocalSubprocessSandbox`) - inherits straight from `Sandbox` and consumers fall back to - `requires_sandbox = _RemoteIOSandbox`. Honest about the layering, - slightly more boilerplate. - -We pick the first for now — keep IO on the base for ergonomic uniformity, -document them as a "convenience surface" rather than a framework contract, -and revisit when a non-IO sandbox kind first appears (tracked in -[`08-decisions-log.md#open-questions`](08-decisions-log.md#open-questions)). -What we explicitly do **not** do under either scheme: add methods that -don't generalize (e.g. `compile_lean_file`) to the base. Those always -live on the subclass and are reached by consumers that import the -subclass directly. - -The same shape governs `Worker` and `Criterion`. Framework-contract -method (`execute` / `evaluate`) on the base. Subclass-specific config in -typed pydantic fields. Subclass-specific methods invisible to the -framework, called only by consumers that imported the subclass. - -## Foundational change A — Worker becomes serializable - -Move `tools` out of `Worker.__init__`. Pass `sandbox` to `execute`. - -```python -# Before -class MiniF2FReactWorker(ReActWorker): - def __init__(self, *, name, model): - super().__init__(name=name, model=model, tools=[], system_prompt=..., max_iterations=30) - - async def execute(self, task, *, context): - sandbox = MiniF2FSandboxManager().get_sandbox(task.task_id) - toolkit = MiniF2FToolkit(sandbox=sandbox, ...) - self.tools = list(toolkit.get_tools()) # MUTATION — the smell - async for item in super().execute(task, context=context): - yield item - -# After — framework interface only: -class Worker(BaseModel, ABC): - """The entire framework-level worker interface. - - Knows about: task, context, sandbox. - Knows nothing about: tools, toolkits, prompts, models, iterations, - agents, LLMs, or any other per-strategy concept. - """ - - @abstractmethod - async def execute( - self, task: Task, *, context: WorkerContext, sandbox: Sandbox, - ) -> AsyncGenerator[WorkerStreamItem, None]: ... -``` - -That's the whole framework contract. Everything else — including how -`ReActWorker` chooses to organize its config — is **implementation detail -of `ergon_builtins`, not part of the authoring API.** - -**`sandbox` is non-optional.** The framework guarantees every worker -receives a live `Sandbox`. Every task picks a real, concrete template -(`"lean"`, `"swebench"`, `"research-e2b"`, `"python-3.13"`, …). The type -contract is uniform: `execute()` always gets a sandbox, no `None`-checks -anywhere in worker code. - -**The discipline that matters:** the framework provides the **primitive** -(`sandbox: Sandbox`). The worker decides the **strategy** (ReAct? tree -search? single-shot? hand-coded? multi-agent?). The framework does not -mention tools, toolkits, prompts, models, agents, LLMs, or any other -shape that pre-supposes "agents look like X." Other worker strategies can -look completely unlike `ReActWorker` and be first-class — they implement -`execute(task, *, context, sandbox)` and that's it. - -### How `ReActWorker` reorganizes (informative — not a framework concern) - -This subsection describes how the existing `ReActWorker` in -`ergon_builtins` ends up shaped after the smell is removed. It is **not -a public-API contract** — `ergon_builtins` owns this and can change it. -Sketched here only because it determines what benchmark authors actually -type when they configure a ReAct-style worker. - -The smell was that `ReActWorker` subclasses mutated `self.tools` inside -`execute()` because tools need a sandbox to construct. With `sandbox` -now passed to `execute()`, tool construction can move into the worker's -own internals. There are several reasonable shapes; the one we'd ship -collapses per-benchmark subclasses entirely: - -```python -# In ergon_builtins.workers.baselines.react_worker — module-private: -class _Toolkit(BaseModel, ABC): - """Internal to react_worker. Authors don't import the base directly; - they import concrete toolkits like MiniF2FToolkit.""" - @abstractmethod - def build_tools(self, sandbox: Sandbox, task: Task) -> list[AgentTool]: ... - -class ReActWorker(Worker): - name: str - model: str | None - system_prompt: str | None - max_iterations: int - _toolkit: _Toolkit # _type-discriminated, JSON round-trips - - async def execute(self, task, *, context, sandbox): - tools = self._toolkit.build_tools(sandbox, task) - # ... pydantic-ai agent loop with `tools` ... - -# In ergon_builtins.benchmarks.minif2f.toolkit — what already exists, -# made pydantic-serializable: -class MiniF2FToolkit(_Toolkit): - lean_version: str = "4.7.0" - ask_stakeholder: bool = False - - def build_tools(self, sandbox, task): - # uses self.lean_version + sandbox to build pydantic_ai Tool list - ... -``` - -Author code becomes one `ReActWorker` instance per role, no per-benchmark -worker subclasses anywhere: - -```python -ReActWorker(name="prover-1", model="openai:gpt-4o", system_prompt="...", - max_iterations=30, - toolkit=MiniF2FToolkit(lean_version="4.7.0")) - -ReActWorker(name="patcher", model="openai:gpt-4o", system_prompt="...", - max_iterations=50, - toolkit=SWEBenchToolkit(repo_url="...", timeout_s=600)) -``` - -`MiniF2FReactWorker`, `SWEBenchReActWorker`, `GDPEvalReActWorker` all -**delete entirely** — they were "ReAct + tool list" combinations -masquerading as Worker subclasses. With `toolkit` as a field, the -combination is just an instance. - -Other worker strategies in `ergon_builtins` (a hypothetical -`TreeSearchWorker`, `SingleShotWorker`, …) are free to ignore the -`_Toolkit` convention and shape their config however makes sense for -them. The framework neither sees nor cares. - -## Foundational change B — Sandbox becomes a typed `Sandbox` subclass per kind - -`Task.sandbox: Sandbox` is non-optional. The author picks the concrete -sandbox subclass that matches the task's environment kind: - -```python -Task(sandbox=LeanSandbox(lean_version="4.7.0")) -Task(sandbox=PythonSandbox(pip_packages=("pandas", "requests"))) -Task(sandbox=ResearchE2BSandbox()) -Task(sandbox=LocalDockerSandbox(image="my-team/eval:latest")) -``` - -The base `Sandbox` class is `ABC` and not directly instantiable. Authors -must commit to a concrete kind — there's no `Sandbox(template="...")` -escape hatch. The class identity *is* the dispatch key; no parallel -template-string registry exists. - -**No `"none"` / no-op sandbox.** A sandbox that satisfies the type -contract while no-op'ing `run_command` is a dangerous lie — it makes every -call site look uniform while making behavior surprising. The honest -options are: - -- For tasks that genuinely use an environment, pick the real sandbox - subclass that matches (`LeanSandbox`, `PythonSandbox`, etc.). -- For LLM-only or pure-orchestration tasks that we genuinely don't want - to spin a container for: **out of scope for this redesign**. Land it - later via a generic-Docker `DefaultPythonSandbox` (basic Python, no - special setup) subclass — at least then `sandbox.run_command(...)` - actually works rather than pretending to. - -Until that generic subclass lands, every existing benchmark already has a -real sandbox it uses, so non-optional `Task.sandbox: Sandbox` is currently -satisfiable for the in-tree benchmarks without inventing the no-op. - -Per-benchmark sandbox managers (`MiniF2FSandboxManager`, -`SWEBenchSandboxManager`, etc.) **delete entirely**. Each becomes a -`Sandbox` subclass that owns its own `provision()` / `terminate()` — -see [`03-runtime.md#sandbox-provisioning`](03-runtime.md#sandbox-provisioning) -for the full shape. Heterogeneous DAGs work natively because each Task -carries its own typed `Sandbox` subclass. - -## Foundational change C — `Experiment` lifts into the public API - -`Experiment` is the composition root authors construct around a benchmark -definition. The benchmark (via normal Python composition) produces Tasks -that already carry their workers, sandboxes, and evaluators. Today -`Experiment` lives in `ergon_core.core.domain.experiments` and is -imported into author code from there — a non-obvious path that pretends -the type is internal when in fact every benchmark constructor calls it -directly. This redesign **moves the class definition to -`ergon_core/api/experiment.py`** and deletes -`ergon_core/core/domain/experiments/experiment.py` — no re-export -indirection. The single import path is `from ergon_core.api import -Experiment`, in line with `Benchmark`, `Task`, `Worker`, `Sandbox`, etc. - -What lives where after the lift: - -| Concern | Lives in | -|---|---| -| `Experiment` class definition (the type itself) | `ergon_core/api/experiment.py` (this redesign) | -| `ExperimentValidationService` (cross-component rules engine) | `ergon_core/core/domain/experiments/validation.py` (unchanged location) — operates on the public `Experiment` instance | -| `DefinitionHandle` (return shape of `persist_definition`) | `ergon_core/core/domain/experiments/handles.py` (unchanged) — internal; held as `Experiment._persisted` PrivateAttr | -| `definition_writer.py`, `service.py`, `launch.py` | `ergon_core/core/application/experiments/` (unchanged) — import `Experiment` from `ergon_core.api` | - -The class itself becomes a Pydantic `BaseModel` so it can host the -`requires_sandbox` `@model_validator(mode="after")` — see -[`08-decisions-log.md`](08-decisions-log.md) "Worker → Sandbox -compatibility checking" for why the validator lives here. The -runtime-only `_persisted` reference becomes a `PrivateAttr` (same -pattern as `Task._task_id`): - -```python -# ergon_core/api/experiment.py -class Experiment(BaseModel): - """Composition root for a benchmark definition. Tasks carry their - worker/sandbox/evaluator objects directly; Experiment owns the - benchmark as a whole plus run-level metadata.""" - model_config = {"frozen": False, "arbitrary_types_allowed": True} - - benchmark: Benchmark - name: str | None = None - description: str | None = None - # First-class authoring-metadata fields. Anything the framework - # *reads* (dashboard listing, audit, denormalized indexed columns - # per 02-persistence-layer.md §3) lives here, not in `metadata`. - # `metadata` is for opaque author-provided tags only. - created_by: str | None = None - metadata: Mapping[str, Any] = Field(default_factory=dict) - - _persisted: DefinitionHandle | None = PrivateAttr(default=None) - - @model_validator(mode="after") - def _validate_sandbox_compatibility(self) -> "Experiment": - """For each task, check `task.worker.requires_sandbox` and every - directly bound evaluator/criterion's `requires_sandbox` against - `task.sandbox`. - Raises `SandboxKindMismatch` (canonical signature in - "Public exceptions" above) at construction time, before - `persist_definition` ever runs.""" - ... # walk benchmark.tasks and inspect each task's direct objects - return self -``` - -The `validate()` instance method that today wraps -`ExperimentValidationService` stays as-is — the cross-component rules -that aren't about `requires_sandbox` (task slugs unique, dependency slugs -resolve, etc.) keep living in the service layer and get called by -`definition_writer` before persisting. The pydantic -`@model_validator` only owns the type-driven sandbox check; everything -else stays in the explicit service so the rules are inspectable in one -place and don't run on every `Experiment(...)` construction in tests. - -The old `Experiment.workers`, `Experiment.evaluators`, and -`Experiment.assignments` pools delete. The modularity they were trying -to provide moves into normal Python composition: benchmark factories can -accept different `Worker` / `Evaluator` objects and construct otherwise -identical `Task` graphs for cohort runs. The core model stays object-first -instead of exposing a mini dependency-injection container. - -## Things that fall out for free - -- **`TaskSpec` dies.** Single `Task` class with `_task_id` PrivateAttr (see above). -- **`SandboxSpec` and per-benchmark `*SandboxManager` classes die.** One - abstract `Sandbox` base + one concrete subclass per environment kind - (`LeanSandbox`, `PythonSandbox`, `LocalDockerSandbox`, …); each subclass - owns its own `provision()` / `terminate()`. No allocator, no template - registry, no string dispatch — `type(sandbox)` *is* the dispatch key. -- **`WorkerSpec` dies.** `Task.worker` carries a pre-constructed `Worker` - instance, which is now a serializable pydantic model. -- **`ComponentRegistry`, `register_builtins`, `registry.publish`, - `ComponentCatalogEntry` table, `ComponentCatalogService` all die.** The - worker's class identity travels with each `Task.worker` as a `_type` discriminator - in JSON: `{"_type": "ergon_builtins.workers...:ReActWorker", "name": ..., ...}`. - The container does `import_component_string(json["_type"]).model_validate(json)`. - Slugs survive only at the CLI surface as a static `BUILTIN_WORKERS = {slug: - import_path}` dict in `ergon_builtins`. -- **Per-benchmark `MiniF2FReactWorker`/`GDPEvalReactWorker`/`SWEBenchReactWorker` - delete entirely.** They were `ReActWorker` + a tool list pretending to - be a class hierarchy. With `ReActWorker` taking `toolkit` as a field - (see "How `ReActWorker` reorganizes" above), each benchmark just - configures a `ReActWorker(toolkit=BenchmarkToolkit(...))` instance — no - Worker subclass needed. The existing `MiniF2FToolkit`, - `SWEBenchToolkit`, `GDPEvalToolkit` classes (already in tree) become - pydantic-serializable and lose their `__init__(sandbox=...)` — - `sandbox` moves to a `build_tools(sandbox, task)` method. The - benchmark-specific environment bits (Lean install, repo clone) move - out of the worker entirely and into the relevant `Sandbox` subclass's - `provision()` (`LeanSandbox.provision()` does the Lean install, - `SWEBenchSandbox.provision()` does the repo clone) — that's a sandbox - concern, not a worker concern. -- **The whole `definition_task_id` / `node_id` / `task_id` confusion - collapses to a single `task_id`** (see - [`02-persistence-layer.md#identifier-model`](02-persistence-layer.md#identifier-model-two-tables-one-identity)). - Born once at experiment-define (or spawn) time and carried unchanged - through the pipeline. `WorkerContext` shrinks to the things that aren't - on the task: `run_id`, `execution_id`, `definition_id`. -- **`CriterionContext` shrinks dramatically.** Its 12 proxy methods - (`run_command`, `read_resource`, `write_file`, …) become "you have a - `sandbox: Sandbox` and a `task: Task`, use them directly." The - `_runtime` PrivateAttr that `CriterionContext` invented for its own use - is exactly the pattern `Sandbox` now uses; the context shrinks to a - pure data carrier. - -## `TaskSpec` — resolved by the unified pattern - -The earlier draft of this redesign left this as an open question with three -options. The unified `PrivateAttr`-for-runtime-state pattern subsumes it: -**there is one `Task`. Definition-time, `_task_id` is `None`; the public -`task_id` property raises if accessed. Materialization-time, the framework -sets `_task_id`; `task.task_id` returns the UUID.** No `TaskSpec` class, no -inheritance, no `task_id: UUID | None` propagation through downstream types. - -The compile-time non-null guarantee from the previous "two-class" design -becomes a runtime invariant guarded by the property accessor. The narrowness -of the materialization callsite (one in `worker_execute.py`, one in the -dynamic-subtask path) makes this safe — author code can never reach a -materialized `Task` through a path that didn't go through the framework. - -## Sandbox capability surface — locked `[v2: locked]` (Q24) - -The exact methods the base `Sandbox` class exposes, locked at workshop. -Future PRs that want to add a method to the base must amend this section -first. - -### Framework contract (must be implemented; framework calls these) - -| Method | Signature | Notes | -|---|---|---| -| `provision` | `async def provision(self) -> None` | Abstract. Subclass spins up the underlying environment and attaches `self._runtime`. | -| `terminate` | `async def terminate(self) -> None` | Defaults to no-op. Override for environments that need teardown. | - -That's the framework's actual contract — only these two are called by -`SandboxLifecycleHub`. Everything else on the base is a convenience for -worker / criterion authors. - -### Convenience surface (every IO-backed sandbox implements these) - -| Method | Signature | Purpose | -|---|---|---| -| `run_command` | `async def run_command(self, cmd: str \| Sequence[str], *, timeout: int \| None = None) -> CommandResult` | Run a shell command. Single-string is split per shell rules; `Sequence[str]` is exec-style. | -| `write_file` | `async def write_file(self, path: str, content: bytes \| str) -> None` | Write file at `path` inside the sandbox. `str` is UTF-8 encoded. | -| `read_file` | `async def read_file(self, path: str) -> bytes` | Read file from `path` inside the sandbox. | -| `list_files` | `async def list_files(self, path: str) -> list[str]` | Directory listing under `path`. Returns flat names, not absolute paths. | - -These are documented as "convenience surface, not framework contract" — -the framework never calls them itself. They are inherited by every -sandbox subclass via the `_runtime` proxy. The day a sandbox kind that -genuinely cannot implement them appears (e.g. a hypothetical -`LocalSubprocessSandbox` with no remote IO), they get promoted onto a -`_RemoteIOSandbox` intermediate base; until then they live on the base -for ergonomic uniformity. - -### Properties - -| Property | Type | Purpose | -|---|---|---| -| `output_path` | `str` (field, defaults `"/workspace/final_output/"`) | Convention path for "task output" files. Overridable per subclass / instance. Replaces the v1 hardcoded path in worker code AND the resource publisher. | -| `is_live` | `bool` | True iff `provision()` has run and `_runtime` is attached. | -| `sandbox_id` | `str` (raises if not live) | Debug/tracing handle from the underlying runtime. | -| `env` | `dict[str, str]` (field) | Environment variables passed at provision time. Authors set; subclass `provision()` reads. | -| `timeout_seconds` | `int \| None` (field) | Default per-command timeout if not overridden by `run_command`'s `timeout=`. | -| `requires_network` | `bool` (field) | Authoring hint; subclass `provision()` decides whether to enable network. | - -### Explicitly NOT on the base (kind-specific affordances) - -The following are deliberately left for subclasses to add as needed: - -- `compile_lean_file(path: str) -> CompileResult` — `LeanSandbox` only. -- `apply_patch(diff: str) -> PatchResult` — `SWEBenchSandbox` only. -- `execute_code(code: str, language: str) -> SandboxResult` — generic - Python/REPL sandboxes only; in shell-based sandboxes use - `run_command(["python", "-c", code])` directly. -- `read_resource(name: str) -> bytes` / - `read_resource_by_id(id: UUID) -> bytes` — these are NOT on the - sandbox; they live on `RunResourceRepository` (cross-cutting between - sandbox and run state). Workers/criteria reach the resource repo via - `WorkerContext.resources(...)` or by direct import. Sandbox is for - per-sandbox filesystem; resource repo is for cross-task content - storage. The two boundaries stay separate. -- `upload_files(files: list[dict]) -> None` — folded into the resource - repo's materialise-into-sandbox helper. Not a sandbox method per se; - it's a sandbox + resource-repo composition. - -### Migration from v1's CriterionRuntime - -v1's `CriterionRuntime` Protocol had `upload_files`, `execute_code`, -`cleanup`, `read_resource`, `read_resource_by_id` in addition to the -file/command methods. v2's split: - -| v1 `CriterionRuntime` method | v2 home | -|---|---| -| `run_command` | `Sandbox.run_command` (base) | -| `write_file` | `Sandbox.write_file` (base) | -| `read_file` *(implicit)* | `Sandbox.read_file` (base) | -| `list_files` *(implicit)* | `Sandbox.list_files` (base) | -| `upload_files` | `RunResourceRepository.materialise_into(sandbox, ...)` (cross-cutting helper) | -| `execute_code` | subclass-specific (e.g. `PythonSandbox.execute_code`); not on base | -| `cleanup` | folded into `Sandbox.terminate` (rename) | -| `read_resource` | `RunResourceRepository.read_by_name` | -| `read_resource_by_id` | `RunResourceRepository.read_by_id` | - -The split preserves "Sandbox = per-sandbox filesystem and process -runtime" as the base contract, with `RunResourceRepository` owning the -run-tier content store. Criteria that need both reach for both — -`Sandbox` via `context.task.sandbox` (live in the eval worker per Δ.5), -repo via `context.resources(...)` or direct import. - -## Criterion class signature — locked `[v2: locked]` - -The `Criterion.evaluate(...)` method keeps the v1 signature unchanged. -The earlier `[P4]` annotation on `criterion.py` in the file-tree -section ("evaluate(..., sandbox: Sandbox)") is **superseded** by this -lock — the sandbox flows in via `context.task.sandbox`, not as a -separate positional parameter. Rationale: - -- The object-bound `Task` (Foundational change A) makes `task.sandbox` - the canonical place to find the sandbox at evaluation time. -- Per Δ.5, the eval worker's `Sandbox.from_definition(sandbox_id=...)` - call attaches a live `_runtime` to `task.sandbox` before - `criterion.evaluate(context)` runs, so `context.task.sandbox.run_command(...)` - is the live IO entry point. -- Keeping the v1 single-arg signature avoids a churn-cascade across - every in-tree criterion subclass. - -### Signature - -```python -from ergon_core.api.criterion.context import CriterionContext -from ergon_core.api.criterion import CriterionOutcome - - -class Criterion(BaseModel, ABC): - """Atomic evaluation unit. - - Subclasses override `evaluate` and access the live sandbox via - `context.task.sandbox` (a `Sandbox` instance with its `_runtime` - attached by the eval worker's reconnect-by-sandbox_id call). - """ - - type_slug: ClassVar[str] - required_packages: ClassVar[list[str]] = [] - - @abstractmethod - async def evaluate(self, context: CriterionContext) -> CriterionOutcome: - """Run one atomic evaluation against the provided context. - - `context.task` carries the inflated Task with a live sandbox - attached (per Δ.5). `context.worker_result` carries the - WorkerOutput from the just-finished worker run. - """ - ... - - @classmethod - def from_definition(cls, criterion_json: TaskDefinitionJson) -> "Criterion": - """Reconstruct a concrete Criterion subclass from its persisted - JSON via the `_type` discriminator. Raises `ValueError` if - `_type` is missing or non-string.""" - ... -``` - -### CriterionContext shape (v2) - -`CriterionContext` becomes a **pure data carrier** — its v1 runtime -proxy methods (`run_command`, `read_resource`, `write_file`, …) are -dropped. The sandbox proxies move to `Sandbox` itself (per the -capability surface lock). `CriterionContext` carries: - -```python -class CriterionContext(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - run_id: UUID - task_id: UUID - execution_id: UUID - task: Task # task.sandbox is live in the eval worker - worker_result: WorkerOutput - metadata: dict[str, Any] = Field(default_factory=dict) - # _runtime PrivateAttr is gone — sandbox proxies live on Sandbox. -``` - -`sandbox_id` is no longer a separate field — it's available as -`context.task.sandbox.sandbox_id` once the runtime is attached. - -### What this kills - -- `CriterionRuntime` Protocol and its `with_runtime(...)` constructor on - `CriterionContext` (replaced by sandbox proxies on `Sandbox`). -- The `sandbox: Sandbox` positional parameter that the file-tree - `[P4]` annotation hinted at — same access via `context.task.sandbox`. -- The earlier reading of `02-persistence-layer.md` and - `06-inngest-event-contracts.md` pseudocode that suggested - `evaluator.evaluate(task=..., worker_output=...)` as a positional-arg - form. Reconciled: the EvaluationService internally constructs a - `CriterionContext` and calls `criterion.evaluate(context)`. - -### How `Rubric.evaluate` interacts - -`Rubric` (an `Evaluator` subclass) iterates its `criteria`, calls each -`criterion.evaluate(context)`, and aggregates the `CriterionOutcome` -results into an `EvaluatorResult`. The Rubric.evaluate signature is the -same shape: `async def evaluate(self, context: CriterionContext) -> EvaluatorResult`. - -## Worker / Criterion non-pydantic runtime state — locked `[v2: locked]` (Q25) - -`Worker` and `Criterion` are pydantic models. Their public fields are -serializable (round-trip through `_type` discriminator). But some -workers and criteria need *non-serializable* runtime state — HTTP -clients, model resolvers, in-memory caches. - -The locked pattern for v2: - -### Allowed - -- **Construct fully at author init time.** Author writes - `MyWorker(name="foo", model="gpt-4")`. All state needed by `execute()` - either lives in public pydantic fields (config) or is built fresh - inside `execute()` each call. -- **Classmethod factories.** `Worker.from_config(...)` / `Worker.from_env()` - are fine — they're explicit constructors that produce a fully-wired - instance. The factory does whatever expensive setup is needed (HTTP - client warmup, credential lookup, …) before returning. -- **PrivateAttr set explicitly inside a classmethod / inside `execute`.** - If a worker holds a non-serializable handle (e.g. a `ModelResolver`), - the field is a `PrivateAttr` and the only way it gets populated is by - an explicit assignment inside a factory or inside `execute()`. - -### Disallowed - -- **`model_post_init` for runtime state.** Pydantic's `model_post_init` - runs implicitly after every construction (including - `model_validate(...)` from JSON). Using it to lazy-init runtime - resources means every JSON deserialization (e.g. cross-process - reconstruction in `worker_execute`) tries to spin up an HTTP client - before the runtime is ready. This is a class of v1 bug we're - explicitly avoiding. -- **Lazy-init via `@property` that mutates state.** Same problem: - hidden, time-varying behavior tied to method access rather than - explicit lifecycle. - -### Pattern in code - -```python -class HttpClientWorker(Worker): - """Worker that calls an external HTTP API.""" - name: str - model: str - api_base: str - - # NO model_post_init. NO @property that builds clients on first read. - - async def execute(self, task, *, context, sandbox): - # Build the client fresh each execute() call. Cheap (httpx - # client construction is microseconds); honest about lifecycle - # (client lifetime == single execute() call); easy to test - # (no global state). - async with httpx.AsyncClient(base_url=self.api_base) as client: - ... -``` - -```python -class CachedResolverWorker(Worker): - """Worker that needs a long-lived model resolver.""" - name: str - model: str - - _resolver: ModelResolver | None = PrivateAttr(default=None) - - @classmethod - def from_config(cls, *, name: str, model: str) -> "CachedResolverWorker": - """Explicit factory — populates _resolver at construction.""" - instance = cls(name=name, model=model) - object.__setattr__(instance, "_resolver", ModelResolver(model)) - return instance - - async def execute(self, task, *, context, sandbox): - if self._resolver is None: - # Author called `CachedResolverWorker(...)` directly instead - # of `from_config(...)`; that's an authoring error worth - # surfacing. NOT silently building one here. - raise RuntimeError( - "CachedResolverWorker constructed without resolver; use " - "CachedResolverWorker.from_config(...)" - ) - ... -``` - -### Why this is locked - -The "lazy-init via `model_post_init`" anti-pattern crashed v1 in -several places (sandbox creds resolved at construction time before env -was wired; HTTP clients built during Inngest event-handler boot before -the event loop was ready). v2 says: **state lifecycle is explicit, in -`__init__` or `from_config`, never `model_post_init`.** The cost is -slightly more boilerplate; the win is no more "why is this client -trying to connect on import?" debugging sessions. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/02-persistence-layer.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/02-persistence-layer.md deleted file mode 100644 index c65334e96..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/02-persistence-layer.md +++ /dev/null @@ -1,524 +0,0 @@ -# 02 — Persistence and identity - -> How the typed objects from [`01-api-surface.md`](01-api-surface.md) move -> between Python and Postgres, how the runtime reconstructs them, and the -> two-table identity model that underpins both static tasks and dynamic -> spawning. See [`03-runtime.md`](03-runtime.md) for what the rollout -> container does with the result and -> [`04-walkthrough.md`](04-walkthrough.md) for the concrete trace. - -> **What v2 changed.** The §3–§5 sections (two-tier persistence layout, -> read boundary, dynamic-subtask shape) are new in v2 and lock in the -> audit findings from -> [`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md). -> §1 (`from_definition` convention) and §2 (two-table identity model) -> are carried forward unchanged. Sections marked `[v2: added]` are new; -> everything else is the v1 design intent that holds up. - -## Persistence: typed repos at the boundary, `from_definition` on the class - -The job body should never deserialize an authoring class directly. Two -project conventions converge on this: - -1. **Reads from Postgres go through a typed repository.** That convention - already exists in tree (`DefinitionRepository.task_with_instance`, - etc.). The repository owns the SQLModel session, returns typed row - objects, and is the only layer that touches `session.get`. -2. **Class reconstruction lives on the class.** The - `import_component_string + model_validate` dance (which every - serializable authoring class needs) is framework infrastructure, not - job-body logic. Each authoring class exposes a `from_definition` - classmethod that owns it. - -```python -# In ergon_core.api — base classes carry the convention: - -from pydantic import JsonValue - - -type TaskDefinitionJson = dict[str, JsonValue] -"""Serialized form of a Task/Worker/Sandbox/etc — `_type`-discriminated -JSON written to `run_graph_nodes.task_json` and the matching definition -columns. Field names are NOT enforced by the type (the discriminator -dispatch in `from_definition` does that). The value side IS typed via -pydantic's `JsonValue`, so sticking a `datetime` or `UUID` object into -the snapshot fails at typecheck time instead of at JSON-serialization -time. The alias is the named boundary every `from_definition` -classmethod accepts.""" - - -class Worker(BaseModel, ABC): - @classmethod - def from_definition(cls, worker_json: TaskDefinitionJson) -> "Worker": - """Reconstruct a concrete Worker subclass from its persisted JSON. - Discovers the subclass via the `_type` discriminator. Raises - `ValueError` if `_type` is missing or non-string — there is no - soft-default to a base class, because doing so would silently - produce the wrong Worker subclass at runtime.""" - worker_type = worker_json.get("_type") - if not isinstance(worker_type, str): - raise ValueError( - f"Worker snapshot is missing the required `_type` " - f"discriminator (got {type(worker_type).__name__})." - ) - WorkerCls = import_component_string(worker_type) - return WorkerCls.model_validate(worker_json) - -class Task(BaseModel, Generic[PayloadT]): - # Persisted direct bindings. Worker/evaluator/sandbox objects all - # round-trip through their own `_type` discriminators inside task_json. - worker: Worker - sandbox: Sandbox - evaluators: tuple[Evaluator, ...] = () - - # Runtime-only state. Not in JSON; populated by `from_definition` below. - # The public accessor (`task.task_id`) lives on the class itself and is - # defined in 01-api-surface.md. - _task_id: UUID | None = PrivateAttr(default=None) - - @classmethod - def from_definition( - cls, - task_json: TaskDefinitionJson, - *, - task_id: UUID, - ) -> "Task": - """Reconstruct a Task fully inflated for runtime use. Combines - the two concerns the framework never wants separately: - - 1. Deserialize the persisted JSON via the `_type` discriminator. - 2. Bind the per-run identity (`_task_id`). - - Framework-internal: only the graph repository calls this (see - `RunGraphNodeView` below). Author code constructs `Task(...)` - directly and never touches `from_definition`. - - Raises `ValueError` if `_type` is missing or non-string — there - is no soft-default to base `Task`, because doing so would - silently drop the authored worker/sandbox/evaluator bindings. - """ - task_type = task_json.get("_type") - if not isinstance(task_type, str): - raise ValueError( - f"Task snapshot is missing the required `_type` " - f"discriminator (got {type(task_type).__name__})." - ) - TaskCls = import_component_string(task_type) - instance = TaskCls.model_validate(task_json) - # PrivateAttr on a non-frozen BaseModel is settable directly; - # object.__setattr__ kept for symmetry with frozen-Sandbox patterns. - object.__setattr__(instance, "_task_id", task_id) - return instance - -class Sandbox(BaseModel, ABC): - @classmethod - def from_definition(cls, sandbox_json: TaskDefinitionJson) -> "Sandbox": - sandbox_type = sandbox_json.get("_type") - if not isinstance(sandbox_type, str): - raise ValueError( - f"Sandbox snapshot is missing the required `_type` " - f"discriminator (got {type(sandbox_type).__name__})." - ) - SandboxCls = import_component_string(sandbox_type) - return SandboxCls.model_validate(sandbox_json) - -# Same shape on Criterion, Rubric, Benchmark. -``` - -`from_definition` takes a `TaskDefinitionJson` (the JSON from the row), -**not** the typed row itself — that would create an `api → persistence` -dependency in the wrong direction. It should not take a -worker/evaluator pool: those objects are part of `Task` itself now. The -guideline: after `from_definition`, no field on the returned object -should still be a "go look this up over there" reference. - -The boundary type is `dict[str, JsonValue]`, not a Pydantic model and -not a `TypedDict`. The shape *after* discriminator dispatch lives on -the concrete subclass; mirroring it in a parallel typed structure -would duplicate the schema and rot. The boundary type asserts only -what's honest at the boundary: "JSON, value-side typed, shape TBD by -the discriminator." - -There is **no `_materialize` classmethod** on `Task` (or any other -class). `from_definition` is the single framework-internal entry -point that goes from "raw JSON + resolution context" to "fully usable -runtime instance." This avoids the two-step `model_validate` → -`_materialize` shape an earlier draft had, where it was unclear which -method owned identity binding. - -The job body shouldn't see a `TaskDefinitionJson` either — the repo is -the bridge, and it returns *fully inflated typed objects*, never raw JSON. -The graph repo returns a typed `RunGraphNodeView` with the `Task` already -inflated: - -```python -class RunGraphNodeView(BaseModel): - """Typed view of one run_graph_nodes row + its inflated Task.""" - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - run_id: UUID - task_id: UUID - parent_task_id: UUID | None - status: NodeStatus - task: Task # already inflated by from_definition - # ...other row fields -``` - -Pydantic (frozen) rather than `@dataclass(frozen=True)` for project -consistency: every typed object the repo emits is a `BaseModel`, including -the inflated `Task` it wraps. The view itself never round-trips through -JSON (it's a runtime read result, not a persisted shape), so the choice is -about uniform construction/validation ergonomics with its `Task` payload, -not serialization. - -The job body becomes orchestration only — typed objects throughout, no -dicts visible: - -```python -node = graph_repo.node(session, run_id=payload.run_id, - task_id=payload.task_id) - -task = node.task -worker = task.worker -# task.evaluators is already a typed tuple because it lives in task_json. -``` - -`Task.from_definition` is called *once*, *inside* `graph_repo.node(...)`, -and never appears in the job body. It's a framework-internal classmethod -(authors construct via `Task(...)`, not via `from_definition`) — the -`TaskDefinitionJson` parameter is honest about being the typing boundary -between PG's untyped JSON column and the typed authoring hierarchy, and -that boundary lives entirely inside the repo. No `session.get` in sight; -no `import_component_string` in sight; no binding-key chase at the -iteration site (downstream code reads `node.task.worker` and -`node.task.evaluators` as typed objects); no static/dynamic branching. -The two infrastructure concerns — -*how do I fetch this from PG?* and *how do I rebuild this object from -JSON?* — each have one canonical home, and each authoring class is -reconstructed at most once per job. - -## Identifier model: two tables, one identity - -The runtime separates *what was authored* from *what is happening*, and -treats `task_id` as a single stable identity that flows from the first -table to the second. - -### Two-table split - -| Table | Mutability | Holds | Lifetime | -|---|---|---|---| -| `experiment_definition_tasks` | **immutable** after experiment-define | `task_json` (including task worker, sandbox, evaluators) + dependencies | written once at experiment-define; never mutated | -| `run_graph_nodes` | **mutable** during a run | per-run task state + inline `task_json` snapshot | populated by copy at run-launch; grown by the runtime as workers spawn dynamic children; settled as tasks complete | - -**Definitions never change after the experiment is defined.** Runs are -independent snapshots, fully self-contained on `run_graph_nodes` after -launch — definition mutations (if we ever allowed any) cannot affect -in-flight runs because the run carries its own copy. - -### Identity preservation: `task_id` is the single canonical id - -`task_id` is born exactly once — at experiment-define time for static -tasks, at spawn time for dynamic tasks — and the same value flows -unchanged through the entire pipeline: - -``` - ┌─ static tasks ─────────────────────────────┐ - │ │ -ExperimentDefinitionTask.id → run_graph_nodes.task_id │ - (literal copy at run-launch) │ - │ - ┌─ dynamic tasks ────────────────────────────┤ - │ │ - uuid4() at spawn → run_graph_nodes.task_id │ - (no definition row exists) │ - │ - task.task_id (PrivateAttr) ─────────────────────────→┘ - (Same value, set by Task.from_definition.) -``` - -`(run_id, task_id)` is the **composite primary key** of -`run_graph_nodes`. Edges, executions, and any other rows that point at a -node use composite FKs into `(run_id, task_id)`. The same `task_id` value -recurs across N runs of the same definition — uniqued per row by `run_id`. - -| Layer | Identifier | Born at | Cardinality | -|---|---|---|---| -| Experiment | `definition_id` (`experiment_definitions.id`) | experiment-define | 1 per defined experiment | -| Task | `task_id` | experiment-define (static) OR spawn (dynamic) | N per definition; reused once per run | -| Run | `run_id` (`runs.id`) | run-launch | 1 per invocation | -| Per-run task | `(run_id, task_id)` | run-launch (static) OR spawn (dynamic) | composite PK on `run_graph_nodes` | -| Per-attempt | `execution_id` | first attempt + each retry | N per `(run_id, task_id)` | - -### What this kills - -- **`definition_task_id` as a separate column.** It existed because - `run_graph_nodes.id` was a freshly-generated UUID per row, distinct from - the definition row's `id`. Now `task_id` *is* the definition's id (for - static tasks); no separate FK column needed. The discriminator for - "static vs. dynamic" is `parent_task_id IS NULL` *or* whether a row in - `experiment_definition_tasks` exists with this id — both are O(1) checks. -- **`node_id`.** Was the runtime per-row UUID. The composite - `(run_id, task_id)` does its job; nothing references it anymore. -- **The "definition vs. runtime task identity" mental model.** There's - one task identity, full stop. It just lives in two places (the - immutable definition row, and the mutable per-run row that copied it). - -### Impact - -- **Authors** never write any IDs. -- **Worker / criterion code** reads `task.task_id` (and `context.execution_id` - for retries). The value is the *same* UUID the definition row has — no - conceptual jump between layers. -- **Runtime / persistence code** uses `(run_id, task_id)` as the canonical - row key. Cross-run analytics ("how did `<task_id>` perform across all - runs?") is `WHERE task_id = X`; per-run lookup ("what's task X doing in - run Y?") is `WHERE (run_id, task_id) = (Y, X)`. Both natural. - -## §3 — Two-tier persistence layout `[v2: added]` - -v1 carried three persistence tiers — `ExperimentRecord` (telemetry / -authoring metadata), `ExperimentDefinition` (immutable definition), -`RunGraphNode` (mutable run state). The audit -([`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md)) -found `ExperimentRecord` was a thin wrapper around `ExperimentDefinition` -that duplicated authoring fields and added no behavior. v2 collapses it. - -### The two tiers - -| Tier | Tables | Lifetime | Mutability | -|---|---|---|---| -| **Definition tier** | `experiment_definitions`, `experiment_definition_tasks`, `experiment_definition_edges` | Born once at `persist_definition`; one row per defined experiment | Immutable after write | -| **Run tier** | `runs`, `run_graph_nodes`, `run_graph_edges`, `run_graph_annotations`, `run_graph_mutations`, `task_executions` | Born at `launch_run`; mutable for the lifetime of the run; settled when run completes | Mutable | - -`run_graph_annotations` and `run_graph_mutations` together form the WAL -that keeps the live run-graph reconstructable from any point in time -(see v1's design discussion — unchanged). - -### `experiment_definitions` columns (collapsed) - -The columns formerly split across `ExperimentRecord` and -`ExperimentDefinition` collapse onto `experiment_definitions`: - -```sql -CREATE TABLE experiment_definitions ( - id UUID PRIMARY KEY, -- definition_id - -- Authoring metadata (was ExperimentRecord) ↓ - name TEXT NOT NULL, - description TEXT, - metadata JSONB NOT NULL DEFAULT '{}'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - created_by TEXT, -- user / agent who defined - -- Definition payload (was ExperimentDefinition) ↓ - benchmark_json JSONB NOT NULL, -- _type-discriminated Benchmark - experiment_json JSONB NOT NULL -- _type-discriminated Experiment frame -); -``` - -Authoring metadata moves directly onto `Experiment`: - -```python -class Experiment(BaseModel): - benchmark: Benchmark - name: str | None = None - description: str | None = None - metadata: Mapping[str, Any] = Field(default_factory=dict) - # ... validator unchanged from 01-api-surface.md -``` - -`name` / `description` / `metadata` round-trip through `experiment_json` -*and* are denormalized into dedicated columns for indexable lookup -(`SELECT id FROM experiment_definitions WHERE name = ...` should not -require JSON path queries). - -### `runs` columns - -```sql -CREATE TABLE runs ( - id UUID PRIMARY KEY, -- run_id - definition_id UUID NOT NULL REFERENCES experiment_definitions(id), - status run_status NOT NULL, -- enum: pending, running, succeeded, failed, cancelled - started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - completed_at TIMESTAMPTZ, - -- Telemetry per run (was on ExperimentRecord, moves to run-scope) ↓ - triggered_by TEXT, - metadata JSONB NOT NULL DEFAULT '{}'::jsonb -- per-run overrides, tags -); -``` - -Run-scoped telemetry (who triggered, when started, completion timestamp) -lives on `runs`, not on `experiment_definitions`. The collapse is only -about `ExperimentRecord` — `Run`-tier telemetry stays where it was. - -### What this kills - -In addition to the v1 deletions in §2 "What this kills": - -- **`ExperimentRecord` table.** Folded into `experiment_definitions`. -- **`experiment_record_id` FKs anywhere they appeared.** Replaced by - `definition_id`. -- **The `ExperimentService.create_experiment_record()` step in the - authoring path.** Becomes a single `persist_definition(experiment)`. - -### Why the collapse is safe - -`ExperimentRecord` was only ever populated by `persist_definition`, only -ever read by `launch_run` and the dashboard list endpoint, and never -mutated after creation. It had a 1:1 relationship with -`ExperimentDefinition`. The audit confirmed there is no use case where -the two needed independent lifecycles — the second tier was -infrastructure overhead, not a domain distinction. - -## §4 — Read boundary: runtime reads only run-tier tables `[v2: added]` - -A class of v1 bugs ("the runtime is reading the definition row for the -sandbox subclass when it should be reading the snapshot in the run row") -is eliminated by a single rule: - -> **After `prepare_run` copies the graph from definition into run-tier -> tables, all subsequent runtime code reads exclusively from run-tier -> tables.** No fall-through path may resolve a `Sandbox`, `Worker`, -> `Evaluator`, or `Task` payload by reading from `experiment_definitions` -> or `experiment_definition_tasks`. - -### Code-level expression - -The rule is encoded in three places: - -1. **`graph_repo.node(session, run_id, task_id)`** is the only call site - the runtime uses to fetch a Task during a run. It joins - `run_graph_nodes` *only* — no join into `experiment_definition_tasks` - even when the row is a static-task copy. The `task_json` it returns - is the run-tier copy, end of story. -2. **`Task.from_definition(...)` is invoked exclusively by - `graph_repo.node`** (and by the dynamic-subtask spawn path that - writes a fresh `run_graph_nodes` row). Workers, evaluators, and - `worker_execute` never call `from_definition` directly; they read - typed `node.task` from the repo view. -3. **No method named `_prepare_definition` exists.** The `prepare_run` - service copies definition rows into run rows once, at run-launch, and - from that point forward `definition_*` tables are not read by any - code under `core.application.*` except `definition_writer.py` - (write-only, at experiment-define time). - -### Architecture-guard test - -The boundary is enforced by a guard test in -[`07-test-strategy.md`](07-test-strategy.md): - -```python -def test_runtime_does_not_read_definition_tables() -> None: - """Static check: no module under core/application/runtime imports - DefinitionRepository, ExperimentDefinitionTask, or any other - definition-tier ORM class.""" -``` - -### Why the boundary matters - -Without it, the runtime can silently disagree with itself: -`worker_execute` resolves the sandbox from the run-tier copy, -`prepare_subtask` resolves it from the definition-tier copy, and a -benign-looking edit to either tier produces drift only one path sees. -Forcing all reads through run-tier means the snapshot taken at run-launch -is the **only** truth the runtime knows about. - -## §5 — Dynamic subtasks are graph-native `[v2: added]` - -When a worker calls `context.spawn_task(...)` mid-run, the new task is -born into `run_graph_nodes` only — there is no synthesized -`experiment_definition_tasks` row. - -### Schema implications - -`experiment_definition_tasks` rows have `task_id UUID PRIMARY KEY` -(non-null, present for every static task). `run_graph_nodes` rows have -`task_id UUID` (always non-null, since identity exists for all tasks) -and an optional join target — the discriminator `static vs. dynamic` is: - -```sql --- A dynamic task is a run_graph_nodes row whose task_id has no matching --- row in experiment_definition_tasks for the same definition_id. -SELECT n.run_id, n.task_id -FROM run_graph_nodes n -JOIN runs r ON r.id = n.run_id -LEFT JOIN experiment_definition_tasks d - ON d.experiment_definition_id = r.definition_id - AND d.task_id = n.task_id -WHERE d.task_id IS NULL; -``` - -For lookup performance, `run_graph_nodes` carries an `is_dynamic` -boolean column (denormalized at insert time) so this question doesn't -need a join. - -### What the spawn path does - -```python -# Inside WorkerContext.spawn_task — framework-internal: -new_task_id = uuid4() -graph_repo.insert_node( - session, - run_id=self.run_id, - task_id=new_task_id, - parent_task_id=self.task_id, - task_json=spec.to_definition(), # the spawned-task spec, _type-discriminated - is_dynamic=True, - status=NodeStatus.PENDING, -) -graph_repo.insert_edge( - session, - run_id=self.run_id, - parent_task_id=self.task_id, - child_task_id=new_task_id, - kind=EdgeKind.PARENT_CHILD, -) -# No write to experiment_definition_tasks. None. -``` - -Subsequent reads of the spawned task go through the same -`graph_repo.node(...)` path as static tasks — by §4's rule, the runtime -doesn't know or care whether the row was originally copied from a -definition or born dynamically. - -### What this kills - -- The `materialize_dynamic_subtask_definition` path in v1, which wrote a - synthetic row into `experiment_definition_tasks` so dynamic tasks - could be looked up via the same definition-tier query as static ones. - Replaced by: there's only one query path, and it goes through - `run_graph_nodes`. -- The `_prepare_definition` runtime helper that hydrated `Task` from - `experiment_definition_tasks` rows for *both* static-copy and - dynamic-synthetic cases. Deleted. - -### Identity is still preserved - -A dynamic task's `task_id` is generated at spawn time (`uuid4()`) and -that same `task_id` is what `node.task.task_id` returns to worker code. -Cross-run analytics on dynamic tasks works the same way as for static -tasks — the only thing different is whether a row exists in -`experiment_definition_tasks` for the same `task_id`. - -## §6 — Decisions locked at workshop `[v2: locked]` - -The workshop resolved the open questions §3–§5 surfaced. For -provenance: - -- **Indexed columns on `experiment_definitions`** — **locked: dedicated - columns + JSONB.** Per §3, `name` and `description` are dedicated - `TEXT` columns; `metadata` is `JSONB`. Dashboard queries hit indexed - columns; ad-hoc tags live in `metadata`. -- **`is_dynamic` boolean denormalization** — **locked: carry the - boolean.** Per §5, `run_graph_nodes.is_dynamic BOOLEAN NOT NULL` - avoids a left-join on every node read. Cheap to maintain - (denormalized at insert time, never updated). -- **Definition versioning** — **locked: stripped.** No `version` column - on `experiment_definitions`. Each `persist_definition` call produces - a fresh definition row with a new `definition_id`. Authors who want - "v2 of a benchmark" create a new definition with a new `name` (or - reuse the same `name` — `name` is not a unique key). The v1 - `(name, version)` compound key concept is gone. -- **Run-graph WAL retention** — **locked: keep forever (for v2 - launch).** `run_graph_mutations` and `run_graph_annotations` are - retained without expiry. Storage is not the bottleneck at current - scale; revisit if/when row counts get painful and tier into S3 or - drop oldest. Tracked as a non-blocking follow-up. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/03-runtime.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/03-runtime.md deleted file mode 100644 index ffb31887e..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/03-runtime.md +++ /dev/null @@ -1,745 +0,0 @@ -# 03 — Runtime - -> What happens inside the rollout container: how a `Sandbox` subclass -> gets provisioned, how the lifecycle hub tracks live sandboxes, and -> what `WorkerContext` exposes to a running worker — the single public -> runtime surface for v1, with the curated single-target methods most -> workers need. Toolkits or other consumers that need batch ops or -> CLI-tier control reach into the internal services -> (`TaskManagementService` / `TaskInspectionService` / -> `RunResourceRepository`) directly today; promoting those to the -> public surface is deferred (see -> [`09-implementation-plan.md`](09-implementation-plan.md) "What's deferred" and -> [`08-decisions-log.md`](08-decisions-log.md) "Layered public runtime -> API" under Alternatives considered). -> -> See [`01-api-surface.md`](01-api-surface.md) for the -> typed surface this consumes, -> [`02-persistence-layer.md`](02-persistence-layer.md) for -> how the typed objects arrive, and -> [`04-walkthrough.md`](04-walkthrough.md) for the end-to-end trace. - -## Sandbox provisioning - -There is no allocator, no template registry, no string-keyed dispatch. -Each `Sandbox` subclass owns its own `provision()` and `terminate()`. -The framework calls them through the `SandboxLifecycleHub` (defined in -the next section) so retries, quotas, and shutdown work correctly: - -```python -# Inside worker_execute (the canonical path — orchestrator function): -sandbox = await lifecycle_hub.acquire(task.sandbox, run_id=run_id, task_id=task_id) -# acquire(...) calls task.sandbox.provision() (or reattaches an existing -# live sandbox on a retry — see below). After it returns, task.sandbox -# IS sandbox (same instance, _runtime now attached). The persisted -# run_task_executions row gets sandbox_id stamped onto it so eval -# workers can re-attach. -try: - # 1. Run the worker. Collect the terminal WorkerOutput from the stream. - final_output: WorkerOutput | None = None - async for chunk in worker.execute(task, context=ctx, sandbox=sandbox): - if isinstance(chunk, WorkerOutput): - final_output = chunk - yield chunk - await graph_repo.persist_worker_output( - run_id=run_id, task_id=task_id, - execution_id=execution_id, output=final_output, - ) - - # 2. Fan out one Inngest invocation per evaluator. The parent SUSPENDS - # until every invocation returns — sandbox stays alive through - # gather. Each evaluate_task_run gets a thin id-only payload; the - # eval worker re-loads task via the same graph_repo.node(...) and - # re-attaches the sandbox by sandbox_id read off the execution row. - await asyncio.gather(*[ - ctx.step.invoke( - f"eval-{i}", - evaluate_task_run, - TaskEvaluateRequest( - run_id=run_id, - task_id=task_id, - execution_id=execution_id, - evaluator_index=i, - ), - ) - for i in range(len(task.evaluators)) - ]) -finally: - # 3. Release once gather has returned. This worker_execute job is - # the sole acquirer/releaser for (run_id, task_id); eval workers - # only ATTACH (build a local _runtime handle to the same external - # sandbox) and DETACH (drop the local handle, never terminate). - # Sandbox lifetime is bounded by this try/finally — never crosses - # job boundaries unsupervised. - await lifecycle_hub.release(sandbox) -``` - -Author code never imports `SandboxLifecycleHub`. The base -`Sandbox.provision()` / `terminate()` *are* the public contract that -each subclass implements; the hub is the single framework caller of -them. If you're writing a new sandbox subclass you implement -`provision()`; you don't think about the hub. - -### Synchronous fanout, single sandbox owner `[v2: clarified]` - -The snippet above is canonical. Three properties of the v2 design lock -in: - -1. **Two Inngest function shapes per task: an orchestrator - (`worker_execute`) and a per-evaluator worker - (`evaluate_task_run`).** The orchestrator is one event per task; - evaluators fan out one event per evaluator. The orchestrator - suspends on `asyncio.gather(*[ctx.step.invoke(...)])` until every - eval invocation returns. The full event taxonomy is specified in - [`06-inngest-event-contracts.md`](06-inngest-event-contracts.md). -2. **`worker_execute` is the sole sandbox-lifetime owner for - `(run_id, task_id)`.** `acquire` happens once at the top, `release` - happens once in `finally` after `gather` returns. Eval workers only - *attach* (build a local `_runtime` handle to the same external - sandbox via `Sandbox.from_definition(json, sandbox_id=...)`) and - *detach* (drop the local handle, never terminate). The `run/cleanup` - Inngest function is a shutdown / failure-recovery backstop — it - sweeps live sandboxes a crashed pod left behind, but is **never** - the primary release path for a successful task. -3. **Evaluators see the same external sandbox the worker used.** - Sandbox state is shared because the external sandbox (e.g. e2b cloud - sandbox) is a single live resource keyed by `sandbox_id`; the local - `_runtime` handles are per-process reconnections to it, not copies. - An evaluator can `cat` the file the worker wrote, run a verification - command, or inspect any sandbox resource — same filesystem, same - process *on the sandbox side*, two Python processes on the Ergon - side. - -This is a deliberate v2 reshape of v1's split, not a collapse of it. -v1 fan-out'd criteria evaluation into a separate Inngest function but -got the lifecycle wrong: release lived in `check_evaluators`, a -sibling job, and per-run cleanup masked the leak. v2 keeps the fanout -(retaining Inngest's per-function retry, concurrency cap, and -observability) and fixes the lifecycle (single owner, bounded by the -orchestrator's `try/finally`, sandbox guaranteed alive through the -`gather`). The reason v1's split looked like it had to go was the -audit's lifecycle finding — that finding is resolved by the -synchronous-fanout shape without giving up the operational properties -of separate Inngest functions. - -The audit decision and the rejected alternatives are recorded in -[`08-decisions-log.md`](08-decisions-log.md) "Locked decisions -inherited from v1 audit" — Δ.4 / Δ.5. The full job-graph (what events -fire when, what's idempotent, how retries behave) lives in -[`06-inngest-event-contracts.md`](06-inngest-event-contracts.md). - -**Why this collapses so far.** Today's `BaseSandboxManager` was a -process-wide singleton conflating four roles: dispatch on benchmark kind, -hold E2B credentials, manage a `dict[task_id, AsyncSandbox]` pool, and -provide per-sandbox operations keyed by `task_id`. The proposed Sandbox -subclass-per-kind absorbs three of them naturally: - -| Concern (today) | Where it lives now | -|---|---| -| "What kind of environment?" | `type(sandbox)` — the class identity | -| "How do I provision this kind?" | `sandbox.provision()` — subclass method | -| "How do I run a command in *this specific* sandbox?" | `sandbox.run_command(cmd)` — binding implicit in object | -| "Where are credentials?" | Read from `settings`/env *inside* `provision()`. **Not** stored as fields (would persist secrets in Postgres). | -| "Where is the live-sandbox pool?" | Optional thin `SandboxLifecycleHub` for retries / quota / shutdown — see below | -| "How do I dispatch by kind?" | No dispatch needed; class identity already tells you | - -**Adding a new sandbox kind** is now: subclass `Sandbox`, implement -`provision()` (and `terminate()` if needed). No registration step. No -template-string registry. The class is the contract; the import path -(stored as `_type` in the persisted JSON) is how cross-process resolution -finds it. - -```python -class WasmSandbox(Sandbox): - wasm_module: str - memory_limit_mb: int = 512 - - async def provision(self) -> None: - runtime = await spawn_wasm_runtime( - module=self.wasm_module, - memory_mb=self.memory_limit_mb, - ) - object.__setattr__(self, "_runtime", runtime) -``` - -That's it — usable in any `Task(sandbox=WasmSandbox(wasm_module="..."))` -immediately. A third-party package can ship its own `WasmSandbox` -without touching `ergon_core` or `ergon_builtins`. - -## `SandboxLifecycleHub` — the small thing that survives - -The one role of the old singleton manager that doesn't fold into the -subclass is **process-wide live-sandbox tracking**, used for: - -- Reconnection across job retries (retried task should re-attach to its - existing live sandbox, not `provision()` a fresh one), -- Process-wide concurrency / quota (cap on simultaneous live sandboxes), -- Graceful-shutdown teardown (terminate every live sandbox when the - container exits). - -These are real concerns but they're tiny and *kind-agnostic*. A small -`SandboxLifecycleHub` instance owned by the rollout container handles -them with three methods: - -```python -class SandboxLifecycleHub: - """Process-wide registry of live sandboxes. Knows nothing about kinds. - - Lives at `ergon_core/core/infrastructure/sandbox/lifecycle.py`. - One instance per rollout container process; constructed at - container startup and disposed at shutdown. - """ - - # In-process map keyed by (run_id, task_id). Survives within a - # single rollout container; does NOT survive pod restart (the - # underlying e2b/Docker handle is rebuilt fresh on the next - # acquire after restart, which is the correct behaviour — a - # restarted pod can't reattach to a sandbox the previous pod - # held). - _live: dict[tuple[UUID, UUID], Sandbox] - - async def acquire( - self, sandbox: Sandbox, *, run_id: UUID, task_id: UUID, - ) -> Sandbox: - """Get a live `Sandbox` for `(run_id, task_id)`. - - Lookup order: - 1. If `(run_id, task_id)` is already in `_live`, return that - instance — same in-process retry, sandbox is already up. - 2. Otherwise call `sandbox.provision()`, register - `_live[(run_id, task_id)] = sandbox`, return it. - - Cross-process retry (Inngest re-fires the worker_execute job - on a new pod) does NOT reattach — the new pod's hub has an - empty `_live`, so step 2 runs and `provision()` creates a - fresh underlying sandbox. The old pod's sandbox leaks until - shutdown — that's the v1 trade-off; the alternative is - out-of-process state (Postgres-backed registry of live - sandbox_ids + e2b reattachment), which is deferred. Same - leak window we have today. - - Raises whatever `sandbox.provision()` raises (no wrapping — - the framework wants the original error chain visible). - """ - - async def release(self, sandbox: Sandbox) -> None: - """Terminate the sandbox and remove it from `_live`. Idempotent - — release on an already-released sandbox is a no-op (lets - worker_execute call it in a `finally:` without race-checking).""" - - async def terminate_all(self) -> None: - """Container-shutdown hook. Calls `release()` on every entry in - `_live`. Best-effort — exceptions are logged and swallowed so - one stuck sandbox can't block shutdown of the rest.""" -``` - -It does **not** know about Lean vs Python vs Wasm. It just calls -`sandbox.provision()` / `sandbox.terminate()` and uses -`(run_id, task_id)` as the registry key. Replaceable for tests with -a one-line stub. Author code never sees it. - -The `(run_id, task_id)` key is deliberate — it's the same composite -identity the runs row uses, so "the sandbox for this task" has one -canonical lookup. No `sandbox_id` plumbing through Inngest payloads -is needed for reattach: the lookup key is already on the job's -payload. The `sandbox.sandbox_id` (from `_runtime.sandbox_id` once -provisioned) is still useful as a debug/tracing handle but is not -the lifecycle key. - -## Worker runtime API: WorkerContext - -Inside `worker.execute(task, *, context, sandbox)`, every interaction -with the run graph (spawning subtasks, cancelling siblings, listing -descendants, looking up resources produced upstream, …) flows through -`WorkerContext`. It is the single public runtime surface for v1. - -`WorkerContext` carries the **curated** set of single-target methods -(spawn / cancel / refine / restart, plus own-scope inspection and -resource discovery) — each one a direct delegate to a method on an -internal application-layer service. Operations that don't fit the -curation rule (batch / predicate / cross-scope inspection / CLI-tier -materialisation) stay on the internal services and are reached by -direct import for the rare consumer that needs them. There is no -public service-class tier between the worker and the implementation. - -### Curation rule for `WorkerContext` methods - -A method is added to `WorkerContext` if and only if: - -- it is **single-target** (operates on exactly one task or one resource — - no batch, no `predicate=` parameter, no scope keyword that fans out), - AND -- it is **high-frequency** (used by ≥2 in-tree workers, or expected to - be used by most workers as a matter of course). - -Everything else lives on the internal services -(`TaskManagementService`, `TaskInspectionService`, -`RunResourceRepository`) only. Workers that need batch / predicate / -materialisation ops import the service directly: - -```python -from ergon_core.core.application.tasks.management import TaskManagementService -# ... -await TaskManagementService(...).cancel_all_matching(predicate=...) -``` - -That escape hatch is the same import path `ergon_builtins.tools.*` and -`ergon_cli.commands.workflow` use today. We don't enforce a public / -internal boundary on it for v1 — see -[`08-decisions-log.md`](08-decisions-log.md) "Layered public runtime -API" under Alternatives considered for the rationale and what we'd -need to see to revisit. - -When in doubt: leave it off `WorkerContext`. Adding to the facade is a -deliberate API commitment; *not* adding is reversible. - -### The v1 WorkerContext surface - -```python -class WorkerContext(BaseModel): - """Runtime handle for a worker execution. Curated single-target - operations on the run graph; drop to internal services for batch - or CLI-tier ops.""" - - # ── Identity (read-only fields) ──────────────────────────────── - run_id: UUID - task_id: UUID # this worker's task - execution_id: UUID - definition_id: UUID - - # ── Internal services (PrivateAttr; framework-set at job start) ── - _task_mgmt: TaskManagementService = PrivateAttr() - _task_inspect: TaskInspectionService = PrivateAttr() - _resource_repo: RunResourceRepository = PrivateAttr() - - # ── Mutation: single-target, parent-on-child only ─────────────── - async def spawn_task( - self, task: Task, *, depends_on: tuple[UUID, ...] = (), - ) -> SpawnedTaskHandle: - """Spawn one child task under this worker. The task must already - carry its concrete `task.worker`, `task.sandbox`, and - `task.evaluators` object bindings. - - Fire-and-forget only in v1: returns a `SpawnedTaskHandle` with - the new `task_id` immediately, parent continues. Workers that - need to wait for a child poll via `context.get_task(handle.task_id)` - in a loop — or, more commonly, fan out a batch of children and - examine results once all are terminal. Synchronous - `await_completion=True` semantics (parent's sandbox lifecycle - during the wait, Inngest wait-for-event integration) is - deferred — see [`08-decisions-log.md#future-work`](08-decisions-log.md#future-work). - """ - return await self._task_mgmt.add_subtask( - run_id=self.run_id, parent_task_id=self.task_id, - task=task, depends_on=depends_on, - ) - - async def cancel_task(self, task_id: UUID) -> None: - """Cancel one descendant of this worker's task. Raises - ContainmentViolation if task_id is not a descendant of self.task_id.""" - self._assert_descendant(task_id) - await self._task_mgmt.cancel_task(run_id=self.run_id, task_id=task_id) - - async def refine_task(self, task_id: UUID, *, description: str) -> None: - """Update the description of one descendant. Allowed on any - status except RUNNING. Pairs with restart_task.""" - self._assert_descendant(task_id) - await self._task_mgmt.refine_task( - run_id=self.run_id, task_id=task_id, description=description, - ) - - async def restart_task(self, task_id: UUID) -> None: - """Reset one terminal descendant back to PENDING and re-dispatch.""" - self._assert_descendant(task_id) - await self._task_mgmt.restart_task(run_id=self.run_id, task_id=task_id) - - # ── Inspection: own-scope, single-call ────────────────────────── - def subtasks(self) -> Iterable[SubtaskInfo]: - """Direct children of this worker's task.""" - return self._task_inspect.list_subtasks( - run_id=self.run_id, parent_task_id=self.task_id, - ) - - def descendants(self, *, max_depth: int = 3) -> Iterable[SubtaskInfo]: - """BFS over this worker's subtree, up to max_depth.""" - return self._task_inspect.descendants( - run_id=self.run_id, parent_task_id=self.task_id, max_depth=max_depth, - ) - - def get_task(self, task_id: UUID) -> SubtaskInfo: - """Fetch one task in this run by id. Allowed targets: - `self.task_id` (this worker's own task) or any descendant of - `self.task_id`. Anything outside that subtree raises - `ContainmentViolation` — same rule as the mutation methods, - for consistency. Reading your own row is the common case - (e.g. when you need `instance_key` to construct a child Task); - reading a descendant is how you poll a child's status.""" - if task_id != self.task_id: - self._assert_descendant(task_id) - return self._task_inspect.get_subtask( - run_id=self.run_id, task_id=task_id, - ) - - def resources( - self, *, scope: Literal["own", "children", "descendants", "run"] = "own", - ) -> Iterable[RunResourceView]: - """Resources produced in the requested scope, newest first.""" - # Per-scope dispatch onto the existing RunResourceRepository methods. - ... -``` - -That's it for `WorkerContext`. `plan_subtasks` (batch), -`cancel_all_matching` (batch + predicate), `get_resource_by_content_hash` -(rare lookup), `next_actions` (CLI-shaped) — all stay on the internal -services. Workers that need them import the service explicitly. - -`SubtaskInfo` and `RunResourceView` (and the corresponding service -classes' other return types) are the same internal DTOs the services -return today, **with `node_id` fields renamed to `task_id`** as -part of the schema migration (see -[`../2026-05-08-authoring-api-redesign/05-migration.md` §14.C](../2026-05-08-authoring-api-redesign/05-migration.md#14c--internal-dto-reshape)). v1 -doesn't promote them into `ergon_core.api.types` — that's a follow-up -to do alongside any future "promote the internal services to the -public API" work. - -### `_assert_descendant` and `ContainmentViolation` - -The mutation methods (`cancel_task`, `refine_task`, `restart_task`) -and the inspection method `get_task` (when the target ≠ `self.task_id`) -all run a containment check before delegating. The check is a single -synchronous SQL query against `run_graph_nodes.parent_task_id`, -implemented as a recursive CTE so depth is bounded by the database, -not by Python: - -```python -class WorkerContext(BaseModel): - ... - - def _assert_descendant(self, task_id: UUID) -> None: - """Raise ContainmentViolation if `task_id` is not in the subtree - rooted at self.task_id. Sync — issues one CTE query against - run_graph_nodes via the inspection service. Self is *not* a - descendant of self; callers that want to allow self must check - explicitly (see get_task).""" - if not self._task_inspect.is_descendant( - run_id=self.run_id, - ancestor_task_id=self.task_id, - candidate_task_id=task_id, - ): - raise ContainmentViolation( - target=task_id, - ancestor=self.task_id, - run_id=self.run_id, - ) -``` - -`TaskInspectionService.is_descendant(run_id, *, ancestor_task_id, -candidate_task_id) -> bool` is the new method this requires — -implemented as a Postgres recursive CTE walking -`parent_task_id`, with an early-exit `WHERE candidate_task_id = ANY(...)`. -The same CTE shape backs `descendants(...)` and `is_descendant(...)` -so there's one query template, two callers. (Today's -`graph.traversal.descendants` is a Python BFS; v1 keeps it as a -fallback for SQLite tests but uses the CTE in Postgres — the -mechanical change is folded into step 16a.) - -`ContainmentViolation` is a public exception in `ergon_core.api.errors` -— signature pinned in -[`01-api-surface.md`](01-api-surface.md#public-exceptions). Inherits -from `RuntimeError` (not bare `Exception`) so `except RuntimeError:` -blocks behave reasonably. - -Workers should not generally catch this — hitting it is a logic bug -in the worker, not a recoverable runtime condition. The public class -exists so toolkits that wrap `WorkerContext` methods in `try/except` -have a typed exception to surface in their structured tool result. - -### Framework-side WorkerContext construction - -Author code never constructs a `WorkerContext` — the framework wires -one up at the top of every `worker_execute` job and passes it into -`worker.execute(...)`. The construction has two phases because the -PrivateAttrs that carry service references aren't part of the -public field schema: - -```python -# Inside worker_execute (framework code, not author code): -context = WorkerContext._for_job( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - definition_id=payload.definition_id, - task_mgmt=task_mgmt_service, - task_inspect=task_inspect_service, - resource_repo=resource_repo, -) -``` - -`WorkerContext._for_job` is the framework-only constructor. It builds -the public-fields instance and uses `object.__setattr__` to populate -the service PrivateAttrs in one call: - -```python -@classmethod -def _for_job( - cls, - *, - run_id: UUID, task_id: UUID, execution_id: UUID, definition_id: UUID, - task_mgmt: TaskManagementService, - task_inspect: TaskInspectionService, - resource_repo: RunResourceRepository, -) -> "WorkerContext": - """Framework-only constructor. Authors never call this.""" - instance = cls( - run_id=run_id, task_id=task_id, - execution_id=execution_id, definition_id=definition_id, - ) - object.__setattr__(instance, "_task_mgmt", task_mgmt) - object.__setattr__(instance, "_task_inspect", task_inspect) - object.__setattr__(instance, "_resource_repo", resource_repo) - return instance -``` - -Same shape as `Task.from_definition` (two-phase: pydantic validation -of public fields, then PrivateAttr injection). Same reason: the -framework needs to wire runtime state that doesn't belong in the -JSON schema; the constructor convention is "public-fields ctor → -classmethod that adds runtime state." - -### Containment - -`WorkerContext`'s mutation methods (`cancel_task`, `refine_task`, -`restart_task`) check that the target `task_id` is a descendant of -`self.task_id` before delegating, raising `ContainmentViolation` -otherwise. This kills a class of bug that today's -`SubtaskLifecycleToolkit` documents as a TODO: - -> "cancel_task, refine_task, and get_subtask accept a node_id from the -> LLM and do not yet verify containment (i.e. that the target is a -> descendant of parent_node_id). The service layer checks status guards -> but not subtree membership." - -**The containment check lives on `WorkerContext`, not on the internal -services.** A consumer that imports `TaskManagementService` directly -opts out of the check (the service trusts the (run_id, task_id) it -receives). That's fine for the workflow CLI (operating with explicit -admin intent) and for any toolkit that knows what it's doing; it's the -same shape as today. - -### Deciding what to call - -| Use case | Where | -|---|---| -| "Spawn one child task." | `WorkerContext.spawn_task(...)` | -| "Cancel one specific descendant by id." | `WorkerContext.cancel_task(...)` | -| "Update one descendant's description." | `WorkerContext.refine_task(...)` | -| "Restart one terminal descendant." | `WorkerContext.restart_task(...)` | -| "List my direct children." | `WorkerContext.subtasks()` | -| "List BFS up to depth N." | `WorkerContext.descendants(max_depth=N)` | -| "Look up a task by id (must be in my subtree)." | `WorkerContext.get_task(id)` | -| "List resources I / my children / my subtree / the run produced." | `WorkerContext.resources(scope=...)` | -| "Cancel every running descendant." | `from ... import TaskManagementService; await svc.cancel_all_running_in_subtree(...)` | -| "Atomically create a sub-DAG of 5 tasks with deps." | `from ... import TaskManagementService; await svc.plan_subtasks(...)` | -| "Look up resource by content hash." | `from ... import RunResourceRepository; await repo.get_by_content_hash(...)` | -| "Materialize a resource into my sandbox." | `from ... import RunResourceRepository, SandboxResourcePublisher; ...` | - -The bottom four are exactly the import shape `ergon_builtins.tools.*` -and `ergon_cli.commands.workflow` use today. - -### Where the implementation lives - -| WorkerContext method group | Backing implementation | -|---|---| -| `spawn_task` / `cancel_task` / `refine_task` / `restart_task` | `ergon_core/core/application/tasks/management.py` (`TaskManagementService`) | -| `subtasks` / `descendants` / `get_task` | `ergon_core/core/application/tasks/inspection.py` (`TaskInspectionService`) | -| `resources(scope=...)` | `ergon_core/core/application/resources/` (`RunResourceRepository`) | -| (process-wide) `SandboxLifecycleHub` | `ergon_core/core/infrastructure/sandbox/lifecycle.py` | - -The application-layer modules are reachable by direct import — there -is no boundary test enforcing public-only access. That deliberate -porousness is the v1 simplification; see -[`08-decisions-log.md`](08-decisions-log.md) for what we'd need to see -before adding the firewall. - -## Dynamic spawning: what changes (almost nothing) - -Workers can spawn child tasks at runtime — fan-out research questions, -recursive decomposition, tree-search candidates, etc. The two-table -identity model (see -[`02-persistence-layer.md#identifier-model`](02-persistence-layer.md#identifier-model-two-tables-one-identity)) -makes this a small delta on the static path rather than a parallel -codepath. **The `worker_execute` job body is identical for static and -dynamic tasks**; only the row's *origin* differs. - -### What a worker sees - -`WorkerContext.spawn_task(...)` (signature in -[`#the-v1-workercontext-surface`](#the-v1-workercontext-surface) above) -is the v1 path for spawning a single child. Workers that need to -materialise a sub-DAG (multiple children with dependencies between -them) in a single transaction import `TaskManagementService` and call -`plan_subtasks` directly — that batch op stays on the internal service -by the curation rule. - -**One sandbox + one worker per task is invariant in v1.** A spawned task -is just a new task — the framework provisions a fresh sandbox via -`task.sandbox.provision()` exactly the way it does for static tasks. The -parent and child have independent sandbox runtimes, independent -filesystems, independent lifecycles. There is no API surface for sharing -or reusing the parent's sandbox (see -[`08-decisions-log.md#future-work`](08-decisions-log.md#future-work) on -multi-agent / multi-sandbox patterns). - -`task.worker` is a concrete, serializable `Worker` object. There is no -separate `worker=` argument and no experiment-level worker pool to -validate against. That is an intentional object-first choice: if a -child task has different execution behavior, the task says so directly. -The modularity story moves up into Python factories that construct -variants of the task graph, not down into string bindings in the runtime -model. - -**v1 is fire-and-forget only.** `spawn_task` returns a -`SpawnedTaskHandle` carrying the new `task_id` immediately; the -parent continues. Workers that want to wait on a child's result -poll `context.get_task(handle.task_id)` until its status is terminal -and then read its output. A synchronous `await_completion=True` -mode (parent blocks until the child terminates, child output -returned inline) is deferred — see -[`08-decisions-log.md#future-work`](08-decisions-log.md#future-work) -for the open questions (parent's sandbox lifecycle during the wait, -Inngest wait-for-event integration, interaction with workers -holding in-memory state). - -If a worker genuinely needs to spawn something, it constructs a full -`Task` with its own `worker`, `sandbox`, and `evaluators`. That makes the -dynamic child identical in shape to a static task: no inherited worker, -no hidden evaluator pool, no separate sandbox parameter. - -### What the framework does on `spawn_task` - -```python -async def spawn_task(self, task, *, depends_on=()): - # 1. Allocate a fresh task_id — no definition row to inherit from. - new_task_id = uuid4() - - # 2. Insert one run_graph_nodes row. The worker, sandbox, and - # evaluators are whatever task carries — there are no separate - # runtime parameters or pool lookups. - await graph_repo.insert_dynamic_node( - run_id=self.run_id, - task_id=new_task_id, - parent_task_id=self.task_id, # links child to parent - task_json=task.model_dump(), # full pydantic JSON, _type discriminator + worker + sandbox + evaluators + payload - depends_on=depends_on, # composite-FK edges into (run_id, dep_task_id) - ) - - # 3. Dispatch worker_execute for (run_id, new_task_id). Same Inngest - # event as static. Fire-and-forget — return the handle and let - # the parent decide whether/how to wait via get_task(). - return await dispatch_worker_execute(self.run_id, new_task_id) -``` - -### What `worker_execute` sees (unchanged) - -```python -node = graph_repo.node(session, run_id=payload.run_id, - task_id=payload.task_id) # RunGraphNodeView, .task pre-inflated - -task = node.task -worker = task.worker -# Identical to the static path. The job body cannot tell the difference, -# and does not need to. The underlying task_json was either copied from -# experiment_definition_tasks at run-launch (static) or written inline at -# spawn (dynamic) — same shape, same reconstruction (inside graph_repo). -``` - -This is the payoff of making `Task` object-bound: dynamic spawning adds -**zero new branches** to the runtime. The run row stores a full task -snapshot; `worker_execute` inflates the task and calls `task.worker`. -One new method on `WorkerContext` writes a row; the rest of the system -already handles it. - -### Identity recap for dynamic tasks - -| Property | Static | Dynamic | -|---|---|---| -| `task_id` born at | experiment-define | spawn (`uuid4()`) | -| `experiment_definition_tasks` row exists? | yes | no | -| `run_graph_nodes` row written by | `run_started` (copy from defs) | `WorkerContext.spawn_task` (inline) | -| `parent_task_id` | NULL (or static parent if any) | parent's `task_id` | -| `task_json` on the row | copied at run-launch | written at spawn | -| Worker JSON store | inline in `task_json.worker` | inline in `task_json.worker` | -| Sandbox provisioning | fresh runtime via `task.sandbox.provision()` | identical — fresh runtime, never shared with parent | -| `worker_execute` reads | `run_graph_nodes` row only | identical | -| `from_definition` shape | identical | identical | - -### Constraints - -1. **No static dependent on a dynamic task.** Static dependencies are - declared at experiment-define time and reference - `experiment_definition_tasks.id` values that exist at that time. - Dynamic task ids don't exist at define time, so any static-on-dynamic - edge would block forever. Validator at experiment-define rejects this - trivially (static deps reference task slugs from `build_instances`, - dynamic tasks have no define-time slug); runtime check on edge - insertion prevents anyone smuggling one in. Static-on-static, - dynamic-on-static, and dynamic-on-dynamic all work normally. - -2. **Dynamic children may carry their own evaluators.** Most spawned - children pass `evaluators=()` because they're internal strategy - steps — the parent task is what gets scored, not its sub-decisions. - But if a child should be independently scored, it carries the - concrete evaluator objects directly, exactly like a static task. - -3. **One sandbox + one worker per task — no exceptions in v1.** A - spawned child is a new task in every respect: it carries its own - `task.sandbox` instance and the lifecycle hub provisions a fresh - runtime for it. Parent and child filesystems / runtimes / lifecycles - are fully independent. We are not shipping any opt-in sharing, - reattachment, or snapshot/clone primitive in this redesign. Sharing is - genuinely tricky (concurrency on the runtime, lifecycle coupling, - filesystem mutation races, cross-process reattachment, interaction - with future synchronous-wait semantics), and any future work on it - will be designed together with the broader "multiple agents per task" - question rather than smuggled in as an addendum here. See - [`08-decisions-log.md#future-work`](08-decisions-log.md#future-work). - -### What this surfaces that needs a follow-up - -- **Backpressure / runaway spawning.** A buggy worker can fork-bomb a - run. Need configurable per-run / per-parent / depth caps. Probably a - small `GraphSpawnGovernor` analogous to `SandboxLifecycleHub`. Out of - scope for this redesign; flagged in - [`08-decisions-log.md#open-questions`](08-decisions-log.md#open-questions). -- **Cancellation cascade.** Already handled by - `TaskManagementService.cancel_task` (descendants cancelled). Generalizes - cleanly to whatever depth dynamic spawning produces. - -### What this kills in the existing code - -- The current slug-based worker resolution (`registry.workers[slug]`) in - `add_subtask` dies with the registry; a spawned task carries the - concrete `Task.worker` object directly. -- The "subtask has no `task_payload`/`sandbox`/`evaluators`" - implicit-inheritance pattern dies. Every spawned `Task` carries its own - config explicitly, same as a static `Task`. There is no longer a - privileged "subtask" subtype with implicit-inheritance semantics — a - dynamic task is just a `Task` written to the runs row by a worker - instead of by the run-launcher. -- **Containment-check TODOs in `subtask_lifecycle_toolkit.py` close** - *for tools that route through `WorkerContext`.* Today the toolkit - notes that `cancel_task`, `refine_task`, and `get_subtask` accept a - `node_id` from the LLM and don't verify it's a descendant of the - manager's task. The new `WorkerContext.cancel_task` / - `refine_task` / `get_task` enforce that check before delegating, so - any toolkit that switches to the facade methods inherits the check. - Toolkits that keep importing `TaskManagementService` directly are on - their own for containment, same as today. - -(`TaskManagementService`, `TaskInspectionService`, and -`RunResourceRepository` themselves stay as they are, including their -existing public-when-imported availability to `ergon_builtins.tools.*` -and `ergon_cli.commands.workflow`. Promoting them to a public service -tier with an enforced boundary is deferred — see -[`08-decisions-log.md`](08-decisions-log.md) "Layered public runtime -API".) diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/04-walkthrough.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/04-walkthrough.md deleted file mode 100644 index f452a25ec..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/04-walkthrough.md +++ /dev/null @@ -1,750 +0,0 @@ -# 04 — Walkthrough: linear DAG, 4 tasks, 4 workers - -> Concrete trace through the proposed model — author writes a script, -> definition lands in Postgres, container loads and runs. **This is the -> single source of truth for "what running end-to-end looks like."** Other -> docs in this folder describe shapes; this doc describes flow. -> -> Smells the trace surfaces are flagged in -> [`08-decisions-log.md#smells-the-walkthrough-surfaces`](08-decisions-log.md#smells-the-walkthrough-surfaces). - -## Stage 1 — author API (in the user's process) - -```python -from collections.abc import Mapping - -from ergon_core.api import ( - Benchmark, Task, Sandbox, Worker, Criterion, Evaluator, Rubric, - CriterionOutcome, Experiment, WeightedCriterion, -) -from ergon_builtins.workers import ReActWorker -from ergon_builtins.toolkits import ( - ResearchToolkit, PythonCodingToolkit, ReadOnlyFilesystemToolkit, -) -from ergon_builtins.sandboxes import ( - ResearchE2BSandbox, PythonSandbox, # concrete Sandbox subclasses -) -# Note: toolkit and sandbox subclass imports come from ergon_builtins. -# `ergon_core.api` exports the abstract `Sandbox` base; concrete kinds -# live with the templates / setup logic that defines them. - -class FourStepBench(Benchmark): - type_slug = "four-step" - - def __init__( - self, *, - workers: Mapping[str, Worker], - default_evaluator: Evaluator, - ) -> None: - self.workers = workers - self.default_evaluator = default_evaluator - - def build_instances(self): - return { - "sample-1": [ - Task( - task_slug="research", - instance_key="sample-1", - description="Research the topic; write findings.md.", - worker=self.workers["researcher"], - sandbox=ResearchE2BSandbox(), - evaluators=(self.default_evaluator,), - ), - Task( - task_slug="code", - instance_key="sample-1", - description="Implement based on findings.md.", - worker=self.workers["coder"], - sandbox=PythonSandbox(pip_packages=("pandas", "requests")), - parent_task_slug="research", - dependency_task_slugs=("research",), - evaluators=(self.default_evaluator,), - ), - Task( - task_slug="review", - instance_key="sample-1", - description="Review the implementation; write review.md.", - worker=self.workers["reviewer"], - sandbox=PythonSandbox(), - parent_task_slug="code", - dependency_task_slugs=("code",), - evaluators=(self.default_evaluator,), - ), - Task( - task_slug="summarize", - instance_key="sample-1", - description="Summarize the prior three steps.", - worker=self.workers["summarizer"], - # Summarizer needs to read the prior tasks' resources; uses - # PythonSandbox so it can list and read the staged files. - # See decisions-log smell #1 about how cross-task data - # actually arrives in this sandbox. - sandbox=PythonSandbox(), - parent_task_slug="review", - dependency_task_slugs=("review",), - evaluators=(self.default_evaluator,), - ), - ] - } - -class AlwaysPass(Criterion): - type_slug = "always-pass" - slug: str = "always-pass" - description: str = "Always passes (placeholder)." - - async def evaluate(self, *, task, worker_output, context, sandbox): - return CriterionOutcome(slug=self.slug, name=self.slug, - score=1.0, passed=True) - -default_evaluator = Rubric( - name="default", - criteria=[WeightedCriterion(criterion=AlwaysPass(), weight=1.0)], -) - -experiment = Experiment( - benchmark=FourStepBench( - workers={ - "researcher": ReActWorker( - name="researcher", model="openai:gpt-4o", - system_prompt="...", max_iterations=20, - toolkit=ResearchToolkit(max_search_results=10), - ), - "coder": ReActWorker( - name="coder", model="openai:gpt-4o", - system_prompt="...", max_iterations=30, - toolkit=PythonCodingToolkit(), - ), - "reviewer": ReActWorker( - name="reviewer", model="anthropic:claude-sonnet-4", - system_prompt="...", max_iterations=15, - toolkit=PythonCodingToolkit(), # same toolkit, different model + prompt - ), - "summarizer": ReActWorker( - name="summarizer", model="openai:gpt-4o-mini", - system_prompt="...", max_iterations=5, - toolkit=ReadOnlyFilesystemToolkit(), - ), - }, - default_evaluator=default_evaluator, - ), -) -experiment.validate() -handle = ExperimentService().persist_definition(experiment) -result = await ExperimentService().run_experiment( - ExperimentRunRequest(experiment_id=handle.definition_id, wait=True), -) -``` - -**Author writes zero IDs.** Tasks get `task_id` at materialization, sandboxes -get `_runtime` at allocation, the framework owns identity throughout. - -## Stage 2 — persisted to Postgres (in the API process) - -`experiment.persist_definition()` decomposes into rows. Sketched as JSON -(real columns are typed pydantic-validated JSONB): - -```sql --- ExperimentDefinition (one row) -{ - id: <def_uuid>, - benchmark_json: { - "_type": "myproj.bench:FourStepBench", - "...": "benchmark config, if any" - } -} - --- ExperimentDefinitionInstance (one row, instance_key="sample-1") -{ id: <inst_uuid>, experiment_definition_id: <def_uuid>, instance_key: "sample-1" } - --- ExperimentDefinitionTask (4 rows) -[ - { id: <dtid_research>, instance_id: <inst_uuid>, task_slug: "research", - task_json: { - "_type": "ergon_core.api:Task", - "task_slug": "research", - "instance_key": "sample-1", - "description": "Research the topic; write findings.md.", - "worker": { - "_type": "ergon_builtins.workers:ReActWorker", - "name": "researcher", - "model": "openai:gpt-4o", - "system_prompt": "...", - "max_iterations": 20, - "toolkit": {"_type": "ergon_builtins.toolkits:ResearchToolkit", - "max_search_results": 10} - }, - "sandbox": {"_type": "ergon_builtins.sandboxes:ResearchE2BSandbox", - "env": {}, "timeout_seconds": null, "requires_network": true}, - "evaluators": [{ - "_type": "ergon_core.api:Rubric", - "name": "default", - "criteria": [{ - "_type": "ergon_core.api:WeightedCriterion", - "weight": 1.0, - "criterion": {"_type": "myproj.bench:AlwaysPass", - "slug": "always-pass", "description": "..."} - }] - }], - "parent_task_slug": null, - "dependency_task_slugs": [], - "task_payload": {} - } - }, - { id: <dtid_code>, ..., task_slug: "code", ..., parent: "research", deps: ["research"] }, - { id: <dtid_review>, ..., task_slug: "review", ..., parent: "code", deps: ["code"] }, - { id: <dtid_summary>, ..., task_slug: "summarize", ..., parent: "review", deps: ["review"] }, -] -``` - -**Note:** no `ComponentCatalogEntry`, no `worker_slug` indirection, no -experiment-level worker/evaluator/assignment tables. Each task row -self-describes every object it needs via `_type`: task, worker, toolkit, -sandbox, evaluator, criterion. `_task_id` (PrivateAttr) is naturally absent -from the dump; ditto `_runtime` on Sandbox. - -## Stage 3 — run launch (in the API process) - -`run_experiment` creates one `RunRecord` and dispatches to Inngest: - -```sql -RunRecord { id: <run_uuid>, definition_id: <def_uuid>, status: "running", ... } -``` - -The `run_started` Inngest function then **copies** every -`ExperimentDefinitionTask` row into `run_graph_nodes`, preserving each -task's `id` as the per-run `task_id`, inlining the `task_json` snapshot, -and deriving edges from `dependency_task_slugs`. - -```sql -run_graph_nodes (4 rows; (run_id, task_id) is the composite PK) -[ - { run_id: <run_uuid>, task_id: <dtid_research>, parent_task_id: null, - task_json: { ...full Task JSON, including worker + sandbox + evaluators... }, - status: "pending", … }, - { run_id: <run_uuid>, task_id: <dtid_code>, parent_task_id: <dtid_research>, - task_json: {...}, status: "blocked", … }, - { run_id: <run_uuid>, task_id: <dtid_review>, parent_task_id: <dtid_code>, - task_json: {...}, status: "blocked", … }, - { run_id: <run_uuid>, task_id: <dtid_summary>, parent_task_id: <dtid_review>, - task_json: {...}, status: "blocked", … }, -] -``` - -Three important properties of this copy: - -1. **Identity preservation.** `task_id` is *literally* the definition - row's `id`. No fresh UUID is generated for static tasks at run-launch. - The same `task_id` will recur in every future run of this definition - (uniqued per row by `run_id`). -2. **Self-contained task snapshot.** `task_json` is copied inline so the - run is independent of definition mutation (we don't allow it, but if - we ever did, in-flight runs would be unaffected). -3. **Worker/evaluator JSON is part of the task snapshot.** This is the - object-first API choice: the runtime doesn't chase binding keys or - load pools. If authors want five cohort variants with different - workers, they build five `Experiment` objects with normal Python - factory functions that pass different worker objects into the - benchmark constructor. - -Then for the root task (`research`, no dependencies), the runtime fires -`sandbox_setup` then `worker_execute` with payload -`(run_id, task_id=<dtid_research>)`. - -## Stage 4 — `worker_execute` for one task (in the rollout container) - -```python -# Inngest job payload carries: run_id, definition_id, task_id, execution_id. -# definition_task_id and node_id are gone; task_id is the canonical -# identity that points at the (run_id, task_id) row directly. -# definition_id may stay for logging/lookup, but worker/evaluator pools -# are no longer loaded; task_json is self-contained. - -with get_session() as session: - # ── 1. Repo calls return fully inflated typed objects — no dicts. ── - # graph_repo.node returns a typed RunGraphNodeView whose `.task` is the - # already-inflated Task — Task.from_definition is called inside the repo - # and never in the job body. - node = graph_repo.node(session, run_id=payload.run_id, - task_id=payload.task_id) # RunGraphNodeView - - # ── 2. Read the directly bound worker. ── - task = node.task # already inflated - worker = task.worker # already inflated - - # task.sandbox is a typed Sandbox subclass with _runtime=None - # (e.g. for the research task, task.sandbox is a ResearchE2BSandbox). - # task.task_id returns payload.task_id; task.evaluators is the typed - # tuple of Evaluators bound directly to this task; worker is the - # fully-reconstructed ReActWorker (with its toolkit field) identical - # to the author's original instance. The job body cannot tell whether the underlying - # task_json arrived via "copied at run-launch from a definition" or - # "written inline by a worker that spawned this child" — and it does - # not need to. The dict→Task conversion is fully encapsulated in the repo. - -# ── 3. Provision the sandbox via the lifecycle hub. ── -sandbox = await lifecycle_hub.acquire( - task.sandbox, run_id=payload.run_id, task_id=node.task_id, -) -# Internally calls task.sandbox.provision() (or reuses an in-process -# retry attempt). task.sandbox._runtime is now attached; task.sandbox -# IS sandbox. See 03-runtime.md "SandboxLifecycleHub" for the exact -# reattach semantics (in-process only in v1; cross-process retry -# provisions fresh). - -# ── 4. Build the runtime context (framework-only ctor). ── -# WorkerContext carries `task_id` as a public field so workers can read -# `context.task_id` symmetrically with `task.task_id` (same value); -# author code that wants identity has both surfaces. The framework -# wires the service PrivateAttrs via `_for_job`. -# See 03-runtime.md "Framework-side WorkerContext construction". -context = WorkerContext._for_job( - run_id=payload.run_id, - task_id=node.task_id, - execution_id=payload.execution_id, - definition_id=payload.definition_id, - task_mgmt=task_mgmt_service, # constructed once per job - task_inspect=task_inspect_service, - resource_repo=resource_repo, -) -# NOTE: no sandbox_id here — that's on `sandbox` (passed separately). - -try: - # ── 5. Run the worker. ── - async for chunk in worker.execute(task=task, context=context): - await persist_event(chunk) - await persist_worker_output(execution_id=payload.execution_id, output=worker_output) - - # ── 6. Synchronously fan out one Inngest invocation per evaluator. - # Each ctx.step.invoke suspends worker_execute until that - # evaluate_task_run invocation returns, so the sandbox stays - # alive throughout. The payload is id-only; the eval worker - # reloads task state via the same graph_repo.node call we used - # in stage 4, this time passing sandbox_id so task.sandbox is - # re-attached live in the eval worker's process. ── - await asyncio.gather(*[ - ctx.step.invoke( - f"eval-{i}", - evaluate_task_run, - TaskEvaluateRequest( - run_id=payload.run_id, - task_id=node.task_id, - execution_id=payload.execution_id, - evaluator_index=i, - ), - ) - for i in range(len(task.evaluators)) - ]) -finally: - # ── 7. Settle the task; tear down sandbox; mark dependents unblocked. - # Release runs only after every step.invoke has returned. Eval - # workers never reach this branch — they only call sandbox.detach(), - # dropping their local _runtime while leaving the external - # sandbox alive for any sibling invocation still running. ── - await lifecycle_hub.release(sandbox) # calls sandbox.terminate() -``` - -The whole job is **typed-node load → provision → execute → release**. -No `session.get` in the job body, no -`import_component_string + model_validate` dance, no `Task.from_definition` -call in the job body either (it's invoked once inside `graph_repo.node`), -no registry lookup, no slug indirection, no per-benchmark sandbox manager -subclass dispatch, no template string lookup, no static/dynamic branching, -no `dict[str, Any]` ever crossing the repo boundary into the job body. Two -infrastructure concerns each have one canonical home: PG access *and* -JSON-to-typed reconstruction in the repos, and the public API surface -the job body sees is uniformly typed. - -## Stage 5 — what runs next - -`research` settles with `status: "completed"`. The runtime checks dependents -of `<task_uuid_research>` — finds `<task_uuid_code>`, sees its only -unsatisfied dep is now satisfied, transitions to `pending`, fires -`worker_execute` for the `coder` binding. The job's stage-4 step 3 calls -`PythonSandbox.provision()` (since `task.sandbox` is a `PythonSandbox` -this time, not a `ResearchE2BSandbox`). Same seven-step sequence. Repeat -for `review` and `summarize`. - -`summarize` is technically an LLM-only task — but it picks `PythonSandbox` -in this walkthrough so it can read the prior tasks' resources from the -sandbox. (Genuinely zero-environment tasks are out of scope for this -redesign; they'll be covered when a generic `DefaultPythonSandbox` lands. -There is no no-op `Sandbox` subclass — one that satisfies the type -contract while no-op'ing `run_command` is a type-system lie we're -explicitly not shipping.) - -## Stage 6 — example consumer tools (the runtime API in practice) - -The 5-stage trace above shows the runtime calling `worker.execute(...)`. -What does the worker *do* inside `execute`? It uses `WorkerContext`'s -curated single-target methods for the common case, and drops to the -internal services in `core.application.*` directly for the rest. Two -example consumers, both grounded in the in-tree codebase. - -### Consumer A — simple typed tool (the 90% case) - -A manager worker that wants to fan out one subtask per source URL. Uses -**only** `WorkerContext` — single-target spawn, the child task is fully -object-bound, and the framework enforces containment. - -```python -# ergon_builtins/tools/spawn_research_subtask.py -from typing import Literal -from pydantic import BaseModel -from ergon_core.api import Task, WorkerContext - -class SpawnResearchSubtaskSuccess(BaseModel): - kind: Literal["success"] = "success" - task_id: str - status: str - model_config = {"frozen": True} - -class ToolFailure(BaseModel): - kind: Literal["failure"] = "failure" - error: str - model_config = {"frozen": True} - -type SpawnResult = SpawnResearchSubtaskSuccess | ToolFailure - - -def make_spawn_research_subtask_tool(*, context: WorkerContext): - """Build an LLM-callable tool that spawns one research subtask under - the calling worker's task. Containment is a framework concern — the - tool just constructs a fully-bound Task and calls context.spawn_task.""" - - async def spawn_research_subtask( - url: str, focus: str, - ) -> SpawnResult: - """Spawn a single researcher subtask for one URL. - - Args: - url: The source URL the subtask should research. - focus: One-line description of what the subtask should extract. - """ - try: - handle = await context.spawn_task( - Task( - task_slug=f"research-{hash(url) & 0xffff:04x}", - instance_key=context.get_task(context.task_id).instance_key, - description=f"Research {url}: {focus}", - worker=ReActWorker( - name="researcher", - model="openai:gpt-4o", - system_prompt="Research this URL and write findings.", - max_iterations=20, - toolkit=ResearchToolkit(max_search_results=10), - ), - sandbox=ResearchE2BSandbox(), - evaluators=(), - ), - ) - return SpawnResearchSubtaskSuccess( - task_id=str(handle.task_id), status="pending", - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - return ToolFailure(error=str(exc)) - - return spawn_research_subtask -``` - -What's worth noticing: - -- **The child task is fully bound.** The worker object, sandbox, and - evaluators live on the `Task`; `spawn_task` has no separate worker - parameter and no experiment-level pool validation. -- **No subtree-membership check.** The framework enforces it inside - `context.spawn_task` (and the other `WorkerContext` mutation - methods). Toolkits stop carrying a TODO they had no way to actually - solve. -- **No `ergon_core.core.*` imports.** Single-target spawn is on the - curated surface; the tool only needs `ergon_core.api`. - -### Consumer B — power tool that drops to the internal service (the escape hatch) - -A manager worker that needs to atomically plan a sub-DAG of dependent -research tasks (cycles detected up-front, all-or-nothing). `plan_subtasks` -fails the `WorkerContext` curation rule (it's a batch op), so the -toolkit imports `TaskManagementService` directly. Same import path -`ergon_builtins/tools/subtask_lifecycle_toolkit.py` uses today; v1 -doesn't try to relocate it. - -```python -# ergon_builtins/tools/plan_research_subtasks_toolkit.py -from typing import Literal -from pydantic import BaseModel -from ergon_core.api import WorkerContext - -# v1: the batch op stays on the internal service. -# Promote when there's a real third-party consumer. -from ergon_core.core.application.tasks.management import TaskManagementService -from ergon_core.core.application.tasks.models import SubtaskSpec # slug-based - -class PlanSubtasksSuccess(BaseModel): - kind: Literal["success"] = "success" - task_ids: dict[str, str] # slug → task_id - model_config = {"frozen": True} - -class PlanFailure(BaseModel): - kind: Literal["failure"] = "failure" - error: str - model_config = {"frozen": True} - - -def make_plan_research_subtasks_tool( - *, - context: WorkerContext, - task_mgmt: TaskManagementService, - research_worker: Worker, -): - """Build a tool that atomically plans a sub-DAG of research tasks. - Drops below WorkerContext because plan_subtasks is a batch op (fails - the curation rule of single-target + high-frequency).""" - - async def plan_research_subtasks( - plan: list[dict], # [{slug, description, depends_on}] - ) -> PlanSubtasksSuccess | PlanFailure: - """Atomically materialise a sub-DAG. Cycles, duplicate slugs, - and dependency references are all rejected up-front.""" - try: - specs = [ - SubtaskSpec( - task=Task( - task_slug=item["slug"], - instance_key=context.get_task(context.task_id).instance_key, - description=item["description"], - worker=research_worker, - sandbox=ResearchE2BSandbox(), - evaluators=(), - ), - depends_on=list(item.get("depends_on", [])), - ) - for item in plan - ] - # plan_subtasks is the internal service method. It already - # exists and works today; v1 doesn't need to wrap it in a - # public service class. - result = await task_mgmt.plan_subtasks( - run_id=context.run_id, parent_task_id=context.task_id, - specs=specs, - ) - return PlanSubtasksSuccess( - task_ids={slug: str(tid) for slug, tid in result.items()}, - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - return PlanFailure(error=str(exc)) - - return plan_research_subtasks -``` - -What's worth noticing: - -- **The internal service import is the v1 escape hatch.** Same shape - `ergon_builtins/tools/subtask_lifecycle_toolkit.py`, - `ergon_builtins/tools/graph_toolkit.py`, and - `ergon_cli/commands/workflow.py` all use today. -- **Containment is on the toolkit, not the framework.** When you - bypass `WorkerContext`, you bypass its containment check. For - toolkits operating on their own `(run_id, task_id)` scope (the - common case) that's fine; for toolkits accepting target ids from an - LLM, validate against `context.get_task(target_id)` first or do the - descendant check yourself. -- **No public service class involved.** An earlier design wrapped this - call site in `GraphMutator.for_worker(context).plan_subtasks(...)` — - same shape, one extra public class, no v1 consumer that needed it. - Promote when one shows up. - -## Blast radius — files affected - -What the trace above implies in code. Phase tags ([P1] / [P2] / [P3] / -[P4]) match [`09-implementation-plan.md`](09-implementation-plan.md). Paths are real as of -the current tree; verify against your branch before touching. - -This is the **file-level inventory** of what changes; for the *why* -behind each ADD/DELETE pairing — i.e. "we're deleting `X` because we're -adding `Y` in the public API" — see the -[Core deduplication audit](../2026-05-08-authoring-api-redesign/05-migration.md#core-deduplication-audit). -The [Deletion checklist](../2026-05-08-authoring-api-redesign/05-migration.md#deletion-checklist-reviewer) -gives the `git grep` invocations the reviewer runs to verify nothing's -left dangling. - -``` -ergon/ -├── ergon_core/ergon_core/ -│ ├── api/ # public authoring surface -│ │ ├── __init__.py # MODIFY: drop TaskSpec, ComponentRegistry exports; add Sandbox, WeightedCriterion, SpawnedTaskHandle, new exception types [P1-P4] -│ │ ├── registry.py # DELETE: ComponentRegistry replaced by _type discriminator + import_component_string [P1] -│ │ ├── benchmark/ -│ │ │ ├── benchmark.py # MODIFY: add Benchmark.from_definition classmethod [P1] -│ │ │ └── task.py # MODIFY: add worker: Worker, sandbox: Sandbox, evaluators: tuple[Evaluator, ...], _task_id PrivateAttr, task_id property, from_definition classmethod; delete TaskSpec [P3] -│ │ ├── criterion/ -│ │ │ ├── criterion.py # MODIFY: add Criterion.from_definition [P1]; evaluate signature gains sandbox: Sandbox [P4] -│ │ │ └── context.py # MODIFY: drop proxy methods; CriterionContext becomes pure data [P4] -│ │ ├── rubric/ -│ │ │ ├── rubric.py # MODIFY: add Rubric.from_definition [P1]; add WeightedCriterion wrapper [P4] -│ │ │ └── evaluator.py # MODIFY: add Evaluator.from_definition [P1] -│ │ ├── sandbox/ # ADD: new public-API folder for the abstract base [P2] -│ │ │ ├── __init__.py # ADD: re-export Sandbox base -│ │ │ ├── sandbox.py # ADD: Sandbox(BaseModel, ABC) + _runtime PrivateAttr + abstract provision/terminate + from_definition + IO proxy methods -│ │ │ └── runtime.py # ADD: SandboxRuntime protocol (the "live" backing object) -│ │ ├── worker/ -│ │ │ ├── worker.py # MODIFY: become BaseModel + ABC; add execute(task, *, context, sandbox); add from_definition [P1]; sandbox kwarg becomes non-optional [end of P2] -│ │ │ ├── context.py # MODIFY: WorkerContext becomes BaseModel; add _task_mgmt / _task_inspect / _resource_repo PrivateAttrs; add curated single-target methods (spawn_task in step 16, then cancel_task/refine_task/restart_task + subtasks/descendants/get_task/resources in step 16a) — each delegates directly to the internal service [P3] -│ │ │ └── results.py # MODIFY: add WorkerStreamItem (discriminated union); add SpawnedTaskHandle (.task_id, .wait()) — return type of WorkerContext.spawn_task [P3] -│ │ └── experiment.py # ADD: canonical home of class Experiment — Pydantic BaseModel + requires_sandbox @model_validator(mode="after"); _persisted DefinitionHandle PrivateAttr. Lifted out of core.domain.experiments (audit #6) [P1] -│ ├── core/ -│ │ ├── application/ -│ │ │ ├── components/ # DELETE: whole folder dies [P1] -│ │ │ │ └── catalog.py # DELETE: ComponentCatalogService → replaced by Worker/Task/etc.from_definition -│ │ │ ├── experiments/ -│ │ │ │ ├── definition_writer.py # MODIFY: persist Worker pydantic JSON (not slug+recipe); persist Task with sandbox subclass [P1-P3] -│ │ │ │ ├── service.py # MODIFY: static-on-dynamic dep validator [P3]. Note: Worker/Criterion `requires_sandbox` validation moves onto Experiment as a `@model_validator(mode="after")` (see 08-decisions-log.md "Worker → Sandbox compatibility checking"), so it fails at construction in the author's process — service-layer validation no longer needs to repeat it. -│ │ │ │ └── repository.py # MODIFY: add graph_repo.node returning RunGraphNodeView with task.worker/task.evaluators already inflated [P3] -│ │ │ ├── jobs/ -│ │ │ │ └── worker_execute.py # MODIFY: RunGraphNodeView read, worker = task.worker, no import_component_string in body; inject TaskManagementService/TaskInspectionService/RunResourceRepository into WorkerContext at job start [P3] -│ │ │ ├── tasks/ -│ │ │ │ ├── execution.py # MODIFY: rebuilt around RunGraphNodeView; no TaskSpec; no slug-based registry resolution [P3] -│ │ │ │ ├── management.py # MODIFY: add_subtask gains a Task-shaped path (taking task: Task directly; task carries worker/sandbox/evaluators) for WorkerContext.spawn_task [P3]. NOT made import-private — toolkits and CLI keep importing it directly. -│ │ │ │ └── inspection.py # MODIFY: minor — methods that WorkerContext exposes (list_subtasks, descendants, get_subtask) get whatever signature tweaks the facade needs; the rest stays as-is [P3] -│ │ │ └── resources/ # MODIFY (light): nothing forced; whatever scope dispatch WorkerContext.resources(scope=...) needs gets added [P3] -│ │ ├── domain/ -│ │ │ └── experiments/ -│ │ │ ├── experiment.py # DELETE: class Experiment lifts to ergon_core/api/experiment.py (audit #6) [P1] -│ │ │ ├── worker_spec.py # DELETE: WorkerSpec dies; Task.worker carries Worker instances directly (audit #2) [P1] -│ │ │ ├── handles.py # unchanged — DefinitionHandle stays internal (return shape of persist_definition; held as Experiment._persisted PrivateAttr) -│ │ │ ├── validation.py # MODIFY: ExperimentValidationService keeps the cross-component rules engine; THINS to drop checks now done by *.from_definition / Experiment.@model_validator [P1] -│ │ │ └── __init__.py # MODIFY: drop Experiment + WorkerSpec re-exports; keep DefinitionHandle [P1] -│ │ ├── infrastructure/ -│ │ │ ├── inngest/handlers/ -│ │ │ │ ├── worker_execute.py # MODIFY: payload reshape — add definition_id, drop node_id [P3] -│ │ │ │ ├── persist_outputs.py # MODIFY: composite (run_id, task_id) refs [P3] -│ │ │ │ ├── evaluate_task_run.py # MODIFY: thin id-only payload, reload via graph_repo.node(..., sandbox_id=...), call criterion.evaluate directly, detach in finally [P4] -│ │ │ │ ├── cleanup_cancelled_task.py # MODIFY: composite-FK refs [P3] -│ │ │ │ └── cancel_orphan_subtasks.py # MODIFY: composite-FK refs [P3] -│ │ │ └── sandbox/ -│ │ │ ├── manager.py # DELETE: BaseSandboxManager + DefaultSandboxManager → per-Sandbox-subclass provision() [P2] -│ │ │ ├── lifecycle.py # MODIFY (or REPLACE): becomes SandboxLifecycleHub — small kind-agnostic acquire/release/terminate_all [P2] -│ │ │ ├── resource_publisher.py # MODIFY: read sandbox via Sandbox subclass, not BaseSandboxManager [P2] -│ │ │ ├── instrumentation.py # MODIFY: hook the new Sandbox lifecycle calls [P2] -│ │ │ └── event_sink.py # MODIFY: same — hooked into new lifecycle [P2] -│ │ └── persistence/ -│ │ ├── components/ # DELETE: whole folder dies [P1] -│ │ │ └── models.py # DELETE: ComponentCatalogEntry table → drop and recreate; no replacement (worker JSON lives inside task_json.worker, _type discriminator carries class identity) -│ │ ├── definitions/ -│ │ │ └── models.py # MODIFY: ExperimentDefinitionTask gains task_json column containing worker/sandbox/evaluators; ExperimentDefinitionWorker/ExperimentDefinitionTaskAssignment become unnecessary for authoring path [P1-P3] -│ │ └── graph/ -│ │ ├── models.py # MODIFY: RunGraphNode PK becomes composite (run_id, task_id); add task_json; drop id/node_id/definition_task_id/description/task_slug/instance_key/assigned_worker_slug; RunGraphEdge becomes composite-FK both sides; RunGraphAnnotation/Mutation likewise [P3] -│ │ └── status_conventions.py # MODIFY: any node-id-keyed helpers become (run_id, task_id)-keyed [P3] -│ └── tests/ -│ └── unit/ -│ ├── api/test_public_api_imports.py # MODIFY: drop TaskSpec/ComponentRegistry; add Sandbox, Experiment, SpawnedTaskHandle [P1-P3] -│ ├── api/test_task_spec_contract.py # DELETE: TaskSpec dies [P3] -│ ├── api/test_worker_contract.py # MODIFY: new execute signature, sandbox kwarg [P1] -│ ├── architecture/test_public_api_boundaries.py # MODIFY: new boundary set [P1-P3] -│ ├── architecture/test_public_api_target_structure.py # MODIFY: same -│ ├── registry/test_component_registry.py # DELETE [P1] -│ ├── registry/test_catalog_backed_registry_resolution.py # DELETE [P1] -│ ├── registry/test_react_factories.py # MODIFY (or DELETE): per-benchmark *ReactWorker subclasses die [P1] -│ ├── registry/test_builtin_pairings.py # MODIFY or DELETE: no registry/pool worker resolution remains [P1, P3] -│ ├── runtime/test_worker_execute_stream_contract.py # MODIFY: RunGraphNodeView read; worker = task.worker [P3] -│ ├── runtime/test_experiment_definition_writer.py # MODIFY: Worker pydantic JSON; sandbox-on-Task [P1-P2] -│ ├── runtime/test_experiment_definition_service.py # MODIFY: drop requires_sandbox cases (now an Experiment model_validator — covered by experiment unit tests) [P2] -│ └── sandbox/test_sandbox_reconnect.py # MODIFY: SandboxLifecycleHub-based reconnect [P2] -│ -├── ergon_builtins/ergon_builtins/ -│ ├── registry.py # DELETE [P1] -│ ├── registry_core.py # DELETE [P1] -│ ├── registry_data.py # DELETE [P1] — or trim to BUILTIN_WORKERS = {slug: import_path} for CLI surface only -│ ├── registry_local_models.py # MODIFY: keep LLM-backend registration; drop component-registration plumbing [P1] -│ ├── sandboxes/ # ADD: new module — per-kind Sandbox subclasses [P2] -│ │ ├── __init__.py # ADD -│ │ ├── _e2b_base.py # ADD: _E2BBackedSandbox shared parent (E2B client construction) -│ │ ├── lean.py # ADD: LeanSandbox(_E2BBackedSandbox) — owns the Lean install -│ │ ├── python.py # ADD: PythonSandbox(_E2BBackedSandbox) — pip install -│ │ ├── swebench.py # ADD: SWEBenchSandbox(_E2BBackedSandbox) — repo clone -│ │ ├── research_e2b.py # ADD: ResearchE2BSandbox(_E2BBackedSandbox) -│ │ └── gdpeval.py # ADD: GDPEvalSandbox(_E2BBackedSandbox) -│ ├── toolkits/ # ADD or MOVE: one home for concrete _Toolkit subclasses [P1] -│ │ ├── __init__.py # ADD -│ │ ├── minif2f.py # MOVE from benchmarks/minif2f/toolkit.py: become pydantic, lose __init__(sandbox), gain build_tools(sandbox, task) -│ │ ├── swebench.py # MOVE from benchmarks/swebench_verified/toolkit.py: same -│ │ ├── gdpeval.py # MOVE from benchmarks/gdpeval/toolkit.py: same -│ │ └── research_rubrics.py # MOVE from benchmarks/researchrubrics/toolkit_types.py + tools/research_rubrics_toolkit.py -│ ├── tools/ # NOT TOUCHED in this redesign: -│ │ ├── subtask_lifecycle_toolkit.py # unchanged — keeps `from ergon_core.core.application.tasks.management import TaskManagementService` (the v1 escape hatch). Containment-check TODO does NOT close here; only WorkerContext.cancel_task / refine_task / get_task enforce it. Migrate this module to the curated facade in a follow-up. -│ │ ├── graph_toolkit.py # unchanged — same v1 escape hatch. -│ │ └── workflow_cli_tool.py # unchanged — same v1 escape hatch. -│ ├── workers/baselines/ -│ │ └── react_worker.py # MODIFY: ReActWorker takes _toolkit: _Toolkit field; _Toolkit ABC lives module-private here; no per-benchmark subclasses [P1] -│ └── benchmarks/ -│ ├── minif2f/ -│ │ ├── sandbox_manager.py # DELETE: → sandboxes/lean.py [P2] -│ │ ├── toolkit.py # DELETE: → toolkits/minif2f.py [P1] -│ │ ├── worker_factory.py # MODIFY: returns ReActWorker(toolkit=MiniF2FToolkit(...)); no MiniF2FReactWorker subclass [P1] -│ │ ├── benchmark.py # MODIFY: build_instances() returns Mapping[str, Sequence[Task]] with sandbox=LeanSandbox(...) on each Task [P2-P3] -│ │ └── __init__.py # MODIFY: drop MiniF2FReactWorker re-export -│ ├── swebench_verified/ -│ │ ├── sandbox_manager.py # DELETE: → sandboxes/swebench.py [P2] -│ │ ├── sandbox_manager_support.py # DELETE: → folded into sandboxes/swebench.py -│ │ ├── toolkit.py # DELETE: → toolkits/swebench.py [P1] -│ │ ├── worker_factory.py # MODIFY: returns ReActWorker(toolkit=SWEBenchToolkit(...)) [P1] -│ │ ├── benchmark.py # MODIFY: sandbox=SWEBenchSandbox(...) on each Task [P2-P3] -│ │ └── criterion.py # MODIFY: evaluate(..., sandbox: Sandbox) directly [P4] -│ ├── researchrubrics/ -│ │ ├── sandbox_manager.py # DELETE: → sandboxes/research_e2b.py [P2] -│ │ ├── toolkit_types.py # DELETE: → toolkits/research_rubrics.py [P1] -│ │ ├── worker_factory.py # MODIFY: returns ReActWorker(toolkit=ResearchRubricsToolkit(...)) [P1] -│ │ ├── benchmark.py # MODIFY: sandbox=ResearchE2BSandbox(...) per Task [P2-P3] -│ │ └── vanilla.py # MODIFY: drop slug-based factory [P1] -│ └── gdpeval/ -│ ├── sandbox.py # DELETE: GDPEvalSandboxManager → sandboxes/gdpeval.py [P2] -│ ├── sandbox_utils.py # DELETE or MOVE: helpers fold into sandboxes/gdpeval.py [P2] -│ ├── toolkit.py # DELETE: → toolkits/gdpeval.py [P1] -│ ├── worker_factory.py # MODIFY: returns ReActWorker(toolkit=GDPEvalToolkit(...)) [P1] -│ └── benchmark.py # MODIFY: sandbox=GDPEvalSandbox(...) per Task [P2-P3] -│ -├── ergon_cli/ergon_cli/ -│ └── commands/ -│ └── workflow.py # NOT TOUCHED — keeps existing imports of TaskManagementService / TaskInspectionService / WorkflowService internals. Migration to a public service tier is deferred (see 09-implementation-plan.md "What's deferred"). -│ -└── docs/ - ├── architecture/ # MODIFY (post-acceptance): update 01_public_api.md, cross_cutting/sandbox_lifecycle.md, etc. (see 09-implementation-plan.md "On acceptance") - └── rfcs/active/2026-05-08-authoring-api-redesign/ # this folder — graduates to accepted/ post-migration -``` - -### Headline counts - -| Action | ergon_core | ergon_builtins | ergon_cli | tests | total | -|---|---|---|---|---|---| -| ADD (new files) | 4 (`api/sandbox/{__init__,sandbox,runtime}.py` + `api/experiment.py` — last is the Experiment lift; not a new shell, the class definition itself moves here) | 11 (`sandboxes/*`, `toolkits/*`) | — | — | 15 | -| MODIFY | ~20 | ~9 (workers/baselines + benchmark dirs) | — | ~10 | ~39 | -| DELETE | 6 (`api/registry.py`, `application/components/`, `persistence/components/`, `infrastructure/sandbox/manager.py` partial, `core/domain/experiments/experiment.py`, `core/domain/experiments/worker_spec.py`) | 12 (`registry*.py`, per-benchmark `sandbox_manager.py` + `toolkit.py` + gdpeval `sandbox*.py`) | — | 3 | ~21 | - -### What's notably *not* on the list - -- `ergon_builtins/observability/`, `ergon_builtins/models/`, - `ergon_builtins/common/`, `ergon_builtins/evaluators/criteria/` — - unaffected. These don't touch the registry, the sandbox manager, or - the worker construction surface. -- `ergon_builtins/tools/` — **unaffected in this PR.** An earlier - draft migrated `subtask_lifecycle_toolkit.py`, `graph_toolkit.py`, - and `workflow_cli_tool.py` to consume new public service classes; - that's deferred. -- `ergon_cli/commands/workflow.py` — **unaffected in this PR.** Same - reason — deferred along with the public service tier. -- `ergon_core/core/persistence/telemetry/`, `.../saved_specs/`, - `.../imports/`, `.../context/`, `.../shared/` — unaffected. They - reference runs/tasks via foreign key but don't carry component - identity or sandbox state. -- `ergon_ingestion/` (external-run import) — unaffected. Writes runs - straight to the persistence layer below the public API. (Already - flagged in [`09-implementation-plan.md`](09-implementation-plan.md).) -- `ergon_core/core/rest_api/` — mostly unaffected; rollout/run-listing - endpoints continue to read from the same persistence rows. The one - spot that may need a touch is wherever `node_id` leaked into a - response shape (none expected, but worth grepping during P3). -- `ergon_core/core/application/workflows/service.py` — unaffected. - `WorkflowService` stays as today; the workflow CLI keeps importing - it. - -If you find a file the trace implies should change but it's not on this -list, that's a real omission — flag it in -[`08-decisions-log.md`](08-decisions-log.md) and add it here. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/05-cli-authoring-interface.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/05-cli-authoring-interface.md deleted file mode 100644 index 907c5c9ed..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/05-cli-authoring-interface.md +++ /dev/null @@ -1,247 +0,0 @@ -# 05 — CLI authoring interface - -> What `ergon define` and `ergon run` actually do, after the v1 -> divergence is repaired. The short answer: the CLI is a composition -> convenience, not a parallel persistence path. It builds an `Experiment` -> from a registered factory and calls the same `persist_definition` -> entry point the public Python API uses. There is no second slug-based -> persistence flow. -> -> See [`01-api-surface.md`](01-api-surface.md) for the public types, -> [`02-persistence-layer.md`](02-persistence-layer.md) for the -> persistence model, and -> [`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md) -> for the v1 audit finding that motivated this doc. - -## Why this doc exists - -v1 grew two parallel ways to persist an experiment definition: - -- **Public Python API.** Author writes a benchmark factory, constructs - `Experiment(benchmark=..., ...)`, calls - `persist_definition(experiment)`. Hits - `experiment_definitions` + `experiment_definition_tasks` + - `experiment_definition_edges`. Used by the dashboard, by tests, and by - any agent that wants to programmatically launch a run. -- **CLI slug path.** `ergon define <benchmark-slug>` resolved a slug - through `ergon_builtins.benchmarks._registry`, called a - `_persist_single_sample_workflow_definition` helper, wrote rows into a - `saved_specs` table that **no other code in the system reads**. Used - by no one in production; tested only by tests of the CLI command - itself. - -The CLI path was non-functional in the literal sense: a definition -written by the CLI could not be launched as a run, because the launcher -reads from `experiment_definitions`, not from `saved_specs`. The v1 -audit categorized `saved_specs` as write-only dead infrastructure and -the CLI define path as a parallel non-functional code branch. - -v2 unifies. The CLI **builds** an `Experiment` and **delegates** to the -public persistence path. There is one persistence flow. - -## Architectural model - -``` - ┌─ public Python API path ──────────────────────────┐ - │ │ -author code ─► Experiment(...) ─► persist_definition(experiment) ─► - │ - ▼ - experiment_definitions - + experiment_definition_tasks - + experiment_definition_edges - ▲ - │ - ┌─ CLI path ─────────────────────────────────────────┤ - │ │ -$ ergon define <slug> ─► _BUILTIN_BENCHMARKS[slug]() ─► │ - build Experiment(...) ─► │ - persist_definition(experiment) ────┘ -``` - -There is one persistence implementation. The CLI is a convenience that -selects which `Experiment` to build via a slug. - -## CLI entry points - -### `ergon define <benchmark-slug> [--name NAME] [--description TEXT]` - -What it does: - -1. Resolve `<benchmark-slug>` to a benchmark factory: - ```python - _BUILTIN_BENCHMARKS: dict[str, Callable[[], Benchmark]] = { - "minif2f-react-baseline": lambda: MiniF2FBenchmark(...), - "swebench-react-baseline": lambda: SWEBenchBenchmark(...), - # ... - } - ``` - Static dict in `ergon_builtins`, no dynamic lookup, no registry - table. The slug is purely a CLI ergonomics aid. -2. Build a `Benchmark` instance by calling the factory. -3. Wrap it in an `Experiment`: - ```python - experiment = Experiment( - benchmark=benchmark, - name=name or f"{slug}-{datetime.utcnow().isoformat()}", - description=description, - metadata={"created_by": "cli", "slug": slug}, - ) - ``` -4. Call the same `persist_definition(experiment)` the public Python API - uses. -5. Print the new `definition_id` so the user can `ergon run - <definition_id>` next. - -```python -# Inside ergon_cli.commands.define — full implementation: -def define( - slug: str, - *, name: str | None = None, description: str | None = None, -) -> None: - factory = _BUILTIN_BENCHMARKS.get(slug) - if factory is None: - raise typer.BadParameter( - f"Unknown benchmark slug: {slug!r}. " - f"Known slugs: {sorted(_BUILTIN_BENCHMARKS)}" - ) - - experiment = Experiment( - benchmark=factory(), - name=name or _default_name(slug), - description=description, - metadata={"created_by": "cli", "slug": slug}, - ) - handle = persist_definition(experiment) - typer.echo(f"defined {handle.definition_id}") -``` - -That's the entire CLI define command. **No `_persist_single_sample_workflow_definition`, -no `saved_specs` writes, no second resolution path.** - -### `ergon run <definition-id> [--metadata KEY=VALUE]...` - -What it does: - -1. Take a `definition_id` (UUID) — the same id `ergon define` printed. - This is the only handle a run needs; benchmark slugs are *not* a - valid input here. (Rationale: a slug-input run would re-resolve the - factory and could pick up different code than was persisted; pinning - to `definition_id` means runs always execute the exact persisted - payload.) -2. Call `launch_run(definition_id, run_metadata=...)` — the same entry - point the public Python API uses. -3. Print the new `run_id` so the user can stream logs / poll status. - -```python -def run( - definition_id: UUID, - *, metadata: list[str] = (), -) -> None: - run_metadata = dict(_parse_kv(item) for item in metadata) - handle = launch_run(definition_id, metadata=run_metadata) - typer.echo(f"launched run {handle.run_id} (definition {definition_id})") -``` - -### No `ergon run <slug>` shortcut `[v2: locked out]` - -The "define and run in one step" shortcut is deliberately *not* -provided. `ergon run` accepts a `definition_id` (UUID) only. - -Rationale: shortcuts that conflate define and run muddy the mental -model — users start to think "running a slug" is one operation when it -is actually two (a write to `experiment_definitions` followed by a -write to `runs`). Two-step is honest about what happened: the slug -*became* a definition, and the definition *became* a run. The two-line -shell invocation `defn=$(ergon define <slug>) && ergon run "$defn"` is -the canonical compose pattern. - -## What CLI does *not* do - -### No second persistence path - -There is no `saved_specs` table, no -`_persist_single_sample_workflow_definition` helper, no -`SavedSpecRepository`. The v1 audit identified these as write-only -dead infrastructure. Deletion lives in -[`09-implementation-plan.md`](09-implementation-plan.md). - -### No slug-based registry table in the database - -The `BUILTIN_BENCHMARKS` dict in `ergon_builtins` is **in-process -Python state**, not a database table. The slug is a CLI argument and -nothing else; it does not survive into `experiment_definitions` as a -foreign key (it goes into `metadata.slug` as a free-form string for -human reference only). - -This means: third-party benchmark packages don't register slugs -through a database insert. They ship their own CLI plugin that -contributes to a different `BUILTIN_BENCHMARKS`-shaped dict, or users -build their `Experiment` programmatically and skip the slug entirely. - -### No CLI-side validation - -`Experiment.@model_validator(mode="after")` already enforces -`requires_sandbox` compatibility at construction time -([01-api-surface.md "Foundational change C"](01-api-surface.md#foundational-change-c--experiment-lifts-into-the-public-api)). -`ExperimentValidationService` already enforces cross-component rules -during `persist_definition`. The CLI does not duplicate either. A -malformed slug factory raises at `factory()` call time; the user sees -the original Python error, not a wrapped CLI error. - -## Composition convenience, not abstraction - -The slug system in v2 is *deliberately* unfancy: - -- It's a `dict[str, Callable[[], Benchmark]]`. -- It lives in one Python module (`ergon_builtins.benchmarks._registry`). -- It is mutated only by being edited. -- It is not pluggable across packages without users editing the dict. - -If a user wants more than this — e.g. "every benchmark in my private -package shows up in `ergon define --list`" — they import their -benchmark module and call `persist_definition(...)` directly. The CLI -is for the in-tree benchmark catalogue only. - -This design is reachable from "the CLI was non-functional" because the -non-functionality came from the CLI doing *too much*: it tried to be a -parallel persistence interface with its own resolution semantics. v2's -CLI does *less*, which is why it works. - -## Testing - -The architecture-guard test in -[`07-test-strategy.md`](07-test-strategy.md) enforces: - -```python -def test_no_saved_specs_imports() -> None: - """ergon_core, ergon_cli, ergon_builtins must not import any - `saved_specs` symbol — the package is deleted as of v2.""" - -def test_cli_define_routes_through_persist_definition() -> None: - """ergon_cli.commands.define must call persist_definition, not - any helper named *_persist_single_sample_workflow_definition* - (deleted) or any saved-specs write path (deleted).""" -``` - -The CLI's own behavioural test asserts that after `ergon define -<slug>`, the new `definition_id` is loadable by `launch_run` — -catching any regression that re-introduces a parallel persistence -path. - -## Decisions locked at workshop `[v2: locked]` - -- **Plugin slug registration** — **locked: deferred.** Third-party - packages don't register slugs through any framework hook. They call - `persist_definition(experiment)` directly. The CLI is for in-tree - benchmarks only. -- **`ergon run --override`** — **locked: not allowed.** Overrides - always produce a fresh `experiment_definitions` row via a new - `ergon define`. `definition_id` is 1:1 with what was actually - executed; no exceptions. -- **Cohort runs from CLI** — **locked: deferred.** v2 ships - one-run-per-invocation. `--replicas N` is a follow-up if a real - workload demands it. -- **`ergon launch <slug>` convenience** — **locked out: rejected.** - Per "No `ergon run <slug>` shortcut" above; explicit two-step - preserves the mental model. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/06-inngest-event-contracts.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/06-inngest-event-contracts.md deleted file mode 100644 index 23b4462f7..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/06-inngest-event-contracts.md +++ /dev/null @@ -1,473 +0,0 @@ -# 06 — Inngest event contracts - -> The complete event taxonomy for v2: every event the runtime fires, -> what payload it carries, who produces it, who consumes it, how -> fan-out works, and what idempotency / retry semantics apply. -> -> Supersedes v1's [`05-migration.md` §14.B "Inngest payloads"](../2026-05-08-authoring-api-redesign/05-migration.md). -> The headline difference: **one Inngest function per task, not two**. -> See [`03-runtime.md`](03-runtime.md) "Synchronous fanout" for why. -> -> See [`04-walkthrough.md`](04-walkthrough.md) for the trace of these -> events firing in order, and -> [`02-persistence-layer.md`](02-persistence-layer.md) for the row -> writes each event triggers. - -## The job graph - -``` - ┌──────────────────┐ - API: launch_run() ───► │ workflow/started │ - └────────┬──────────┘ - │ - │ on receive: prepare_run service - │ (1) copy definition→run rows - │ (2) for each ready task, fire task/ready - ▼ - ┌──────────────────┐ - │ task/ready │ (one per ready task) - └────────┬──────────┘ - │ - │ on receive: dispatcher - │ fires task/worker-execute - ▼ - ┌──────────────────────┐ - │ task/worker-execute │ (the work) - └────────┬──────────────┘ - │ - ┌────────────success──┴──failure──┐ - ▼ ▼ - ┌──────────────────┐ ┌──────────────────┐ - │ task/completed │ │ task/failed │ - └────────┬──────────┘ └────────┬──────────┘ - │ │ - │ on receive (both): │ - │ advance_run service │ - │ - mark this task terminal │ - │ - mark dependents ready ──────┘ - │ - if no more pending, fire workflow/completed - ▼ - ┌──────────────────┐ - │ task/ready │ (fan-out for newly-ready tasks) - └──────────────────┘ - - ┌──────────────────┐ - run reaches terminal │ workflow/ │ - state ───► │ completed │ - └────────┬──────────┘ - │ - │ on receive: run/cleanup (best-effort) - ▼ - ┌──────────────────┐ - │ run/cleanup │ (sandbox sweep) - └──────────────────┘ -``` - -## Events - -For every event below: producer, consumer, payload, idempotency key, -retry policy. - -### `workflow/started` - -| Property | Value | -|---|---| -| **Producer** | `launch_run(definition_id, *, metadata)` — public API | -| **Consumer** | `prepare_run` Inngest function | -| **Idempotency key** | `run_id` (assigned by `launch_run` before dispatch; same key on retry) | -| **Retries** | up to 3 with exponential backoff | -| **Side effects on consume** | (1) copy `experiment_definition_tasks` → `run_graph_nodes` for this `run_id`; (2) copy `experiment_definition_edges` → `run_graph_edges`; (3) mark `runs.status = RUNNING`; (4) for each task with no incoming edges, fire `task/ready` | - -```python -class WorkflowStartedPayload(BaseModel): - run_id: UUID - definition_id: UUID - metadata: Mapping[str, Any] -``` - -`prepare_run` is the only place definition-tier tables are read by the -runtime path (per [`02-persistence-layer.md` §4 read -boundary](02-persistence-layer.md)). After `prepare_run` returns, no -subsequent Inngest function reads from definition tables. - -### `task/ready` - -| Property | Value | -|---|---| -| **Producer** | `prepare_run` (initial fan-out); `advance_run` (dependency-driven fan-out after a task completes) | -| **Consumer** | `dispatch_task` Inngest function | -| **Idempotency key** | `(run_id, task_id, generation)` where `generation` increments on restart_task | -| **Retries** | up to 3 | -| **Side effects on consume** | fire `task/worker-execute` with the same `(run_id, task_id, execution_id)` | - -```python -class TaskReadyPayload(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID # fresh UUID per attempt - generation: int = 0 # incremented by restart_task -``` - -The reason this is a separate event from `task/worker-execute`: it's the -fan-out point. `prepare_run` and `advance_run` both fire `task/ready` -events; the dispatcher consumes them and fires the actual work events. -Decoupling means restart-task / requeue logic only has to fire -`task/ready`, not understand the worker-execute payload shape. - -### `task/worker-execute` - -The orchestrator event. Reshapes v1's pair: `worker_execute` is still -one Inngest function per task, but it now **synchronously invokes** -`evaluate_task_run` once per evaluator via `ctx.step.invoke`. Eval -runs as a separate Inngest function (own retry, own concurrency cap, -own observability slug); the parent `gather`s on every invocation -before releasing the sandbox. - -| Property | Value | -|---|---| -| **Producer** | `dispatch_task` (in response to `task/ready`) | -| **Consumer** | `worker_execute` Inngest function | -| **Idempotency key** | `(run_id, task_id, execution_id)` | -| **Retries** | up to N (configurable per task; default 1 — workers are expensive) | -| **Side effects on consume** | (1) acquire sandbox via `SandboxLifecycleHub` and stamp `sandbox_id` onto `run_task_executions`; (2) run `worker.execute()`; (3) persist `worker_output` to `run_graph_nodes`; (4) `asyncio.gather(*[ctx.step.invoke(evaluate_task_run, …) for each evaluator])` — sandbox stays alive through gather; (5) release sandbox in `finally` after gather returns; (6) fire `task/completed` or `task/failed` | - -```python -class TaskWorkerExecutePayload(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID - definition_id: UUID # for WorkerContext convenience -``` - -The function body is the canonical sequence in -[`03-runtime.md` "Synchronous fanout"](03-runtime.md). Concretely: - -```python -@inngest_function(event="task/worker-execute", retries=N, concurrency=...) -async def worker_execute(ctx, event: TaskWorkerExecutePayload) -> None: - node = await graph_repo.node(session, run_id=event.run_id, task_id=event.task_id) - task = node.task - - sandbox = await lifecycle_hub.acquire( - task.sandbox, run_id=event.run_id, task_id=event.task_id, - ) - # Stamp sandbox_id onto the execution row so eval workers can attach. - await task_execution_repo.set_sandbox_id( - execution_id=event.execution_id, sandbox_id=sandbox.sandbox_id, - ) - try: - worker_ctx = WorkerContext._for_job( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, - definition_id=event.definition_id, - task_mgmt=..., task_inspect=..., resource_repo=..., - ) - final_output: WorkerOutput | None = None - async for chunk in task.worker.execute(task, context=worker_ctx): - if isinstance(chunk, WorkerOutput): - final_output = chunk - await graph_repo.persist_stream_chunk( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, chunk=chunk, - ) - await graph_repo.persist_worker_output( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, output=final_output, - ) - - # Synchronous fanout: each ctx.step.invoke suspends the parent - # until that eval invocation returns. asyncio.gather lets all - # evals run in parallel against the same live external sandbox. - # The sandbox stays alive through gather because the parent is - # still in its try block. - await asyncio.gather(*[ - ctx.step.invoke( - f"eval-{i}", - evaluate_task_run, - TaskEvaluateRequest( - run_id=event.run_id, - task_id=event.task_id, - execution_id=event.execution_id, - evaluator_index=i, - ), - ) - for i in range(len(task.evaluators)) - ]) - - await graph_repo.mark_task_succeeded( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, output=final_output, - ) - await fire_event("task/completed", TaskCompletedPayload( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, - )) - except Exception as exc: - await graph_repo.mark_task_failed( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, error=str(exc), - ) - await fire_event("task/failed", TaskFailedPayload( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, error=str(exc), - )) - raise # let Inngest see the failure for retry/observability - finally: - # Always reached — sandbox release is bounded by this function's - # lifetime, never delegated to a sibling job. Eval workers only - # attach/detach; only the orchestrator terminates. - await lifecycle_hub.release(sandbox) -``` - -### `task/evaluate` - -Per-evaluator fanout target. Receives a thin id-only payload; reloads -task state via the same `graph_repo.node(...)` the orchestrator used. - -| Property | Value | -|---|---| -| **Producer** | `worker_execute` via `ctx.step.invoke` | -| **Consumer** | `evaluate_task_run` Inngest function | -| **Idempotency key** | `(run_id, task_id, execution_id, evaluator_index)` | -| **Retries** | up to 3 (judge LLMs are flaky; per-function retry is the point of keeping the fanout) | -| **Concurrency** | function-level cap (e.g. 50) for global eval throttling independent of `worker_execute` concurrency | -| **Side effects on consume** | (1) load task via `graph_repo.node` with `sandbox_id` from execution row, producing a Task with live `_runtime` attached; (2) pick `task.evaluators[payload.evaluator_index]`; (3) load `worker_output` from `run_graph_nodes`; (4) call `evaluator.evaluate(task, worker_output)`; (5) persist outcome; (6) detach (drop local `_runtime`, never terminate the external sandbox) | - -```python -class TaskEvaluateRequest(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID - evaluator_index: int # position in task.evaluators -``` - -Note what is **not** in the payload: `sandbox_id` (read off -`run_task_executions`), `evaluator_type` / `binding_key` (recovered -from `task.evaluators[index]`), `task_payload` (carried in -`task_json`), `worker_output` (loaded from `run_graph_nodes`). The -payload is identifiers only; everything else is a side-channel lookup -against persisted state, which means restart and retry are trivially -correct. - -```python -@inngest_function(event="task/evaluate", retries=3, concurrency=50) -async def evaluate_task_run(ctx, event: TaskEvaluateRequest) -> None: - execution = await task_execution_repo.get(event.execution_id) - node = await graph_repo.node( - session, - run_id=event.run_id, - task_id=event.task_id, - sandbox_id=execution.sandbox_id, # makes task.sandbox live - ) - task = node.task - output = await graph_repo.load_worker_output( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, - ) - evaluator = task.evaluators[event.evaluator_index] - - try: - outcome = await evaluator.evaluate(task=task, worker_output=output) - await graph_repo.persist_evaluation( - run_id=event.run_id, task_id=event.task_id, - execution_id=event.execution_id, outcome=outcome, - ) - finally: - # Drop the local _runtime handle. The external sandbox keeps - # running — termination is the orchestrator's job. - await task.sandbox.detach() -``` - -### `task/completed` - -| Property | Value | -|---|---| -| **Producer** | `worker_execute` (success path) | -| **Consumer** | `advance_run` Inngest function | -| **Idempotency key** | `(run_id, task_id, execution_id)` | -| **Retries** | up to 5 (advance_run is a small DB transaction; tolerant of more retries) | -| **Side effects on consume** | (1) for each downstream task whose deps are now all satisfied, fire `task/ready`; (2) if no pending tasks remain in run, fire `workflow/completed` | - -```python -class TaskCompletedPayload(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID -``` - -Note: `worker_output` is **not** in the payload — it's persisted to -`run_graph_nodes` and read from there if a downstream needs it. -Keeping payloads small keeps Inngest replay cheap. - -### `task/failed` - -| Property | Value | -|---|---| -| **Producer** | `worker_execute` (terminal failure path) | -| **Consumer** | `advance_run` Inngest function | -| **Idempotency key** | `(run_id, task_id, execution_id)` | -| **Retries** | up to 5 | -| **Side effects on consume** | (1) walk the spawn subtree (recursive `parent_task_id` walk) of the failed task; mark every descendant FAILED. (2) leave dependency-dependents (tasks whose `depends_on` includes the failed task) at PENDING — they are never dispatched and never marked. (3) leave non-descendants alone — they continue running. (4) re-evaluate run terminal state (no RUNNING tasks AND no dispatchable PENDING tasks ⇒ fire `workflow/completed`). | - -```python -class TaskFailedPayload(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID - error: str # truncated to 4KB; full error is in run_graph_nodes.last_error - failure_class: Literal["worker_error", "criterion_error", "sandbox_error", "timeout", "cancelled"] -``` - -`failure_class` is a coarse classification that lets `advance_run` log -the right way without re-parsing the error string. It does **not** -gate whether descendants cascade — the cascade is unconditional on -spawn-subtree, never on dependency-dependents. - -#### Failure semantics — the four-axis lock `[v2: locked]` - -The workshop resolved this explicitly. v2's failure model: - -| Axis | Behavior | -|---|---| -| **Spawn-children of failed task** (lifecycle-coupled, via `parent_task_id`) | Cascade FAILED. Recursive walk — grandchildren, great-grandchildren also marked FAILED. Their sandboxes are released by the same `lifecycle_hub.terminate_all_for_run` backstop that handles any orphaned sandbox. | -| **Dependency-dependents of failed task** (data-coupled, via `depends_on`) | Stay **PENDING** indefinitely. Never dispatched, never marked, never cancelled. They show up in the final run snapshot as PENDING — that's the signal "this task was blocked by a failed upstream." | -| **Non-descendants** (independent subtrees) | Continue executing happily. Other parallel tasks finish normally. | -| **`runs.status` final** | `SUCCEEDED` iff every task ended SUCCEEDED. Otherwise `FAILED`. PENDING-stuck tasks at run-end count as "did not succeed" ⇒ run is FAILED. | - -Why the spawn/dependency asymmetry: spawn-relationships are -*lifecycle-coupled* — when the parent dies, the child has no reason -to exist (it was created to serve the parent). Dependency-relationships -are *data-coupled* — task B needs A's output but is otherwise -independent; there's no lifecycle reason to mark B failed when A -failed (B never started, has no sandbox, holds no resources). - -Marking dependency-dependents as PENDING (rather than CANCELLED or -SKIPPED) preserves a useful operator affordance: an admin can manually -restart the failed upstream and the run will pick up the -dependency-dependents naturally without an explicit "unblock" step. -Marking them CANCELLED would force a manual reset before retry. - -**Run termination check** (run by `advance_run` after every task -terminal): - -```python -def is_run_terminal(run_id: UUID) -> bool: - """A run is terminal when no task is RUNNING and no PENDING task - is dispatchable (every PENDING task has at least one FAILED or - PENDING dep — i.e. is permanently blocked).""" - if any_task_running(run_id): - return False - for task in pending_tasks(run_id): - if all_deps_succeeded(task): - return False # this task can still be dispatched - return True -``` - -When `is_run_terminal` flips to True, `advance_run` fires -`workflow/completed` with `final_status = SUCCEEDED` if all tasks -SUCCEEDED, else `FAILED`. - -### `workflow/completed` - -| Property | Value | -|---|---| -| **Producer** | `advance_run` (when all tasks are terminal) | -| **Consumer** | `run_cleanup` Inngest function | -| **Idempotency key** | `run_id` | -| **Retries** | up to 5 | -| **Side effects on consume** | (1) call `lifecycle_hub.terminate_all_for_run(run_id)` as a backstop sweep — should be a no-op in normal cases since each `worker_execute` released its own sandbox; (2) mark `runs.completed_at` | - -```python -class WorkflowCompletedPayload(BaseModel): - run_id: UUID - final_status: Literal["succeeded", "failed", "cancelled"] -``` - -### `run/cleanup` - -`run/cleanup` is **not a separate event** in v2. v1 had it as a -distinct event with its own handler; the audit found it duplicated -work `worker_execute`'s `finally` block already does. v2 folds the -cleanup logic into `workflow/completed`'s `run_cleanup` consumer. - -If a use case for explicit cleanup (re-running a sweep on a stuck -run) emerges, it can be reintroduced as a manual admin-triggered -event. None of the current code paths need it. - -## Idempotency in detail - -Every event consumer is idempotent on its primary key: - -- `prepare_run` keyed on `run_id`: re-firing `workflow/started` for a - run already in `RUNNING` state is a no-op (no double row writes). -- `dispatch_task` keyed on `(run_id, task_id, execution_id)`: - re-firing `task/ready` for an already-dispatched execution is a - no-op. -- `worker_execute` keyed on `(run_id, task_id, execution_id)`: - re-firing for an already-completed execution is a no-op (status - guard checks before doing real work). -- `advance_run` keyed on `(run_id, task_id, execution_id)`: - re-firing `task/completed` for an already-advanced row is a no-op. -- `run_cleanup` keyed on `run_id`: idempotent by construction. - -This means at-least-once delivery semantics from Inngest are safe. - -## Retries vs. restarts - -Two distinct concepts: - -| Concept | Trigger | What happens | -|---|---|---| -| **Retry** | Inngest re-fires the same event after a transient failure (network blip, pod kill) | Same `execution_id`. The consumer's idempotency guard either no-ops (work already done) or resumes (work was interrupted). Sandbox is re-acquired (cross-pod) or reattached (same-pod). | -| **Restart** | User or worker calls `restart_task` via `WorkerContext` or admin CLI | Fresh `execution_id`. New row appended to `task_executions`. New `task/ready` event fired. Old execution's outputs are kept for audit but a fresh sandbox is provisioned. | - -A retry never produces a new `execution_id`. A restart always does. - -## Concurrency & fan-out - -`worker_execute` events are processed concurrently up to a -per-Inngest-function concurrency cap. For a benchmark with 4 ready -tasks and a cap of 4, all four `worker_execute` functions run in -parallel. The `SandboxLifecycleHub` lives per-pod; if the four are -sharded across pods, four separate hubs each manage their own -sandboxes. This is fine — sandboxes are keyed by `(run_id, task_id)` -which is unique per event. - -For a benchmark with 100 ready tasks and a cap of 8, Inngest queues -the surplus and processes them as workers free up. No code in the -`worker_execute` body manages concurrency; it's all Inngest config. - -## Diff vs. v1 - -| v1 | v2 | Why | -|---|---|---| -| `task/worker-execute` + `task/evaluate` (fire-and-forget; release in `check_evaluators`) | `task/worker-execute` orchestrates + `task/evaluate` via synchronous `ctx.step.invoke` + release in orchestrator's `finally` | Audit's finding was the **lifecycle**, not the topology: fire-and-forget eval lost sandbox ownership. Synchronous fanout fixes that without giving up Inngest-level retry/concurrency/observability for eval. | -| `EvaluateTaskRunRequest` payload (definition-coupled, multi-field) | `TaskEvaluateRequest` (id-only: `run_id, task_id, execution_id, evaluator_index`) | Thin contract; everything else side-channels from persisted state | -| `evaluate_task_run.py` Inngest module (definition-row reader, registry-driven) | `evaluate_task_run.py` reshaped (run-tier reader via `graph_repo.node`, no registry) | Same function name and slug; reshaped body | -| `run/cleanup` as an event | folded into `workflow/completed`'s consumer | Same handler, no need for a separate event | -| `terminate_sandbox_by_id(sandbox_id)` (no-op stub) | (deleted) | `SandboxLifecycleHub.release(sandbox)` is the canonical path; no string-id-keyed termination needed | - -## Decisions locked at workshop `[v2: locked]` - -- **`task/failed` cascade semantics** — **locked.** Spec'd in full in - the "Failure semantics — the four-axis lock" subsection of - `task/failed` above. Spawn-subtree cascades FAILED; - dependency-dependents stay PENDING; non-descendants continue; - run.status is `FAILED` if anything didn't succeed. v1's strict - semantics are preserved with the addition of explicit - PENDING-blocking for dependency-dependents. -- **`workflow/completed` payload** — **locked: minimal.** Payload - carries `run_id` and `final_status` only. Consumers needing - summary data read from `runs` and `run_graph_nodes` directly. One - source of truth, smaller replays. -- **Retry policy** — **locked: framework-only.** Retry counts live - on the Inngest function definition. They are not exposed on - `Task` or `Experiment`. Authors who want different retry behavior - for different task kinds either (a) accept the framework default, - or (b) raise the discussion as a follow-up RFC if a real workload - surfaces the need. -- **Stream-chunk persistence** — **locked: annotation-WAL.** - `WorkerStreamItem` chunks are persisted as `run_graph_annotations` - rows. No dedicated `task_execution_stream` table; annotations are - already "things attached to a node at a moment in time" and chunks - fit that shape. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md deleted file mode 100644 index 4d359686d..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md +++ /dev/null @@ -1,583 +0,0 @@ -# 07 — Test strategy - -> The test surface that distinguishes "v2 implementation matches v2 -> spec" from "v2 has the same kinds of drift v1 had". The audit failure -> mode v1 hit was: tests passed, the framework appeared to work, the -> non-functional code paths went undetected because nothing exercised -> them or guarded against their reintroduction. v2's test strategy -> closes that hole. -> -> Three kinds of tests, each owning a different invariant class: -> architecture-guard tests, the walkthrough integration test, and the -> regression net for v1 audit findings. -> -> See [`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md`](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md) -> for the full set of v1 findings; this doc spec'es the tests that -> would have caught each, and now keep them caught. - -## Test layers - -| Layer | Owns | Runs | Failure means | -|---|---|---|---| -| **Architecture guards** | "no module imports X"; "every public class has Y"; "deleted symbols are deleted" | unit suite, on every PR | a structural rule was violated | -| **XFail ledgers (executable program)** | landing-PR-keyed `xfail(strict=True)` cases for every v2 invariant and every v1 dead path; markers removed by the PR that lands the invariant | unit suite, on every PR | an invariant landed early without ledger update (XPASS) or regressed (FAIL) | -| **Walkthrough smoketests (observable effects)** | unit-level effect tests that drive a public entry point and assert resulting DB/event state matches the spec — outcomes only, never call-graph mocks | unit suite, on every PR | spec drifted from behaviour at the unit level | -| **Walkthrough integration** | end-to-end flow from author code through worker_execute to terminal state matches the canonical `04-walkthrough.md` step-by-step | integration suite, on every PR | runtime behaviour drifted from the spec | -| **Regression net (v1 audit)** | every concrete bug class the v1 audit found has a test that fails when reintroduced | unit suite | the same v1 bug is back | -| **Standard unit + integration tests** | per-module behavior of `prepare_run`, `worker_execute`, `WorkerContext`, etc. | unit + integration | normal regression | - -Standard tests are out of scope for this doc — they're the same kind -the v1 implementation already had. This doc focuses on the layers v1 -was *missing* that allowed drift to ship. - -## §0 — The four-file ledger pattern - -The v2 implementation program is encoded as four executable ledger files -landed in PR 0 and PR 1 alongside the existing transition guard. Each -later PR removes the appropriate `xfail` markers as it lands its -invariant; PR 11 verifies that all four ledgers are empty of pending -markers. Together they act as the machine-readable twin of the prose PR -plan in [`09-implementation-plan/`](09-implementation-plan/). - -| File | Lands in | Asserts | Marker scheme | -|---|---|---|---| -| [`test_v2_transition_ledger.py`](09-implementation-plan/01-pr-00-transition-ledger.md) | PR 0 | Old symbols are still allowed (negative) | One row per symbol with `owner_pr` / `deletion_pr` | -| [`test_v2_final_state_ledger.py`](09-implementation-plan/01-pr-00-transition-ledger.md) | PR 0 | v2 invariants must hold at the end (positive) | `pytest.mark.xfail(reason="PR N", strict=True)` — markers in `_XFAIL_BY_NAME` | -| [`test_dead_path_audit.py`](09-implementation-plan/01-pr-00-transition-ledger.md) | PR 0 | v1 dead paths stay callerless until deleted | `pytest.mark.xfail(strict=True)` — markers in `_XFAIL_BY_SYMBOL` | -| [`test_repository_layer_conventions.py`](09-implementation-plan/01b-pr-0-5-repository-standard.md) | PR 0.5 | Repository class/file/method shape per § 0.5 | `pytest_collection_modifyitems` applies markers from `_KNOWN_VIOLATORS` | -| [`test_repository_companion_files.py`](09-implementation-plan/01b-pr-0-5-repository-standard.md) | PR 0.5 | `errors.py` + DTO location per § 0.5 | Same | -| [`test_no_dead_repository_methods.py`](09-implementation-plan/01b-pr-0-5-repository-standard.md) | PR 0.5 | Every public repo method has a production caller | `pytest.mark.xfail` — markers in `_KNOWN_UNUSED_FOR_NOW` | -| [`test_walkthrough_smoketest.py`](09-implementation-plan/02-pr-01-run-tier-task-snapshot.md) | PR 1 | Observable effects of the v2 happy path | `pytest.mark.xfail` per case, removed by landing PR | -| [`test_identity_invariants.py`](09-implementation-plan/02-pr-01-run-tier-task-snapshot.md) | PR 1 | `task_id`/`execution_id`/`sandbox_id` flow per §2 | `pytest.mark.xfail` per case, removed by landing PR | - -**`strict=True` is load-bearing.** It surfaces unexpected passes — if a -refactor lands an invariant ahead of its scheduled PR, the ledger fails -and the author must update the marker dict in the same commit. Without -`strict`, drift between code and ledger is silent. - -**Why effect-based smoketests, not call-graph mocks.** The audit failure -mode v1 hit ("this helper was supposed to be called, wasn't") is -tempting to catch with `mock.assert_called_with(...)` tests. We -deliberately don't. Call-graph mocks test how code is structured, not -whether it works; the moment someone extracts a helper or moves logic -between layers, the test breaks even though behaviour is unchanged. -Effect-based smoketests drive the public entry point and observe the -resulting database / event-bus state — they survive refactors, and they -read like prose for an engineer learning the topology. - -**Why xfail(strict=True), not skip or markers-as-todo.** A skipped -test provides no signal until someone manually unskips it; an -`xfail(strict=True)` is *active* — every CI run rechecks whether the -invariant has been satisfied early, and an unexpected pass fails the -build. The ledger is the v2 program's progress meter, automated. - -**The completion bar.** The v2 program is complete when both -`_XFAIL_BY_NAME` and `_XFAIL_BY_SYMBOL` are empty dicts and every case -in the walkthrough smoketest and identity invariants is unmarked. PR 11 -adds two explicit assertions -(`test_no_v2_invariants_are_still_xfailed`, -`test_no_dead_paths_are_still_xfailed`) that fail the build until both -dicts are empty. - -## §0.5 — Repository layer standard - -Every data-access repository in `ergon_core/core/**` follows the same -file layout and naming conventions. The standard is enforced by three -architecture-guard tests that land in **PR 0.5** -(`01b-pr-0-5-repository-standard.md`); subsequent PRs flip the xfails -for the current violators. - -### Package shape - -``` -<package>/ - __init__.py # explicit __all__ re-exports - repository.py # the Repository class(es); data access only - models.py # SQLModel tables + Pydantic request/response DTOs - errors.py # custom exception classes (only if package raises) - service.py # optional: use-case orchestrator composing repo calls -``` - -### The 10 rules - -1. **Class name ends with `Repository`** (e.g. `WorkflowGraphRepository`, - not `GraphRepo` or `Graph`). -2. **File is `repository.py` (singular)**, even when multiple Repository - classes share a file. The current `telemetry/repositories.py` is a - known violator fixed by PR 1. -3. **Methods take `session` as the first non-self positional arg.** - Consistent across the codebase already; the rule pins it. -4. **Write methods are async; reads may be sync.** Write-prefix - heuristic: methods starting with `add_`, `update_`, `remove_`, - `delete_`, `set_`, `create_`, `append_`, `insert_`, `persist_` are - writes. Async reads are required only when the method performs - genuine I/O — `graph_repo.node` after PR 5 attaches a live sandbox, - which is the textbook case. -5. **Repositories never call `session.commit()`.** Transactions belong - to the caller (the service or job body), not the data layer. -6. **Repositories never import from `core.infrastructure.*`.** The - data layer stays framework-agnostic so it can be reused in training - pipelines, replay systems, and test harnesses that don't run inside - Inngest. See `graph/errors.py` docstring for the rationale. -7. **Repositories return typed views or domain objects, never - `dict[str, Any]`.** The boundary type for serialized JSON is - `TaskDefinitionJson = dict[str, JsonValue]` (see PR 2); after - inflation, the repo returns the typed Pydantic object. -8. **Custom exceptions live in `errors.py`.** If any file in the - package raises something beyond stdlib, `errors.py` must exist. - Generic `raise ValueError(...)` in a repo is a typed-error gap - — callers can't catch specifically. The current - `experiments/repository.py` raises `ValueError`; PR 7 fixes it. -9. **Request/response DTOs (Pydantic BaseModels) live in `models.py`, - not in `repository.py`.** The current `telemetry/repositories.py` - defines `CreateTaskEvaluation` inline; PR 4 moves it. -10. **Every public Repository method must have a non-test production - caller.** The dead-method audit catches v1's `Worker.from_buffer` - failure mode scoped to the layer where it matters. - -### Why scope dead-code detection to the repository layer - -Tree-wide dead-code tooling (e.g. `vulture`) has decent precision but -trips repeatedly on Inngest functions registered by string, Pydantic -validators (`@field_validator`, `@model_serializer`), pytest fixtures, -and dynamic dispatch. The signal-to-noise tanks and the whitelist -drifts. - -A hand-rolled "every public Repository method has a production caller" -test is scoped tightly to the layer where the v1 audit found real dead -helpers. It catches 80% of the value at 10% of the cost. If we later -want broader dead-code detection, a focused `vulture` config + `ruff` -rules is the path — but that's a separate decision. - -## §0.6 — No type-checker circumventors - -The v2 program treats type-laundering as a code smell. Three patterns -are banned in production code under `ergon_core`, `ergon_builtins`, -`ergon_cli`: - -1. **`getattr(obj, "attribute_name", default)`** — string-based - attribute access the type checker cannot reason about. If the - attribute might not exist, the typing contract is wrong, not the - call site. Fix the source: add the field with a typed default, or - wrap the foreign object in a `Protocol`. - -2. **`hasattr(obj, "attribute_name")`** — same shape, same reasoning. - Use `isinstance(obj, SomeProtocol)` or split call sites by type. - -3. **Untyped `dict.get("key")` chains for *structured* values** — e.g. - `created_by=str(d.get("x")) if "x" in d else None`. If the framework - cares about the field, it belongs as a first-class typed model - field. `dict[str, Any]` metadata is for opaque user-provided - payloads, not for fields the framework reads. - -### Narrow exemptions (each requires a `# typing:` comment) - -- **Dynamic qualname walking.** The `_import_component(path)` helper - walks a string like `"module.sub:Class.Inner"` to resolve a class. - The `getattr(obj, part)` call inside the walk is genuinely - string-based because the path components are user-controlled - discriminator values. Each line carries a - `# typing: dynamic qualname walk` comment. - -- **External library type-erasure boundaries.** When the foreign object - comes from a third-party SDK whose interface we don't control (e2b - `AsyncSandbox`, etc.), wrap it in a `Protocol` defined in - `ergon_core/api/...` and pass *typed* objects across our layers. A - `getattr` on the raw SDK object inside the wrapper is acceptable - with a `# typing: <library> SDK boundary` comment. - -- **Test code asserting symbol absence.** `assert not hasattr(Cls, "x")` - in the dead-symbol guards is the load-bearing form of "this symbol - is gone." Tests are exempt by path. - -### Architecture guard - -`ergon_core/tests/unit/architecture/test_no_type_circumventors.py` -lands in PR 0 (alongside the transition / final-state ledgers) and -greps production code for `getattr(` and `hasattr(`. Each hit must be -either: - -- Inside a test file (path-exempt). -- Adjacent to a `# typing:` comment naming the exemption category. -- In `_KNOWN_EXEMPTIONS` with a one-line justification and a landing - PR (mirror of `_XFAIL_BY_NAME`). - -PR 11 asserts `_KNOWN_EXEMPTIONS` is empty of "to be removed" entries -— only the truly-legitimate exemptions remain. - -## §1 — Architecture-guard tests - -These run at unit-test speed (no DB, no sandbox) and enforce static -properties of the source tree. They live at -`ergon_core/tests/unit/architecture/`. - -### `test_public_api_target_structure.py` — the public surface - -Already exists in v1 in skeleton form. v2 hardens it. - -Asserts: - -- `ergon_core.api` re-exports exactly the types in - [`01-api-surface.md`](01-api-surface.md) "Two surfaces, one public - package" — no more, no less. -- Every type in `ergon_core.api` either *is* a `BaseModel` subclass - with a `from_definition` classmethod, *or* is one of the listed - exception classes. -- `WorkerContext` has exactly the curated method set listed in - [`03-runtime.md` "The v1 WorkerContext surface"](03-runtime.md). - Adding a method without a corresponding RFC update fails the test. - -### `test_runtime_does_not_read_definition_tables.py` — the read boundary - -Enforces [`02-persistence-layer.md` §4](02-persistence-layer.md): - -```python -def test_runtime_does_not_import_definition_orm() -> None: - """No module under ergon_core/core/application/runtime/ may import - DefinitionRepository, ExperimentDefinitionTask, or any other - definition-tier ORM class. The runtime reads exclusively from - run-tier tables. - - Allowed exception: prepare_run.py — it's the boundary that copies - definition→run at run-launch. - """ - forbidden = { - "DefinitionRepository", - "ExperimentDefinitionTask", - "ExperimentDefinition", - "ExperimentDefinitionEdge", - } - runtime_root = Path("ergon_core/core/application/runtime/") - for py_file in runtime_root.rglob("*.py"): - if py_file.name == "prepare_run.py": - continue - text = py_file.read_text() - for symbol in forbidden: - assert symbol not in text, ( - f"{py_file} imports {symbol}; runtime must read run-tier only" - ) -``` - -The test is intentionally textual (substring match on imports) rather -than AST-based — false positives are unlikely (these names are -distinctive) and false negatives mean someone deliberately routed -around the rule, which the next code review should catch. - -### `test_no_deleted_symbols.py` — the deletion floor - -Every symbol the v2 audit deleted has a corresponding "this symbol -must not exist" assertion. The list: - -```python -DELETED_SYMBOLS = { - # v2 deletions per 09-implementation-plan.md - "ergon_core.core.persistence.saved_specs": "package", - "_persist_single_sample_workflow_definition": "function", - "Worker.from_buffer": "classmethod", - "CriterionExecutor": "Protocol", - "InngestCriterionExecutor": "Protocol", - "_prepare_definition": "function", - "definition_task_id": "column", # checked via Alembic schema - "ExperimentRecord": "ORM class", # checked via SQLAlchemy registry - "EvaluateTaskRunRequest": "dataclass", # replaced by TaskEvaluateRequest - "terminate_sandbox_by_id": "function", -} -# evaluate_task_run is intentionally NOT here — it survives reshaped per Δ.4. -KEPT_RESHAPED_SYMBOLS = { - "evaluate_task_run": "Inngest function (thin id-only payload)", - "TaskEvaluateRequest": "payload class", -} - -def test_deleted_symbols_stay_deleted() -> None: - """For every entry in DELETED_SYMBOLS, ensure no module under - ergon_core, ergon_cli, ergon_builtins still defines or imports it.""" -``` - -Each entry has its own targeted assertion that picks the right -inspection technique (import probe for packages/classes, source-text -search for functions, schema introspection for columns/tables). - -### `test_cli_define_routes_through_persist_definition.py` — the CLI path - -Enforces [`05-cli-authoring-interface.md`](05-cli-authoring-interface.md): - -```python -def test_cli_define_calls_persist_definition() -> None: - """ergon_cli.commands.define.define() must call - persist_definition(experiment), not any helper function. - - Implementation: monkey-patch persist_definition with a recording - spy, invoke define() against a known slug, assert the spy was - called exactly once with an Experiment instance whose benchmark - matches the slug factory's output. - """ - -def test_cli_define_does_not_write_to_saved_specs() -> None: - """Stronger: invoke define() against an in-memory test database - and assert the saved_specs table either does not exist (preferred) - or has zero rows after the call (acceptable transitional state). - """ -``` - -### `test_inngest_evaluate_task_run_thin_payload.py` — fanout shape - -Enforces [`06-inngest-event-contracts.md`](06-inngest-event-contracts.md) -Δ.4 / Δ.5. The Inngest function registry **must** contain a function -handling `task/evaluate` events, and it must use the thin `TaskEvaluateRequest` -payload (not v1's `EvaluateTaskRunRequest`): - -```python -def test_evaluate_task_run_is_registered_with_thin_payload() -> None: - """The per-evaluator fanout target survives in v2 per Δ.4. It must - be registered, and its payload must be the id-only TaskEvaluateRequest. - """ - from ergon_core.runtime.inngest import registered_functions - events = {f.event for f in registered_functions} - assert "task/evaluate" in events, ( - "evaluate_task_run must remain registered — see Δ.4 in " - "08-decisions-log.md" - ) - payload_types = {f.payload_type.__name__ for f in registered_functions - if f.payload_type is not None} - assert "TaskEvaluateRequest" in payload_types - assert "EvaluateTaskRunRequest" not in payload_types # v1 payload deleted - - -def test_worker_execute_fans_out_via_step_invoke() -> None: - """Static check on worker_execute.py: must use ctx.step.invoke to - fan out to evaluate_task_run, wrapped in asyncio.gather, with the - sandbox release in finally.""" - body = read_source("ergon_core/core/application/jobs/worker_execute.py") - assert "ctx.step.invoke" in body - assert "evaluate_task_run" in body - assert "asyncio.gather" in body - assert "finally:" in body -``` - -### `test_sandbox_release_path.py` — the lifecycle owner - -Enforces [`03-runtime.md` "Cross-job sandbox lifetime"](03-runtime.md): - -```python -def test_worker_execute_releases_sandbox_in_finally() -> None: - """Static check on worker_execute.py: the function body must - contain a try/finally where the finally clause calls - lifecycle_hub.release(sandbox). - - AST-based — locate the worker_execute function definition, walk - to its try/finally, assert release(sandbox) is in the finally - suite. (Brittle to refactoring but exactly what we want for an - architecture rule that's load-bearing.) - """ - -def test_no_other_release_callsites() -> None: - """No module other than worker_execute.py and lifecycle.py itself - may call lifecycle_hub.release(...). Guards against - re-introduction of v1's per-run cleanup-as-primary-release path. - """ -``` - -## §2 — Walkthrough integration test - -The single source of truth for "what running end-to-end looks like" is -[`04-walkthrough.md`](04-walkthrough.md). v2 ships *one* integration -test that executes that exact walkthrough end-to-end — author code, -real Postgres (via test container), real `worker_execute` body, real -`SandboxLifecycleHub`, fake `Sandbox` subclass with deterministic -behavior. If this test passes, the framework matches its spec. - -```python -# tests/integration/test_walkthrough.py - -@pytest.mark.integration -async def test_walkthrough_end_to_end(test_db, fake_sandbox_runtime) -> None: - """Step-by-step replication of 04-walkthrough.md. - - Author-side: - - Construct the same MiniBenchmark with 4 tasks, 1 criterion, fake sandbox. - - Wrap in Experiment. - - Call persist_definition(experiment). - - Framework-side: - - Assert one row written to experiment_definitions. - - Assert four rows written to experiment_definition_tasks. - - Assert dependency edges match. - - Runtime: - - Call launch_run(definition_id). - - Drive the Inngest event loop synchronously (test driver). - - Assert events fire in the expected order: - workflow/started → 1× prepare_run - task/ready × 1 (root task only — others depend) - task/worker-execute × 1 - task/completed × 1 - task/ready × 3 (newly-ready dependents) - ... etc., until all 4 tasks complete - workflow/completed × 1 - - Persistence: - - For each task, assert run_graph_nodes row reaches RUNNING then - SUCCEEDED, with task_json matching the definition copy. - - For each task, assert exactly one task_executions row. - - For each task, assert exactly one criterion_outcomes row. - - Sandbox lifecycle: - - Assert acquire was called exactly 4 times (one per task). - - Assert release was called exactly 4 times. - - Assert each release happened AFTER the criterion outcome was - persisted for the same (run_id, task_id). - - Assert NO release was called from the run_cleanup path - (cleanup is a no-op when worker_execute did its job). - - Final state: - - runs.status == SUCCEEDED - - all 4 run_graph_nodes are SUCCEEDED - """ -``` - -This test is verbose (it asserts O(20) properties), and that's -exactly the point — it pins the *flow*, not just the outcome. A -reviewer can mechanically diff the asserted sequence against -[`04-walkthrough.md`](04-walkthrough.md) and verify they match. Any -drift between spec and behavior shows up as a test failure with a -clear "expected event N, got event M" message. - -### Variants - -The same test scaffold parameterises four variants: - -1. **Happy path** (above) — all 4 tasks succeed. -2. **Failure cascade** — task 2 fails; assert task 3 (which depends - on task 2) is cancelled; assert run terminates with status FAILED. -3. **Dynamic spawn** — task 1 spawns task 1.a; assert 1.a is born - into `run_graph_nodes` only (no `experiment_definition_tasks` row); - assert it runs through the same worker_execute path; assert - sandbox lifecycle is independent of parent. -4. **Restart task** — after task 1 succeeds, call - `restart_task(task_1_id)`; assert a new `execution_id` is minted - and a new sandbox is acquired/released. - -Each variant is a parameterized version of the base test; the -parameterisation makes the variant-specific assertions explicit and -the shared assertions (sandbox lifecycle, no double-release) shared. - -## §3 — Regression net for v1 audit findings - -For every concrete bug class the v1 audit found, a unit test exists -that *fails when the bug is reintroduced*. The list maps 1:1 to v1 -audit's `08-cleanup-audit.md` findings. - -```python -# tests/unit/regression/test_v1_audit_findings.py - -def test_no_double_path_to_persist_definition() -> None: - """v1 had _persist_single_sample_workflow_definition writing to - saved_specs in parallel with persist_definition writing to - experiment_definitions. There must be exactly one persistence - function reachable from authoring code.""" - -def test_run_graph_node_holds_inline_task_json() -> None: - """v1 'works in spirit' but in practice the runtime resolved - sandbox/worker by reading task_json from - experiment_definition_tasks instead of run_graph_nodes. After - prepare_run, the run row's task_json must be self-sufficient.""" - -def test_dynamic_subtask_has_no_definition_row() -> None: - """v1 wrote a synthetic experiment_definition_tasks row when a - worker spawned a subtask. v2 does not.""" - -def test_sandbox_released_after_inline_criteria() -> None: - """v1 acquired sandbox in worker_execute, then split criteria into - a separate Inngest function that did not have access to the - sandbox at all. The release happened on a separate cleanup path - that ran on every event-fire including ones that should not have - released. v2: sandbox released in worker_execute's finally, - after the synchronous fanout's asyncio.gather returns.""" - -def test_worker_has_no_from_buffer_constructor() -> None: - """v1 had Worker.from_buffer(buffer: bytes) for protocol-buffer - inflation. No callers ever existed. Deleted in v2.""" - -def test_no_criterion_executor_protocol() -> None: - """v1 defined a CriterionExecutor Protocol with a single - implementation. The indirection added no value; deleted in v2. - Criteria are called directly via evaluator.evaluate().""" - -def test_definition_task_id_column_does_not_exist() -> None: - """v1 had a definition_task_id column on run_graph_nodes that - duplicated task_id. Schema reset removes it.""" - -def test_experiment_record_table_does_not_exist() -> None: - """v1 had ExperimentRecord and ExperimentDefinition as separate - tables with 1:1 lifecycle. Schema reset collapses them.""" -``` - -Each test is small (5-15 lines) and named so a regression failure -points directly at the v1 audit finding it guards. - -## §4 — What's deliberately not tested - -Per [`08-decisions-log.md`](08-decisions-log.md), the v2 design -accepts these tradeoffs and does not test them: - -- **No public/internal boundary on `core.application.*` imports.** - Toolkits and the workflow CLI are allowed to import services - directly. There is no architecture guard preventing this; if a - future refactor wants to enforce a public/internal boundary, it - adds the guard then. Until then, the porousness is intentional. -- **No exhaustive payload-roundtrip test for every event.** The - walkthrough integration test exercises every event in the happy - path and the failure cascade; that's enough. We don't need a - per-event JSON-schema-validation test for each. -- **No fuzz testing of `_type` discriminator resolution.** Pydantic's - validation handles malformed JSON; we trust it. - -## §5 — Running the suite - -```sh -# Architecture guards (fast) -pytest ergon_core/tests/unit/architecture/ - -# Regression net (fast) -pytest ergon_core/tests/unit/regression/ - -# Walkthrough integration (slow — needs Postgres test container) -pytest -m integration ergon_core/tests/integration/test_walkthrough.py - -# Everything -pytest ergon_core/ -``` - -CI runs all three layers on every PR. The walkthrough integration test -is the slowest (~30s with test container spinup) and is the gating -test for "merge to main." - -## Decisions locked at workshop `[v2: locked]` - -- **Architecture-guard implementation** — **locked: textual.** - `test_no_other_release_callsites` and similar guards stay as - substring-style scans. We don't take the `grimp` dependency - speculatively; revisit if/when a textual guard misses a real - regression. -- **Walkthrough as living document** — **locked: hand-written - pytest test that mirrors the walkthrough.** No markdown-to-test - generator. The hand-written version forces a human to read both - files and check they match — the point is the synchronization - pressure, not the automation. -- **Regression-net scope** — **locked: 8 load-bearing findings - only** (the list in §3). The other 24 fix-plan items from the v1 - audit are "delete this dead function" deletions whose - reintroduction would be caught by the deletion-checklist tests - in §1; no separate regression test per item. -- **Per-PR test budget** — **locked: run all 4 walkthrough - variants sequentially.** ~2 min is acceptable for v2 launch. - Revisit only if cumulative test wall time crosses 10 min. - -### Follow-up after v2 lands - -The v1 integration and end-to-end test suites need refresh after -v2 ships — they exercise paths v2 deletes (separate -`evaluate_task_run`, `saved_specs` writes, `_prepare_definition`). -That refresh is **not in v2's scope**; it's a follow-up PR sized -against whatever the test suite looks like once v2 is merged. -Tracked here so it doesn't get lost. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/08-decisions-log.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/08-decisions-log.md deleted file mode 100644 index e3049d119..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/08-decisions-log.md +++ /dev/null @@ -1,611 +0,0 @@ -# 08 — Decisions log - -> Living document. Update as decisions resolve, smells get triaged, and -> future work crystallizes into follow-up redesigns. The reference docs -> ([`01-api-surface.md`](01-api-surface.md), -> [`02-persistence-layer.md`](02-persistence-layer.md), -> [`03-runtime.md`](03-runtime.md)) are the *current* shape; this doc -> captures the *why* and the *what's still open*. -> -> **Two parts:** -> -> - **Locked decisions inherited from v1 audit** — the eight -> architectural deltas that distinguish v2 from v1. Each links to -> the doc where the decision is structurally encoded. -> - **Alternatives considered** (and below: smells, open questions, -> future work) — long-form material carried forward from v1's -> `06-decisions-log.md`. Useful as reading for "why v2 looks like -> this, not like X." - -## Locked decisions inherited from v1 audit `[v2: added]` - -The v1 audit -([`../2026-05-08-authoring-api-redesign/08-cleanup-audit.md` §0](../2026-05-08-authoring-api-redesign/08-cleanup-audit.md)) -crystallised eight architectural decisions as load-bearing constraints -on v2. They are restated here verbatim from -[`00-readme.md` "What changed from v1"](00-readme.md), with the -rationale for each. - -### Δ.1 — `ExperimentRecord` collapses into `ExperimentDefinition` - -**Decision.** Persistence is two-tier (`ExperimentDefinition` ⊕ run -tier), not three. The fields `ExperimentRecord` carried (`name`, -`description`, `metadata`, `created_at`, `created_by`) move directly -onto `ExperimentDefinition` (and onto the `Experiment` Pydantic model -as authoring fields). - -**Rationale.** The v1 `ExperimentRecord` table had a 1:1 relationship -with `ExperimentDefinition`, was only ever populated by -`persist_definition`, and was only read by `launch_run` and the -dashboard list endpoint. The two never had independent lifecycles. -The audit found no use case requiring the second tier; it was -infrastructure overhead. - -**Encoded in.** [`02-persistence-layer.md` §3](02-persistence-layer.md). - -### Δ.2 — Runtime reads only run-tier tables - -**Decision.** After `prepare_run` copies definition rows into run-tier -tables at run-launch, no subsequent runtime code reads from -`experiment_definitions` or `experiment_definition_tasks`. There is -no `_prepare_definition` fall-through path. - -**Rationale.** v1 had the runtime resolving `Sandbox` / `Worker` -subclass instances by reading the original definition rows, even -when run-tier had a `task_json` snapshot of the same data. This -allowed silent disagreement between paths and made "what was actually -run" hard to reconstruct from the run-tier alone. v2 makes the -snapshot taken at run-launch the only truth the runtime knows. - -**Encoded in.** [`02-persistence-layer.md` §4](02-persistence-layer.md); -guarded by a test in [`07-test-strategy.md`](07-test-strategy.md) §1. - -### Δ.3 — Dynamic subtasks are graph-native - -**Decision.** Tasks spawned at runtime by `WorkerContext.spawn_task` -live exclusively in `run_graph_nodes`. There is no synthesised -`experiment_definition_tasks` row. - -**Rationale.** v1 wrote a synthetic definition row for every dynamic -task so that lookup queries could use the same join pattern as static -tasks. The audit found this added a definition-tier write path with -no read consumer (definitions are immutable for static tasks; dynamic -tasks have no second run to be reused for). v2 makes -`run_graph_nodes` the single home for dynamic tasks; lookup -discriminates with a left-join check (`is_dynamic` denormalized for -performance). - -**Encoded in.** [`02-persistence-layer.md` §5](02-persistence-layer.md). - -### Δ.4 — Synchronous fanout with thin payload contract - -**Decision.** Two Inngest function shapes per task: an orchestrator -(`worker_execute`, one per task) and a per-evaluator worker -(`evaluate_task_run`, one per evaluator). The orchestrator does -`await asyncio.gather(*[ctx.step.invoke(evaluate_task_run, …) for i in -range(len(task.evaluators))])` — synchronous fanout. Eval payload is -`(run_id, task_id, execution_id, evaluator_index)` only; everything -else (task config, sandbox_id, worker_output, evaluator instance) is -recovered from persisted state via the same `graph_repo.node(...)` -the orchestrator used. The live sandbox is rebuilt on the eval side -via `Sandbox.from_definition(sandbox_json, sandbox_id=...)`, which -attaches a fresh local `_runtime` to the same external sandbox. - -**Rationale.** The v1 audit's finding was a **lifecycle** problem -(sandbox release in a sibling job; cleanup masking the leak), not a -**topology** problem. The first cut of v2 read the audit as -"eliminate the second Inngest function" and proposed inline criteria, -which would have given up Inngest's per-function retry, -per-function concurrency cap, and per-function observability slug — -all of which the project's debug workflow relies on (CLAUDE.md -documents Inngest GraphQL function-failure queries as the primary -telemetry). Synchronous fanout keeps the operational properties while -fixing the lifecycle: the orchestrator's `try/finally` bounds sandbox -lifetime, eval workers only attach/detach, and the gather guarantees -the sandbox is alive for every eval invocation. The thin payload -contract follows from making eval re-read state via `graph_repo.node` -rather than receiving denormalized fields — which means restart, -retry, and replay all become trivially correct. - -**Rejected alternative — pure inline criteria.** Earlier draft of -this RFC. Operational properties (retry / concurrency / observability -per-eval) lost; debug workflow degraded. Rejected when the second -look showed synchronous fanout could fix the lifecycle without giving -up those properties. - -**Rejected alternative — durable lifecycle hub with reference -counting.** Would allow asynchronous (fire-and-forget) eval fanout -with a durable coordinator releasing the sandbox after every eval -completes. v1 had a primitive version of this; the lifecycle was -hard to test under retry/replay. Synchronous fanout is the same -operational behavior with a much simpler invariant (sandbox alive -iff the orchestrator is in its try block) and no new durable service. - -**Encoded in.** [`03-runtime.md` "Synchronous fanout"](03-runtime.md); -[`06-inngest-event-contracts.md`](06-inngest-event-contracts.md) -`task/worker-execute` and `task/evaluate`; guarded by a test in -[`07-test-strategy.md`](07-test-strategy.md) §1. - -### Δ.5 — Cross-job sandbox lifetime, single owner, bounded by parent's try/finally - -**Decision.** `worker_execute` is the sole sandbox-lifetime owner for -each `(run_id, task_id)`. `acquire` happens at the top, `release` -happens in the `finally` block after `asyncio.gather` returns. Eval -workers only **attach** (build a process-local `_runtime` handle to -the same external sandbox via `Sandbox.from_definition(..., -sandbox_id=...)`) and **detach** (drop the local handle in `finally`, -never terminate the external sandbox). The `run/cleanup` Inngest -function is a backstop sweep, never the primary release path. - -**Rationale.** v1 split acquire into `worker_execute` and release -into a sibling cleanup path; the result was sandbox leaks when the -event sequence diverged. v2's invariant — "exactly one acquire, -exactly one release, both in `worker_execute`'s try/finally; eval -workers may attach but never terminate" — is testable as both a -static property (grep `worker_execute.py` for the acquire/release -calls) and a runtime property (the walkthrough's eval-before-release -ordering assertion). The synchronous gather is what makes the bound -hold: the orchestrator cannot reach its `finally` until every eval -has returned, so the sandbox is guaranteed alive throughout. - -**Encoded in.** [`03-runtime.md` "Synchronous fanout"](03-runtime.md); -guarded by a test in [`07-test-strategy.md`](07-test-strategy.md) §1. - -### Δ.6 — Schema reset, not incremental drops - -**Decision.** Wipe the v1 Alembic migration history. Generate one -fresh "initial schema" migration that matches v2's two-tier model. -No "drop a column" or "drop a table" migration chain that walks v1's -mistakes back step by step. - -**Rationale.** v1 has not shipped to production; the migration chain -exists only in a worktree. Walking it back via individual drop -migrations would keep historical schema artefacts in the chain -forever (Alembic's `down_revision` graph is forever). v2's clean -slate is cheaper and more honest. - -**Encoded in.** [`02-persistence-layer.md` §3](02-persistence-layer.md); -[`09-implementation-plan.md`](09-implementation-plan.md). - -### Δ.7 — Deletions - -**Decision.** v2 deletes the following symbols outright (none are -deprecated, none get a transition window, none survive in any form): - -- `ergon_core.core.persistence.saved_specs` — package, write-only -- `_persist_single_sample_workflow_definition` — function, no readers of its writes -- `Worker.from_buffer` — classmethod, no callers -- `CriterionExecutor` Protocol — single trivial impl -- `_prepare_definition` — runtime helper, parallel to `_prepare_graph_native` -- `definition_task_id` column on `run_graph_nodes` — duplicates `task_id` -- `ExperimentRecord` ORM class + table — collapsed (Δ.1) -- `InngestCriterionExecutor` and the `CriterionExecutor` Protocol — reshaped `evaluate_task_run` calls `criterion.evaluate(...)` directly with no executor indirection -- `terminate_sandbox_by_id` — no-op stub - -**Kept and reshaped (not deleted) — see Δ.4:** - -- `evaluate_task_run` Inngest function — survives as the per-evaluator fanout target; body is rewritten to read run-tier state via `graph_repo.node` and attach the sandbox by id, not as a v1 deletion candidate -- `EvaluateTaskRunRequest` payload class — replaced by `TaskEvaluateRequest` (id-only); the v1 dataclass is deleted but the *role* survives - -**Rationale.** Per-symbol justifications in the v1 audit. The common -thread: each was either a parallel non-functional path, an unused -abstraction, or an indirection layer with one trivial impl. - -**Encoded in.** [`09-implementation-plan.md`](09-implementation-plan.md); -guarded by per-symbol tests in -[`07-test-strategy.md`](07-test-strategy.md) §1. - -### Δ.8 — CLI is composition convenience - -**Decision.** `ergon define <slug>` builds an `Experiment` from a -registered factory and calls the same `persist_definition` the public -Python API uses. There is no second slug→definition persistence -flow, no `saved_specs` table, no -`_persist_single_sample_workflow_definition` helper. - -**Rationale.** v1's CLI define wrote rows to a table no other code -read; it was a parallel persistence path that produced -non-functional definitions. v2 makes the CLI a thin layer over the -public Python API. - -**Encoded in.** [`05-cli-authoring-interface.md`](05-cli-authoring-interface.md); -guarded by a test in [`07-test-strategy.md`](07-test-strategy.md) §1. - -## Alternatives considered - -### Keep `WorkerSpec`, just rename it to `WorkerBinding` - -Cosmetic. Doesn't address the underlying smell (workers can't be -pre-constructed because of `self.tools = ...` mutation). User would still -encounter the binding/instance distinction; the rename only makes the name -slightly more honest. Rejected — fixes naming without fixing structure. - -### Keep the registry, just fix the catalog UX - -Make `register_builtins` automatic via a decorator at class-definition time. -Removes the manual bootstrap step but leaves the parallel slug-as-identity -layer mirroring Python's import system. Rejected — Python already provides a -global string identity for any class (`module:qualname`); a parallel registry -is duplicate machinery unless we want versioning/aliasing, which we don't -appear to be using. - -### One `Task`, identity in `WorkerContext.task_id` only - -Discussed and rejected — `task.task_id` reads more naturally; tasks should -own their own ID; the value object should be self-identifying. The unified -`PrivateAttr` pattern preserves `task.task_id` as the access path, so this -alternative is moot. - -### Two-class `TaskSpec` / `Task` (with or without inheritance) - -Earlier draft of this redesign proposed keeping the two-class split, -optionally with `class Task(TaskSpec)` inheritance to remove field -duplication. Superseded by the unified `PrivateAttr` pattern, which -collapses to one class without losing the "definition-time has no ID, -runtime does" invariant. Inheritance approach kept as a fallback if the -PrivateAttr pattern proves too magic in code review. - -### Generic phantom-typed `Task[Phase]` with `Defined`/`Live` markers - -```python -Task[Defined] # task_id is structurally absent -Task[Live] # task_id present -``` - -Considered. Pydantic's relationship with phantom generics is rough — -runtime validation of phantom params requires custom validators, the type -system only enforces at static-check time, and pydantic v2 doesn't preserve -phantom params through `model_dump`/`model_validate`. The PrivateAttr -pattern gets the same "two phases of one type" semantics with simpler -machinery. - -### "Live" subclass at runtime: `LiveSandbox(Sandbox)` - -Same conceptual split as today's Spec/Live but with inheritance instead of -sibling classes. Better than today (gives `LiveSandbox <: Sandbox` -substitutability) but still two classes for one concept. Author code now -has to know "do I receive a `Sandbox` or a `LiveSandbox`" — the latter -which only exists at runtime. Loses the "one class, you always know which -one you're holding" property of the PrivateAttr pattern. - -### Per-benchmark sandbox managers stay; `Task.sandbox` is just a slug - -Half-measure. Keeps the per-benchmark manager subclasses -(`MiniF2FSandboxManager`, `SWEBenchSandboxManager`, …) but lets a Task -pick which one. Rejected because the per-benchmark managers are mostly -per-template setup scripts wearing the wrong hat — the install logic is -template-specific, not benchmark-specific. Better to consolidate into one -generic manager dispatching on template. - -### Tools as a `Benchmark.toolkit_for(sandbox, task)` method - -Considered in the discussion. Right idea but too opinionated for the -framework — it forces every worker to use a "list of pydantic-ai Tool" -abstraction. Workers should be free to use the sandbox however they want -(raw `run_command`, hand-coded shell scripts, no-LLM workflows). Rejected -in favor of the framework giving workers `sandbox: Sandbox` and letting -each worker construct its own tools (or none). - -### Layered public runtime API: `WorkerContext` facade + `GraphMutator` / `GraphInspector` / `ResourceInspector` - -Designed in detail (curation rule, `.for_worker(context)` constructors, -containment-by-construction, public-`SubtaskSpec` shape, public exception -types), then **rolled back as scope creep for v1**. The design moved -three new public classes into `ergon_core.api`, enforced an -`ergon_builtins → ergon_core.core.application.*` import boundary, added -new boundary tests, and required migrating the three in-tree consumers -(`subtask_lifecycle_toolkit.py`, `graph_toolkit.py`, the workflow CLI) -onto the new public surface. ~6 migration steps for purely speculative -external consumers we don't have. The justification was "expose the -escape-hatch as a supported public path" — but until someone outside -this repo asks for it, the escape hatch is one porous import away -(`from ergon_core.core.application.tasks import TaskManagementService`), -which is what `ergon_builtins.tools.*` does today and will continue to -do. `WorkerContext` ships as the single public runtime surface with the -curated facade methods (`spawn_task`, `cancel_task`, `subtasks()`, …); -batch ops, predicate-based ops, and CLI-tier surfaces stay on the -internal services and are reached by direct import when needed. - -When a real third-party consumer arrives — or when an in-tree refactor -genuinely needs the architectural firewall — promote the internal -services to `ergon_core.api`, add the boundary test, and migrate the -three in-tree consumers in a follow-up redesign. The design is preserved -in the git history of this folder and in -[`09-implementation-plan.md`](09-implementation-plan.md)'s "What's deferred" section so the -follow-up doesn't have to rederive it. - -### Fat `WorkerContext` only (predecessor to the rolled-back layered API) - -Put every graph mutation, inspection, and resource discovery method -directly on `WorkerContext` and call it done. ~15-20 methods on a single -class, every worker sees the manager-only mutation surface -(`refine_task`, `restart_task`, `cancel_all_*`) whether they need it or -not. Rejected at the time in favour of the layered API; with the -layered API now also rejected (above), the v1 outcome is *neither* -extreme. `WorkerContext` carries the **curated** facade (single-target + -high-frequency only — ~7 methods) and batch / predicate / advanced ops -stay on the internal services. The "fidelity escape hatch" use case is -served by direct internal-service imports, the same way it works today. - -## Smells the walkthrough surfaces - -These are gaps the abstract design doesn't expose but the concrete trace -in [`04-walkthrough.md`](04-walkthrough.md) does. Each is a real thing we -need to either solve, document as a known-limitation, or punt to a -follow-up redesign. - -**1. Cross-task data handoff is undefined.** `code` task `dependency_task_slugs=("research",)` is a *scheduling* edge — it tells the runtime "wait for research to finish." It does not tell the framework "the coder needs research's output." Today this works because `SandboxResourcePublisher` writes files from one task's sandbox to a shared blob store and the next task's sandbox mounts them. But that's an implicit contract: the benchmark has to know that workers write to `/workspace/final_output/` and other tasks read from there. **Open question: should `Task` or `Worker` declare its inputs/outputs explicitly?** e.g. `Task(..., inputs=("findings.md",), outputs=("solution.py",))` so the runtime can fail-fast if a dependency hasn't produced what its dependent expects. Not in this redesign's scope but worth flagging. - -**2. Heterogeneous sandboxes break naive file passing.** `research` uses `research-e2b`, `code` uses `python-3.13`. Files written in E2B aren't visible in the Python sandbox unless the runtime explicitly stages them. `SandboxResourcePublisher` already does this via Postgres-backed `RunResource` blobs, but it relies on the worker writing to `/workspace/final_output/` — a sandbox-specific path convention. **The path convention isn't on `Sandbox` or `Task`; it's a hidden contract.** Could be a `Sandbox.output_path: str` field, or a `task.publish_files(...)` helper. Flag for follow-up. - -**3. `instance_key` is repeated four times in `build_instances`.** Every Task in an instance has the same `instance_key`. The outer dict already keys by instance — having `Task.instance_key` is redundant. **Deferred.** Keeping it in v1 avoids coupling this authoring API migration to a benchmark/definition-writer contract change. The cleanup is small, but it touches every `Task(...)` constructor and the materialization path; land it as a follow-up once the object-bound Task model is implemented. - -**4. `assignments` lives on `Experiment` but `dependency_task_slugs` lives on `Task`.** The DAG structure is split across two places. A new contributor reading the experiment can't see "who runs `code`" without cross-referencing two dicts. **Resolved by object-first task binding.** `Task.worker` is now a direct `Worker` object; `Experiment.assignments` deletes. The earlier modularity rationale ("same benchmark, swap the team") is still achievable through normal Python composition: benchmark factories accept different worker objects and construct otherwise identical task graphs. - -**5. `Rubric` is bound at the experiment level, not the task level.** All four tasks share the same `default` Rubric with the same criteria. If you want different criteria per task ("the coder's output gets style-checked, the reviewer's output gets length-checked") you need to declare multiple evaluator binding keys and remember which task uses which. **Resolved by object-first task binding.** `Task.evaluators: tuple[Evaluator, ...]` directly carries the Rubric or any future evaluator kind. Shared rubrics are ordinary Python variables reused across tasks, not framework binding keys. - -**6. `WorkerContext` is now thin enough to question.** ~~After the cleanup it carries `(run_id, definition_id, execution_id)` — three UUIDs and that's it.~~ **Resolved by promoting the curated facade methods directly onto `WorkerContext`.** `WorkerContext` carries the IDs *and* the small high-frequency-use method surface (`spawn_task`, `cancel_task`, `refine_task`, `restart_task`, `subtasks()`, `descendants()`, `get_task()`, `resources(scope=...)`) — each delegating directly to the internal `TaskManagementService` / `TaskInspectionService` / `RunResourceRepository`. It is no longer thin — but it's also not bloated, because the curation rule (single-task ops only; everything batch-y or rare stays on the internal service and is reached by direct import) keeps it bounded. `CriterionContext` remains thin; the question of whether it collapses to kwargs is independent and still open below. - -**7. `Worker` as pydantic model: where does private state live?** A `ReActWorker` might want a lazy-initialized HTTP client or a model resolver cached after first use. The pattern: declare as `PrivateAttr`, initialize in `model_post_init` or lazily inside `execute`. **Document as a worker-author convention** — not a blocker, but the redesign should state which pattern to use so contributors don't reach for `__init__` overrides (which break model_validate round-tripping). - -**8. `_type` discriminator is everywhere.** Every persisted Worker, Task, Sandbox, Benchmark, Evaluator, Criterion carries a `_type: "module:qualname"` field. That's a lot of repetition in the JSON columns. **Could hoist** to a wrapper envelope (`{kind: "worker", impl: "...", config: {...}}`) at the persistence layer, but that adds a layer of indirection on read. I'd live with it — the JSON is debug-friendly, deserialization is one-liner, and you can always grep `"_type":` to find class references in the DB. - -**9. `Benchmark` is still a subclass-only abstraction.** Workers, Tasks, Sandboxes, Criteria are all instances now. `Benchmark` is the lone exception — you subclass it because `build_instances()` is data-generation logic, not declarative config. **Is that right?** Probably yes — benchmarks have real code (HuggingFace dataset loading, jsonl parsing, `task_payload_model` declaration), not just config. But the asymmetry is real and worth naming. Not a blocker. - -**10. Sandbox setup payload is `dict[str, Any]`.** Each template's setup script knows its own payload schema, but the type is opaque at the boundary. **Could parametrize `Sandbox` with a payload type the way `Task` is parametrized with `PayloadT`** — `Sandbox[LeanPayload]` where `LeanPayload` is the template-specific config model. More machinery but tighter authoring. ~~Defer.~~ **Resolved by Phase 2.** Sandbox is now a typed subclass per kind; subclass-specific config lives as typed pydantic fields on the subclass (e.g. `LeanSandbox(lean_version=...)`). There is no `setup_payload: dict[str, Any]` anywhere. Smell #10 is closed. - -The first two are known limitations to document around resource handoff. -`instance_key` is a follow-up cleanup, not a blocker for this redesign. -The rest are follow-up redesigns or known-limitations to document. - -## Open questions - -- ~~**Is the PrivateAttr pattern too magic?**~~ — *resolved at workshop: - no, keep the pattern.* Author code never needs `task_id` at - definition time; the framework is the only consumer. Error message - on premature access names the materialization callsite, which makes - the failure self-debugging. Soft followup: code-review sanity check - the first time a contributor hits the error. -- ~~**Where does `Experiment` live in the public API?**~~ — *resolved - (re-export pattern rolled back).* `Experiment` is the composition - root authors construct around a benchmark definition — every benchmark - constructor calls it directly, and it sits alongside `Benchmark`, - `Task`, `Worker`, `Sandbox` in the authoring contract. It belongs in - `ergon_core.api`, not in an internal module that authors happen to - reach into. **The class definition moves to - `ergon_core/api/experiment.py`** as part of step 4, and - `ergon_core/core/domain/experiments/experiment.py` is **deleted**. - An earlier draft of this entry kept the class defined in - `core.domain.experiments` and added a re-export shim in - `api/experiment.py` — that was rejected as - worst-of-both-worlds (the class is publicly imported but lives at - an internal path; new readers have to chase the re-export to find - the real definition). The lift also makes the - `requires_sandbox` `@model_validator(mode="after")` (see "Worker - → Sandbox compatibility checking" below) live where it naturally - belongs — on the public type, in the public package — rather than - being defined on a class whose canonical home is private. - - `DefinitionHandle` and `ExperimentValidationService` stay in - `core/domain/experiments/`: the handle is purely internal (return - shape of `persist_definition`, held as `Experiment._persisted` - `PrivateAttr` for the runtime side), and the validation service is - a service that operates on the public `Experiment` instance from - `definition_writer` — services keep living next to the application - layer that calls them. See - [`01-api-surface.md`](01-api-surface.md#foundational-change-c--experiment-lifts-into-the-public-api) - for the lifted-class sketch and - [`09-implementation-plan.md`](09-implementation-plan.md) audit row #6 + step 4 for the - callsite migration. Boundary tests in - `tests/unit/api/test_public_api_imports.py`, - `tests/unit/architecture/test_public_api_boundaries.py`, and - `tests/unit/architecture/test_public_api_target_structure.py` - treat `Experiment`, `WeightedCriterion`, and `Sandbox` as part of - the public surface, and assert that the old - `core.domain.experiments.experiment` path no longer exists. -- ~~**`"none"` template semantics**~~ — *resolved by deletion.* No - `"none"` template ships in this redesign. A sandbox that satisfies the - type contract while no-op'ing `run_command` is a lie that makes call - sites look uniform while making behavior surprising. Genuinely - zero-environment tasks become possible later via a generic-Docker - template (basic Python, no special setup) where - `sandbox.run_command(...)` actually works. Until then, - `Task.sandbox: Sandbox` is satisfiable by every in-tree benchmark with - the templates we already have. -- ~~**`Sandbox` IO methods on the base — contract or convenience?**~~ — - *resolved at workshop: keep on base, documented as convenience.* - `run_command` / `write_file` / `read_file` / `list_files` live on the - `Sandbox` base. They're "convenience surface" — the framework never - calls them. The day a non-IO sandbox kind appears (e.g. a hypothetical - `LocalSubprocessSandbox`), promote them to a `_RemoteIOSandbox` - intermediate base; until then, base for ergonomic uniformity. Locked - surface enumerated in [`01-api-surface.md` "Sandbox capability surface — locked"](01-api-surface.md). -- ~~**Worker → Sandbox compatibility checking — where does the check - live?**~~ — *resolved.* The check runs in a pydantic - `@model_validator(mode="after")` on `Experiment`, since that's the - composition root that can walk the benchmark's task list after the - author has constructed the whole graph. For each task, the validator - walks `task.worker.requires_sandbox` and any - `requires_sandbox` declared by the directly bound - `task.evaluators` / criteria, then checks `isinstance(task.sandbox, X)` - for each declared `X`. Mismatches raise - `SandboxKindMismatch(task_id=..., component=..., required=..., actual=...)` - (signature pinned in - [`01-api-surface.md`](01-api-surface.md#public-exceptions)) - before `Experiment.__init__` returns, so a misconfigured experiment - fails at construction in the author's process — never reaches - `persist_definition` or any rollout. Workers/criteria with - `requires_sandbox: ClassVar[type[Sandbox]] = Sandbox` (the default) - pass trivially. -- ~~**Backward compatibility window**~~ — *resolved: one cohesive break, - no deprecation cycle.* Ergon has no external benchmark contributors yet, - so there's nothing to deprecate against. The redesign ships as a single - PR that lands all four phases together; no `WorkerSpec` shim release, - no `Worker.__init__(tools=...)` warning period, no transitional - `Sandbox | None` parameter. Local PG carries no production data so all - schema changes are drop-and-recreate rather than data migrations. See - [`09-implementation-plan.md`](09-implementation-plan.md) for the consolidated plan. -- ~~**Per-agent tool customization**~~ — *resolved.* `ReActWorker` takes - a `toolkit` field; benchmark authors instantiate - `ReActWorker(toolkit=BenchmarkToolkit(...), ...)`. `_Toolkit` is - module-private to `ergon_builtins`, never promoted to - `ergon_core.api`, so the framework doesn't pre-shape every future - worker strategy around the assumption that "agents have toolkits." - Per-benchmark Worker subclasses delete entirely; the variation lives - in the `toolkit` instance. -- ~~**Where does the runtime graph-edit API live?**~~ — *resolved (twice): - WorkerContext is the only public runtime surface for v1.* The framework - today scatters runtime mutation across `WorkerContext` (just IDs), - `TaskManagementService` and `TaskInspectionService` (in - `ergon_core.core.application.tasks`, technically internal but freely - imported by `ergon_builtins.tools.*` and the workflow CLI). An earlier - resolution promoted three public service classes - (`GraphMutator` / `GraphInspector` / `ResourceInspector`) to - `ergon_core.api` and enforced the boundary; that was rolled back as - scope creep — see ["Layered public runtime API"](#layered-public-runtime-api-workercontext-facade--graphmutator--graphinspector--resourceinspector) - in Alternatives considered for the rationale. **Final v1 shape:** - `WorkerContext` carries the curated facade methods (single-target + - high-frequency only; see curation rule below), each delegating - directly to the internal services. The internal services stay where - they are; toolkits and the CLI keep their existing - `from ergon_core.core.application import …` imports. No boundary test, - no public service classes. Promote later when a real third-party - consumer arrives. -- ~~**Curation rule for `WorkerContext` facade methods**~~ — *still applies.* - An operation gets a `WorkerContext` facade method **iff** it is (a) - single-target (operates on exactly one task or one resource — no - batch / no `predicate` parameter), and (b) used by ≥2 in-tree workers - or expected to be used by most. Everything else stays only on the - internal `TaskManagementService` / `TaskInspectionService` / - `RunResourceRepository`. This keeps `WorkerContext` bounded (~7 - methods at v1) while the internal services carry however many - granular ops the CLI / advanced toolkits need without bloating what - every worker sees in its IDE autocomplete. When in doubt, *don't* add - to `WorkerContext` — the escape-hatch is one direct import away. -- ~~**`Sandbox` capability surface**~~ — *resolved at workshop: locked - in [`01-api-surface.md` "Sandbox capability surface — locked"](01-api-surface.md).* - Base = `provision`, `terminate`, `run_command`, `write_file`, - `read_file`, `list_files`, `output_path`, `is_live`, `sandbox_id`, `env`, - `timeout_seconds`, `requires_network`. Cross-cutting resource methods - (`read_resource`, `upload_files`) belong on `RunResourceRepository`, - not on Sandbox. Subclass-specific affordances (`compile_lean_file`, - `apply_patch`, `execute_code`) stay on the subclass. -- ~~**Pydantic-model `Worker` and `Criterion` — non-pydantic state?**~~ - — *resolved at workshop: explicit construction, NOT `model_post_init`.* - Allowed patterns: (a) construct fully at author init time; - (b) classmethod factories like `Worker.from_config(...)` that - populate `PrivateAttr`s explicitly; (c) build per-call inside - `execute()`. Disallowed: `model_post_init`-based lazy init, - `@property` that mutates state. Locked in - [`01-api-surface.md` "Worker / Criterion non-pydantic runtime state"](01-api-surface.md). -- ~~**`Task.sandbox` config vs `Task.task_payload`**~~ — *resolved by - spec.* `task_payload` is benchmark-domain data the worker reasons - about; `sandbox` is environment kind + provisioning config. Sandbox - subclasses carry their own typed fields (e.g. - `LeanSandbox(lean_version="4.7.0", mathlib_revision="...")`). If a - sandbox needs benchmark-specific config to provision itself, that - field lives on the sandbox subclass; `task_payload` does not leak - into provisioning. Boundary doctest TODO: add a small example to - [`02-persistence-layer.md`](02-persistence-layer.md) §3 to lock this - in (low priority, follow-up commit). -- ~~**Cross-task data handoff**~~ *(surfaced by walkthrough smell #1)* — - *resolved at workshop: deferred.* Today this is an implicit contract - between dependent benchmarks and `SandboxResourcePublisher`. Adding - `Task.inputs` / `Task.outputs` declaration is a real follow-up; v2 - defers it so the persistence/runtime cleanup can land first. Tracked - in "Future work" below. -- ~~**Sandbox output-path convention**~~ *(surfaced by walkthrough smell #2)* - — *resolved at workshop: added.* `Sandbox.output_path: str` is now - a base field, defaulting `/workspace/final_output/` per - [`01-api-surface.md`](01-api-surface.md). Subclasses override per - environment kind. Worker code and the resource publisher both read - from `task.sandbox.output_path`; no more hardcoding. -- ~~**Task worker binding**~~ *(surfaced by walkthrough smell #4)*: - **resolved by direct object binding.** `Task.worker: Worker` replaces - `Experiment.assignments` and any string `Task.assigned_worker` - half-step. Cohort variants reuse the same benchmark shape through - Python factories that pass different workers into task construction. -- ~~**Per-task criteria / evaluators**~~ *(surfaced by walkthrough smell #5)*: - **resolved by direct object binding.** `Task.evaluators: - tuple[Evaluator, ...]` replaces experiment-level evaluator pools and - `evaluator_binding_keys`. -- ~~**Should `WorkerContext` and `CriterionContext` collapse?**~~ *(surfaced - by walkthrough smell #6)* — *resolved at workshop: keep both, no - collapse.* `WorkerContext` carries the curated runtime facade - (`spawn_task`, `cancel_task`, inspection, resource discovery). - `CriterionContext` stays as a typed data carrier — having a named - type for the runtime context bag is worth the small typing tax; - it's also extensible (future fields can be added without changing - every `evaluate()` signature). -- ~~**Backpressure for dynamic spawning**~~ — *resolved at workshop: - deferred.* A buggy worker can still fork-bomb a run; v2 doesn't ship - the `GraphSpawnGovernor` (per-run / per-parent / depth caps). - Tracked in "Future work" below; revisit when a real workload - surfaces the need. -- ~~**`await_completion=True` semantics**~~ — *resolved by deferral.* - v1 ships `spawn_task` as fire-and-forget only — the kwarg doesn't - exist on the v1 surface. Parent workers that need to wait poll - `context.get_task(handle.task_id)` until the child's status is - terminal. Synchronous "block until child returns its WorkerOutput" - semantics is genuinely tricky (parent's sandbox lifecycle during - the wait, Inngest wait-for-event integration, interaction with - workers holding in-memory conversational state) and trying to - decide it now would either ship something brittle or stall this - PR. Add it in a follow-up once a real consumer demands it; the - follow-up gets to pick "hold sandbox" vs "yield + reacquire" with - a concrete workload to validate against. Listed under future work. -- ~~**Drop `Task.instance_key`**~~ *(surfaced by walkthrough smell #3)* — - *resolved at workshop: deferred.* The redundancy is real (the outer - `Mapping[instance_key, Sequence[Task]]` already keys by instance); - but dropping it touches the benchmark/definition-writer contract which - is out of scope for v2's persistence/runtime cleanup. Land in a - follow-up after v2 ships. - -## PR 5 implementation notes - -**`_type` serializers on `Task` and `Sandbox`.** The PR 5 plan specified -`Worker` getting a `_type`-injecting `model_serializer` (documented in Task -2b). The plan assumed `Task.model_dump()` and `Sandbox.model_dump()` would -also carry `_type`, but never explicitly called out adding serializers to -those classes. When the round-trip test was written it immediately surfaced -that `_task_to_definition_json(Task(...))` produced JSON without `_type`, -which would crash `Task.from_definition` on re-inflation. Both serializers -were added in the PR 5 Task 5 commit. This is consistent with the `Worker` -and `Evaluator` pattern; the plan just omitted calling it out explicitly. - -**`CriterionContext` proxy method removal deferred to PR 11.** The PR 5 plan -(and `01-api-surface.md` §"CriterionContext shape (v2)") said PR 5 removes -the 12 runtime proxy methods from `CriterionContext`, drops `_runtime`, and -drops `sandbox_id`. This was not done — criteria in PRs 6/10a/10b/10c still -use `context.run_command(...)` via the legacy proxy and need it to remain -available during migration. PR 11 Task 1.3 does the removal once all -criterion bodies have been migrated to `context.task.sandbox.run_command(...)`. - -**`Criterion.from_definition` deferred to PR 11.** Specified in the PR 5 plan, -not landed. No current code path hits it — `Rubric.criteria` is -`exclude=True` so criteria never appear in serialized JSON — but it's needed -for completeness before PR 11's final audit. Added as PR 11 Task 1.4. - -## Future work - -Deliberately deferred so this redesign can land cleanly without having -to design them: - -- **Synchronous `spawn_task(..., await_completion=True)`.** v1 ships - fire-and-forget only. A future redesign decides between - "parent holds sandbox" (simple, expensive) and "parent yields + - reacquires via lifecycle hub" (cost-efficient, complicated by - in-memory worker state) once a real consumer constrains the choice. -- **Multiple agents per task / sandbox sharing.** Today's invariant is - *one worker per task*; this redesign adds *one sandbox per task*. The - two invariants together rule out: parent and child sharing a sandbox, - multiple workers collaborating in one task, snapshot/clone semantics - for fast fan-out from a common base state. Each is a real ergonomic - win for some workload (tactic search, multi-agent debate, branching - exploration), and several touch overlapping mechanism (sandbox - reattachment, ref-counted lifecycle, concurrency on the runtime, - per-task event attribution). Designing them piecemeal would produce - conflicting primitives. Deferred to a single follow-up redesign scoped - as *"multiple agents per task"* — that redesign will decide whether to - ship any of them and, if so, will pick one shared mechanism. Until - then: spawning a task creates a new task with a new sandbox, full stop. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan.md deleted file mode 100644 index eeee25c93..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan.md +++ /dev/null @@ -1,101 +0,0 @@ -# 09 — Implementation plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this program PR-by-PR. The -> detailed plan lives in the sibling folder -> [`09-implementation-plan/`](09-implementation-plan/). - -**Goal:** Ship the v2 authoring/runtime redesign as a chain of runnable, -reviewable, depth-first PRs instead of one broad cut. - -**Architecture:** The accepted v2 target from `01`-`08` is unchanged: -object-bound authoring, run-tier task snapshots, run-tier-only runtime -reads, synchronous-fanout criteria orchestrated by `worker_execute`, graph-native dynamic subtasks, -typed sandbox subclasses, and CLI composition over the Python API. The -implementation path uses temporary internal bridges so `main` stays -runnable after every PR. - -**Tech Stack:** Python 3.11+, Pydantic v2, SQLModel, Alembic, Inngest, -pytest, SQLite unit tests, Postgres walkthrough integration. - ---- - -## Program Docs - -Read these in order: - -| # | File | Owns | -|---|---|---| -| 00 | [`00-program.md`](09-implementation-plan/00-program.md) | Strategy, churn budget, ownership lanes, bridge ledger, PR sequence | -| 01 | [`01-pr-00-transition-ledger.md`](09-implementation-plan/01-pr-00-transition-ledger.md) | First guardrails: transition ledger and old-symbol inventory | -| 01b | [`01b-pr-0-5-repository-standard.md`](09-implementation-plan/01b-pr-0-5-repository-standard.md) | Repository layer standard and dead-method audit | -| 02 | [`02-pr-01-run-tier-task-snapshot.md`](09-implementation-plan/02-pr-01-run-tier-task-snapshot.md) | Additive run-tier `task_json` / `is_dynamic` foundation | -| 03 | [`03-pr-02-typed-run-node-boundary.md`](09-implementation-plan/03-pr-02-typed-run-node-boundary.md) | `from_definition` methods and `RunGraphNodeView` | -| 04 | [`04-pr-03-worker-execute-typed-node.md`](09-implementation-plan/04-pr-03-worker-execute-typed-node.md) | Flip worker execution prep to typed run nodes | -| 05 | [`05-pr-04-inline-criteria.md`](09-implementation-plan/05-pr-04-inline-criteria.md) | Synchronous-fanout criteria via `ctx.step.invoke` and sandbox release ownership (filename retained for stable links; content describes fanout, not inline) | -| 06 | [`06-pr-05-object-bound-api.md`](09-implementation-plan/06-pr-05-object-bound-api.md) | Public object-bound API plus definition-writer bridges | -| 07 | [`07-pr-06-minif2f-vertical.md`](09-implementation-plan/07-pr-06-minif2f-vertical.md) | First builtin vertical: MiniF2F object-bound path | -| 08 | [`08-pr-07-persistence-collapse.md`](09-implementation-plan/08-pr-07-persistence-collapse.md) | Collapse experiment metadata onto definitions behind bridges | -| 09 | [`09-pr-08-cli-composition.md`](09-implementation-plan/09-pr-08-cli-composition.md) | CLI define/run through `Experiment` and canonical launch | -| 10 | [`10-pr-09-dynamic-subtasks.md`](09-implementation-plan/10-pr-09-dynamic-subtasks.md) | Graph-native dynamic subtasks and containment facade | -| 11 | [`11-pr-10a-swebench.md`](09-implementation-plan/11-pr-10a-swebench.md) | PR 10a — SWEBench vertical (object-bound `Task`, `SWEBenchSandbox`, shared `ManagerBackedSandboxRuntime` adapter) | -| 11b | [`11b-pr-10b-researchrubrics.md`](09-implementation-plan/11b-pr-10b-researchrubrics.md) | PR 10b — ResearchRubrics vertical (Pydantic `JudgeCriterion`) | -| 11c | [`11c-pr-10c-gdpeval.md`](09-implementation-plan/11c-pr-10c-gdpeval.md) | PR 10c — GDPEval vertical + builtins cleanup (registry-import shrink and no-registry architecture guard) | -| 12 | [`12-pr-11-deletion-final-schema.md`](09-implementation-plan/12-pr-11-deletion-final-schema.md) | Delete bridges and regenerate final v2 schema | -| 13 | [`13-pr-12-walkthrough-ci.md`](09-implementation-plan/13-pr-12-walkthrough-ci.md) | Walkthrough integration and CI hardening | - -## Executable Program Twin - -The PR plan is mirrored by five `xfail(strict=True)` ledger files landed -in PR 0 / PR 1. Each subsequent PR removes its xfail markers; PR 11 -asserts both marker dicts are empty as a hard completion-bar guard. See -[`07-test-strategy.md` §0](07-test-strategy.md) for the four-file ledger -pattern and the per-PR flip schedule in -[`09-implementation-plan/00-program.md` "Ledger Files"](09-implementation-plan/00-program.md). - -## Completion Bar - -The program is complete only when: - -- `test_v2_final_state_ledger.py::_XFAIL_BY_NAME` is an empty dict. -- `test_dead_path_audit.py::_XFAIL_BY_SYMBOL` is an empty dict. -- Runtime reads no definition-tier task rows after prepare-run. -- `worker_execute` synchronously fans evaluators out via - `ctx.step.invoke` to `evaluate_task_run`, gathers, then releases its - sandbox in `finally`. -- `evaluate_task_run` takes the thin id-only payload - `(run_id, task_id, execution_id, evaluator_index)` and rebuilds the - live sandbox via `Sandbox.from_definition(json, sandbox_id=...)`. -- Dynamic subtasks write only to `run_graph_nodes`. -- In-tree builtins construct object-bound `Task` objects. -- CLI define/run delegates to canonical Python persistence and launch. -- `TaskSpec`, `WorkerSpec`, `ComponentRegistry`, `saved_specs`, - `ExperimentRecord`, `definition_task_id`, `CriterionExecutor`, - `InngestCriterionExecutor`, and `Worker.from_buffer` are gone. - (`evaluate_task_run` and the eval-payload class are **kept and - reshaped**, not deleted — see Δ.4 / Δ.5 in `08-decisions-log.md`.) -- Walkthrough integration passes happy path, failure cascade, dynamic - spawn, and restart variants — including per-eval `step.invoke` - observability through the synchronous fanout. - -## What's Deferred - -These remain out of scope for the v2 implementation program: - -- Public `GraphMutator` / `GraphInspector` / `ResourceInspector` service - classes. -- Synchronous `spawn_task(..., await_completion=True)`. -- Multiple agents per task or sandbox sharing. -- Cross-task data handoff via first-class `Task.inputs` / `Task.outputs`. -- `GraphSpawnGovernor` fork-bomb backpressure. -- `Task.instance_key` removal. -- Cohort runs from the CLI. - -## On Acceptance - -After PR 12 lands, graduate the accepted parts of this folder into -`docs/architecture/`, archive the active RFC folder, and update -`docs/architecture/01_public_api.md` plus -`docs/architecture/cross_cutting/sandbox_lifecycle.md` to point at the -shipped v2 surfaces. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md deleted file mode 100644 index 270b09507..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md +++ /dev/null @@ -1,275 +0,0 @@ -# Authoring API Redesign v2 Program Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Ship the v2 redesign through small, runnable PRs with explicit -bridges and deletion gates. - -**Architecture:** Add the new run-tier and object-bound paths beside the -old slug/definition paths, flip one vertical at a time, then delete the old -paths. The final architecture remains the accepted v2 spec from `01`-`08`. - -**Tech Stack:** Python 3.11+, Pydantic v2, SQLModel, Alembic, Inngest, -pytest, SQLite unit tests, Postgres integration tests. - ---- - -## Why This Program Exists - -The old plan was still a big cut. It began by resetting schema and then -changed public API, persistence identity, runtime event shape, sandbox -lifecycle, builtins, and CLI semantics on the same long-running branch. -That makes failures ambiguous and makes review expensive. - -This program is intentionally depth-first: - -- add new data where old code can ignore it; -- add new typed boundaries beside old callers; -- flip one runtime path; -- prove one builtin vertical; -- migrate remaining callers; -- delete bridges. - -## Destructive Step: Migration Reset In PR 11 - -**Before merging PR 11**, every machine with an Ergon dev DB must run -`alembic downgrade base` and tear down any persistent Docker volumes. PR 11 -deletes the entire `ergon_core/migrations/versions/` directory and replaces -27+ revisions with one new `00000000_initial_v2.py` that has -`down_revision = None`. The workshop locked "no prod data" for v2 launch; -the same decision applies to contributors' local databases — there is no -data preservation path. See PR 11 § "Developer-facing downgrade step" for -the exact developer instructions and the CI invariant that locks the head -count to one. - -## Churn Budget - -| Area | Likely churn | -|---|---:| -| Core API + runtime + persistence | 6k-10k changed lines | -| Tests, fixtures, architecture guards | 3k-6k changed lines | -| Migrations / final schema reset | 1k+ changed lines | -| Builtins and CLI migration | 2k-4k changed lines | -| **Total PR chain** | **10k-18k changed lines** | - -PRs above 2.5k non-generated changed lines must be split unless the excess -is mechanical deletion. - -## Ownership Lanes - -| Lane | Owns | Must not own | -|---|---|---| -| API | `ergon_core.api`, public exceptions, Pydantic round trips | Inngest orchestration | -| Persistence | ORM, Alembic, repositories, `RunGraphNodeView` | Worker/evaluator behavior | -| Runtime | jobs, Inngest registry, `worker_execute`, lifecycle ownership | Builtin benchmark factories | -| Sandbox | `api/sandbox`, `core/infrastructure/sandbox`, concrete builtins sandboxes | Definition/run identity | -| Builtins | benchmark constructors, workers, toolkits, one vertical at a time | Core migrations | -| CLI | `ergon_cli.commands.experiment`, slug factories, command tests | Public API design | -| Tests | architecture guards, regression net, walkthrough harness | Production implementation | - -## Bridge Ledger - -| Bridge | Introduced | New default | Deleted | -|---|---|---|---| -| Run-tier `task_json` beside definition payload | PR 1 | PR 3 | PR 11 removes old fields | -| `graph_repo.node(...).task` beside `_prepare_definition` | PR 2 | PR 3 | PR 11 | -| Synchronous-fanout criteria beside legacy `evaluate_task_run` body | PR 4 | PR 4 | PR 11 prunes legacy reads; `evaluate_task_run` survives reshaped | -| Object-bound `Task` beside `TaskSpec` | PR 5 | PR 6 for MiniF2F, PR 10 globally | PR 11 | -| Public `Experiment` beside domain `Experiment` | PR 5 | PR 8 | PR 11 | -| `Sandbox` subclasses beside `BaseSandboxManager` | PR 5/6 | PR 10 | PR 11 | -| Definition metadata beside `ExperimentRecord` | PR 7 | PR 7 | PR 11 | -| CLI slug factory beside `saved_specs` path | PR 8 | PR 8 | PR 11 | - -## Ledger Files (executable program twin) - -Four landing-PR-keyed `xfail(strict=True)` ledgers land in PR 0 / PR 1 -alongside the existing transition ledger. Each later PR is responsible -for removing the markers for the invariants it lands; PR 11 asserts both -marker dicts are empty as a completion-bar guard. See -[`07-test-strategy.md` §0](../07-test-strategy.md) for the rationale and -the no-call-graph-mocks rule. - -| File | Lands in | Marker store | Final-state owner | -|---|---|---|---| -| `test_v2_transition_ledger.py` | PR 0 | `TRANSITIONAL_SYMBOLS` | PR 11 replaces with deletion guards | -| `test_v2_final_state_ledger.py` | PR 0 | `_XFAIL_BY_NAME` | PR 11 empties the dict and adds completion-bar guard | -| `test_dead_path_audit.py` | PR 0 | `_XFAIL_BY_SYMBOL` | PR 11 empties the dict and adds completion-bar guard | -| `test_no_type_circumventors.py` | PR 0 | `_KNOWN_EXEMPTIONS` | PR 11 empties the dict and adds completion-bar guard | -| `test_repository_layer_conventions.py` | PR 0.5 | `_KNOWN_VIOLATORS` (file-shared) | PR 11 empties and removes the xfail on `test_no_repository_violators_remain` | -| `test_repository_companion_files.py` | PR 0.5 | `_KNOWN_VIOLATORS` (same dict, shared) | PR 11 empties (same) | -| `test_no_dead_repository_methods.py` | PR 0.5 | `_KNOWN_UNUSED_FOR_NOW` | PR 11 empties and removes the xfail on `test_no_dead_repository_methods_remain` | -| `test_walkthrough_smoketest.py` | PR 1 | per-test `@xfail` | PR 11 asserts no XFAIL remains | -| `test_identity_invariants.py` | PR 1 | per-test `@xfail` | PR 11 asserts no XFAIL remains | - -Per-PR ledger updates (which markers each PR removes): - -| PR | Final-state entries flipped | Dead-path entries flipped | Smoketest/identity flipped | Repo-standard entries flipped | -|---|---|---|---|---| -| 1 | — | — | — | `telemetry/repositories.py` rename (filename-singular case) | -| 2 | — | — | `test_task_id_propagates_into_runtime_task_instance` | — | -| 3 | `worker_execute_imports_only_run_tier` | `_prepare_definition` | `test_worker_execute_reads_task_from_run_tier_only` | — | -| 4 | `evaluate_task_run_uses_thin_payload`, `check_evaluators_is_unregistered` | — | 3 smoketest + 2 identity cases | `CreateTaskEvaluation` moves to `telemetry/models.py` | -| 5 | `task_has_no_model_post_init` | `_worker_from_payload_bridge`, `_DetachableSandboxBridge` | — | — | -| 7 | — | — | `test_persist_definition_writes_only_intended_tables` | `experiments/errors.py` added; ValueError raises replaced | -| 8 | — | `_persist_single_sample_workflow_definition` | — | — | -| 9 | `materialize_dynamic_subtask_definition_is_gone` | `materialize_dynamic_subtask_definition` | `test_dynamic_spawn_writes_only_to_run_graph_nodes`, `test_dynamic_task_id_has_no_definition_row` | — | -| 11 | all remaining entries | all remaining entries | `test_run_completion_releases_every_acquired_sandbox` | all remaining `_KNOWN_UNUSED_FOR_NOW` entries; remove xfail on `test_no_repository_violators_remain` and `test_no_dead_repository_methods_remain` | - -PR 11 also runs the **Structural Simplification Audit** (10 named -candidates) — see [`12-pr-11-deletion-final-schema.md`](12-pr-11-deletion-final-schema.md) -§ Task 5. The audit produces a KEEP/RENAME/INLINE/SPLIT decision per -candidate in the PR description; it is a hand-checked review pass, not -a pytest assertion. - -## PR Sequence - -| PR | Name | Runnable after merge | Primary invariant | -|---:|---|---|---| -| 0 | Transition ledger | yes | Old paths are inventoried and guarded | -| 0.5 | Repository layer standard | yes | Repository conventions are documented and enforced; current violators xfailed | -| 1 | Run-tier task snapshot | yes | Run nodes can carry self-contained task JSON | -| 2 | Typed run-node boundary | yes | Repo can inflate typed tasks from run-tier JSON | -| 3 | Worker-execute typed node | yes | Worker execution prep uses run-tier typed node | -| 4 | Synchronous-fanout criteria | yes | Worker-execute orchestrates eval via `ctx.step.invoke` + gather; owns sandbox release in `finally` | -| 5 | Object-bound API | yes | New authoring objects serialize beside old adapters | -| 6 | MiniF2F vertical | yes | One builtin uses the v2 shape end to end | -| 7 | Persistence collapse | yes | New definitions launch without `ExperimentRecord` | -| 8 | CLI composition | yes | CLI define output is launchable by canonical launch | -| 9 | Dynamic subtasks | yes | Spawned tasks are graph-native | -| 10a | SWEBench vertical | yes | SWEBench uses object-bound `Task`/`Sandbox`; lands shared `ManagerBackedSandboxRuntime` adapter | -| 10b | ResearchRubrics vertical | yes | ResearchRubrics uses object-bound `Task`/`Sandbox`; `JudgeCriterion` becomes Pydantic | -| 10c | GDPEval vertical + builtins cleanup | yes | GDPEval uses object-bound `Task`/`Sandbox`; registry-import shrink + no-registry guard cover all four migrated benchmarks | -| 11 | Deletion + final schema | yes | Old paths are gone | -| 12 | Walkthrough + CI | yes | Canonical integration variants pass | - -## PR 10 Sub-PR Sequencing - -PRs 10a, 10b, 10c are **three independently-reviewable GitHub PRs**, -not a single PR with three internal tasks. Reviewers focus on one -benchmark at a time; the 2.5k-line churn budget applies per sub-PR. - -| Sub-PR | Lands when | Depends on | Cross-cutting role | -|---|---|---|---| -| 10a — SWEBench | After PR 9 is on `main` | Lands the shared `ManagerBackedSandboxRuntime` adapter at `ergon_builtins/sandboxes/_manager_backed.py` | Adapter is imported by 10b/10c | -| 10b — ResearchRubrics | After PR 9; independent of 10a | Imports the adapter (if 10a already merged) — otherwise Step 0 instructs the author to create the adapter inline | Adds Pydantic `JudgeCriterion` | -| 10c — GDPEval + cleanup | **Must merge last** of the three | Imports the adapter; runs the no-registry guard across all four migrated benchmarks (minif2f, swebench_verified, researchrubrics, gdpeval) | Registry shrink + architecture guard | - -The three sub-PRs do **not** depend on each other sequentially in -content — only the no-registry guard in 10c requires the other two to -be merged so its parametrized `_MIGRATED_BENCHMARKS` list passes for -every entry. If 10b lands before 10a, 10b's Step 0 lands the adapter; -if 10a lands first, 10b just imports it. - -## PR Description Template - -Every PR must include: - -```markdown -### V2 Slice Ledger - -Invariant landed: - -Bridge code introduced: - -Old path still intentionally alive: - -Deletion gate: - -Tests added or updated: - -Modules owned by this PR: -``` - -## Parallelization Rules - -Safe in parallel: - -- PR 0 audit work and PR 1 schema exploration. -- PR 4 synchronous-fanout criteria draft after PR 2 defines `RunGraphNodeView`. -- PR 8 CLI draft after PR 5 defines public `Experiment`. -- PR 12 walkthrough draft after PR 6 proves one vertical path. - -Do not parallelize changes to these files: - -- `ergon_core/ergon_core/core/application/jobs/worker_execute.py` -- `ergon_core/ergon_core/core/application/experiments/definition_writer.py` -- `ergon_core/ergon_core/core/application/graph/repository.py` -- `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- `ergon_cli/ergon_cli/commands/experiment.py` - -## Final Deleted Symbols - -PR 11 must remove all of: - -**v1 public API and abstractions:** -- `ergon_core.api.registry.ComponentRegistry` -- `ergon_core.api.benchmark.task.TaskSpec` -- `ergon_core.core.domain.experiments.worker_spec.WorkerSpec` -- `ergon_core.core.domain.experiments.Experiment` (domain — public version in `ergon_core.api.experiment` replaces it) -- `Worker.from_buffer` -- `Worker.validate` (renamed to `validate_runtime_deps` in PR 5) - -**Dead packages:** -- `ergon_core.core.persistence.saved_specs` -- `ergon_core.core.application.components` (ComponentCatalogService and friends) -- `ergon_core.core.application.experiments.repository.DefinitionRepository` - -**Deprecated Inngest jobs (absorbed into worker_execute by PR 4):** -- `ergon_core.core.application.jobs.execute_task` -- `ergon_core.core.application.jobs.sandbox_setup` -- `ergon_core.core.application.jobs.persist_outputs` -- `ergon_core.core.application.jobs.check_evaluators` - -**Runtime DTOs:** -- `EvaluateTaskRunRequest` (replaced by `TaskEvaluateRequest`, the id-only payload) -- `PreparedTaskExecution.node_id`, `.definition_task_id`, `.worker_type`, `.assigned_worker_slug`, `.model_target` -- `_prepare_legacy_definition`, `_worker_from_payload_bridge`, `_DetachableSandboxBridge` - -**Schema (composite PK collapse):** -- `RunGraphNode.id` (composite PK becomes `(run_id, task_id)`) -- `RunGraphNode.definition_task_id` -- `RunGraphNode.parent_node_id` → renamed `parent_task_id` -- `RunGraphEdge.source_node_id` / `target_node_id` → renamed `source_task_id` / `target_task_id` -- `RunGraphEdge.definition_dependency_id` -- `RunTaskExecution.node_id` (renamed `task_id`), `.definition_task_id` -- `RunTaskEvaluation.node_id`, `.definition_task_id` -- `ergon_core.core.persistence.telemetry.models.ExperimentRecord` (table) - -**Evaluation:** -- `CriterionExecutor`, `InngestCriterionExecutor` -- `terminate_sandbox_by_id` - -**Kept and reshaped (PR 11 must NOT delete these):** - -- `evaluate_task_run` Inngest function — survives as the per-evaluator - fanout target. Body is rewritten in PR 4 to take the id-only - `TaskEvaluateRequest`, reload state via `graph_repo.node`, attach the - sandbox via `Sandbox.from_definition(sandbox_id=...)`, call - `evaluator.evaluate(...)` directly (no `CriterionExecutor`), then - detach. PR 11 only deletes the *v1 body and v1 payload class*, not - the function/slug itself. - -## Verification Commands - -Run these at the end of every PR unless the PR-specific doc narrows the set: - -```bash -uv run pytest ergon_core/tests/unit/architecture -q -uv run pytest ergon_core/tests/unit/api -q -uv run pytest ergon_core/tests/unit/runtime -q -``` - -PRs touching builtins also run: - -```bash -uv run pytest ergon_builtins/tests/unit -q -``` - -PRs touching CLI also run: - -```bash -uv run pytest ergon_cli/tests -q -``` diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01-pr-00-transition-ledger.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01-pr-00-transition-ledger.md deleted file mode 100644 index 920b88c56..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01-pr-00-transition-ledger.md +++ /dev/null @@ -1,986 +0,0 @@ -# PR 0 — Transition Ledger And Guard Scaffolding - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add guardrails that make temporary old paths explicit during the -v2 transition, and land the executable spec of the v2 final state as a set -of `xfail(strict=True)` invariants whose markers each subsequent PR removes. - -**Architecture:** Three static, source-text architecture guards land -together: - -1. **Transition ledger** (`test_v2_transition_ledger.py`) — *negative* - ledger: "these old symbols still exist and that's intentional". -2. **Final-state ledger** (`test_v2_final_state_ledger.py`) — *positive* - ledger: "these v2 invariants must hold at the end; each is - `xfail(strict=True)` with its landing-PR reason until that PR flips - the marker off". `strict=True` so an unexpected pass also fails CI — - if a later PR lands an invariant early, we want to know. -3. **Dead-path audit** (`test_dead_path_audit.py`) — every symbol/package - the v1 audit flagged as "wired up to nothing" has a `grep_callers` - test asserting it stays callerless until its deletion PR. - -Together, the three files act as a single-glance progress meter for the -whole program: the count of `xfail` markers shrinks monotonically as PRs -land. The PR ledger and bridge ledger in `00-program.md` stay as the -human-readable narrative; the test files are the machine-readable twin. - -**Tech Stack:** pytest, pathlib, source-text architecture checks, pytest -`xfail(strict=True)`. - ---- - -## Files - -**Create:** - -```text -ergon_core/tests/unit/architecture/test_v2_transition_ledger.py -ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py -ergon_core/tests/unit/architecture/test_dead_path_audit.py -``` - -**Modify:** - -```text -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md -``` - -## Current State - -These old symbols are present and intentionally transitional: - -```text -TaskSpec -WorkerSpec -ComponentRegistry -BaseSandboxManager -ExperimentRecord -EvaluateTaskRunRequest -evaluate_task_run -CriterionExecutor -InngestCriterionExecutor -saved_specs -definition_task_id -Worker.from_buffer -terminate_sandbox_by_id -``` - -## Target State For This PR - -The symbols remain in production code, but they are recorded in a test-owned -ledger. The ledger gives reviewers a single place to see whether a symbol is -still allowed or has become forbidden. - -## Task 1: Add Transition Ledger Test - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_v2_transition_ledger.py` - -- [x] **Step 1: Write the guard file** - -```python -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[4] -SEARCH_ROOTS = ( - ROOT / "ergon_core", - ROOT / "ergon_builtins", - ROOT / "ergon_cli", -) - - -@dataclass(frozen=True) -class TransitionalSymbol: - name: str - owner_pr: str - deletion_pr: str - allowed_reason: str - - -TRANSITIONAL_SYMBOLS = ( - TransitionalSymbol( - name="TaskSpec", - owner_pr="PR 5", - deletion_pr="PR 11", - allowed_reason="old benchmark definitions still return TaskSpec", - ), - TransitionalSymbol( - name="WorkerSpec", - owner_pr="PR 5", - deletion_pr="PR 11", - allowed_reason="old Experiment composition still binds worker specs", - ), - TransitionalSymbol( - name="ComponentRegistry", - owner_pr="PR 5", - deletion_pr="PR 11", - allowed_reason="registry remains while old builtins are unmigrated", - ), - TransitionalSymbol( - name="BaseSandboxManager", - owner_pr="PR 6", - deletion_pr="PR 11", - allowed_reason="sandbox subclass migration is incremental", - ), - TransitionalSymbol( - name="ExperimentRecord", - owner_pr="PR 7", - deletion_pr="PR 11", - allowed_reason="read models are migrated after collapsed definitions land", - ), - TransitionalSymbol( - name="EvaluateTaskRunRequest", - owner_pr="PR 4", - deletion_pr="PR 11", - allowed_reason=( - "v1 multi-field payload replaced by TaskEvaluateRequest " - "(id-only) when PR 4 reshapes evaluate_task_run; the import " - "shim survives until cleanup." - ), - ), - TransitionalSymbol( - name="CriterionExecutor", - owner_pr="PR 4", - deletion_pr="PR 11", - allowed_reason=( - "Protocol kept compiling during PR 4 reshape; the reshaped " - "evaluate_task_run calls criterion.evaluate(...) directly so " - "no executor indirection remains in production code paths." - ), - ), - TransitionalSymbol( - name="InngestCriterionExecutor", - owner_pr="PR 4", - deletion_pr="PR 11", - allowed_reason=( - "Concrete impl of the Protocol; survives alongside " - "CriterionExecutor until PR 11 deletes both." - ), - ), - TransitionalSymbol( - name="saved_specs", - owner_pr="PR 8", - deletion_pr="PR 11", - allowed_reason="CLI define moves before persistence package deletion", - ), - TransitionalSymbol( - name="definition_task_id", - owner_pr="PR 1", - deletion_pr="PR 11", - allowed_reason="old runtime identity survives until task_id becomes canonical", - ), - TransitionalSymbol( - name="from_buffer", - owner_pr="PR 11", - deletion_pr="PR 11", - allowed_reason="dead Worker constructor is deleted in the cleanup PR", - ), - TransitionalSymbol( - name="terminate_sandbox_by_id", - owner_pr="PR 4", - deletion_pr="PR 11", - allowed_reason="old cleanup path remains until worker_execute owns release", - ), -) - - -def _hits(symbol: str) -> list[str]: - hits: list[str] = [] - for root in SEARCH_ROOTS: - for path in root.rglob("*.py"): - text = path.read_text() - if symbol in text: - hits.append(str(path.relative_to(ROOT))) - return sorted(hits) - - -EXEMPT_DIR_PARTS: frozenset[str] = frozenset( - {"tests", "migrations", "__pycache__"} -) - - -def _production_hits(symbol: str) -> list[str]: - """Return production-code hits for a symbol, excluding tests and migrations.""" - - hits: list[str] = [] - for root in SEARCH_ROOTS: - for path in root.rglob("*.py"): - if EXEMPT_DIR_PARTS.intersection(path.parts): - continue - text = path.read_text() - if symbol in text: - hits.append(str(path.relative_to(ROOT))) - return sorted(hits) - - -def test_transitional_symbols_are_explicitly_ledgered() -> None: - missing: list[str] = [] - for symbol in TRANSITIONAL_SYMBOLS: - if not _hits(symbol.name): - missing.append( - f"{symbol.name} disappeared; update this ledger and the deletion docs " - f"for {symbol.deletion_pr}" - ) - assert missing == [] - - -# Symbols that PR 11 expects to delete. If production code starts using any -# legacy term that is NOT in TRANSITIONAL_SYMBOLS, the v2 program has -# regressed — every transitional path must be named in the ledger. -LEGACY_PRODUCTION_TERMS: frozenset[str] = frozenset( - { - "TaskSpec", - "WorkerSpec", - "ComponentRegistry", - "BaseSandboxManager", - "ExperimentRecord", - "EvaluateTaskRunRequest", - "CriterionExecutor", - "InngestCriterionExecutor", - "saved_specs", - "definition_task_id", - "from_buffer", - "terminate_sandbox_by_id", - # evaluate_task_run is intentionally NOT here — it survives reshaped per Δ.4. - } -) - - -def test_no_unledgered_legacy_term_appears_in_production_code() -> None: - """Real check: a legacy term may live in production code only if ledgered.""" - - ledgered = {symbol.name for symbol in TRANSITIONAL_SYMBOLS} - offenders: list[str] = [] - for term in LEGACY_PRODUCTION_TERMS: - if term in ledgered: - continue - hits = _production_hits(term) - if hits: - offenders.append( - f"{term} appears in production code at {hits} but is not in " - f"TRANSITIONAL_SYMBOLS. Either add it to the ledger with a " - f"deletion_pr, or remove the production references." - ) - assert offenders == [], "\n".join(offenders) -``` - -- [x] **Step 2: Run the guard** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_v2_transition_ledger.py -q -``` - -Expected: pass while old symbols are still present. - -- [x] **Step 3: Commit** *(deferred to Task 5 combined commit)* - -```bash -git add ergon_core/tests/unit/architecture/test_v2_transition_ledger.py \ - docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan -git commit -m "test: add v2 transition ledger guard" -``` - -## Task 2: Add Final-State Ledger (xfail invariants) ✅ - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` - -The final-state ledger is the executable form of the v2 deletion list and -the run-tier read boundary. One parametrized test, one case per invariant. -Each case starts `xfail(reason="<landing PR>: ...", strict=True)`; the PR -that lands the invariant removes the marker entry from `_XFAIL_BY_NAME`. - -`strict=True` causes a case to fail CI if it passes without an xfail -marker (an invariant landed early without the ledger update) AND if it -fails without an xfail marker (an invariant regressed after landing). The -ledger entries below are the load-bearing v2 invariants from -[`02-persistence-layer.md`](../02-persistence-layer.md) §4 and -[`00-readme.md`](../00-readme.md) Δ.4 / Δ.7. - -- [x] **Step 1: Write the guard file** - -```python -"""Executable spec of the v2 final state. - -Each FinalStateAssertion is one architecture invariant that must hold once -the v2 program is complete. Invariants that have not landed yet are marked -xfail(strict=True) with their landing PR. Removing a marker is the -landing-PR signal — CI will flag any case that passes without a marker -(invariant landed early) or fails without a marker (invariant regressed). -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Callable - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOTS = ( - ROOT / "ergon_core" / "ergon_core", - ROOT / "ergon_builtins" / "ergon_builtins", - ROOT / "ergon_cli" / "ergon_cli", -) -EXEMPT_PARTS: frozenset[str] = frozenset({"tests", "migrations", "__pycache__"}) - - -def _read(relpath: str) -> str: - return (ROOT / relpath).read_text() - - -def _grep_production(symbol: str) -> list[str]: - hits: list[str] = [] - for root in PRODUCTION_ROOTS: - for path in root.rglob("*.py"): - if EXEMPT_PARTS.intersection(path.parts): - continue - if symbol in path.read_text(): - hits.append(str(path.relative_to(ROOT))) - return sorted(hits) - - -@dataclass(frozen=True) -class FinalStateAssertion: - name: str - landing_pr: str - check: Callable[[], None] - reason: str - - -def _assert_no_definition_repository_in_worker_execute() -> None: - text = _read("ergon_core/ergon_core/core/application/jobs/worker_execute.py") - assert "DefinitionRepository" not in text - assert "task_with_instance" not in text - assert "ExperimentDefinitionTask" not in text - - -def _assert_no_prepare_definition_method() -> None: - text = _read("ergon_core/ergon_core/core/application/tasks/execution.py") - assert "_prepare_definition" not in text - assert "_prepare_legacy_definition" not in text - - -def _assert_no_materialize_dynamic_subtask_definition() -> None: - assert _grep_production("materialize_dynamic_subtask_definition") == [] - - -def _assert_no_criterion_executor() -> None: - assert _grep_production("CriterionExecutor") == [] - assert _grep_production("InngestCriterionExecutor") == [] - - -def _assert_no_saved_specs_package() -> None: - assert not (ROOT / "ergon_core/ergon_core/core/persistence/saved_specs").exists() - - -def _assert_evaluate_task_run_takes_thin_payload() -> None: - """Δ.4: evaluate_task_run survives reshaped with TaskEvaluateRequest.""" - - import inspect - - from ergon_core.core.application.jobs.evaluate_task_run import evaluate_task_run - - sig = inspect.signature(evaluate_task_run) - assert "TaskEvaluateRequest" in repr(sig), ( - "evaluate_task_run must take TaskEvaluateRequest after PR 4's reshape" - ) - - -def _assert_run_graph_node_has_no_definition_task_id_column() -> None: - from ergon_core.core.persistence.graph.models import RunGraphNode - - assert "definition_task_id" not in RunGraphNode.model_fields - - -def _assert_task_has_no_model_post_init() -> None: - """CLAUDE.md guardrail: no model_post_init in core public API objects.""" - - from ergon_core.api.benchmark.task import Task - - assert "model_post_init" not in Task.__dict__ - - -def _assert_worker_from_buffer_is_gone() -> None: - from ergon_core.api.worker.worker import Worker - - assert not hasattr(Worker, "from_buffer") - - -def _assert_terminate_sandbox_by_id_is_gone() -> None: - assert _grep_production("terminate_sandbox_by_id") == [] - - -def _assert_no_check_evaluators_registration() -> None: - text = _read("ergon_core/ergon_core/core/infrastructure/inngest/registry.py") - assert "check_evaluators" not in text - - -FINAL_STATE_ASSERTIONS: tuple[FinalStateAssertion, ...] = ( - FinalStateAssertion( - name="worker_execute_imports_only_run_tier", - landing_pr="PR 3", - check=_assert_no_definition_repository_in_worker_execute, - reason="Δ.2: runtime reads only run-tier tables", - ), - FinalStateAssertion( - name="evaluate_task_run_uses_thin_payload", - landing_pr="PR 4", - check=_assert_evaluate_task_run_takes_thin_payload, - reason="Δ.4: per-evaluator fanout takes TaskEvaluateRequest", - ), - FinalStateAssertion( - name="check_evaluators_is_unregistered", - landing_pr="PR 4", - check=_assert_no_check_evaluators_registration, - reason="Δ.4: synchronous fanout replaces check_evaluators dispatch", - ), - FinalStateAssertion( - name="task_has_no_model_post_init", - landing_pr="PR 5", - check=_assert_task_has_no_model_post_init, - reason="CLAUDE.md: no model_post_init in public API objects", - ), - FinalStateAssertion( - name="materialize_dynamic_subtask_definition_is_gone", - landing_pr="PR 9", - check=_assert_no_materialize_dynamic_subtask_definition, - reason="Δ.3: dynamic subtasks are graph-native", - ), - FinalStateAssertion( - name="prepare_definition_helper_is_removed", - landing_pr="PR 11", - check=_assert_no_prepare_definition_method, - reason="Δ.2: no fallback to definition-tier reads", - ), - FinalStateAssertion( - name="criterion_executor_is_removed", - landing_pr="PR 11", - check=_assert_no_criterion_executor, - reason="Δ.7: deletion list", - ), - FinalStateAssertion( - name="saved_specs_package_is_removed", - landing_pr="PR 11", - check=_assert_no_saved_specs_package, - reason="Δ.7: write-only package, no readers", - ), - FinalStateAssertion( - name="run_graph_node_has_no_definition_task_id_column", - landing_pr="PR 11", - check=_assert_run_graph_node_has_no_definition_task_id_column, - reason="Δ.7 + identity model: task_id is the single canonical id", - ), - FinalStateAssertion( - name="worker_from_buffer_is_removed", - landing_pr="PR 11", - check=_assert_worker_from_buffer_is_gone, - reason="Δ.7: dead constructor with no callers", - ), - FinalStateAssertion( - name="terminate_sandbox_by_id_is_removed", - landing_pr="PR 11", - check=_assert_terminate_sandbox_by_id_is_gone, - reason="Δ.7: cleanup path subsumed by worker_execute.finally", - ), -) - - -# When an assertion's landing PR merges, delete its entry from this dict. -# strict=True surfaces unexpected passes — if a later refactor flips an -# invariant green ahead of its landing PR, CI fails and the ledger gets -# the update at the same time. -_XFAIL_BY_NAME: dict[str, str] = { - "worker_execute_imports_only_run_tier": "PR 3 flips worker_execute to run-tier", - "evaluate_task_run_uses_thin_payload": "PR 4 reshapes evaluate_task_run", - "check_evaluators_is_unregistered": "PR 4 removes check_evaluators dispatch", - # task_has_no_model_post_init: already holds in v1 (no `model_post_init` - # on Task today). Asserted every run; PR 5's v2 Task must preserve it. - # materialize_dynamic_subtask_definition_is_gone: the v1 codebase doesn't - # use this exact name. Asserted every run; PR 9 must not reintroduce it. - "prepare_definition_helper_is_removed": "PR 11 deletes legacy prepare path", - "criterion_executor_is_removed": "PR 11 deletes Protocol pair", - "saved_specs_package_is_removed": "PR 11 deletes write-only package", - "run_graph_node_has_no_definition_task_id_column": "PR 11 drops the column", - "worker_from_buffer_is_removed": "PR 11 deletes dead constructor", - "terminate_sandbox_by_id_is_removed": "PR 11 deletes legacy cleanup path", -} - - -def _cases() -> list: - cases = [] - for assertion in FINAL_STATE_ASSERTIONS: - marks = [] - reason = _XFAIL_BY_NAME.get(assertion.name) - if reason is not None: - marks.append( - pytest.mark.xfail( - reason=f"{assertion.landing_pr}: {reason}", strict=True - ) - ) - cases.append(pytest.param(assertion, marks=marks, id=assertion.name)) - return cases - - -@pytest.mark.parametrize("assertion", _cases()) -def test_v2_final_state(assertion: FinalStateAssertion) -> None: - """One check per v2 invariant. Each is xfail(strict=True) until its - landing PR flips the marker off in `_XFAIL_BY_NAME`.""" - - assertion.check() -``` - -- [ ] **Step 2: Run the guard** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py -q -``` - -Expected: every case is XFAIL on PR 0 (none of the v2 cutovers have -landed yet). No XPASS, no FAIL. - -## Task 3: Add Dead-Path Audit ✅ - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_dead_path_audit.py` - -This file is the executable form of "v1 helpers that were wired up to -nothing must stay callerless until their deletion PR removes them." It -is the textual grep counterpart to the deletion list in -[`00-program.md`](00-program.md) "Final Deleted Symbols". Each case is -xfail until the symbol's last production caller is gone. - -- [x] **Step 1: Write the guard file** - -```python -"""Dead-path audit: v1 helpers that were wired up to nothing must stay -callerless until their deletion PR. - -The v1 audit found multiple helpers (saved_specs, Worker.from_buffer, -CriterionExecutor, etc.) that production code never invoked. This file -asserts they stay callerless. Each case is xfail(strict=True) until the -listed landing PR deletes the last production reference. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOTS = ( - ROOT / "ergon_core" / "ergon_core", - ROOT / "ergon_builtins" / "ergon_builtins", - ROOT / "ergon_cli" / "ergon_cli", -) -EXEMPT_PARTS: frozenset[str] = frozenset({"tests", "migrations", "__pycache__"}) - - -def _grep_production_callers(symbol: str) -> list[str]: - hits: list[str] = [] - for root in PRODUCTION_ROOTS: - for path in root.rglob("*.py"): - if EXEMPT_PARTS.intersection(path.parts): - continue - if symbol in path.read_text(): - hits.append(str(path.relative_to(ROOT))) - return sorted(hits) - - -@dataclass(frozen=True) -class DeadPath: - symbol: str - landing_pr: str - audit_note: str - - -DEAD_PATHS: tuple[DeadPath, ...] = ( - DeadPath( - symbol="saved_specs", - landing_pr="PR 11", - audit_note="v1 wrote, nothing read", - ), - DeadPath( - symbol="Worker.from_buffer", - landing_pr="PR 11", - audit_note="constructor with zero callers", - ), - DeadPath( - symbol="CriterionExecutor", - landing_pr="PR 11", - audit_note=( - "Protocol with one trivial impl; reshaped eval calls " - "evaluator.evaluate directly" - ), - ), - DeadPath( - symbol="InngestCriterionExecutor", - landing_pr="PR 11", - audit_note="trivial impl removed with the Protocol", - ), - DeadPath( - symbol="_prepare_definition", - landing_pr="PR 3", - audit_note="renamed to _prepare_legacy_definition in PR 3", - ), - DeadPath( - symbol="_prepare_legacy_definition", - landing_pr="PR 11", - audit_note="transitional name; deleted by PR 11", - ), - DeadPath( - symbol="materialize_dynamic_subtask_definition", - landing_pr="PR 9", - audit_note="synthesized definition row for dynamic spawn; graph-native replaces it", - ), - DeadPath( - symbol="terminate_sandbox_by_id", - landing_pr="PR 11", - audit_note="legacy cleanup path; orchestrator try/finally owns release", - ), - DeadPath( - symbol="_persist_single_sample_workflow_definition", - landing_pr="PR 8", - audit_note="v1 CLI write to saved_specs; replaced by canonical persist_definition", - ), - DeadPath( - symbol="_worker_from_payload_bridge", - landing_pr="PR 5", - audit_note="PR 3 bridge; replaced by task.worker when object-bound API lands", - ), - DeadPath( - symbol="_DetachableSandboxBridge", - landing_pr="PR 5", - audit_note="PR 4 bridge; lifted into Sandbox.detach() base method", - ), - # --- Inngest jobs that PR 4 absorbs into worker_execute --- - DeadPath( - symbol="execute_task", - landing_pr="PR 11", - audit_note=( - "v1 orchestrator that fanned out to sandbox_setup → worker_execute → " - "persist_outputs → check_evaluators; v2 collapses into worker_execute" - ), - ), - DeadPath( - symbol="sandbox_setup", - landing_pr="PR 11", - audit_note="PR 4 acquires inline in worker_execute", - ), - DeadPath( - symbol="persist_outputs", - landing_pr="PR 11", - audit_note="PR 4 persists WorkerOutput inline before fanout", - ), - # --- Components/registry layer that v2 retires --- - DeadPath( - symbol="ComponentCatalogService", - landing_pr="PR 11", - audit_note=( - "Registry-driven runtime resolution; replaced by object-bound " - "task.worker / task.sandbox / task.evaluators" - ), - ), - DeadPath( - symbol="DefinitionRepository", - landing_pr="PR 11", - audit_note=( - "Runtime read path into definition tables; replaced by " - "graph_repo.node reading run-tier task_json" - ), - ), - # --- Renamed symbols whose old name must disappear --- - DeadPath( - symbol="Worker.validate", - landing_pr="PR 11", - audit_note="Renamed to validate_runtime_deps in PR 5", - ), - # --- Domain Experiment retired in favor of public Experiment --- - DeadPath( - symbol="ergon_core.core.domain.experiments.Experiment", - landing_pr="PR 11", - audit_note="Replaced by ergon_core.api.experiment.Experiment in PR 5", - ), - # --- DTO fields that PR 11 drops from PreparedTaskExecution --- - DeadPath( - symbol="PreparedTaskExecution.node_id", - landing_pr="PR 11", - audit_note="Identity collapses to task_id only", - ), - DeadPath( - symbol="PreparedTaskExecution.definition_task_id", - landing_pr="PR 11", - audit_note="Identity collapses to task_id only", - ), - DeadPath( - symbol="PreparedTaskExecution.worker_type", - landing_pr="PR 11", - audit_note="Worker resolved from task.worker after PR 5", - ), - DeadPath( - symbol="PreparedTaskExecution.assigned_worker_slug", - landing_pr="PR 11", - audit_note="Worker resolved from task.worker after PR 5", - ), - # --- Run-graph column collapses --- - DeadPath( - symbol="parent_node_id", - landing_pr="PR 11", - audit_note="Renamed to parent_task_id in PR 11 schema reset", - ), - DeadPath( - symbol="source_node_id", - landing_pr="PR 11", - audit_note="Renamed to source_task_id in PR 11 schema reset", - ), - DeadPath( - symbol="target_node_id", - landing_pr="PR 11", - audit_note="Renamed to target_task_id in PR 11 schema reset", - ), -) - - -# Only symbols that have at least one production caller TODAY appear -# here. Currently-callerless symbols listed in `DEAD_PATHS` (e.g. names -# not yet present in v1, names a later PR introduces, or names already -# cleaned up) are still asserted every run — they just pass without -# xfail. When a transitional symbol is *introduced* by a later PR -# (e.g. `_DetachableSandboxBridge` in PR 4), that PR's flip step adds -# the corresponding entry; the deletion PR removes it. -_XFAIL_BY_SYMBOL: dict[str, str] = { - "CriterionExecutor": "PR 11: Protocol pair deleted", - "InngestCriterionExecutor": "PR 11: Protocol pair deleted", - "_prepare_definition": "PR 3: renamed to _prepare_legacy_definition", - "terminate_sandbox_by_id": "PR 11: orchestrator owns release", - "_persist_single_sample_workflow_definition": "PR 8: CLI uses persist_definition", - # Inngest jobs absorbed into worker_execute by PR 4; deleted by PR 11 - "execute_task": "PR 11: worker_execute is the orchestrator", - "sandbox_setup": "PR 11: worker_execute acquires inline", - "persist_outputs": "PR 11: worker_execute persists inline", - # Registry/definition layer retired by PR 11 - "ComponentCatalogService": "PR 11: object-bound task replaces registry resolution", - "DefinitionRepository": "PR 11: runtime reads run-tier only", - # Schema column renames - "parent_node_id": "PR 11: renamed to parent_task_id", - "source_node_id": "PR 11: renamed to source_task_id", - "target_node_id": "PR 11: renamed to target_task_id", -} - - -def _cases() -> list: - cases = [] - for dp in DEAD_PATHS: - marks = [] - reason = _XFAIL_BY_SYMBOL.get(dp.symbol) - if reason is not None: - marks.append(pytest.mark.xfail(reason=reason, strict=True)) - cases.append(pytest.param(dp, marks=marks, id=dp.symbol)) - return cases - - -@pytest.mark.parametrize("dead_path", _cases()) -def test_dead_path_has_no_production_callers(dead_path: DeadPath) -> None: - callers = _grep_production_callers(dead_path.symbol) - assert callers == [], ( - f"{dead_path.symbol!r} is on the deletion list " - f"(lands {dead_path.landing_pr}; audit note: {dead_path.audit_note}) " - f"but still has production callers: {callers}" - ) -``` - -- [ ] **Step 2: Run the guard** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_dead_path_audit.py -q -``` - -Expected: every case is XFAIL on PR 0. As later PRs delete the last -caller of each symbol, they remove that symbol's entry from -`_XFAIL_BY_SYMBOL` in the same commit. - -## Task 4: Add No-Type-Circumventors Guard ✅ - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_no_type_circumventors.py` - -`getattr(obj, "x", default)` and `hasattr(obj, "x")` are banned in -production code per [`07-test-strategy.md` § 0.6](../07-test-strategy.md). -The guard greps production trees and allows hits only when accompanied -by a `# typing: ...` exemption comment on the same or previous line, or -listed in `_KNOWN_EXEMPTIONS` with a landing PR. - -- [ ] **Step 1: Write the guard** - -```python -"""Ban getattr/hasattr in production code. - -See 07-test-strategy.md § 0.6 for the policy. Exemptions live in -`_KNOWN_EXEMPTIONS`; lines with a `# typing: ...` comment immediately -above or on the same line are allowlisted (the comment names the -exemption category — see the policy doc for the allowed categories). -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOTS = ( - ROOT / "ergon_core" / "ergon_core", - ROOT / "ergon_builtins" / "ergon_builtins", - ROOT / "ergon_cli" / "ergon_cli", -) -EXEMPT_PARTS: frozenset[str] = frozenset({"tests", "migrations", "__pycache__"}) - -_GETATTR_RE = re.compile(r"\bgetattr\s*\(") -_HASATTR_RE = re.compile(r"\bhasattr\s*\(") -# Two exemption forms coexist: -# - `# typing: <category>` — the v2 convention introduced in § 0.6. -# New code should use this. -# - `# slopcop: ignore[no-hasattr-getattr]` — the existing project -# convention. Recognized so this guard composes with the broader -# slopcop linter without forcing a tree-wide retrofit. -_EXEMPTION_RE = re.compile( - r"#\s*(?:typing:|slopcop:\s*ignore\[no-hasattr-getattr\])" -) - - -@dataclass(frozen=True) -class Violation: - path: str - lineno: int - line: str - pattern: str - - -def _scan_file(path: Path) -> list[Violation]: - violations: list[Violation] = [] - lines = path.read_text().splitlines() - for i, line in enumerate(lines): - for pat_name, pat in (("getattr", _GETATTR_RE), ("hasattr", _HASATTR_RE)): - if not pat.search(line): - continue - # Same-line typing: comment exempts it. - if _EXEMPTION_RE.search(line): - continue - # Previous-line typing: comment exempts it too. - if i > 0 and _EXEMPTION_RE.search(lines[i - 1]): - continue - violations.append( - Violation( - path=str(path.relative_to(ROOT)), - lineno=i + 1, - line=line.strip(), - pattern=pat_name, - ) - ) - return violations - - -def _all_violations() -> list[Violation]: - out: list[Violation] = [] - for root in PRODUCTION_ROOTS: - for path in root.rglob("*.py"): - if EXEMPT_PARTS.intersection(path.parts): - continue - out.extend(_scan_file(path)) - return out - - -# Lines that are known violators today and have a landing PR for the -# fix. Mirrors PR 0's _XFAIL_BY_NAME shape — each entry is "(relpath, -# lineno_marker_substring)": "PR N: <reason>". PR 11 asserts the dict -# is empty. -_KNOWN_EXEMPTIONS: dict[tuple[str, str], str] = { - # Populate from the first run; one row per current violator with - # the PR that fixes it. -} - - -def test_no_unexpected_type_circumventors() -> None: - violations = _all_violations() - unexpected: list[Violation] = [] - for v in violations: - key = (v.path, v.line) - # Allow if explicitly listed. - if any( - v.path == k_path and k_marker in v.line - for (k_path, k_marker) in _KNOWN_EXEMPTIONS - ): - continue - unexpected.append(v) - assert unexpected == [], "\n".join( - f"{v.path}:{v.lineno} {v.pattern} {v.line}" for v in unexpected - ) -``` - -- [ ] **Step 2: Run the guard and populate `_KNOWN_EXEMPTIONS`** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_no_type_circumventors.py -q -``` - -The first run lists every current violator. For each, either: - -- Add a `# typing: <category>` comment to the source line (legitimate - exemption — only `dynamic qualname walk`, `<library> SDK boundary`, - or another approved category from § 0.6). -- Add an entry to `_KNOWN_EXEMPTIONS` with the landing PR (every - v2 fix is scheduled in this commit's sibling PRs). - -After this PR lands, the guard passes; later PRs remove entries as -they land their fixes; PR 11 asserts `_KNOWN_EXEMPTIONS == {}` (only -the legitimate `# typing:`-annotated exemptions remain). - -## Task 5: Commit - -- [ ] **Step 1: Stage and commit all four ledgers** - -```bash -git add ergon_core/tests/unit/architecture/test_v2_transition_ledger.py \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_dead_path_audit.py \ - ergon_core/tests/unit/architecture/test_no_type_circumventors.py \ - docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan -git commit -m "test: add v2 architecture ledgers (transition, final-state, dead-path, no-type-circumventors)" -``` - -## PR Ledger - -Invariant landed: old paths are tracked; v2 final-state invariants are -executable; v1 dead paths have callerless guards. - -Bridge code introduced: none. - -Old path still intentionally alive: all old paths listed in -`test_v2_transition_ledger.py`. - -Deletion gate: PR 11 replaces this ledger with deleted-symbol guards. The -final-state ledger and dead-path audit reach zero xfails at PR 11 — PR 11 -verifies this explicitly (see PR 11 § "Verify No XFails Remain"). - -Tests added or updated: `test_v2_transition_ledger.py`, -`test_v2_final_state_ledger.py`, `test_dead_path_audit.py`. - -Modules owned by this PR: architecture tests and docs only. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01b-pr-0-5-repository-standard.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01b-pr-0-5-repository-standard.md deleted file mode 100644 index b4a7bffb9..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/01b-pr-0-5-repository-standard.md +++ /dev/null @@ -1,669 +0,0 @@ -# PR 0.5 — Repository Layer Standard And Guards - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the repository layer's shape explicit and enforceable, so -the v2 deletions and the post-v2 cleanup work against a known standard -rather than five subtly different conventions. - -**Architecture:** Three architecture-guard tests encode the repository -layer's conventions: file/class naming, sync/async split, transaction -ownership, error-typing, contract location, and dead-method audit. PRs -that touch a violator flip the corresponding xfail to green; no -violator stays unfixed past its scheduled landing PR. - -**Tech Stack:** pytest parametrized guards, source-text architecture -checks, the same `xfail(strict=True)` ledger pattern PR 0 introduced. - -This PR sits between PR 0 (transition ledger) and PR 1 (run-tier task -snapshot) because the conventions it pins are load-bearing for every -later PR that touches a repository — particularly PR 2's typed -`graph_repo.node`, PR 4's `WorkerOutputRepository`, and PR 7's -collapsed read paths. - ---- - -## Files - -**Create:** - -```text -ergon_core/tests/unit/architecture/test_repository_layer_conventions.py -ergon_core/tests/unit/architecture/test_repository_companion_files.py -ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py -``` - -**Modify:** - -```text -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md -``` - -## Current State - -Five repository packages, three different shapes: - -| Package | Repo file | errors.py | Notes | -|---|---|---|---| -| `core/application/tasks/` | `repository.py` | ✅ | clean | -| `core/application/experiments/` | `repository.py` | ❌ raises `ValueError` directly | typed-error gap | -| `core/application/resources/` | `repository.py` | ❌ no raises today | OK | -| `core/application/graph/` | `repository.py` | ✅ | clean (35+ methods) | -| `core/persistence/telemetry/` | **`repositories.py` (plural!)** | ❌ | + `CreateTaskEvaluation` DTO leaks into the repo file | - -Additional patterns observed: - -- `WorkflowGraphRepository` enforces "writes async, reads sync" by - convention but nothing pins it. -- `experiments/repository.py` raises `ValueError("ExperimentDefinitionTask - {id} not found")` — generic exception, callers can't catch specifically. -- `telemetry/repositories.py` defines `class CreateTaskEvaluation(BaseModel)` - inline at line 12 — a request DTO living in the repo file instead of - `telemetry/models.py`. -- Several public methods on `TaskExecutionRepository` (e.g. - `latest_for_definition_task`, `task_payload_for_execution`, - `next_attempt_for_definition_task`) may not have callers after the v2 - cutovers land, but nothing catches "method has no production callers" - today — the same audit failure mode v1 had with `Worker.from_buffer`. - -## Target State For This PR - -Three guard test files land. Each enforces one slice of the standard: - -1. **Conventions** — class naming, file naming, method signatures, - sync/async split, no `session.commit()` in repo methods, no - `core.infrastructure` imports. -2. **Companion files** — `errors.py` exists iff package raises; - DTO classes don't live in `repository.py`. -3. **Dead-method audit** — every public Repository method has at least - one non-test caller, with an xfail allowlist mirror of PR 0's - dead-path audit pattern. - -Current violators are xfailed with `strict=True`, keyed to the PR that -fixes each: - -- PR 1: rename `telemetry/repositories.py` → `telemetry/repository.py`. -- PR 4: move `CreateTaskEvaluation` from `telemetry/repository.py` to - `telemetry/models.py`. -- PR 7: add `experiments/errors.py` with typed exceptions - (`DefinitionNotFoundError`, `InstanceNotFoundError`), replace - `ValueError` raises. -- PR 11: deletes methods on `TaskExecutionRepository` whose callers go - away during the v2 cutover. - -## Task 1: Land The Repository Layer Standard - -**Files:** - -- Modify: `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md` - -- [x] **Step 1: Add the standard section** - -Add a new section to `07-test-strategy.md` after § 0 documenting the -10 rules — see Task 1 Step 2 of `07-test-strategy.md` update. The -section is the canonical reference the guard tests cite in error -messages. - -## Task 2: Add `test_repository_layer_conventions.py` - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_repository_layer_conventions.py` - -- [x] **Step 1: Write the guard file** - -```python -"""Repository layer conventions — see 07-test-strategy.md § Repository -layer standard. - -For every Repository class discovered in `ergon_core`, enforce: - - Class name ends with `Repository`. - - Containing file is `repository.py` (singular). - - Public methods take `session` as the first non-self positional arg. - - Write methods are async; read methods may be sync OR async (async - is required only when the method performs genuine I/O like a live - sandbox attach — `graph_repo.node` is the textbook case). - - No `session.commit(` calls inside repository methods (transactions - belong to the caller). - - The repository module does not import from - `ergon_core.core.infrastructure.*` (keeps the data layer framework- - agnostic — see graph/errors.py for the rationale). -""" - -from __future__ import annotations - -import importlib -import inspect -import pkgutil -from collections.abc import Iterator -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOT = ROOT / "ergon_core" / "ergon_core" - - -_WRITE_PREFIXES = ( - "add_", "update_", "remove_", "delete_", "set_", - "create_", "append_", "insert_", "persist_", -) - - -def _discover_repository_classes() -> list[type]: - """Walk `ergon_core` and import every `*/repository.py` and - `*/repositories.py`, returning the classes whose name ends in - `Repository`.""" - - pkg_root = PRODUCTION_ROOT / "core" - classes: list[type] = [] - for path in pkg_root.rglob("repository.py"): - classes.extend(_load_classes_from_file(path)) - # NB: `repositories.py` (plural) is a known violator captured by - # the companion-files guard; we still load it here so its classes - # get audited. - for path in pkg_root.rglob("repositories.py"): - classes.extend(_load_classes_from_file(path)) - return classes - - -def _load_classes_from_file(path: Path) -> list[type]: - rel = path.relative_to(ROOT / "ergon_core") - module_name = ".".join(rel.with_suffix("").parts) - # Strip the leading "ergon_core" since it's already the package root. - if module_name.startswith("ergon_core."): - module_name = module_name - module = importlib.import_module(module_name) - return [ - cls - for name, cls in inspect.getmembers(module, inspect.isclass) - if name.endswith("Repository") and cls.__module__ == module.__name__ - ] - - -REPO_CLASSES: list[type] = _discover_repository_classes() - - -def _is_write_method(name: str) -> bool: - return any(name.startswith(p) for p in _WRITE_PREFIXES) - - -def _public_methods(cls: type) -> Iterator[tuple[str, object]]: - for name, method in inspect.getmembers(cls, inspect.isfunction): - if name.startswith("_"): - continue - yield name, method - - -# --- Naming ------------------------------------------------------------ - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_repository_class_name_ends_in_Repository(cls: type) -> None: - assert cls.__name__.endswith("Repository"), ( - f"{cls.__name__} must end in 'Repository' per the repository " - "layer standard" - ) - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_repository_file_is_singular(cls: type) -> None: - """File must be `repository.py`, not `repositories.py`.""" - - filename = Path(inspect.getfile(cls)).name - assert filename == "repository.py", ( - f"{cls.__name__} lives in {filename}; the standard requires " - "`repository.py` (singular) even when multiple Repository " - "classes share the file." - ) - - -# --- Method signatures ------------------------------------------------- - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_public_methods_take_session_first(cls: type) -> None: - offenders: list[str] = [] - for name, method in _public_methods(cls): - params = list(inspect.signature(method).parameters) - # Skip the implicit `self` / `cls`. - non_self = params[1:] if params and params[0] in {"self", "cls"} else params - if not non_self or non_self[0] != "session": - offenders.append(f"{cls.__name__}.{name}({params!r})") - assert offenders == [], ( - "Repository methods must take `session` as the first non-self " - f"positional arg. Offenders: {offenders}" - ) - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_write_methods_are_async(cls: type) -> None: - """Methods named with a write prefix (`add_`, `update_`, etc.) must - be async. Reads may be sync OR async — async reads are only required - when the method performs genuine I/O (e.g. `graph_repo.node` after - PR 5 attaches a live sandbox).""" - - offenders: list[str] = [] - for name, method in _public_methods(cls): - if _is_write_method(name) and not inspect.iscoroutinefunction(method): - offenders.append(f"{cls.__name__}.{name}") - assert offenders == [], ( - "Write methods must be async (the naming heuristic catches the " - f"common write prefixes). Offenders: {offenders}" - ) - - -# --- Transaction ownership -------------------------------------------- - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_no_commit_inside_repository(cls: type) -> None: - src = inspect.getsource(cls) - assert "session.commit(" not in src, ( - f"{cls.__name__} commits inside the repository — transactions " - "belong to the caller (the service / job body), not the data " - "layer. Remove the `session.commit()` call and let the caller " - "decide when to commit." - ) - - -# --- Layer boundary --------------------------------------------------- - - -@pytest.mark.parametrize("cls", REPO_CLASSES, ids=lambda c: c.__name__) -def test_repository_does_not_import_infrastructure(cls: type) -> None: - """The data layer must stay framework-agnostic so it can be reused - in training pipelines, replay systems, and test harnesses that - don't run inside Inngest. See graph/errors.py docstring for the - rationale.""" - - src = Path(inspect.getfile(cls)).read_text() - assert "ergon_core.core.infrastructure" not in src, ( - f"{cls.__name__}'s module imports from " - "`ergon_core.core.infrastructure`. Repositories must stay " - "framework-agnostic." - ) -``` - -- [x] **Step 2: Run the guard** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_repository_layer_conventions.py -q -``` - -Expected: every case PASS for current repositories EXCEPT the -`test_repository_file_is_singular` case for `TelemetryRepository`, -which fails because it lives in `repositories.py`. Wrap that single -case as `xfail(strict=True, reason="PR 1: telemetry rename")` (see -Task 5 below). - -## Task 3: Add `test_repository_companion_files.py` - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` - -- [x] **Step 1: Write the guard file** - -```python -"""Repository package companion files — see 07-test-strategy.md § -Repository layer standard. - -For every package containing a Repository: - - If any `.py` file other than `errors.py` contains a `raise` line, - the package must have an `errors.py` carrying typed exceptions. - Generic `raise ValueError(...)` in a repository is a typed-error - gap. - - `repository.py` must not define Pydantic BaseModel DTOs (request - or response shapes). Those belong in `models.py`. SQLModel tables - are not BaseModel DTOs — they live in `models.py` anyway. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOT = ROOT / "ergon_core" / "ergon_core" - - -def _discover_repo_packages() -> list[Path]: - packages: list[Path] = [] - for path in (PRODUCTION_ROOT / "core").rglob("repository.py"): - packages.append(path.parent) - for path in (PRODUCTION_ROOT / "core").rglob("repositories.py"): - packages.append(path.parent) - return sorted(set(packages)) - - -REPO_PACKAGES = _discover_repo_packages() - - -_RAISE_RE = re.compile(r"^\s*raise\s+\w", re.MULTILINE) -_BASEMODEL_DEF_RE = re.compile( - r"^class\s+\w+\s*\(\s*BaseModel\b", - re.MULTILINE, -) - - -@pytest.mark.parametrize( - "pkg", REPO_PACKAGES, ids=lambda p: p.relative_to(ROOT).as_posix() -) -def test_package_has_errors_py_if_it_raises(pkg: Path) -> None: - """If anything in the package `raise`s, `errors.py` must exist with - typed exception classes — even one custom exception is better than a - generic `ValueError` because callers can catch it.""" - - raisers: list[str] = [] - for path in pkg.glob("*.py"): - if path.name in {"errors.py", "__init__.py"}: - continue - if _RAISE_RE.search(path.read_text()): - raisers.append(path.name) - if not raisers: - return # no raises, no errors.py needed - assert (pkg / "errors.py").exists(), ( - f"{pkg.relative_to(ROOT)} has raises in {raisers} but no " - "errors.py. Add typed exception classes so callers can catch " - "specifically rather than catching generic ValueError." - ) - - -@pytest.mark.parametrize( - "pkg", REPO_PACKAGES, ids=lambda p: p.relative_to(ROOT).as_posix() -) -def test_repository_file_does_not_define_dtos(pkg: Path) -> None: - """Pydantic DTOs (request / response shapes) live in `models.py`, - not in `repository.py`. SQLModel tables are exempt — they live in - `models.py` anyway by convention, and a BaseModel definition that - extends a SQLModel is unusual enough to flag if it appears in - `repository.py`.""" - - for repo_file in ("repository.py", "repositories.py"): - path = pkg / repo_file - if not path.exists(): - continue - matches = _BASEMODEL_DEF_RE.findall(path.read_text()) - assert matches == [], ( - f"{path.relative_to(ROOT)} defines Pydantic BaseModel " - f"classes inline ({matches}); move them to " - f"{pkg.relative_to(ROOT)}/models.py." - ) -``` - -- [x] **Step 2: Run the guard** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_repository_companion_files.py -q -``` - -Expected violators (each gets an xfail marker in Step 3): - -- `experiments/`: `test_package_has_errors_py_if_it_raises` fails — - `repository.py` raises `ValueError` without an `errors.py`. -- `telemetry/`: `test_repository_file_does_not_define_dtos` fails — - `CreateTaskEvaluation` lives in `repositories.py`. - -- [x] **Step 3: Add the xfail markers** - -Wrap each violator case via parametrize-id targeting: - -```python -_PACKAGE_XFAILS = { - ("core/application/experiments", "test_package_has_errors_py_if_it_raises"): - "PR 7: add experiments/errors.py with DefinitionNotFoundError etc.", - ("core/persistence/telemetry", "test_repository_file_does_not_define_dtos"): - "PR 4: move CreateTaskEvaluation to telemetry/models.py", -} -``` - -Implement the marker application via a pytest hook in the same file -(see PR 0's `_XFAIL_BY_NAME` pattern for the shape). - -## Task 4: Add `test_no_dead_repository_methods.py` - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py` - -- [x] **Step 1: Write the guard file** - -```python -"""Dead-method audit on the repository layer. - -Every public method on every Repository class must have at least one -production caller (non-test file). The audit is scoped tightly to the -repository layer rather than tree-wide because that's where v1's audit -found real dead helpers — going wider has too much false-positive noise -from Inngest registration, Pydantic decorators, etc. - -Methods that are PR-scheduled to land callers later (or to be deleted) -are listed in `_KNOWN_UNUSED_FOR_NOW`, mirroring PR 0's -`_XFAIL_BY_SYMBOL` pattern in test_dead_path_audit.py. -""" - -from __future__ import annotations - -import importlib -import inspect -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOTS = ( - ROOT / "ergon_core" / "ergon_core", - ROOT / "ergon_builtins" / "ergon_builtins", - ROOT / "ergon_cli" / "ergon_cli", -) -EXEMPT_PARTS: frozenset[str] = frozenset({"tests", "migrations", "__pycache__"}) - - -def _discover_repository_classes() -> list[type]: - # Re-use the same discovery as test_repository_layer_conventions — - # in practice, import the helper from there. - from ergon_core.tests.unit.architecture import test_repository_layer_conventions as conv - - return conv.REPO_CLASSES - - -def _all_public_methods() -> list[tuple[type, str]]: - methods: list[tuple[type, str]] = [] - for cls in _discover_repository_classes(): - for name, _ in inspect.getmembers(cls, inspect.isfunction): - if not name.startswith("_"): - methods.append((cls, name)) - return methods - - -def _grep_production_callers( - pattern: str, exclude_path: Path | None = None -) -> list[str]: - hits: list[str] = [] - for root in PRODUCTION_ROOTS: - for path in root.rglob("*.py"): - if EXEMPT_PARTS.intersection(path.parts): - continue - if exclude_path is not None and path == exclude_path: - continue - if pattern in path.read_text(): - hits.append(str(path.relative_to(ROOT))) - return sorted(hits) - - -# Methods that have no production callers AND are expected to land one -# (or be deleted) by a specific PR. Mirrors PR 0's _XFAIL_BY_SYMBOL. -_KNOWN_UNUSED_FOR_NOW: dict[tuple[str, str], str] = { - # Populate with the audit findings from the first run. - # ("TaskExecutionRepository", "latest_for_definition_task"): "PR 11: callers go away with the cutover", - # ("TaskExecutionRepository", "next_attempt_for_definition_task"): "PR 11: callers go away with the cutover", -} - - -def _cases() -> list: - cases = [] - for cls, method in _all_public_methods(): - marks = [] - key = (cls.__name__, method) - reason = _KNOWN_UNUSED_FOR_NOW.get(key) - if reason is not None: - marks.append(pytest.mark.xfail(reason=reason, strict=True)) - cases.append( - pytest.param(cls, method, marks=marks, id=f"{cls.__name__}.{method}") - ) - return cases - - -@pytest.mark.parametrize("cls,method", _cases()) -def test_repository_method_has_a_production_caller( - cls: type, method: str -) -> None: - """Pattern `.method(` catches the common call form. Defining-file is - excluded so the method's own definition doesn't count as its caller.""" - - defining_file = Path(inspect.getfile(cls)) - callers = _grep_production_callers( - f".{method}(", exclude_path=defining_file - ) - assert callers, ( - f"{cls.__name__}.{method} has no production callers. " - "Either add a caller, delete the method, or add to " - "_KNOWN_UNUSED_FOR_NOW with the landing PR." - ) -``` - -- [ ] **Step 2: Run the guard and populate the allowlist** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py -q -``` - -Expected: a handful of FAILs surface today. Add each to -`_KNOWN_UNUSED_FOR_NOW` with the landing PR that either ships a caller -or deletes the method. PR 11's structural simplification audit (§ Task 5 -of `12-pr-11-deletion-final-schema.md`) is the natural deletion gate -for most of these. - -Run again — all FAILs should now be XFAIL. The completion bar mirrors -the other ledgers: PR 11 asserts `_KNOWN_UNUSED_FOR_NOW == {}`. - -## Task 5: Register Initial XFails For Today's Violators - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_repository_layer_conventions.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` - -- [ ] **Step 1: Map known violators to landing PRs** - -```python -_KNOWN_VIOLATORS: dict[tuple[str, str], str] = { - # (test_name, parametrize_id) -> "<PR>: <fix>" - ("test_repository_file_is_singular", "TelemetryRepository"): - "PR 1: rename telemetry/repositories.py -> telemetry/repository.py", - ("test_package_has_errors_py_if_it_raises", - "ergon_core/ergon_core/core/application/experiments"): - "PR 7: add experiments/errors.py with typed DefinitionNotFoundError " - "and replace ValueError raises", - ("test_repository_file_does_not_define_dtos", - "ergon_core/ergon_core/core/persistence/telemetry"): - "PR 4: move CreateTaskEvaluation to telemetry/models.py", -} -``` - -- [ ] **Step 2: Apply via `pytest_collection_modifyitems`** - -In each of the two test files, add: - -```python -def pytest_collection_modifyitems(config, items): # noqa: ARG001 - for item in items: - # item.name is "test_name[param_id]"; split it. - if "[" not in item.name: - continue - test_name, _, param = item.name.partition("[") - param = param.rstrip("]") - reason = _KNOWN_VIOLATORS.get((test_name, param)) - if reason is not None: - item.add_marker(pytest.mark.xfail(reason=reason, strict=True)) -``` - -- [ ] **Step 3: Verify ledger state** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_repository_layer_conventions.py \ - ergon_core/tests/unit/architecture/test_repository_companion_files.py \ - ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py -q -``` - -Expected: all cases PASS or XFAIL. Zero FAIL, zero XPASS. - -## Task 6: Add Completion-Bar Assertions - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_repository_layer_conventions.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` -- Modify: `ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py` - -Mirror PR 11's `test_no_*_are_still_xfailed` pattern. Each ledger gets a -guard that PR 11 will assert against: - -```python -def test_no_repository_violators_remain() -> None: - assert _KNOWN_VIOLATORS == {}, ( - f"Repository layer cleanup incomplete — violators remain: " - f"{sorted(_KNOWN_VIOLATORS)}" - ) - - -def test_no_dead_repository_methods_remain() -> None: - assert _KNOWN_UNUSED_FOR_NOW == {}, ( - f"Repository layer dead-method audit incomplete: " - f"{sorted(_KNOWN_UNUSED_FOR_NOW)}" - ) -``` - -These tests are themselves `xfail(strict=True)` until PR 11 lands — -PR 11 removes the xfail decorator on these two as part of its empty-the- -ledgers task. - -## Task 7: Commit - -```bash -git add ergon_core/tests/unit/architecture/test_repository_layer_conventions.py \ - ergon_core/tests/unit/architecture/test_repository_companion_files.py \ - ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py \ - docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/07-test-strategy.md \ - docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan -git commit -m "test: add repository layer standard guards" -``` - -## PR Ledger - -Invariant landed: the repository layer has a documented, enforced -standard; current violators are listed with their fix PR; the -dead-method audit is scoped to repositories with an xfail allowlist. - -Bridge code introduced: none. - -Old path still intentionally alive: `telemetry/repositories.py` (PR 1 -renames), `CreateTaskEvaluation` inline (PR 4 moves), `ValueError` -raises in `experiments/repository.py` (PR 7 replaces). - -Deletion gate: PR 11 empties `_KNOWN_VIOLATORS` and -`_KNOWN_UNUSED_FOR_NOW`, and removes the `xfail` decorators from -the completion-bar tests in this PR. - -Tests added or updated: three new architecture guards; references -added to `07-test-strategy.md`. - -Modules owned by this PR: architecture tests and docs only. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/02-pr-01-run-tier-task-snapshot.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/02-pr-01-run-tier-task-snapshot.md deleted file mode 100644 index 9d7852316..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/02-pr-01-run-tier-task-snapshot.md +++ /dev/null @@ -1,709 +0,0 @@ -# PR 1 — Run-Tier Task Snapshot Foundation - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Make each `run_graph_nodes` row capable of carrying a full task -snapshot while old runtime readers continue to work. - -**Architecture:** Add `task_json` and `is_dynamic` to run graph nodes. During -run preparation, copy definition task data into `task_json`. Do not remove -`definition_task_id`, `id`, `task_slug`, `description`, or -`assigned_worker_slug` in this PR. - -**Tech Stack:** SQLModel, Alembic additive migration, pytest runtime tests. - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/core/persistence/graph/models.py -ergon_core/ergon_core/core/application/graph/repository.py -ergon_core/ergon_core/core/application/workflows/service.py -ergon_core/ergon_core/core/application/jobs/start_workflow.py -ergon_core/tests/unit/runtime/test_graph_worker_identity.py -ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py -``` - -**Create:** - -```text -ergon_core/migrations/versions/<revision>_add_run_graph_task_json.py -ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py -ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py -ergon_core/tests/unit/runtime/test_identity_invariants.py -``` - -## Current State - -`RunGraphNode` stores dispatch fields: - -```python -id: UUID -run_id: UUID -definition_task_id: UUID | None -instance_key: str -task_slug: str -description: str -assigned_worker_slug: str | None -``` - -Static runtime reconstruction still follows `definition_task_id` back to -`ExperimentDefinitionTask`. Dynamic nodes have no definition row and use the -graph-native path. - -## Target State For This PR - -`RunGraphNode` additionally stores: - -```python -task_json: dict -is_dynamic: bool -``` - -Static nodes receive a snapshot copied from definition tables. Dynamic nodes -can receive a snapshot at creation time. Existing runtime code keeps reading -old fields until later PRs flip it. - -## Task 1: Add Schema Fields - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/graph/models.py` -- Create: `ergon_core/migrations/versions/<revision>_add_run_graph_task_json.py` - -- [x] **Step 1: Update `RunGraphNode`** - -Add imports: - -```python -from sqlalchemy import Boolean -``` - -Add fields after `description`: - -```python - task_json: dict = Field( - default_factory=dict, - sa_column=Column(JSON), - description=( - "Run-tier snapshot of the authored Task. Static nodes copy this " - "from experiment_definition_tasks at prepare-run time; dynamic " - "nodes write it directly at spawn time." - ), - ) - is_dynamic: bool = Field( - default=False, - sa_column=Column(Boolean, nullable=False, default=False), - description="True when this node was spawned during a run.", - ) -``` - -Keep `definition_task_id` unchanged. - -- [x] **Step 2: Add additive migration** - -Create a migration that runs: - -```python -def upgrade() -> None: - op.add_column( - "run_graph_nodes", - sa.Column("task_json", sa.JSON(), nullable=False, server_default="{}"), - ) - op.add_column( - "run_graph_nodes", - sa.Column("is_dynamic", sa.Boolean(), nullable=False, server_default=sa.false()), - ) - op.create_index( - "ix_run_graph_nodes_run_dynamic", - "run_graph_nodes", - ["run_id", "is_dynamic"], - ) - - -def downgrade() -> None: - op.drop_index("ix_run_graph_nodes_run_dynamic", table_name="run_graph_nodes") - op.drop_column("run_graph_nodes", "is_dynamic") - op.drop_column("run_graph_nodes", "task_json") -``` - -- [x] **Step 3: Run model import smoke** - -Run: - -```bash -uv run python -c "from ergon_core.core.persistence.graph.models import RunGraphNode; print(RunGraphNode.model_fields['task_json'].default_factory)" -``` - -Expected: prints a callable/default factory reference and exits 0. - -## Task 2: Copy Definition Task JSON During Prepare - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/graph/repository.py` - -- [x] **Step 1: Add helper near `_node_snapshot` helpers** - -```python -def _definition_task_snapshot( - task: ExperimentDefinitionTask, - *, - instance_key: str, - assigned_worker_slug: str | None, -) -> dict: - return { - "_type": "ergon_core.api.benchmark.task:TaskSpec", - "task_slug": task.task_slug, - "instance_key": instance_key, - "description": task.description, - "parent_task_slug": None, - "dependency_task_slugs": (), - "evaluator_binding_keys": (), - "task_payload": task.task_payload_json, - "_legacy": { - "assigned_worker_slug": assigned_worker_slug, - "definition_task_id": str(task.id), - }, - } -``` - -This is a bridge snapshot, not the final v2 `Task` JSON. PR 5 replaces the -payload shape once object-bound tasks exist. - -- [x] **Step 2: Populate `task_json` when creating static nodes** - -Replace the `RunGraphNode(...)` creation inside `initialize_from_definition` -with the same fields plus: - -```python -task_json=_definition_task_snapshot( - task, - instance_key=instance_key_by_id[task.instance_id], - assigned_worker_slug=worker_by_task.get(task.id), -), -is_dynamic=False, -``` - -- [x] **Step 3: Populate `task_json` in `add_node`** - -Extend `add_node` signature: - -```python - task_json: dict | None = None, - is_dynamic: bool = True, -``` - -Pass into `RunGraphNode(...)`: - -```python -task_json=task_json or {}, -is_dynamic=is_dynamic, -``` - -Existing callers of `add_node` become dynamic by default. If a static caller -exists outside `initialize_from_definition`, pass `is_dynamic=False` -explicitly. - -## Task 3: Add Focused Tests - -**Files:** - -- Create: `ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py` - -- [x] **Step 1: Write static-copy test** - -```python -def test_initialize_from_definition_copies_task_json(session, definition_factory): - definition = definition_factory(task_slug="solve", payload={"problem": "p"}) - repo = WorkflowGraphRepository() - - graph = repo.initialize_from_definition( - session, - run_id=uuid4(), - definition_id=definition.definition_id, - initial_node_status="pending", - initial_edge_status="pending", - task_payload_model=EmptyTaskPayload, - meta=MutationMeta(actor="test", reason="snapshot"), - ) - - row = session.get(RunGraphNode, graph.nodes[0].id) - assert row is not None - assert row.task_json["task_slug"] == "solve" - assert row.task_json["task_payload"] == {"problem": "p"} - assert row.is_dynamic is False -``` - -- [x] **Step 2: Write dynamic insert test** - -```python -@pytest.mark.asyncio -async def test_add_node_can_write_dynamic_task_json(session): - repo = WorkflowGraphRepository() - run_id = uuid4() - payload = { - "_type": "ergon_core.api.benchmark.task:Task", - "task_slug": "child", - "description": "child task", - } - - node = await repo.add_node( - session, - run_id, - task_slug="child", - instance_key="sample-1", - description="child task", - status="pending", - task_json=payload, - is_dynamic=True, - meta=MutationMeta(actor="test", reason="dynamic"), - ) - - row = session.get(RunGraphNode, node.id) - assert row is not None - assert row.task_json == payload - assert row.is_dynamic is True -``` - -- [x] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py -q -``` - -## Task 4: Add Walkthrough Smoketest Skeleton - -**Files:** - -- Create: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` - -The walkthrough smoketest is the **observable-effect** counterpart to the -xfail ledgers from PR 0. Each test drives a public entry point (e.g. -`persist_definition`, `prepare_run`, `worker_execute`) and asserts the -database / event-bus state the spec requires. PR 1 lands the skeleton -plus one passing case (the PR 1 invariant). Every other case is xfail -with strict=True until its landing PR flips it green. - -These are explicitly **not** call-graph mock tests — they observe outcomes -(rows in tables, events on the bus), not call topology. The walkthrough -integration test in PR 12 is the heavier, multi-variant version of the -same pattern; this skeleton is the unit-level seed. - -- [x] **Step 1: Write the smoketest skeleton** - -```python -"""Observable-effect smoketests for the v2 happy path. - -Each test drives a public entry point and asserts database / event state -that the spec requires. NOT call-graph tests — outcomes only. See -07-test-strategy.md § "Why effect-based smoketests, not call-graph mocks". - -Tests for invariants that have not landed are xfail(strict=True) with the -landing PR in the reason. Removing the decorator is the landing-PR signal. -""" - -from __future__ import annotations - -import pytest - -pytestmark = pytest.mark.asyncio - - -async def test_prepare_run_populates_task_json_for_every_node( - session, persisted_definition -): - """PR 1 invariant — must be GREEN after PR 1 lands. - - Every run_graph_nodes row produced by prepare_run carries a non-empty - task_json snapshot. - """ - - from sqlmodel import select - - from ergon_core.core.application.workflows.service import prepare_run - from ergon_core.core.persistence.graph.models import RunGraphNode - - run_id = await prepare_run(session, definition_id=persisted_definition.id) - rows = session.exec( - select(RunGraphNode).where(RunGraphNode.run_id == run_id) - ).all() - - assert rows, "prepare_run produced no nodes" - assert all(row.task_json for row in rows), ( - "every node must carry a self-contained task snapshot" - ) - assert all(row.task_json.get("task_slug") for row in rows) - - -@pytest.mark.xfail( - reason="PR 7: persist_definition collapses ExperimentRecord onto definitions", - strict=True, -) -async def test_persist_definition_writes_only_intended_tables( - session, sample_experiment -): - """PR 7 invariant: persist_definition writes experiment_definitions - plus experiment_definition_tasks. No write to ExperimentRecord, no - write to saved_specs.""" - - from sqlmodel import select - - from ergon_core.core.application.experiments.definition_writer import ( - persist_definition, - ) - from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinition, - ExperimentDefinitionTask, - ) - - persist_definition(session, sample_experiment) - - defs = session.exec(select(ExperimentDefinition)).all() - tasks = session.exec(select(ExperimentDefinitionTask)).all() - assert len(defs) == 1 - assert len(tasks) == len(sample_experiment.benchmark.tasks) - - -@pytest.mark.xfail( - reason="PR 3: worker_execute reads task from run_graph_nodes only", - strict=True, -) -async def test_worker_execute_reads_task_from_run_tier_only( - session, prepared_run, inngest_driver -): - """PR 3 invariant: worker_execute touches no definition-tier table. - - Observed by spying the session for query targets — not by mocking - call paths. The fixture `prepared_run` is added in PR 3 alongside - this assertion turning green. - """ - - pytest.fail("requires PR 3's worker-execute cutover and prepared_run fixture") - - -@pytest.mark.xfail( - reason="PR 4: synchronous fanout via ctx.step.invoke", - strict=True, -) -async def test_worker_execute_emits_one_evaluate_invocation_per_evaluator( - inngest_driver, run_with_two_evaluators -): - """PR 4 invariant: synchronous fanout via ctx.step.invoke.""" - - pytest.fail("requires PR 4's fanout shape") - - -@pytest.mark.xfail( - reason="PR 4: TaskEvaluateRequest is the thin id-only payload", - strict=True, -) -async def test_evaluate_task_run_payload_is_id_only( - inngest_driver, run_with_one_evaluator -): - """PR 4 invariant: TaskEvaluateRequest has exactly four fields: - run_id, task_id, execution_id, evaluator_index.""" - - pytest.fail("requires PR 4's TaskEvaluateRequest") - - -@pytest.mark.xfail( - reason="PR 4: orchestrator try/finally bounds sandbox lifetime through gather", - strict=True, -) -async def test_sandbox_release_happens_after_all_evaluators_complete( - inngest_driver, run_with_two_evaluators -): - """Δ.5: orchestrator's try/finally bounds sandbox lifetime through gather.""" - - pytest.fail("requires PR 4's lifecycle ownership") - - -@pytest.mark.xfail( - reason="PR 9: dynamic subtasks write only to run_graph_nodes", - strict=True, -) -async def test_dynamic_spawn_writes_only_to_run_graph_nodes( - session, running_run, parent_task_context -): - """Δ.3 / PR 9 invariant: dynamic subtasks are graph-native.""" - - pytest.fail("requires PR 9's graph-native dynamic spawn") - - -@pytest.mark.xfail( - reason="PR 11: full v2 lifecycle — every acquire has a release", - strict=True, -) -async def test_run_completion_releases_every_acquired_sandbox( - inngest_driver, run_with_three_tasks -): - """CLAUDE.md guardrail: every sandbox acquire has a release.""" - - pytest.fail("requires the full v2 lifecycle shape") -``` - -The fixtures `persisted_definition`, `sample_experiment`, `prepared_run`, -`inngest_driver`, `run_with_two_evaluators`, etc. are introduced by the -PR that flips the corresponding test green. PR 1 only needs -`persisted_definition` and `session`; the rest are added by their -landing PR. - -- [x] **Step 2: Add the minimal PR 1 fixtures** - -In `ergon_core/tests/unit/runtime/conftest.py` (extend if it exists): - -```python -import pytest -from sqlmodel import Session, SQLModel, create_engine - -from ergon_core.core.application.experiments.definition_writer import ( - persist_definition, -) - - -@pytest.fixture() -def session(): - engine = create_engine("sqlite:///:memory:") - SQLModel.metadata.create_all(engine) - with Session(engine) as session: - yield session - - -@pytest.fixture() -def persisted_definition(session, sample_experiment): - return persist_definition(session, sample_experiment) -``` - -The `sample_experiment` fixture lives where the existing benchmark test -fixtures live; reuse the simplest in-tree benchmark that builds a -non-empty graph. - -- [x] **Step 3: Run the smoketest** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py -q -``` - -Expected on PR 1: one PASS -(`test_prepare_run_populates_task_json_for_every_node`), seven XFAIL, -zero FAIL, zero XPASS. - -## Task 5: Add Identity Invariants Skeleton - -**Files:** - -- Create: `ergon_core/tests/unit/runtime/test_identity_invariants.py` - -Tests the identity-flow invariants from -[`02-persistence-layer.md`](../02-persistence-layer.md) §2: `task_id` is -born once and flows unchanged, `(run_id, task_id)` is the canonical row -key, `execution_id` is the per-attempt id, sandbox identity is preserved -across the worker → evaluate Inngest boundary. - -The PR 1 invariant (task_id flows from definition tasks into -`run_graph_nodes`) is green; later invariants xfail until their landing -PR. - -- [x] **Step 1: Write the skeleton** - -```python -"""Identity-flow invariants from 02-persistence-layer.md §2. - -task_id is born once and flows unchanged. (run_id, task_id) is the -canonical row key. execution_id is the per-attempt id. Sandbox identity -is preserved across the worker → evaluate Inngest boundary. - -These are observable-effect tests, not call-graph tests. -""" - -from __future__ import annotations - -import pytest - -pytestmark = pytest.mark.asyncio - - -async def test_task_id_is_preserved_from_definition_to_run_tier( - session, persisted_definition -): - """PR 1 invariant: the same UUID flows from - experiment_definition_tasks → run_graph_nodes.task_id (via - definition_task_id during the transition; PR 11 collapses to task_id). - """ - - from sqlmodel import select - - from ergon_core.core.application.workflows.service import prepare_run - from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinitionTask, - ) - from ergon_core.core.persistence.graph.models import RunGraphNode - - defn_tasks = session.exec( - select(ExperimentDefinitionTask).where( - ExperimentDefinitionTask.experiment_definition_id - == persisted_definition.id - ) - ).all() - defn_task_ids = {t.id for t in defn_tasks} - - run_id = await prepare_run(session, definition_id=persisted_definition.id) - nodes = session.exec( - select(RunGraphNode).where(RunGraphNode.run_id == run_id) - ).all() - - # During the transition, identity lives in either `id` (run-tier minted) - # or `definition_task_id` (copied from definition). PR 11 collapses to - # `task_id` only. - node_identity = {(n.definition_task_id or n.id) for n in nodes} - assert defn_task_ids == node_identity, ( - f"task_id did not survive prepare_run: definition={defn_task_ids}, " - f"run-tier={node_identity}" - ) - - -@pytest.mark.xfail( - reason="PR 2: graph_repo.node binds task._task_id from the run-tier row", - strict=True, -) -async def test_task_id_propagates_into_runtime_task_instance( - session, persisted_definition -): - """PR 2 invariant: Task.from_definition binds _task_id; reading - `task.task_id` on the inflated instance returns the same UUID the - definition row had.""" - - pytest.fail("requires PR 2's typed run-node boundary") - - -@pytest.mark.xfail( - reason="PR 4: orchestrator stamps sandbox_id on run_task_executions", - strict=True, -) -async def test_sandbox_identity_is_preserved_across_worker_to_evaluate_boundary( - session, inngest_driver, run_with_one_evaluator -): - """Δ.5: the sandbox acquired in worker_execute is the one each - evaluate_task_run invocation attaches to via sandbox_id.""" - - pytest.fail("requires PR 4's persisted sandbox_id contract") - - -@pytest.mark.xfail( - reason="PR 4: execution_id flows through TaskEvaluateRequest payload", - strict=True, -) -async def test_execution_id_is_unique_per_attempt_and_shared_across_evaluators( - session, inngest_driver, run_with_two_evaluators -): - """Two evaluator invocations for the same execution share execution_id; - a retry mints a new one.""" - - pytest.fail("requires PR 4's TaskEvaluateRequest") - - -@pytest.mark.xfail( - reason="PR 9: dynamic task_id is fresh uuid4 with no definition row", - strict=True, -) -async def test_dynamic_task_id_has_no_definition_row( - session, running_run, parent_task_context -): - """Δ.3: dynamic spawn writes only to run_graph_nodes.""" - - pytest.fail("requires PR 9's graph-native spawn") -``` - -- [x] **Step 2: Run the identity tests** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_identity_invariants.py -q -``` - -Expected: one PASS, four XFAIL. - -- [x] **Step 3: Commit** - -```bash -git add ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py \ - ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py \ - ergon_core/tests/unit/runtime/test_identity_invariants.py \ - ergon_core/migrations/versions/*_add_run_graph_task_json.py \ - ergon_core/ergon_core/core/persistence/graph/models.py \ - ergon_core/ergon_core/core/application/graph/repository.py -git commit -m "feat: run-tier task snapshot + walkthrough/identity test skeletons" -``` - -## Task 6: Rename `telemetry/repositories.py` → `telemetry/repository.py` - -**Files:** - -- Move: `ergon_core/ergon_core/core/persistence/telemetry/repositories.py` - → `ergon_core/ergon_core/core/persistence/telemetry/repository.py` -- Modify: every importer of `ergon_core.core.persistence.telemetry.repositories` -- Modify: `ergon_core/tests/unit/architecture/test_repository_layer_conventions.py` - (remove the `TelemetryRepository` entry from `_KNOWN_VIOLATORS`) - -PR 0.5's repository-layer-conventions guard xfails the -`test_repository_file_is_singular[TelemetryRepository]` case until this -rename lands. PR 1 is the natural home because no other PR is heavily -editing `telemetry/repositories.py` first — PR 4 touches it next. - -- [x] **Step 1: Rename the file** - -```bash -git mv ergon_core/ergon_core/core/persistence/telemetry/repositories.py \ - ergon_core/ergon_core/core/persistence/telemetry/repository.py -``` - -- [x] **Step 2: Update importers** - -```bash -rg -l "from ergon_core\.core\.persistence\.telemetry\.repositories" \ - ergon_core ergon_builtins ergon_cli -``` - -Edit each hit to `... .telemetry.repository`. Same for any -`import ergon_core.core.persistence.telemetry.repositories as ...` form. - -- [x] **Step 3: Remove the xfail entry** - -In `test_repository_layer_conventions.py`, delete: - -```python -("test_repository_file_is_singular", "TelemetryRepository"): - "PR 1: rename telemetry/repositories.py -> telemetry/repository.py", -``` - -- [x] **Step 4: Run the guard** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_repository_layer_conventions.py -q -``` - -Expected: the `TelemetryRepository` filename case now PASSes; no XPASS. - -## PR Ledger - -Invariant landed: run graph nodes carry self-contained task snapshots. - -Bridge code introduced: bridge snapshot uses `TaskSpec`-shaped JSON. - -Old path still intentionally alive: `definition_task_id` and definition-tier -runtime lookup. - -Deletion gate: PR 11 deletes old runtime identity fields after PRs 2-10 flip -all consumers. - -Tests added or updated: `test_run_graph_task_snapshot.py`, -`test_walkthrough_smoketest.py` (one passing, seven xfail), -`test_identity_invariants.py` (one passing, four xfail). - -Modules owned by this PR: persistence and graph repository; observable-effect -smoketest skeletons that subsequent PRs will flip green. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/03-pr-02-typed-run-node-boundary.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/03-pr-02-typed-run-node-boundary.md deleted file mode 100644 index 4510d9f7a..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/03-pr-02-typed-run-node-boundary.md +++ /dev/null @@ -1,440 +0,0 @@ -# PR 2 — Typed Run-Node Reconstruction Boundary - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Add a repository method that returns a typed `RunGraphNodeView` -with an inflated `Task` reconstructed from `run_graph_nodes.task_json`. -The method is **async** because PR 5's `Sandbox.from_definition` accepts -an optional `sandbox_id` and may attach a live `_runtime` — both -inflation and attach happen inside `node()` so call sites never see a -half-built Task. - -**Architecture:** Keep old execution prep alive, but create the v2 boundary: -raw JSON stays inside the repository; job bodies receive typed objects. -The async-ness propagates through `Task.from_definition` and every -caller of `WorkflowGraphRepository.node` (`worker_execute`, -`evaluate_task_run`, `TaskExecutionService.prepare`). - -**Tech Stack:** Pydantic v2, SQLModel repository, pytest. - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/api/benchmark/task.py -ergon_core/ergon_core/api/worker/worker.py -ergon_core/ergon_core/api/criterion/criterion.py -ergon_core/ergon_core/api/rubric/rubric.py -ergon_core/ergon_core/api/rubric/evaluator.py -ergon_core/ergon_core/core/application/graph/models.py -ergon_core/ergon_core/core/application/graph/repository.py -ergon_core/tests/unit/api/test_task_spec_contract.py -ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py -``` - -## Current State - -`worker_execute.py` manually builds `Task` by reading `RunGraphNode` and, -for static nodes, following `definition_task_id` back to definition rows. - -## Target State For This PR - -New call: - -```python -node = await graph_repo.node(session, run_id=run_id, task_id=task_id) -task = node.task -# Config-only sandbox: task.sandbox._runtime is None. Caller (or eval -# variant of the call) can pass sandbox_id to get a live runtime: -node_live = await graph_repo.node( - session, run_id=run_id, task_id=task_id, sandbox_id=execution.sandbox_id, -) -``` - -The call reads `run_graph_nodes.task_json`, inflates a `Task`, binds -`task.task_id`, optionally attaches a live sandbox runtime when -`sandbox_id` is passed, and returns no raw dicts to the caller. - -## Task 1: Add Import-Path Reconstruction Helpers - -**Files:** - -- Modify: `ergon_core/ergon_core/api/benchmark/task.py` -- Modify: `ergon_core/ergon_core/api/worker/worker.py` - -- [x] **Step 1: Add import helper and serialized-form type alias to `task.py`** - -```python -from importlib import import_module -from typing import Any -from pydantic import JsonValue, PrivateAttr - - -type TaskDefinitionJson = dict[str, JsonValue] -"""Serialized form of a Task — `_type`-discriminated JSON written to -`run_graph_nodes.task_json`. Field names are NOT enforced by this type -(the discriminator dispatch in `Task.from_definition` does that); only -the value side is typed, so accidentally stuffing a `datetime` or `UUID` -object into the snapshot fails at typecheck time instead of at -JSON-serialization time. The alias is the named boundary every -`from_definition` classmethod accepts.""" - - -def _import_component(path: str) -> type[Any]: - module_name, _, qualname = path.partition(":") - if not module_name or not qualname: - raise ValueError(f"Component _type must be 'module:qualname', got {path!r}") - obj: Any = import_module(module_name) - for part in qualname.split("."): - # typing: dynamic qualname walk — `part` is a user-controlled - # discriminator path component, not a typed attribute name. - obj = getattr(obj, part) - if not isinstance(obj, type): - raise TypeError(f"Component _type {path!r} did not resolve to a class") - return obj -``` - -`TaskDefinitionJson` is intentionally not a Pydantic model and not a -TypedDict. The shape after the discriminator is dispatched lives on the -Task subclass; trying to mirror it in a parallel typed structure would -duplicate the schema and rot. The boundary type only asserts what's -honest at the boundary: "JSON-shaped, value-side typed." - -- [x] **Step 2: Convert `Task` to support runtime identity** - -Change `Task.model_config` from frozen to mutable: - -```python -model_config = {"frozen": False} -``` - -Add: - -```python -_task_id: UUID | None = PrivateAttr(default=None) - -@property -def task_id(self) -> UUID: - if self._task_id is None: - raise RuntimeError( - f"Task {self.task_slug!r} has no task_id; it has not been materialized" - ) - return self._task_id - -@classmethod -async def from_definition( - cls, - task_json: TaskDefinitionJson, - *, - task_id: UUID, - sandbox_id: str | None = None, -) -> "Task": - task_type = task_json.get("_type") - if not isinstance(task_type, str): - raise ValueError( - f"Task snapshot is missing the required `_type` discriminator " - f"(got {type(task_type).__name__}). Every persisted task must " - f"carry `_type` — produced by `model_serializer` on Task " - f"subclasses or by `_definition_task_snapshot` during the PR 1 " - f"bridge. Soft-defaulting to base `Task` here would silently " - f"drop the authored worker/sandbox/evaluator bindings." - ) - TaskCls = _import_component(task_type) - if TaskCls is TaskSpec: - # Transitional bridge: PR 5 replaces TaskSpec snapshots with Task JSON. - spec = TaskSpec.model_validate(task_json) - instance = Task( - task_slug=spec.task_slug, - instance_key=spec.instance_key, - description=spec.description, - parent_task_slug=spec.parent_task_slug, - dependency_task_slugs=spec.dependency_task_slugs, - evaluator_binding_keys=spec.evaluator_binding_keys, - task_payload=spec.task_payload, - ) - else: - instance = TaskCls.model_validate(task_json) - object.__setattr__(instance, "_task_id", task_id) - # sandbox_id is a no-op during PR 2 because TaskSpec snapshots don't - # carry an object-bound sandbox. PR 5 wires this through to - # Sandbox.from_definition once Task.sandbox is non-null. - return instance -``` - -`from_definition` is **async** even in PR 2 to lock in the signature -PR 5 needs. The async-ness is structural; PR 2's body doesn't await -anything yet, but the protocol contract is the v2 final shape. - -During the transition `Task` still has public fields compatible with the -old runtime shape. PR 5 adds object-bound `worker`, `sandbox`, and -`evaluators` and starts honoring the `sandbox_id` parameter. - -- [x] **Step 3: Add `Worker.from_definition` and keep `from_buffer`** - -In `worker.py`, make `Worker` reconstruction available without deleting -the old dead method yet. Reuse the same serialized-form alias for -consistency across components: - -```python -from ergon_core.api.benchmark.task import TaskDefinitionJson as ComponentDefinitionJson - - -@classmethod -def from_definition(cls, worker_json: ComponentDefinitionJson) -> "Worker": - worker_type = worker_json.get("_type") - if not isinstance(worker_type, str): - raise ValueError( - f"Worker snapshot is missing the required `_type` " - f"discriminator (got {type(worker_type).__name__}). Every " - f"persisted worker must carry `_type`." - ) - WorkerCls = _import_component(worker_type) - return cast("Worker", WorkerCls.model_validate(worker_json)) -``` - -Treat `ComponentDefinitionJson` here as the same structural alias as -`TaskDefinitionJson` — different name for readability at the worker -boundary, same underlying `dict[str, JsonValue]`. If a future PR wants -to split the aliases (e.g. add invariants specific to one component -kind), the rename is mechanical. - -`Worker.from_buffer` remains until PR 11. - -## Task 2: Add `RunGraphNodeView` - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/graph/models.py` - -- [x] **Step 1: Add model** - -```python -from ergon_core.api.benchmark import Task - - -class RunGraphNodeView(BaseModel): - model_config = {"frozen": True, "arbitrary_types_allowed": True} - - run_id: UUID - task_id: UUID - node_id: UUID - definition_task_id: UUID | None - parent_node_id: UUID | None - status: str - task: Task - is_dynamic: bool = False -``` - -This view carries both `node_id` and `task_id` during transition. PR 11 -removes `node_id` from the public runtime identity. - -## Task 3: Add `graph_repo.node` - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/graph/repository.py` - -- [x] **Step 1: Import view** - -```python -from ergon_core.api.benchmark import Task -from ergon_core.core.application.graph.models import RunGraphNodeView -``` - -- [x] **Step 2: Add method** - -```python -async def node( - self, - session: Session, - *, - run_id: UUID, - task_id: UUID, - sandbox_id: str | None = None, -) -> RunGraphNodeView: - row = session.exec( - select(RunGraphNode).where( - RunGraphNode.run_id == run_id, - (RunGraphNode.id == task_id) | (RunGraphNode.definition_task_id == task_id), - ) - ).first() - if row is None: - raise NodeNotFoundError(f"Run graph node {task_id} not found in run {run_id}") - - canonical_task_id = row.definition_task_id or row.id - task = await Task.from_definition( - row.task_json, - task_id=canonical_task_id, - sandbox_id=sandbox_id, - ) - return RunGraphNodeView( - run_id=row.run_id, - task_id=canonical_task_id, - node_id=row.id, - definition_task_id=row.definition_task_id, - parent_node_id=row.parent_node_id, - status=row.status, - task=task, - is_dynamic=row.is_dynamic, - ) -``` - -`node()` is async because `Task.from_definition` is async — even in PR 2 -where the await does nothing structural. Every call site needs to be -flipped to `await graph_repo.node(...)` in this PR and PR 3. - -The OR predicate is transitional. PR 11 changes the lookup to exact -`(run_id, task_id)` after schema finalization. - -## Task 4: Tests - -**Files:** - -- Modify: `ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py` - -- [x] **Step 1: Add reconstruction test** - -```python -@pytest.mark.asyncio -async def test_graph_repo_node_inflates_task_from_run_tier(session, run_node_factory): - row = run_node_factory( - task_json={ - "_type": "ergon_core.api.benchmark.task:TaskSpec", - "task_slug": "solve", - "instance_key": "sample-1", - "description": "solve it", - "dependency_task_slugs": [], - "evaluator_binding_keys": [], - "task_payload": {}, - } - ) - - view = await WorkflowGraphRepository().node( - session, - run_id=row.run_id, - task_id=row.definition_task_id or row.id, - ) - - assert view.task.task_slug == "solve" - assert view.task.task_id == (row.definition_task_id or row.id) -``` - -- [x] **Step 2: Add textual boundary test** - -```python -import inspect - -from ergon_core.core.application.graph.repository import WorkflowGraphRepository - - -def test_graph_repo_node_does_not_reference_definition_tier_models() -> None: - """`graph_repo.node` must hydrate Task from run-tier JSON only. - - PR 2's contract is that the runtime read path goes through - run_graph_nodes.task_json and never reaches into definition tables. - This is a textual guard because a subtle import or an inner helper - that delegates to DefinitionRepository would re-open the read path - PR 11 is closing. - """ - - source = inspect.getsource(WorkflowGraphRepository.node) - forbidden = ( - "DefinitionRepository", - "ExperimentDefinitionTask", - "task_with_instance", - "ComponentCatalogService", - ) - offenders = [symbol for symbol in forbidden if symbol in source] - assert offenders == [], ( - f"WorkflowGraphRepository.node references definition-tier symbols " - f"{offenders}; the run-tier read boundary forbids these." - ) -``` - -- [x] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py ergon_core/tests/unit/api/test_task_spec_contract.py -q -``` - -## Task 5: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/runtime/test_identity_invariants.py` - -The PR 1 ledger pre-registered one identity invariant that PR 2 lands. -Remove the corresponding `@pytest.mark.xfail` decorator and verify the -test passes. - -- [x] **Step 1: Remove xfail from `test_task_id_propagates_into_runtime_task_instance`** - -In `test_identity_invariants.py`, delete the decorator: - -```python -@pytest.mark.xfail( - reason="PR 2: graph_repo.node binds task._task_id from the run-tier row", - strict=True, -) -``` - -Replace the `pytest.fail(...)` body with the real assertion: - -```python -async def test_task_id_propagates_into_runtime_task_instance( - session, persisted_definition -): - from ergon_core.core.application.graph.repository import ( - WorkflowGraphRepository, - ) - from ergon_core.core.application.workflows.service import prepare_run - - run_id = await prepare_run(session, definition_id=persisted_definition.id) - repo = WorkflowGraphRepository() - - from sqlmodel import select - - from ergon_core.core.persistence.graph.models import RunGraphNode - nodes = session.exec( - select(RunGraphNode).where(RunGraphNode.run_id == run_id) - ).all() - for row in nodes: - canonical_id = row.definition_task_id or row.id - view = await repo.node(session, run_id=run_id, task_id=canonical_id) - assert view.task_id == canonical_id - assert view.task.task_id == canonical_id -``` - -- [x] **Step 2: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/runtime/test_identity_invariants.py \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_dead_path_audit.py -q -``` - -Expected: two PASS in identity invariants (PR 1 case + the newly-flipped -PR 2 case), the ledgers still XFAIL on every other case. No XPASS. - -## PR Ledger - -Invariant landed: repository can inflate typed tasks from run-tier JSON. - -Bridge code introduced: `TaskSpec` JSON is accepted by -`Task.from_definition`. - -Old path still intentionally alive: `_prepare_definition` and -definition-tier static prep. - -Deletion gate: PR 3 makes worker execution use this view; PR 11 deletes the -legacy prep fallback. - -Tests added or updated: task reconstruction tests and method-source guard. - -Modules owned by this PR: API reconstruction and graph repository boundary. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/04-pr-03-worker-execute-typed-node.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/04-pr-03-worker-execute-typed-node.md deleted file mode 100644 index 1a4fe18f3..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/04-pr-03-worker-execute-typed-node.md +++ /dev/null @@ -1,346 +0,0 @@ -# PR 3 — Worker-Execute Uses Typed Run Nodes - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Make worker execution prepare and run from `graph_repo.node(...).task` -instead of rebuilding static tasks from definition rows. - -**Architecture:** The runtime path prefers the typed run-node boundary from -PR 2. A single legacy fallback remains only for tests or callers that have -not yet been converted. - -**Tech Stack:** Python async jobs, SQLModel repositories, pytest runtime -tests, textual architecture guards. - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/core/application/tasks/execution.py -ergon_core/ergon_core/core/application/workflows/orchestration.py -ergon_core/ergon_core/core/application/jobs/worker_execute.py -ergon_core/ergon_core/core/application/jobs/models.py -ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py -ergon_core/tests/unit/runtime/test_definition_task_payload_typing.py -ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py -``` - -## Current State - -`TaskExecutionService.prepare()` branches: - -```python -if command.node_id is not None: - return await self._prepare_graph_native(command) -return await self._prepare_definition(command) -``` - -`worker_execute.py` then reads `RunGraphNode` by `payload.node_id` and, for -static nodes, calls `DefinitionRepository().task_with_instance(...)`. - -## Target State For This PR - -Preparation returns a payload keyed by run node identity, and -`worker_execute.py` starts with: - -```python -with get_session() as session: - node = await WorkflowGraphRepository().node( - session, - run_id=payload.run_id, - task_id=payload.task_id, - ) -task = node.task -``` - -No static/dynamic branch exists in the job body. The `await` is required -because `WorkflowGraphRepository.node` becomes async in PR 2; the -`get_session()` context manager stays sync (it just opens a DB session) -but the call inside it is awaited. - -## Task 1: Reshape Prepared Execution DTO - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/workflows/orchestration.py` - -- [x] **Step 1: Keep transitional identity fields but make `task_id` canonical** - -Update `PreparedTaskExecution` to include: - -```python -run_id: UUID -definition_id: UUID -task_id: UUID -node_id: UUID -definition_task_id: UUID | None = None -task_slug: str -task_description: str -benchmark_type: str -assigned_worker_slug: str | None = None -worker_type: str | None = None -model_target: str | None = None -execution_id: UUID -``` - -During transition, `task_id` is `definition_task_id` for static nodes and -`node_id` for dynamic nodes. PR 11 removes `node_id` and -`definition_task_id`. - -## Task 2: Prefer Graph-Native Preparation - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/tasks/execution.py` - -- [x] **Step 1: Change `prepare` branch order** - -Replace: - -```python -if command.node_id is not None: - return await self._prepare_graph_native(command) -return await self._prepare_definition(command) -``` - -with: - -```python -return await self._prepare_run_node(command) -``` - -- [x] **Step 2: Add `_prepare_run_node`** - -```python -async def _prepare_run_node(self, command: PrepareTaskExecutionCommand) -> PreparedTaskExecution: - lookup_id = command.node_id or command.task_id - if lookup_id is None: - raise ConfigurationError( - "Task preparation requires node_id or task_id", - run_id=command.run_id, - task_id=None, - ) - with get_session() as session: - view = await self._graph_repo.node(session, run_id=command.run_id, task_id=lookup_id) - node = session.get(RunGraphNode, view.node_id) - if node is None: - raise ConfigurationError( - f"RunGraphNode {view.node_id} not found", - run_id=command.run_id, - task_id=lookup_id, - ) - definition = require_not_none( - session.get(ExperimentDefinition, command.definition_id), - f"Definition {command.definition_id} not found", - ) - execution = RunTaskExecution( - run_id=command.run_id, - node_id=view.node_id, - definition_task_id=view.definition_task_id, - attempt_number=self._task_execution_repo.next_attempt_for_node( - session, command.run_id, view.node_id - ), - status=TaskExecutionStatus.RUNNING, - started_at=utcnow(), - ) - session.add(execution) - session.flush() - await self._graph_repo.update_node_status( - session, - run_id=command.run_id, - node_id=view.node_id, - new_status=graph_status.RUNNING, - meta=MutationMeta(actor="task-execution-service", reason=f"prepare: {execution.id}"), - ) - session.commit() - - await _emit_task_status( - run_id=command.run_id, - node_id=view.node_id, - task_slug=view.task.task_slug, - new_status=graph_status.RUNNING, - old_status=None, - ) - return PreparedTaskExecution( - run_id=command.run_id, - definition_id=command.definition_id, - task_id=view.task_id, - node_id=view.node_id, - definition_task_id=view.definition_task_id, - task_slug=view.task.task_slug, - task_description=view.task.description, - benchmark_type=definition.benchmark_type, - assigned_worker_slug=node.assigned_worker_slug, - worker_type=node.assigned_worker_slug, - model_target=None, - execution_id=execution.id, - ) -``` - -- [x] **Step 3: Rename old method** - -Rename `_prepare_definition` to `_prepare_legacy_definition` and leave it -unused. PR 11 deletes it. Keeping the method available makes rollback -explicit during this PR. - -## Task 3: Update Worker-Execute Job Body - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/worker_execute.py` - -- [x] **Step 1: Remove direct definition imports** - -Delete these imports from the job: - -```python -from ergon_core.core.application.components.catalog import ComponentCatalogService -from ergon_core.core.application.experiments.repository import DefinitionRepository -from ergon_core.core.persistence.graph.models import RunGraphNode -``` - -Add: - -```python -from ergon_core.core.application.graph.repository import WorkflowGraphRepository -``` - -- [x] **Step 2: Replace task construction block** - -Replace the current block that builds `worker`, `node`, `task_payload`, and -`Task(...)` with: - -```python -with get_session() as session: - node = await WorkflowGraphRepository().node( - session, - run_id=payload.run_id, - task_id=payload.task_id, - ) -task = node.task -``` - -Until PR 5, old snapshots do not carry `task.worker`. For this PR keep -worker registry construction in a helper: - -```python -worker = _worker_from_payload_bridge(payload) -``` - -Implement bridge: - -```python -def _worker_from_payload_bridge(payload: WorkerExecuteJobRequest) -> Worker: - catalog = ComponentCatalogService() - with get_session() as session: - return catalog.build_worker( - session, - slug=payload.worker_type, - name=payload.assigned_worker_slug, - model=payload.model_target, - ) -``` - -This bridge is deleted in PR 5 or PR 11 when `task.worker` is always -present. - -## Task 4: Architecture Guard - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py` - -- [x] **Step 1: Add guard** - -```python -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[4] - - -def test_worker_execute_does_not_read_definition_repository() -> None: - text = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/worker_execute.py" - ).read_text() - assert "DefinitionRepository" not in text - assert "task_with_instance" not in text - assert "ExperimentDefinitionTask" not in text -``` - -- [x] **Step 2: Run tests** - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py \ - ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py -q -``` - -## Task 5: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` -- Modify: `ergon_core/tests/unit/architecture/test_dead_path_audit.py` -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` - -This PR flips the worker-execute run-tier-only invariant green. Three -ledger entries must drop their xfail markers in this commit: - -- [x] **Step 1: Remove `worker_execute_imports_only_run_tier` from `_XFAIL_BY_NAME`** - -In `test_v2_final_state_ledger.py`, delete the line: - -```python -"worker_execute_imports_only_run_tier": "PR 3 flips worker_execute to run-tier", -``` - -- [x] **Step 2: Remove `_prepare_definition` from the dead-path xfails** - -In `test_dead_path_audit.py`, delete the line: - -```python -"_prepare_definition": "PR 3: renamed to _prepare_legacy_definition", -``` - -(`_prepare_legacy_definition` stays xfailed until PR 11 deletes the -renamed body.) - -- [x] **Step 3: Flip `test_worker_execute_reads_task_from_run_tier_only`** - -In `test_walkthrough_smoketest.py`, remove the decorator and replace the -`pytest.fail(...)` body with the real assertion. Use a session spy that -records the ORM classes touched during a `worker_execute` invocation; -assert no definition-tier class appears. - -- [x] **Step 4: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_dead_path_audit.py \ - ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py -q -``` - -Expected: the three newly-flipped cases PASS; the remaining cases still -XFAIL; no XPASS, no unexpected FAIL. - -## PR Ledger - -Invariant landed: worker execution uses typed run-node task loading. - -Bridge code introduced: `_worker_from_payload_bridge`. - -Old path still intentionally alive: worker registry payload fields and -legacy prepare method. - -Deletion gate: PR 5 replaces registry worker construction with `task.worker`; -PR 11 deletes legacy prepare. - -Tests added or updated: runtime read boundary guard and worker stream tests. - -Modules owned by this PR: task execution prep and worker-execute read path. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/05-pr-04-inline-criteria.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/05-pr-04-inline-criteria.md deleted file mode 100644 index dc26254f6..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/05-pr-04-inline-criteria.md +++ /dev/null @@ -1,884 +0,0 @@ -# PR 4 — Synchronous-Fanout Criteria And Sandbox Release Ownership - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Make `worker_execute` synchronously fan out per-evaluator -invocations via `ctx.step.invoke(evaluate_task_run, ...)`, wait for them -all via `asyncio.gather`, then release the sandbox in the same job's -`finally` block. Reshape `evaluate_task_run` to take a thin id-only -payload and re-load state via `graph_repo.node`. - -**Architecture:** Two function shapes per task — orchestrator -(`worker_execute`) and per-evaluator worker (`evaluate_task_run`). The -orchestrator owns sandbox lifetime; eval workers only attach to the -external sandbox via `sandbox_id` and detach on completion. Inngest -retries, concurrency caps, and observability slugs apply per-eval — the -operational properties of v1's split are preserved. The lifecycle bug -v1 had (release in a sibling job) is fixed because the orchestrator's -`try/finally` bounds sandbox lifetime through the gather. - -**Tech Stack:** Inngest job functions with `ctx.step.invoke`, existing -evaluation service, pytest lifecycle tests. - ---- - -## Implementation Note — Bridge-Everything Approach (PR 4 → PR 5) - -This plan was originally written against the v2 final-state shape: -`task.evaluators[i]`, `task.sandbox.detach()`, and a `lifecycle_hub` -that owns sandbox `acquire` / `release`. None of those exist in -the codebase at the head of PR 3 — they all land in PR 5 -(object-bound API) or later. - -PR 4 still has to land the **synchronous-fanout invariant** (the bug -fix that motivates the entire reshape: sandbox lifetime bounded by the -orchestrator's `try/finally`, not a sibling job). To do that without -half-building PR 5, every cross-PR dependency is introduced as a named, -greppable bridge that PR 5 deletes: - -| Bridge | Location | Replaced by (PR 5) | -| --- | --- | --- | -| `_evaluator_bridge.resolve_evaluator(session, *, run_id, task, evaluator_index)` | `core/application/jobs/_evaluator_bridge.py` | `task.evaluators[index]` after PR 5 Task 2 lands object-bound evaluators | -| `EvaluationService.evaluate(*, context, evaluator)` — the v2 entry point; the v1 executor-based entry point gets renamed to `evaluate_legacy` to make the dying code carry the awkward name instead of the living code | `core/application/evaluation/service.py` (sibling of `evaluate_legacy`) | PR 11 deletes `evaluate_legacy`, the `CriterionExecutor` Protocol, and `InngestCriterionExecutor` together; `evaluate` keeps the name | -| `terminate_sandbox_by_id(sandbox_id)` called directly inside `worker_execute`'s `finally` (instead of `lifecycle_hub.release(sandbox)`) | existing `core/infrastructure/sandbox/lifecycle.py` helper | when `lifecycle_hub` lands, swap the call site (no API rename needed) | -| Sandbox **acquisition** stays in the existing `sandbox_setup` Inngest function — `worker_execute` does **not** absorb sandbox creation. The orchestrator `execute_task_fn` still invokes `sandbox_setup_fn` first, then `worker_execute_fn` with a stamped `sandbox_id`. `worker_execute`'s `try/finally` only owns the *release* side. | `core/application/jobs/execute_task.py` unchanged | PR 5/6 can either lift release into `lifecycle_hub` or fully merge `sandbox_setup` into `worker_execute` | - -**`_DetachableSandboxBridge` (Task 4) is intentionally *not* introduced -at PR 4.** The plan code expected `sandbox._runtime` on a `Sandbox` -ABC that doesn't exist yet — and would not exist in PR 4 either, -since `task.sandbox` is a PR 5 field. Adding the bridge in PR 4 would -just be a class no caller can invoke. PR 5 introduces `Sandbox`, -`task.sandbox`, the `_runtime` attach, and `Sandbox.detach()` in one -coherent change. PR 4 simply omits the eval-side `detach()` call — -the orchestrator's `try/finally` already bounds external-sandbox -lifetime via `terminate_sandbox_by_id`, so the eval worker has -nothing to release locally. - -**Orchestrator location — `execute_task`, not `worker_execute`.** The -plan code shows `worker_execute` owning `acquire`/`release` and the -fanout. In our codebase `worker_execute` runs *between* -`sandbox_setup` and `persist_outputs` — both are sibling Inngest -functions invoked by `execute_task`. Putting the release in -`worker_execute`'s `finally` would terminate the sandbox before -`persist_outputs` runs (file uploads need the sandbox alive). - -PR 4 therefore lifts the synchronous fanout + release into -`execute_task.py`'s `try/finally` instead. `worker_execute` only: -(a) persists the terminal `WorkerOutput` via `WorkerOutputRepository` -and (b) stamps `sandbox_id` on the execution row via -`TaskExecutionRepository.set_sandbox_id`. The orchestrator's `finally` -calls `terminate_sandbox_by_id(sandbox_id)` after the gather, and -the `check_evaluators` Inngest function is unregistered. The -Task 6 architecture guards target `execute_task.py` accordingly -(the file where the fanout actually lives) — same invariant, more -accurate location. - -**Why bridges instead of waiting for PR 5:** the v1 release-in-sibling- -job bug is a correctness problem (eval workers can run against a -terminated sandbox under retry). Landing the orchestrator-owned -`finally` *now* fixes the bug; the cosmetic shape (`task.evaluators`, -`sandbox.detach()`) follows in PR 5 without changing observable -behavior. Each bridge has a `TODO(PR 5): <delete-when>` comment so -PR 5 can grep them out one-by-one. - -**Test impact:** the Task 6 architecture guards still pass — they -check the *body* of `evaluate_task_run.py` for forbidden strings -(`DefinitionRepository`, `ComponentCatalogService`, -`ExperimentDefinitionTask`). Those strings move into the sibling -`_evaluator_bridge.py`, keeping the eval job body clean and the -final-state shape recognisable. - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/core/application/jobs/worker_execute.py -ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py -ergon_core/ergon_core/core/application/jobs/check_evaluators.py -ergon_core/ergon_core/core/application/jobs/models.py -ergon_core/ergon_core/core/application/evaluation/service.py -ergon_core/ergon_core/core/infrastructure/inngest/registry.py -ergon_core/ergon_core/core/persistence/telemetry/repository.py -ergon_core/tests/unit/runtime/test_worker_execute_sandbox_lifecycle.py -ergon_core/tests/unit/runtime/test_evaluate_task_run_thin_payload.py -ergon_core/tests/unit/runtime/test_failed_task_sandbox_cleanup.py -ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py -``` - -## Current State - -`check_evaluators.py` is fired after `task/completed`. It invokes -`evaluate_task_run` per evaluator (fire-and-forget) and then calls -`terminate_sandbox_by_id`. Sandbox release is owned by a sibling job — -the v1 audit's lifecycle leak. - -`evaluate_task_run.py` (current body) takes `EvaluateTaskRunRequest` -with many fields: `definition_task_id`, `evaluator_id`, -`evaluator_binding_key`, `evaluator_type`, `agent_reasoning`, etc. It -reads definition rows, resolves the evaluator through -`ComponentCatalogService`, and constructs a synthetic `Task`. This -violates the Δ.2 run-tier read boundary every time it runs. - -## Target State For This PR - -`worker_execute.py`: - -```python -sandbox = await lifecycle_hub.acquire(task.sandbox, run_id=..., task_id=...) -await task_execution_repo.set_sandbox_id( - execution_id=payload.execution_id, sandbox_id=sandbox.sandbox_id, -) -try: - output = await consume_worker_stream(...) - persist_worker_output(payload.execution_id, output) - - # Synchronous fanout. Parent suspends until every invoke returns. - await asyncio.gather(*[ - ctx.step.invoke( - f"eval-{i}", - evaluate_task_run, - TaskEvaluateRequest( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - evaluator_index=i, - ), - ) - for i in range(len(task.evaluators)) - ]) -finally: - await lifecycle_hub.release(sandbox) -``` - -`evaluate_task_run.py` (reshaped body): id-only payload, reload state -via `graph_repo.node(..., sandbox_id=...)`, run evaluator, detach. - -`check_evaluators.py` becomes obsolete; PR 11 deletes it. -`terminate_sandbox_by_id` is no longer called by `worker_execute` or -`evaluate_task_run`; PR 11 deletes the helper. - ---- - -## Task 1: Add Thin Eval Payload And Worker Output Persistence - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/repository.py` - -- [ ] **Step 1: Add `TaskEvaluateRequest`** - -```python -from uuid import UUID -from pydantic import BaseModel - - -class TaskEvaluateRequest(BaseModel): - """Thin id-only payload for the per-evaluator Inngest function. - - Every other piece of state — task config, sandbox_id, worker_output, - evaluator instance — is recovered by the receiver via persisted - state lookups. This keeps retries/replays trivially correct. - """ - - run_id: UUID - task_id: UUID - execution_id: UUID - evaluator_index: int -``` - -Keep `EvaluateTaskRunRequest` importable until PR 11; do not yet -remove it from `__all__`. The reshaped `evaluate_task_run` uses the new -payload class; the old class lives as a forwarding shim until cleanup. - -- [ ] **Step 2: Add `set_sandbox_id` and worker-output persistence helpers** - -In `telemetry/repository.py`: - -```python -class TaskExecutionRepository: - ... - - async def set_sandbox_id( - self, - *, - execution_id: UUID, - sandbox_id: str, - ) -> None: - async with self._session() as session: - session.exec( - update(RunTaskExecution) - .where(RunTaskExecution.id == execution_id) - .values(sandbox_id=sandbox_id) - ) - session.commit() -``` - -The column already exists (migration -`925ff225d97e_add_sandbox_id_to_run_task_executions.py`). - -For worker-output persistence/load, the `run_graph_nodes` row already -carries the worker output JSON via `task_json` + the existing -`evaluation_summary` paths. Add explicit `WorkerOutputRepository`: - -```python -class WorkerOutputRepository: - """Persisted worker_output keyed by execution_id, read by eval workers.""" - - async def persist( - self, *, run_id: UUID, task_id: UUID, execution_id: UUID, - output: WorkerOutput, - ) -> None: - async with self._session() as session: - row = RunTaskExecution( - ..., # existing fields - worker_output_json=output.model_dump(mode="json"), - ) - session.merge(row) - session.commit() - - async def load(self, *, execution_id: UUID) -> WorkerOutput: - async with self._session() as session: - row = session.get(RunTaskExecution, execution_id) - if row is None or row.worker_output_json is None: - raise WorkerOutputNotFound(execution_id=execution_id) - return WorkerOutput.model_validate(row.worker_output_json) -``` - -If `RunTaskExecution.worker_output_json` does not exist, add it via an -additive Alembic migration in this PR: - -```python -def upgrade() -> None: - op.add_column( - "run_task_executions", - sa.Column("worker_output_json", sa.JSON(), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("run_task_executions", "worker_output_json") -``` - -## Task 2: Reshape `evaluate_task_run` Body - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` - -- [ ] **Step 1: Replace the function body** - -```python -from datetime import UTC, datetime -import logging - -import inngest - -from ergon_core.api.criterion.context import CriterionContext -from ergon_core.core.application.evaluation.service import EvaluationService -from ergon_core.core.application.graph.repository import WorkflowGraphRepository -from ergon_core.core.application.jobs.models import TaskEvaluateRequest -from ergon_core.core.infrastructure.dashboard.emitters import ( - get_dashboard_emitter, -) -from ergon_core.core.infrastructure.observability.spans import ( - CompletedSpan, - evaluation_task_context, - get_trace_sink, -) -from ergon_core.core.persistence.telemetry.repositories import ( - TaskExecutionRepository, - WorkerOutputRepository, -) -from ergon_core.core.persistence.sessions import get_session - -logger = logging.getLogger(__name__) -_evaluation_persistence = EvaluationService() - - -@inngest.function( - fn_id="evaluate_task_run", - trigger=inngest.TriggerEvent(event="task/evaluate"), - retries=3, - concurrency=inngest.Concurrency(limit=50), -) -async def evaluate_task_run( - ctx: inngest.Context, payload: TaskEvaluateRequest -) -> None: - """Per-evaluator fanout target. Thin id-only payload.""" - - span_start = datetime.now(UTC) - execution = await TaskExecutionRepository().get(payload.execution_id) - if execution is None: - raise ContractViolationError( - f"RunTaskExecution {payload.execution_id} not found", - run_id=payload.run_id, task_id=payload.task_id, - ) - - with get_session() as session: - view = await WorkflowGraphRepository().node( - session, - run_id=payload.run_id, - task_id=payload.task_id, - sandbox_id=execution.sandbox_id, # attaches a live _runtime - ) - task = view.task - output = await WorkerOutputRepository().load(execution_id=payload.execution_id) - evaluator = task.evaluators[payload.evaluator_index] - - # Build a CriterionContext matching the v1-locked Criterion.evaluate - # signature (see 01-api-surface.md § "Criterion class signature — - # locked"). The sandbox is live on context.task.sandbox via the - # graph_repo.node(..., sandbox_id=...) attach above; criteria call - # context.task.sandbox.run_command(...) directly — no separate sandbox - # parameter, no runtime proxies on the context itself. - context = CriterionContext( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - task=task, - worker_result=output, - ) - - try: - # EvaluationService internally iterates evaluator.criteria and - # awaits each `criterion.evaluate(context)` — no CriterionExecutor. - service_result = await _evaluation_persistence.evaluate( - context=context, - evaluator=evaluator, - benchmark_name="", - ) - except Exception as exc: # slopcop: ignore[no-broad-except] - logger.exception( - "evaluate_task_run failed run_id=%s task_id=%s index=%s", - payload.run_id, payload.task_id, payload.evaluator_index, - ) - _evaluation_persistence.persist_failure( - run_id=payload.run_id, - node_id=view.node_id, - task_execution_id=payload.execution_id, - definition_task_id=view.definition_task_id, - evaluator_id=None, # evaluator instance carries identity - evaluator_name=type(evaluator).__name__, - exc=exc, - ) - # Detach so the local _runtime handle is released; the external - # sandbox keeps running — termination is the orchestrator's job. - await task.sandbox.detach() - raise - - persisted = _evaluation_persistence.persist_success( - run_id=payload.run_id, - node_id=view.node_id, - task_execution_id=payload.execution_id, - definition_task_id=view.definition_task_id, - evaluator_id=None, - service_result=service_result, - ) - await get_dashboard_emitter().task_evaluation_updated( - run_id=payload.run_id, - task_id=payload.task_id, - evaluation=persisted.dashboard_dto, - ) - get_trace_sink().emit_span( - CompletedSpan( - name="evaluation.task", - context=evaluation_task_context( - payload.run_id, payload.task_id, - payload.execution_id, evaluator_index=payload.evaluator_index, - ), - start_time=span_start, - end_time=datetime.now(UTC), - attributes={ - "passed": service_result.result.passed, - "score": service_result.result.score, - }, - ) - ) - await task.sandbox.detach() -``` - -Notes: - -- The reshape preserves the function name and Inngest slug. Existing - dashboards / GraphQL queries that filter by - `function.slug == "evaluate_task_run"` keep working. -- `InngestCriterionExecutor` and the `CriterionExecutor` Protocol are - gone from this body — `service.evaluate(...)` calls - `criterion.evaluate(...)` directly because the criterion is already a - fully constructed object on `task.evaluators[i]`. PR 11 deletes the - executor classes. -- `detach()` is the new `Sandbox` protocol method PR 5 adds. PR 4 uses - it via the bridge defined in Task 4 below. - -## Task 3: Make `worker_execute` Fan Out Via `step.invoke` - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/worker_execute.py` - -- [ ] **Step 1: Persist worker output before fanout** - -After the worker stream consumption, persist the terminal `WorkerOutput` -through the new repo: - -```python -await WorkerOutputRepository().persist( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - output=output, -) -``` - -This must commit before the fanout: eval workers load from the same -repo. - -- [ ] **Step 2: Stamp sandbox_id on the execution row** - -Right after acquire: - -```python -await TaskExecutionRepository().set_sandbox_id( - execution_id=payload.execution_id, - sandbox_id=sandbox.sandbox_id, -) -``` - -Eval workers read this when calling `graph_repo.node(..., -sandbox_id=...)` to attach the live runtime. - -- [ ] **Step 3: Replace inline-evaluator block with synchronous fanout** - -```python -import asyncio - -from ergon_core.core.application.jobs.evaluate_task_run import evaluate_task_run -from ergon_core.core.application.jobs.models import TaskEvaluateRequest - - -# ... after worker output persistence: -await asyncio.gather(*[ - ctx.step.invoke( - f"eval-{i}", - evaluate_task_run, - TaskEvaluateRequest( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=payload.execution_id, - evaluator_index=i, - ), - ) - for i in range(len(task.evaluators)) -]) -``` - -The gather is what keeps the sandbox alive: `worker_execute` cannot -reach its `finally` until every `step.invoke` returns. The -`f"eval-{i}"` step IDs make each invocation independently retriable -by Inngest. - -- [ ] **Step 4: Release sandbox in `finally`** - -```python -sandbox_id = sandbox.sandbox_id -try: - output, chunk_count = await _consume_worker_stream(...) - await WorkerOutputRepository().persist(...) - await asyncio.gather(*[ - ctx.step.invoke(...) for i in range(len(task.evaluators)) - ]) -except Exception as exc: - return _worker_failure_result(exc, chunk_count) -finally: - await lifecycle_hub.release(sandbox) -``` - -`lifecycle_hub.release(sandbox)` is the canonical path; it calls -`sandbox.terminate()` which terminates the external sandbox. PR 6 -removes the v1 `terminate_sandbox_by_id` fallback. - -## Task 4: Add `Sandbox.detach()` Stub (Bridge) - -**Files:** - -- Modify: `ergon_core/ergon_core/api/sandbox/sandbox.py` - -PR 5 defines the public `Sandbox` ABC. PR 4 needs `detach()` to exist -before PR 5 lands so the new `evaluate_task_run` body compiles. Add it -as a concrete method on the (still-private) bridge class for now: - -- [ ] **Step 1: Add bridge stub** - -If `Sandbox` ABC does not exist yet (it lands in PR 5), add to -`ergon_core/ergon_core/core/infrastructure/sandbox/runtime.py`: - -```python -from typing import Protocol - - -class _DetachableRuntime(Protocol): - """Bridge-internal Protocol describing the runtime methods PR 4 - needs from a sandbox's `_runtime` handle. - - PR 5 lifts these into a real `SandboxRuntime` Protocol in - `ergon_core.api.sandbox.runtime`. Until then, this Protocol exists - only inside the bridge to give `_DetachableSandboxBridge.detach` - typed access to `close` / `close_local`. - """ - - async def close(self) -> None: ... - async def close_local(self) -> None: ... - - -class _DetachableSandbox(Protocol): - """Bridge-internal Protocol over the Sandbox-ish object the eval - worker has at this point. PR 5's real `Sandbox` ABC satisfies it. - """ - - _runtime: _DetachableRuntime | None - - -class _DetachableSandboxBridge: - """Bridge so PR 4 can call sandbox.detach() before PR 5 lands. - - Matches the loud contract PR 5's Sandbox.detach() ships with — a - detach on a sandbox with no live runtime raises rather than - silently no-oping. Eval workers always attach before they detach - (via graph_repo.node(..., sandbox_id=...)); if this raises, the - attach side broke first and should be debugged at the cause. - """ - - @staticmethod - async def detach(sandbox: _DetachableSandbox) -> None: - runtime = sandbox._runtime - if runtime is None: - raise RuntimeError( - f"{type(sandbox).__name__}.detach() called on a sandbox " - f"with no live runtime. Eval workers must attach before " - f"detaching." - ) - # close_local is on the Protocol from PR 4 onward — every - # manager-backed runtime must implement it for the synchronous- - # fanout eval path to work. If a runtime is encountered without - # it, that's a contract violation we want surfaced (not - # silently downgraded to a full close). - await runtime.close_local() - # The _runtime PrivateAttr is settable on the non-frozen Sandbox - # Pydantic model — object.__setattr__ keeps symmetry with the - # frozen-Sandbox patterns used elsewhere in the API. - object.__setattr__(sandbox, "_runtime", None) -``` - -This bridge is replaced by `Sandbox.detach()` on the base class in -PR 5 (Task 4c). The Protocol definitions also go away — PR 5's -`SandboxRuntime` Protocol absorbs them. Every concrete -`ManagerBackedSandboxRuntime` must implement `close_local` from PR 4 -onward; if a runtime doesn't, the loud failure here surfaces it before -the bridge gets deleted. - -PR 5 lifts this into `Sandbox.detach()` as a base-class method. The -bridge is grep-able by class name for deletion. - -## Task 5: Remove `check_evaluators` Dispatch And Inngest Registration - -**Files:** - -- Modify: `ergon_core/ergon_core/core/infrastructure/inngest/registry.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/check_evaluators.py` - -- [ ] **Step 1: Remove `check_evaluators` send after `task/completed`** - -Where `worker_execute` (or `advance_run`) fires `check_evaluators`, delete -that send. `task/completed` should advance the run only. - -- [ ] **Step 2: Keep `check_evaluators.py` importable** - -Reduce its body to: - -```python -"""Legacy check-evaluators handler. Replaced by synchronous fanout in PR 4. - -This module remains importable until PR 11 deletes it so worktrees that -have not yet rebased can still import the module. The Inngest function -is not registered. -""" - -import inngest - -# Deliberately NOT registered with ALL_FUNCTIONS. -``` - -- [ ] **Step 3: Keep `evaluate_task_run` registered (slug unchanged)** - -`ALL_FUNCTIONS` must continue to include `evaluate_task_run`. Verify -its entry in `registry.py`: - -```python -from ergon_core.core.application.jobs.evaluate_task_run import evaluate_task_run - -ALL_FUNCTIONS = [ - ..., - evaluate_task_run, - ..., -] -``` - -Update the `check_evaluators` entry to be removed: - -```python -# Remove: -from ergon_core.core.infrastructure.inngest.handlers.check_evaluators import ( - check_evaluators, -) -# and the corresponding line in ALL_FUNCTIONS. -``` - -## Task 6: Tests - -**Files:** - -- Modify: `ergon_core/tests/unit/runtime/test_worker_execute_sandbox_lifecycle.py` -- Create: `ergon_core/tests/unit/runtime/test_evaluate_task_run_thin_payload.py` -- Modify: `ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py` - -- [ ] **Step 1: Sandbox release happens AFTER gather returns** - -```python -@pytest.mark.asyncio -async def test_worker_execute_releases_sandbox_after_eval_gather(monkeypatch): - ordering: list[str] = [] - - async def fake_invoke(step_id, fn, payload): - ordering.append(f"invoke-{step_id}") - - async def fake_release(sandbox): - ordering.append("release") - - monkeypatch.setattr( - worker_execute, "_acquire_sandbox", AsyncMock(side_effect=lambda *a, **kw: ordering.append("acquire") or _FakeSandbox()), - ) - monkeypatch.setattr( - "ergon_core.core.application.jobs.worker_execute.lifecycle_hub.release", - fake_release, - ) - - fake_ctx = SimpleNamespace(step=SimpleNamespace(invoke=fake_invoke)) - await worker_execute.run_worker_execute_job(fake_ctx, payload_factory(n_evaluators=3)) - - # acquire → 3 invokes → release, with release strictly last. - assert ordering[0] == "acquire" - assert ordering[-1] == "release" - assert ordering.count("release") == 1 - assert sum(1 for s in ordering if s.startswith("invoke-")) == 3 -``` - -- [ ] **Step 2: Eval thin-payload test** - -```python -@pytest.mark.asyncio -async def test_evaluate_task_run_loads_state_from_run_tier( - session, run_node_factory, task_execution_factory, worker_output_factory, -): - node = run_node_factory( - task_json=_object_bound_task_json(evaluators=[_test_rubric_json()]) - ) - execution = task_execution_factory(node_id=node.id, sandbox_id="sbx-123") - worker_output_factory(execution_id=execution.id, final_text="hello") - - # Stub the sandbox attach so we don't hit e2b in unit tests: - with patch.object(Sandbox, "_bind_runtime", AsyncMock()) as bind: - await evaluate_task_run.fn( - ctx=SimpleNamespace(), - payload=TaskEvaluateRequest( - run_id=node.run_id, - task_id=node.task_id, - execution_id=execution.id, - evaluator_index=0, - ), - ) - - # Sandbox bound to the stamped id, then detached at the end. - bind.assert_awaited_once_with("sbx-123") - # Evaluation persisted: - rows = session.exec( - select(RunTaskEvaluation).where(RunTaskEvaluation.run_id == node.run_id) - ).all() - assert len(rows) == 1 -``` - -- [ ] **Step 3: Architecture guard** - -```python -def test_evaluate_task_run_uses_thin_payload_and_run_tier_read() -> None: - body = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py" - ).read_text() - # Thin payload only: - assert "TaskEvaluateRequest" in body - assert "EvaluateTaskRunRequest" not in body - # No definition-tier reads: - assert "DefinitionRepository" not in body - assert "ExperimentDefinitionTask" not in body - # No registry-based evaluator resolution: - assert "ComponentCatalogService" not in body - # Uses the same run-tier loader the orchestrator uses: - assert "WorkflowGraphRepository" in body - assert ".node(" in body - - -def test_worker_execute_fans_out_via_step_invoke() -> None: - body = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/worker_execute.py" - ).read_text() - assert "ctx.step.invoke" in body - assert "evaluate_task_run" in body - # asyncio.gather over the invocations: - assert "asyncio.gather" in body - # Sandbox release in finally, NOT in a sibling job: - assert "finally:" in body - assert "terminate_sandbox_by_id" not in body -``` - -- [ ] **Step 4: Run focused tests** - -```bash -uv run pytest \ - ergon_core/tests/unit/runtime/test_worker_execute_sandbox_lifecycle.py \ - ergon_core/tests/unit/runtime/test_evaluate_task_run_thin_payload.py \ - ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py -q -``` - -Expected: pass; release strictly after gather; eval reads only run-tier. - -## Task 7: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` -- Modify: `ergon_core/tests/unit/runtime/test_identity_invariants.py` -- Move: `CreateTaskEvaluation` from `telemetry/repository.py` to - `telemetry/models.py` (see step below). - -PR 4 lands four invariants pre-registered in the ledgers: - -- [ ] **Step 1: Remove the PR 4 entries from `_XFAIL_BY_NAME`** - -In `test_v2_final_state_ledger.py`, delete: - -```python -"evaluate_task_run_uses_thin_payload": "PR 4 reshapes evaluate_task_run", -"check_evaluators_is_unregistered": "PR 4 removes check_evaluators dispatch", -``` - -- [ ] **Step 2: Flip three smoketest cases** - -In `test_walkthrough_smoketest.py`, remove the `@pytest.mark.xfail` from -each of: - -- `test_worker_execute_emits_one_evaluate_invocation_per_evaluator` -- `test_evaluate_task_run_payload_is_id_only` -- `test_sandbox_release_happens_after_all_evaluators_complete` - -Replace each `pytest.fail(...)` body with the real assertion. The -`inngest_driver` fixture lands here too — it's the test driver that -records `ctx.step.invoke` calls and exposes -`inngest_driver.step_invocations_for_function("evaluate_task_run")`. - -- [ ] **Step 3: Flip two identity invariants** - -In `test_identity_invariants.py`, remove the `@pytest.mark.xfail` from: - -- `test_sandbox_identity_is_preserved_across_worker_to_evaluate_boundary` -- `test_execution_id_is_unique_per_attempt_and_shared_across_evaluators` - -Implement the real bodies — both lean on `run_task_executions.sandbox_id` -being stamped by the orchestrator (this PR Task 3 Step 2) and on -`TaskEvaluateRequest` carrying `execution_id` + `evaluator_index`. - -- [ ] **Step 4: Move `CreateTaskEvaluation` to `telemetry/models.py`** - -PR 0.5's repository-companion-files guard xfails the -`test_repository_file_does_not_define_dtos[.../telemetry]` case until -the DTO moves out of the repo file. PR 4 is already editing -`telemetry/repository.py` to add `set_sandbox_id` and the -`WorkerOutputRepository` — folding the DTO move in keeps the touch -surface bounded. - -In `ergon_core/ergon_core/core/persistence/telemetry/repository.py`, -delete: - -```python -class CreateTaskEvaluation(BaseModel): - ... -``` - -Add the same class to -`ergon_core/ergon_core/core/persistence/telemetry/models.py`, and -update `telemetry/__init__.py` (or every importer) to re-export from -the new location. Run: - -```bash -rg "CreateTaskEvaluation" ergon_core ergon_builtins ergon_cli -``` - -to confirm every import path is updated. - -Remove the corresponding entry from `_KNOWN_VIOLATORS` in -`test_repository_companion_files.py`: - -```python -("test_repository_file_does_not_define_dtos", - "ergon_core/ergon_core/core/persistence/telemetry"): - "PR 4: move CreateTaskEvaluation to telemetry/models.py", -``` - -- [ ] **Step 5: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_repository_companion_files.py \ - ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py \ - ergon_core/tests/unit/runtime/test_identity_invariants.py -q -``` - -Expected: PR 4 ledger entries gone; the corresponding smoketest, -identity, and repository-companion cases PASS; remaining cases still -XFAIL. - -## PR Ledger - -Invariant landed: worker_execute orchestrates evaluation through -synchronous fanout; sandbox release bounded by the orchestrator's -try/finally; eval is a per-function Inngest target with thin id-only -payload. - -Bridge code introduced: `_DetachableSandboxBridge` (deleted by PR 5 -into `Sandbox.detach()`). - -Old path still intentionally alive: `EvaluateTaskRunRequest` import -shim, `check_evaluators.py` (importable, unregistered), `CriterionExecutor` -/ `InngestCriterionExecutor` (no longer used; PR 11 deletes). - -Deletion gate: PR 11 deletes the executor classes, -`EvaluateTaskRunRequest`, `check_evaluators.py`, and -`terminate_sandbox_by_id`. `evaluate_task_run` itself stays — the slug, -the function, the registration. Only the v1 body and v1 payload class -go. - -Tests added or updated: release-after-gather ordering, eval -thin-payload, runtime-read boundary guards for both jobs. - -Modules owned by this PR: worker-execute orchestration shape, eval -function body, Inngest registration, execution-row sandbox_id -stamping, worker-output persistence. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/06-pr-05-object-bound-api.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/06-pr-05-object-bound-api.md deleted file mode 100644 index 6d43b8c77..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/06-pr-05-object-bound-api.md +++ /dev/null @@ -1,1182 +0,0 @@ -# PR 5 — Object-Bound Public API And Definition Writer Bridge - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Add the v2 public authoring objects while keeping old -`TaskSpec`/`WorkerSpec` benchmarks writable. - -**Architecture:** Introduce `Sandbox`, object-bound `Task`, serializable -`Worker`, public `Experiment`, and a definition-writer bridge that accepts -both old and new benchmark output shapes. - -**Tech Stack:** Pydantic v2, public API exports, definition writer tests. - ---- - -## Files - -**Create:** - -```text -ergon_core/ergon_core/api/experiment.py -ergon_core/ergon_core/api/sandbox/__init__.py -ergon_core/ergon_core/api/sandbox/runtime.py -ergon_core/ergon_core/api/sandbox/sandbox.py -``` - -**Modify:** - -```text -ergon_core/ergon_core/api/__init__.py -ergon_core/ergon_core/api/benchmark/task.py -ergon_core/ergon_core/api/benchmark/benchmark.py -ergon_core/ergon_core/api/worker/worker.py -ergon_core/ergon_core/api/criterion/criterion.py -ergon_core/ergon_core/api/rubric/rubric.py -ergon_core/ergon_core/api/rubric/evaluator.py -ergon_core/ergon_core/api/errors.py -ergon_core/ergon_core/core/application/experiments/definition_writer.py -ergon_core/ergon_core/core/domain/experiments/validation.py -ergon_core/tests/unit/api/ -ergon_core/tests/unit/runtime/test_experiment_definition_service.py -``` - -## Current State - -`Experiment` lives under `core.domain.experiments` and binds: - -```python -workers: Mapping[str, WorkerSpec] -evaluators: Mapping[str, Evaluator] -assignments: Mapping[str, str | Sequence[str]] -``` - -`TaskSpec` carries evaluator binding keys but not concrete objects. - -## Target State For This PR - -New authoring code can write: - -```python -Experiment( - benchmark=BenchThatReturnsTasks(dataset_path="fixtures/minif2f.jsonl"), - name="mini", - description="MiniF2F smoke definition", - metadata={"created_by": "test"}, -) -``` - -and benchmark instances return: - -```python -Task( - task_slug="solve", - instance_key="sample-1", - description="Prove the theorem.", - worker=ReActWorker( - name="solver", - model="openai:gpt-4o-mini", - system_prompt="Write a Lean proof.", - max_iterations=8, - ), - sandbox=LeanSandbox(lean_version="4.7.0"), - evaluators=(Rubric(name="default", criteria=()),), -) -``` - -Old `TaskSpec` benchmarks still persist through a bridge. - -## Task 1: Add Sandbox API - -**Files:** - -- Create: `ergon_core/ergon_core/api/sandbox/runtime.py` -- Create: `ergon_core/ergon_core/api/sandbox/sandbox.py` -- Create: `ergon_core/ergon_core/api/sandbox/__init__.py` -- Modify: `ergon_core/ergon_core/api/errors.py` - -- [ ] **Step 0: Add `SandboxNotLiveError` to `api/errors.py`** - -```python -class SandboxNotLiveError(RuntimeError): - """Raised when a Sandbox method that requires a live runtime is - called on a sandbox whose `_runtime` is None. - - This is *deliberately* loud rather than a silent no-op. The v1 - sandbox-lifecycle audit found that silent-skip semantics on - detach/terminate masked double-release and release-before-acquire - bugs. v2 surfaces these immediately at the call site. - - Cases that raise: - - `Sandbox.terminate()` called before `provision()` succeeded. - - `Sandbox.terminate()` called twice. - - `Sandbox.detach()` called before `_bind_runtime()`. - - `Sandbox.detach()` called twice. - - `task.sandbox.run_command(...)` (or any IO) on a config-only sandbox. - - Lifecycle owners (worker_execute) and eval workers must track - sandbox state explicitly; silently no-oping here only hides bugs. - """ -``` - -Export from `api/__init__.py` alongside `Sandbox`. - -- [ ] **Step 1: Add protocol** - -```python -from collections.abc import Sequence -from typing import Protocol - - -class SandboxRuntime(Protocol): - sandbox_id: str - - async def run_command(self, cmd: str | Sequence[str], *, timeout: int | None = None): ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def read_file(self, path: str) -> bytes: ... - async def list_files(self, path: str) -> list[str]: ... - async def close(self) -> None: ... -``` - -- [ ] **Step 2: Add base class** - -```python -class Sandbox(BaseModel, ABC): - model_config = {"frozen": False, "arbitrary_types_allowed": True} - - env: dict[str, str] = Field(default_factory=dict) - timeout_seconds: int | None = None - requires_network: bool = False - output_path: str = "/workspace/final_output/" - _runtime: SandboxRuntime | None = PrivateAttr(default=None) - - @abstractmethod - async def provision(self) -> None: - """Provision a fresh external sandbox; attach _runtime to it.""" - - @abstractmethod - async def _bind_runtime(self, sandbox_id: str) -> None: - """Re-attach _runtime to an EXISTING external sandbox by id. - - Called by Sandbox.from_definition when sandbox_id is passed. - Authors implement this to connect to an already-running sandbox - (e.g. e2b's AsyncSandbox.connect(sandbox_id)) rather than - provision a new one. - """ - - @classmethod - async def from_definition( - cls, - sandbox_json: TaskDefinitionJson, - *, - sandbox_id: str | None = None, - ) -> "Sandbox": - """Inflate a Sandbox from JSON, optionally attached live. - - - If sandbox_id is None, return a config-only sandbox - (_runtime = None). Caller can later call provision() to make - it live, or pass to authors who only need config (e.g. a - static benchmark loader). - - If sandbox_id is passed, attach to the running sandbox - before returning. The returned instance is fully live; - callers can immediately use run_command / write_file / etc. - """ - - sandbox_type = sandbox_json.get("_type") - if not isinstance(sandbox_type, str): - raise ValueError( - f"Sandbox snapshot is missing the required `_type` " - f"discriminator (got {type(sandbox_type).__name__}). Every " - f"persisted sandbox must carry `_type`. Soft-defaulting " - f"would silently produce the wrong Sandbox subclass." - ) - SandboxCls = _import_component(sandbox_type) - instance = cast("Sandbox", SandboxCls.model_validate(sandbox_json)) - if sandbox_id is not None: - await instance._bind_runtime(sandbox_id) - return instance - - async def terminate(self) -> None: - """Terminate the EXTERNAL sandbox AND drop the local handle. - - Called only by the lifecycle owner (worker_execute). Eval - workers must use detach() instead. Raises - `SandboxNotLiveError` if called on a sandbox that has no live - runtime — this catches double-terminate and "terminate before - acquire" programming errors, which are exactly the lifecycle - bugs the v1 audit found. The caller knows whether a sandbox - is live; do not soft-fail on this. - """ - if self._runtime is None: - raise SandboxNotLiveError( - f"{type(self).__name__}.terminate() called on a sandbox " - f"with no live runtime. Likely double-terminate or " - f"terminate-before-acquire — both are lifecycle bugs." - ) - await self._runtime.close() - object.__setattr__(self, "_runtime", None) - - async def detach(self) -> None: - """Drop the local _runtime handle; DO NOT terminate the external sandbox. - - Called by eval workers in their finally block after - criterion.evaluate(). The external sandbox keeps running so - other eval invocations and worker_execute's final release can - still access it. Raises `SandboxNotLiveError` if called on a - sandbox that has no live runtime — eval workers always attach - before they detach, so a detach without a runtime is a - programming error worth surfacing. - - `SandboxRuntime.close_local` is part of the protocol after - PR 5 — runtimes that don't implement it fail at protocol - conformance, not silently here. - """ - if self._runtime is None: - raise SandboxNotLiveError( - f"{type(self).__name__}.detach() called on a sandbox " - f"with no live runtime. Eval workers must attach before " - f"detaching — see Sandbox.from_definition(sandbox_id=...)." - ) - await self._runtime.close_local() - object.__setattr__(self, "_runtime", None) - - @property - def is_live(self) -> bool: - return self._runtime is not None - - def _require_runtime(self) -> SandboxRuntime: - if self._runtime is None: - raise SandboxNotLiveError(type(self).__name__) - return self._runtime -``` - -The `_bind_runtime` / `detach` pair is the public surface a custom -`Sandbox` author has to think about. `from_definition` is framework -code that dispatches on the optional `sandbox_id`. The `SandboxRuntime` -protocol gets a `close_local()` method alongside `close()`: - -```python -class SandboxRuntime(Protocol): - sandbox_id: str - - async def run_command(self, cmd, *, timeout=None): ... - async def write_file(self, path: str, content: bytes) -> None: ... - async def read_file(self, path: str) -> bytes: ... - async def list_files(self, path: str) -> list[str]: ... - async def close(self) -> None: ... # terminate external + close local - async def close_local(self) -> None: ... # close local only; leave external alive -``` - -For e2b-backed runtimes: `close()` calls `manager.terminate(...)` AND -the SDK's `sandbox.close()`. `close_local()` calls only the SDK's -`sandbox.close()` to drop the gRPC stream / TCP connection on this -process, leaving the cloud sandbox running for the next attach. - -Add IO proxy methods on `Sandbox` so workers and criteria can call -`task.sandbox.write_file(...)` directly. Each method forwards to the -backing `SandboxRuntime` after checking `_require_runtime()`: - -```python - async def run_command( - self, - cmd: str | Sequence[str], - *, - timeout: int | None = None, - ) -> CommandResult: - runtime = self._require_runtime() - effective_timeout = timeout if timeout is not None else self.timeout_seconds - return await runtime.run_command(cmd, timeout=effective_timeout) - - async def write_file(self, path: str, content: bytes | str) -> None: - runtime = self._require_runtime() - payload = content.encode() if isinstance(content, str) else content - await runtime.write_file(path, payload) - - async def read_file(self, path: str) -> bytes: - runtime = self._require_runtime() - return await runtime.read_file(path) - - async def list_files(self, path: str | None = None) -> list[str]: - runtime = self._require_runtime() - return await runtime.list_files(path or self.output_path) - - @property - def sandbox_id(self) -> str: - return self._require_runtime().sandbox_id -``` - -`CommandResult` is the existing DTO returned by -`SandboxRuntime.run_command`; export it from -`ergon_core.api.sandbox.runtime` alongside the protocol so test fixtures -in PR 12 can construct one directly. The `timeout_seconds` fallback is -load-bearing for the v1-audit regression where evaluators issued -sandbox commands with no timeout — see -[`08-decisions-log.md`](../08-decisions-log.md) "Sandbox IO methods on -base". - -## Criterion / Evaluator signature note - -Per [`01-api-surface.md` § Criterion class signature — locked](../01-api-surface.md), -PR 5 keeps the v1 `Criterion.evaluate(self, context: CriterionContext) -> CriterionOutcome` -signature unchanged. The earlier file-tree annotation suggesting -`evaluate(..., sandbox: Sandbox)` is superseded — the sandbox is -accessed via `context.task.sandbox` (live in the eval worker per Δ.5). - -What PR 5 *does* change: - -- `CriterionContext` loses its runtime proxy methods (`run_command`, - `read_resource`, `write_file`, ...) — those now live on `Sandbox`. -- `CriterionContext._runtime` PrivateAttr is removed. -- `CriterionContext.sandbox_id` is removed — read via - `context.task.sandbox.sandbox_id` if needed. -- `Criterion.from_definition(criterion_json: TaskDefinitionJson)` - classmethod added (mirrors Task / Worker / Sandbox). -- `Rubric.evaluate(context)` aggregates per-criterion outcomes. - -No criterion subclass in tree needs a method-signature rewrite — -only the criterion bodies that previously called -`context.run_command(...)` need to switch to -`context.task.sandbox.run_command(...)`. PR 6 (MiniF2F) is the first -real-world conversion; PR 10a/10b/10c carry the rest. - -## Task 2: Add Object-Bound Task Fields - -**Files:** - -- Modify: `ergon_core/ergon_core/api/benchmark/task.py` - -- [ ] **Step 1: Add fields to `Task`** - -```python -worker: Worker | None = None -sandbox: Sandbox | None = None -evaluators: tuple[Evaluator, ...] = () -``` - -They are nullable only in this PR so `TaskSpec` bridge snapshots can still -inflate. PR 11 makes worker and sandbox non-null. - -- [ ] **Step 2: Update `Task.from_definition` to branch on snapshot shape and thread `sandbox_id`** - -Replace the PR 2 body. `Task.from_definition` is async (already locked -in PR 2) and now awaits `Sandbox.from_definition(...)` so the optional -`sandbox_id` flows through. - -```python -import logging - -from ergon_core.api.criterion.criterion import Criterion -from ergon_core.api.rubric.evaluator import Evaluator -from ergon_core.api.rubric.rubric import Rubric -from ergon_core.api.sandbox.sandbox import Sandbox -from ergon_core.api.worker.worker import Worker - - -logger = logging.getLogger(__name__) - - -@classmethod -async def from_definition( - cls, - task_json: TaskDefinitionJson, - *, - task_id: UUID, - sandbox_id: str | None = None, -) -> "Task": - task_type = task_json.get("_type") - if not isinstance(task_type, str): - raise ValueError( - f"Task snapshot is missing the required `_type` discriminator " - f"(got {type(task_type).__name__}). Every persisted task must " - f"carry `_type` — produced by `model_serializer` on Task " - f"subclasses or by `_definition_task_snapshot` during the PR 1 " - f"bridge." - ) - TaskCls = _import_component(task_type) - - if TaskCls is TaskSpec or "_legacy" in task_json: - # Bridge path: PR 1 wrote TaskSpec-shaped JSON for static nodes. - # TaskSpec snapshots carry no object-bound sandbox, so sandbox_id - # cannot be honored. Log a warning rather than silently ignoring, - # because a non-None sandbox_id on the legacy branch is a strong - # signal that a legacy snapshot reached an object-bound caller - # (likely an unmigrated builtin) — exactly the kind of drift the - # v1 audit was designed to surface. - if sandbox_id is not None: - logger.warning( - "Task.from_definition: sandbox_id=%r passed for a " - "TaskSpec/legacy snapshot (task_id=%s); cannot attach " - "a live sandbox to a TaskSpec. Likely a legacy benchmark " - "reached an object-bound code path — migrate the " - "benchmark to return Task instances.", - sandbox_id, task_id, - ) - spec_json = {k: v for k, v in task_json.items() if k != "_legacy"} - spec = TaskSpec.model_validate(spec_json) - instance = Task( - task_slug=spec.task_slug, - instance_key=spec.instance_key, - description=spec.description, - parent_task_slug=spec.parent_task_slug, - dependency_task_slugs=spec.dependency_task_slugs, - evaluator_binding_keys=spec.evaluator_binding_keys, - task_payload=spec.task_payload, - ) - else: - # Object-bound path: validate the Task subclass directly, then - # re-inflate each nested component through its own discriminator. - instance = cast("Task", TaskCls.model_validate(task_json)) - if isinstance(task_json.get("worker"), dict) and instance.worker is None: - object.__setattr__( - instance, "worker", Worker.from_definition(task_json["worker"]) - ) - if isinstance(task_json.get("sandbox"), dict) and instance.sandbox is None: - object.__setattr__( - instance, - "sandbox", - await Sandbox.from_definition( - task_json["sandbox"], - sandbox_id=sandbox_id, - ), - ) - elif instance.sandbox is not None and sandbox_id is not None: - # Sandbox was already model_validate-ed but is config-only; - # attach the runtime now. - await instance.sandbox._bind_runtime(sandbox_id) - elif instance.sandbox is None and sandbox_id is not None: - # Caller wants a live sandbox but the snapshot carries no - # Sandbox to attach to. Silent fall-through here would - # produce a Task whose sandbox is None — every subsequent - # `task.sandbox.run_command(...)` then explodes with a - # confusing AttributeError far from the cause. Loud fail - # here instead. - raise ValueError( - f"sandbox_id={sandbox_id!r} passed to Task.from_definition " - f"but task snapshot has no sandbox to attach to " - f"(task_id={task_id}, _type={task_type!r}). The eval-side " - f"call site expects a live sandbox; the snapshot must " - f"carry one." - ) - evaluators_json = task_json.get("evaluators") or () - if evaluators_json and not instance.evaluators: - inflated: list[Evaluator] = [] - for ev_json in evaluators_json: - ev_type_raw = ev_json.get("_type") - if not isinstance(ev_type_raw, str): - raise ValueError( - f"Evaluator snapshot in task {task_id} is missing " - f"the required `_type` discriminator " - f"(got {type(ev_type_raw).__name__})." - ) - ev_type = _import_component(ev_type_raw) - if issubclass(ev_type, Rubric): - inflated.append(Rubric.model_validate(ev_json)) - elif issubclass(ev_type, Criterion): - inflated.append(Criterion.from_definition(ev_json)) - else: - inflated.append(ev_type.model_validate(ev_json)) - object.__setattr__(instance, "evaluators", tuple(inflated)) - - object.__setattr__(instance, "_task_id", task_id) - return instance -``` - -`Task.from_definition` is the only entry point that propagates -`sandbox_id`. Callers of `graph_repo.node(..., sandbox_id=...)` get a -live sandbox; callers who pass nothing get a config-only sandbox. The -bridge path is deleted in PR 11 along with `TaskSpec`. - -## Task 2b: Convert Worker To Pydantic BaseModel - -PR 6 expects `ReActWorker` to subclass a Pydantic `Worker`. Today `Worker` is -an `ABC` with a hand-rolled `__init__` (see -`ergon_core/ergon_core/api/worker/worker.py`). This step lands that conversion. - -**Files:** - -- Modify: `ergon_core/ergon_core/api/worker/worker.py` - -- [ ] **Step 1: Replace the ABC base with a Pydantic ABC** - -Replace the existing class body with: - -```python -from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, Mapping -from importlib import import_module -from typing import Any, ClassVar, cast - -from pydantic import BaseModel, ConfigDict, Field - -from ergon_core.api.benchmark.task import Task -from ergon_core.api.errors import DependencyError -from ergon_core.api.worker.context import WorkerContext -from ergon_core.api.worker.results import WorkerOutput -from ergon_core.core.domain.generation.context_parts import ContextPartChunk -from ergon_core.core.infrastructure.dependencies import check_packages - -WorkerStreamItem = ContextPartChunk | WorkerOutput - - -class Worker(BaseModel, ABC): - """Base class for all workers. Pydantic-serializable.""" - - model_config = ConfigDict(arbitrary_types_allowed=True, frozen=False) - - type_slug: ClassVar[str] - required_packages: ClassVar[list[str]] = [] - install_hint: ClassVar[str] = "" - - # ClassVar declaring the Sandbox subclass a Worker requires. Default - # is the base `Sandbox` (accepts any kind); concrete Worker - # subclasses override to narrow (e.g. `LeanReActWorker.requires_sandbox - # = LeanSandbox`). Validated at `Experiment` construction time; - # see _validate_sandbox_compatibility in api/experiment.py. - requires_sandbox: ClassVar[type["Sandbox"]] = Sandbox # forward ref OK - - name: str - model: str | None = None - metadata: dict[str, Any] = Field(default_factory=dict) - - @abstractmethod - async def execute( - self, - task: Task, - *, - context: WorkerContext, - ) -> AsyncGenerator[WorkerStreamItem, None]: - """Run the worker, yielding context chunks and a terminal WorkerOutput.""" - raise NotImplementedError - - @classmethod - def from_definition(cls, worker_json: TaskDefinitionJson) -> "Worker": - worker_type = worker_json.get("_type") - if not isinstance(worker_type, str): - raise ValueError( - f"Worker snapshot is missing the required `_type` " - f"discriminator (got {type(worker_type).__name__}). Every " - f"persisted worker must carry `_type`." - ) - WorkerCls = _import_component(worker_type) - return cast("Worker", WorkerCls.model_validate(worker_json)) - - def validate_runtime_deps(self) -> None: - """Check that runtime dependencies are available.""" - errors = check_packages( - self.required_packages, - f"Worker '{self.type_slug}'", - ) - if errors: - parts = [*errors] - if self.install_hint: - parts.append(f"Install with: {self.install_hint}") - raise DependencyError("\n".join(parts)) -``` - -Notes: - -- `Worker.validate(...)` is renamed to `validate_runtime_deps(...)` because - Pydantic v2 reserves `validate` on `BaseModel`. Update the two known - callers (`benchmark_loader.py`, `experiment.py`) in the same commit. -- `Worker.from_buffer` is intentionally **not** carried over; the textual - ledger from PR 0 still names it, and PR 11 deletes the transitional ledger - row. Any subclass that overrode it must move its state into - `model_post_init` or an explicit factory classmethod (see CLAUDE.md - "Do not use `model_post_init` to assemble core public API objects" — prefer - the explicit factory). - -- [ ] **Step 2: Add `_type` discriminator and import helper** - -Add at the top of the module (sibling to `from_definition`): - -```python -def _import_component(path: str) -> type[Any]: - module_name, _, qualname = path.partition(":") - if not module_name or not qualname: - raise ValueError(f"Worker _type must be 'module:qualname', got {path!r}") - obj: Any = import_module(module_name) - for part in qualname.split("."): - # typing: dynamic qualname walk — `part` is a user-controlled - # discriminator path component, not a typed attribute name. - obj = getattr(obj, part) - if not isinstance(obj, type): - raise TypeError(f"Worker _type {path!r} did not resolve to a class") - return obj -``` - -Add a `_type` computed serialization field consistent with `Task`: - -```python -@model_serializer(mode="wrap") -def _serialize(self, handler): - payload = handler(self) - payload["_type"] = f"{type(self).__module__}:{type(self).__qualname__}" - return payload -``` - -- [ ] **Step 3: Subclass smoke test** - -```python -def test_worker_baseclass_is_pydantic_and_serializes_type() -> None: - class _Echo(Worker): - type_slug = "echo" - - async def execute(self, task, *, context): # noqa: ARG002 - yield WorkerOutput(final_text="ok") - - serialized = _Echo(name="e", model=None).model_dump() - assert serialized["_type"].endswith(":_Echo") - rebuilt = Worker.from_definition(serialized | {"_type": serialized["_type"]}) - assert rebuilt.name == "e" -``` - -Run: - -```bash -uv run pytest ergon_core/tests/unit/api -k "worker_baseclass_is_pydantic" -q -``` - -## Task 3: Add Public Experiment - -**Files:** - -- Create: `ergon_core/ergon_core/api/experiment.py` -- Modify: `ergon_core/ergon_core/api/__init__.py` - -- [ ] **Step 1: Add model** - -```python -class Experiment(BaseModel): - model_config = {"arbitrary_types_allowed": True} - - benchmark: Benchmark - name: str | None = None - description: str | None = None - # First-class authoring-metadata fields. Anything the framework - # reads (dashboard listing, audit, denormalized indexed columns) - # lives here, not in `metadata`. `metadata` is for opaque - # author-provided tags. - created_by: str | None = None - metadata: dict[str, Any] = Field(default_factory=dict) - _persisted: DefinitionHandle | None = PrivateAttr(default=None) - - @model_validator(mode="after") - def _validate_sandbox_compatibility(self) -> "Experiment": - for tasks in self.benchmark.build_instances().values(): - for task in tasks: - if not isinstance(task, Task): - continue - if task.worker is None or task.sandbox is None: - continue - required = type(task.worker).requires_sandbox - if not isinstance(task.sandbox, required): - raise SandboxKindMismatch( - task_id=task.task_id if task._task_id else uuid4(), - component=type(task.worker).__name__, - required=required, - actual=type(task.sandbox), - ) - return self -``` - -Use a generated UUID only for the error context during transition because -definition-time tasks do not yet have stable IDs in memory. - -- [ ] **Step 2: Export it** - -Add `Experiment`, `Sandbox`, and `SandboxRuntime` to -`ergon_core.api.__all__`. - -## Task 4: Definition Writer Bridge - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/experiments/definition_writer.py` - -- [ ] **Step 1: Add serializer** - -```python -def _task_to_definition_json(task: Task | TaskSpec) -> dict: - if isinstance(task, Task): - return task.model_dump(mode="json") - return { - "_type": "ergon_core.api.benchmark.task:TaskSpec", - **task.model_dump(mode="json"), - "_legacy": True, - } -``` - -- [ ] **Step 2: Write task JSON into definition rows** - -In `definition_writer.py`, replace the existing task row construction -inside `persist_definition` (look for the loop that builds -`ExperimentDefinitionTask(...)`) with the snippet below. The new -`task_json` column lands here, not in PR 7 — PR 7 only adds metadata to -the parent `ExperimentDefinition` row: - -```python -from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinitionTask, -) - - -def _persist_task_rows( - session: Session, - *, - definition_id: UUID, - instance_id: UUID, - instance_key: str, - tasks: Sequence[Task | TaskSpec], -) -> None: - for task in tasks: - task_json = _task_to_definition_json(task) - row = ExperimentDefinitionTask( - definition_id=definition_id, - instance_id=instance_id, - instance_key=instance_key, - task_slug=task.task_slug, - description=task.description, - task_payload_json=task_json.get("task_payload", {}), - task_json=task_json, - ) - session.add(row) -``` - -If `ExperimentDefinitionTask.task_json` does not exist yet, add it as part -of this PR with an additive Alembic migration (mirror the PR 1 migration -shape): - -```python -def upgrade() -> None: - op.add_column( - "experiment_definition_tasks", - sa.Column( - "task_json", - sa.JSON(), - nullable=False, - server_default="{}", - ), - ) - - -def downgrade() -> None: - op.drop_column("experiment_definition_tasks", "task_json") -``` - -The bridge serializer must be named `_task_to_definition_json` so PR 11 -can grep-and-delete it as a single symbol when only `Task` remains. - -## Task 4c: Lift PR 4 Bridges Into Object-Bound API - -PR 4 landed the synchronous-fanout invariant against the *current* -codebase, which still lacks `task.evaluators`, `task.sandbox`, and the -`lifecycle_hub` acquire/release primitive. To avoid half-building -PR 5 inside PR 4, every cross-PR dependency was introduced as a named, -greppable bridge. PR 5 lifts each one (see PR 4 plan -`05-pr-04-inline-criteria.md` § "Implementation Note — Bridge-Everything -Approach" for the full catalogue): - -| Bridge introduced by PR 4 | Where it lived | PR 5 replacement | -| --- | --- | --- | -| `_evaluator_bridge.resolve_evaluator` | `core/application/jobs/_evaluator_bridge.py` | `task.evaluators[index]` (lifted in step 3 below) | -| `EvaluationService.evaluate(*, context, evaluator)` (the v2 entry point; legacy executor-based entry was renamed to `evaluate_legacy` in PR 4) | `core/application/evaluation/service.py` | unchanged at PR 5 — PR 11 deletes `evaluate_legacy`, the `CriterionExecutor` Protocol, and `InngestCriterionExecutor` together | -| `terminate_sandbox_by_id` called from `worker_execute`'s `finally` | `core/infrastructure/sandbox/lifecycle.py` (existing helper) | optional at PR 5 — when `lifecycle_hub` lands, swap the call site to `lifecycle_hub.release(sandbox)`; if PR 5 doesn't add the hub, leave it for the PR that does | - -**`_DetachableSandboxBridge` is *not* in PR 4.** The PR 4 plan -originally specified it, but the bridge could not have a real caller -without `task.sandbox` and a sandbox `_runtime` — both PR 5 fields. -PR 5 adds `Sandbox`, `task.sandbox`, the `_runtime` attach, and -`Sandbox.detach()` in one coherent change; the steps below introduce -`Sandbox.detach()` from scratch instead of "lifting" a bridge. - -### Step 1: Wire `Sandbox.detach()` into `evaluate_task_run` - -PR 4 deliberately omitted the eval-side `detach()` call because no -`task.sandbox` existed to call it on. Now that `Sandbox` is a real -ABC and `task.sandbox` is bound (Task 1 + Task 2 above), wire the -detach call into the eval body so the local `_runtime` handle is -released as soon as criteria finish — the external sandbox stays -alive (the orchestrator's `try/finally` owns termination). - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` -- Delete: `ergon_core/ergon_core/core/infrastructure/sandbox/runtime.py:_DetachableSandboxBridge` - -- [ ] **Step 1: Add the detach call to the eval body** - -```python -# After (PR 5): -await task.sandbox.detach() -``` - -Place it on both the success and failure paths in -`evaluate_task_run.py` so the local `_runtime` is always released — -the orchestrator's `terminate_sandbox_by_id` keeps owning the external -sandbox. - -- [ ] **Step 2: Verify no stale bridge references** - -```bash -rg "_DetachableSandboxBridge" ergon_core ergon_builtins -``` - -Expected: empty (the bridge was never introduced — confirm nothing -attempted to import it). - -### Step 3: Retire `_evaluator_bridge.py` - -PR 4 routed evaluator lookup through a sibling bridge module so -`evaluate_task_run.py`'s body stayed free of `ComponentCatalogService` -and `DefinitionRepository` (the architecture guard checks the body of -the job file only, not its imports). Now that `Task` carries -`evaluators: tuple[Evaluator, ...]` directly (Task 2 above), the -bridge is dead weight. - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` -- Delete: `ergon_core/ergon_core/core/application/jobs/_evaluator_bridge.py` - -- [ ] **Step 3a: Replace bridge call with the object-bound field** - -```python -# Before (PR 4): -from ergon_core.core.application.jobs._evaluator_bridge import resolve_evaluator -evaluator = await resolve_evaluator(session, task, payload.evaluator_index) - -# After (PR 5): -evaluator = task.evaluators[payload.evaluator_index] -``` - -- [ ] **Step 3b: Delete the bridge module** - -`git rm ergon_core/ergon_core/core/application/jobs/_evaluator_bridge.py` -then run: - -```bash -rg "_evaluator_bridge|resolve_evaluator" ergon_core ergon_builtins -``` - -Expected: empty. - -- [ ] **Step 3c: Add a runtime-read-boundary guard** - -Append to `tests/unit/architecture/test_runtime_read_boundaries.py`: - -```python -def test_evaluate_task_run_uses_object_bound_evaluators() -> None: - text = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py" - ).read_text() - assert "_evaluator_bridge" not in text, ( - "PR 5 retires the PR 4 evaluator resolution bridge — pick " - "evaluators off task.evaluators[index] directly." - ) - assert "task.evaluators[" in text, ( - "PR 5 binds evaluators directly to the Task; the eval worker " - "must dispatch on task.evaluators[index]." - ) -``` - -## Task 4b: Prefer `task.worker`; Keep A Narrow Legacy Fallback While Benchmarks Migrate - -PR 3 introduced `_worker_from_payload_bridge` in `worker_execute.py`. PR 5 -makes `task.worker` canonical, but the legacy benchmarks still return -`TaskSpec` instances (no inline worker) — they get migrated to object-bound -`Task` in PR 6 (minif2f), PR 10a (swebench), PR 10b (researchrubrics) and -PR 10c (gdpeval). Retiring the bridge outright in PR 5 would break the e2e -smoke runs on every legacy benchmark. - -The discipline: `worker_execute.py`'s body reads `task.worker` first and -**only** falls through to the legacy registry-slug construction when -`task.worker is None` (the explicit legacy-snapshot signal). The fallback -lives in a sibling module so the eval-body architecture guard stays clean. - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/jobs/worker_execute.py` -- Create: `ergon_core/ergon_core/core/application/jobs/_legacy_worker_bridge.py` -- Modify: `ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py` - -- [ ] **Step 1: Replace bridge call with `task.worker` + named legacy fallback** - -Delete the in-body `_worker_from_payload_bridge`. Move the -`ComponentCatalogService.build_worker(...)` call into a sibling module: - -```python -# core/application/jobs/_legacy_worker_bridge.py -"""TaskSpec-snapshot fallback for ``worker_execute``. - -TODO(PR 11): delete this module. PR 5 made ``task.worker`` canonical; -this fallback only fires when the snapshot has no inline worker — i.e. -when a benchmark hasn't been migrated yet. Each benchmark migration -(PR 6 minif2f, PR 10a swebench, PR 10b researchrubrics, -PR 10c gdpeval) removes one entry from the legacy-shape set. PR 11 -(Δ.7) deletes the file once every benchmark returns ``Task``. - -The module is grep-able by class/function name so each migration PR -can confirm progress. -""" -... -def legacy_worker_from_payload(payload) -> Worker: - """Reconstruct a Worker from the v1 registry-slug payload fields. - Only called by `worker_execute` when `task.worker is None`.""" - ... -``` - -In `worker_execute.py`: - -```python -worker = task.worker -if worker is None: - # Legacy TaskSpec path — benchmark hasn't migrated to object-bound - # Task yet. PR 6/10a/10b/10c migrate the builtins; PR 11 deletes - # this fallback and its sibling module. - from ergon_core.core.application.jobs._legacy_worker_bridge import ( - legacy_worker_from_payload, - ) - worker = legacy_worker_from_payload(payload) -worker.validate_runtime_deps() -``` - -- [ ] **Step 2: Extend runtime read boundary guard** - -The guard checks the *body* of `worker_execute.py` only — the sibling -module is exempt by design (same pattern as PR 4's `_evaluator_bridge`). - -```python -def test_worker_execute_prefers_task_worker_over_legacy_bridge() -> None: - text = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/worker_execute.py" - ).read_text() - assert "ComponentCatalogService" not in text, ( - "worker_execute body must not import the registry directly — " - "any legacy fallback lives in _legacy_worker_bridge.py." - ) - # `_worker_from_payload_bridge` is the PR 3 in-body name; the PR 5 - # legacy fallback is a sibling-module function and must not appear - # as a module-level def here. - assert "def _worker_from_payload_bridge" not in text - assert "task.worker" in text, ( - "PR 5 prefers task.worker; the legacy fallback only fires " - "when task.worker is None." - ) -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py \ - ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py -q -``` - -Expected: both pass; legacy bridge symbols absent. - -## Task 5: Tests - -**Files:** - -- Modify: `ergon_core/tests/unit/api/test_public_api_imports.py` -- Modify: `ergon_core/tests/unit/runtime/test_experiment_definition_service.py` - -- [ ] **Step 1: Add import test** - -```python -def test_v2_public_api_exports_authoring_objects() -> None: - from ergon_core.api import Experiment, Sandbox, Task, Worker - - assert Experiment is not None - assert Sandbox is not None - assert Task is not None - assert Worker is not None -``` - -- [ ] **Step 2: Add definition-writer dual-shape test** - -```python -import pytest -from sqlmodel import select - -from ergon_core.api import Benchmark, Experiment, Task -from ergon_core.api.benchmark import BenchmarkRequirements, TaskSpec -from ergon_core.core.application.experiments.definition_writer import ( - persist_definition, -) -from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinitionTask, -) -from tests.unit.runtime._test_workers import EchoWorker, EchoSandbox - - -class _LegacyBenchmark(Benchmark): - benchmark_type = "test-legacy" - requirements = BenchmarkRequirements() - - def build_instances(self): - return { - "default": ( - TaskSpec( - task_slug="legacy", - instance_key="sample-1", - description="legacy task", - evaluator_binding_keys=("default",), - ), - ) - } - - -class _ObjectBoundBenchmark(Benchmark): - benchmark_type = "test-object-bound" - requirements = BenchmarkRequirements() - - def build_instances(self): - return { - "default": ( - Task( - task_slug="object", - instance_key="sample-1", - description="object-bound task", - worker=EchoWorker(name="echo", model=None), - sandbox=EchoSandbox(), - evaluators=(), - ), - ) - } - - -@pytest.mark.asyncio -async def test_definition_writer_persists_both_legacy_and_object_bound(session): - legacy_handle = persist_definition( - Experiment(benchmark=_LegacyBenchmark(), name="legacy") - ) - object_handle = persist_definition( - Experiment(benchmark=_ObjectBoundBenchmark(), name="object-bound") - ) - - legacy_row = session.exec( - select(ExperimentDefinitionTask).where( - ExperimentDefinitionTask.definition_id == legacy_handle.definition_id - ) - ).one() - object_row = session.exec( - select(ExperimentDefinitionTask).where( - ExperimentDefinitionTask.definition_id == object_handle.definition_id - ) - ).one() - - # Bridge path: TaskSpec snapshot is marked _legacy and carries - # binding keys but no inline objects. - assert legacy_row.task_json["_type"].endswith(":TaskSpec") - assert legacy_row.task_json.get("_legacy") is True - assert "worker" not in legacy_row.task_json - assert legacy_row.task_json["evaluator_binding_keys"] == ["default"] - - # Object-bound path: full _type discriminators for every component. - assert object_row.task_json["_type"].endswith(":Task") - assert "_legacy" not in object_row.task_json - assert object_row.task_json["worker"]["_type"].endswith(":EchoWorker") - assert object_row.task_json["sandbox"]["_type"].endswith(":EchoSandbox") - assert isinstance(object_row.task_json["evaluators"], list) -``` - -`EchoWorker` and `EchoSandbox` go in `tests/unit/runtime/_test_workers.py` -as minimal Pydantic implementations — keep them in the same module so other -PRs reuse them. If the helper module does not yet exist, create it in this -PR. - -- [ ] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_core/tests/unit/api ergon_core/tests/unit/runtime/test_experiment_definition_service.py -q -``` - -## Task 6: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` -- Modify: `ergon_core/tests/unit/architecture/test_dead_path_audit.py` - -PR 5 introduces the object-bound public API and retires the two bridges -PR 3 / PR 4 introduced (`_worker_from_payload_bridge`, -`_DetachableSandboxBridge`). - -- [ ] **Step 1: Remove `task_has_no_model_post_init` from `_XFAIL_BY_NAME`** - -In `test_v2_final_state_ledger.py`, delete: - -```python -"task_has_no_model_post_init": "PR 5 introduces object-bound Task", -``` - -- [ ] **Step 2: Remove the `_DetachableSandboxBridge` entry from `_XFAIL_BY_SYMBOL`** - -In `test_dead_path_audit.py`, delete: - -```python -"_DetachableSandboxBridge": "PR 5: lifted into Sandbox.detach()", -``` - -(That symbol was never introduced — see PR 5 Task 4c. It's gone because -nothing ever called it.) - -**Don't** remove the `_worker_from_payload_bridge` entry yet — Task 4b -keeps a narrow legacy fallback in `_legacy_worker_bridge.py` while the -benchmark builtins migrate (PR 6 / PR 10a / PR 10b / PR 10c). Replace -the entry's reason string instead: - -```python -"_worker_from_payload_bridge": "PR 11: deleted when every benchmark " -"returns object-bound Task (last gate after PR 10c).", -``` - -The in-body name is still dead (the architecture guard enforces that), -so the entry stays — it just points at PR 11 instead of PR 5. - -- [ ] **Step 3: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_dead_path_audit.py -q -``` - -Expected: three more cases PASS; remaining cases still XFAIL. - -## PR Ledger - -Invariant landed: object-bound authoring exists and persists beside old -TaskSpec authoring. - -Bridge code introduced: nullable `Task.worker`/`Task.sandbox`, -`_task_to_definition_json`, legacy `TaskSpec` validation path, and a new -sibling-module legacy worker fallback at -`core/application/jobs/_legacy_worker_bridge.py` (see Task 4b for the -rationale — benchmark builtins still return `TaskSpec`). - -Bridge code retired: the *in-body* `_worker_from_payload_bridge` is gone -from `worker_execute.py`. The runtime-read guard forbids -`ComponentCatalogService` in the body. The fallback path moved into -`_legacy_worker_bridge.py` and only fires when `task.worker is None`. - -Old path still intentionally alive: `TaskSpec`, `WorkerSpec`, assignments, -registry lookups, and the legacy worker fallback above. - -Deletion gate: - -- PR 6 (minif2f), PR 10a (swebench), PR 10b (researchrubrics), - PR 10c (gdpeval) each migrate one benchmark family from `TaskSpec` to - object-bound `Task`. After each migration, that benchmark's tasks - carry `task.worker` and never hit the legacy fallback. -- PR 11 deletes `_legacy_worker_bridge.py` and the `if worker is None:` - branch in `worker_execute.py` once every benchmark has migrated. - -Tests added or updated: public API exports and dual-shape definition writer. - -Modules owned by this PR: public API and definition writer. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07-pr-06-minif2f-vertical.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07-pr-06-minif2f-vertical.md deleted file mode 100644 index c8c9a0c4a..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07-pr-06-minif2f-vertical.md +++ /dev/null @@ -1,404 +0,0 @@ -# PR 6 — MiniF2F V2 Vertical - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Convert one real builtin benchmark, MiniF2F, to the object-bound -v2 authoring shape end to end. - -**Architecture:** Reuse existing MiniF2F sandbox/toolkit logic, but move the -sandbox kind into a `LeanSandbox(Sandbox)` class and have benchmark tasks -carry concrete worker/sandbox/evaluator objects. - -**Tech Stack:** Pydantic workers, builtin benchmark factories, E2B-backed -sandbox bridge, pytest. - ---- - -## Files - -**Create:** - -```text -ergon_builtins/ergon_builtins/sandboxes/__init__.py -ergon_builtins/ergon_builtins/sandboxes/lean.py -ergon_builtins/ergon_builtins/toolkits/__init__.py -ergon_builtins/ergon_builtins/toolkits/minif2f.py -``` - -**Modify:** - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py -ergon_builtins/ergon_builtins/benchmarks/minif2f/worker_factory.py -ergon_builtins/ergon_builtins/benchmarks/minif2f/rubric.py -ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -ergon_builtins/tests/unit/ -ergon_core/tests/unit/runtime/test_experiment_definition_service.py -``` - -## Current State - -MiniF2F returns `TaskSpec` and relies on worker/sandbox registry binding: - -```python -TaskSpec( - task_slug="prove", - instance_key="sample-1", - description="Prove theorem sample-1.", - evaluator_binding_keys=("default",), -) -``` - -## Target State For This PR - -MiniF2F returns: - -```python -Task( - task_slug="prove", - instance_key="sample-1", - description="Prove theorem sample-1.", - worker=ReActWorker( - name="solver", - model="openai:gpt-4o-mini", - system_prompt="Write a Lean proof.", - max_iterations=8, - toolkit=MiniF2FToolkit(), - ), - sandbox=LeanSandbox(lean_version="4.7.0"), - evaluators=(mini_f2f_rubric(),), -) -``` - -## Task 1: Add `LeanSandbox` - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/sandboxes/lean.py` - -- [ ] **Step 1: Implement class** - -`BaseSandboxManager.create` returns an `AsyncSandbox` (E2B SDK) and stores -it keyed by `task_id` for later `get_sandbox(task_id)` access. The adapter -needs to remember `task_id` so every IO method can look the sandbox up. - -```python -import inspect -from uuid import UUID, uuid4 - -from ergon_core.api.sandbox import Sandbox -from ergon_builtins.benchmarks.minif2f.sandbox_manager import ( - MiniF2FSandboxManager, -) - - -class LeanSandbox(Sandbox): - """Lean 4 sandbox for MiniF2F. Wraps the legacy E2B manager during PR 6.""" - - lean_version: str = "4.7.0" - e2b_template: str = "ergon-minif2f-v1" - requires_network: bool = False - output_path: str = "/workspace/final_output/" - - async def provision(self) -> None: - manager = MiniF2FSandboxManager() - sandbox_key = uuid4() - await manager.create(task_id=sandbox_key, envs=self.env) - live_sandbox = manager.get_sandbox(sandbox_key) - if live_sandbox is None: - raise RuntimeError( - f"MiniF2FSandboxManager.create returned but no sandbox is " - f"registered for task_id={sandbox_key}" - ) - runtime = _ManagerBackedSandboxRuntime( - manager=manager, - sandbox=live_sandbox, - sandbox_key=sandbox_key, - ) - object.__setattr__(self, "_runtime", runtime) -``` - -The `task_id` here is a sandbox-cache key, not the `Task.task_id` — the -manager uses it as a dictionary key. PR 10 extracts this adapter into -`ergon_builtins/ergon_builtins/sandboxes/_manager_backed.py` for reuse. - -- [ ] **Step 2: Add runtime adapter** - -`BaseSandboxManager` exposes `create / get_sandbox / upload_file / -list_files / terminate`. It does **not** expose `run_command` or -`read_file` directly — those are E2B SDK methods on the `AsyncSandbox` -itself. The adapter goes through the live sandbox handle: - -```python -from collections.abc import Sequence -from typing import Protocol -from uuid import UUID - -from ergon_core.api.sandbox.runtime import CommandResult - - -class _E2BSandboxHandle(Protocol): - """Typed view over the parts of e2b's `AsyncSandbox` we depend on. - - The e2b SDK doesn't ship a Protocol we can import; we define one - here at the boundary so the rest of the adapter stays typed. - `sandbox_id`, `commands.run`, and `files.read|write` are stable - parts of the e2b SDK surface circa 2026. - """ - - sandbox_id: str - commands: "_E2BCommands" - files: "_E2BFiles" - - -class _E2BCommands(Protocol): - async def run(self, cmd: str, *, timeout: int | None = None): ... - - -class _E2BFiles(Protocol): - async def read(self, path: str) -> bytes: ... - async def write(self, path: str, content: bytes) -> None: ... - - -class _ManagerBackedSandboxRuntime: - """Adapter from BaseSandboxManager + AsyncSandbox to SandboxRuntime.""" - - def __init__( - self, - *, - manager, - sandbox: _E2BSandboxHandle, - sandbox_key: UUID, - ) -> None: - self._manager = manager - self._sandbox = sandbox - self._sandbox_key = sandbox_key - # e2b's AsyncSandbox always carries `sandbox_id`; the Protocol - # makes that contract explicit so no getattr fallback is needed. - # If a non-conforming handle slips in, AttributeError surfaces - # immediately rather than masquerading as a stringified UUID. - self.sandbox_id: str = sandbox.sandbox_id - - async def run_command( - self, - cmd: str | Sequence[str], - *, - timeout: int | None = None, - ) -> CommandResult: - rendered = cmd if isinstance(cmd, str) else " ".join(cmd) - result = await self._sandbox.commands.run(rendered, timeout=timeout) - return CommandResult( - exit_code=result.exit_code, - stdout=result.stdout or "", - stderr=result.stderr or "", - ) - - async def write_file(self, path: str, content: bytes) -> None: - # Manager.upload_file expects (task_id, local_path, sandbox_path). - # We have bytes — use the underlying SDK directly for parity with - # the v1 path. PR 10 normalizes this on the shared adapter. - await self._sandbox.files.write(path, content) - - async def read_file(self, path: str) -> bytes: - return await self._sandbox.files.read(path) - - async def list_files(self, path: str) -> list[str]: - return await self._manager.list_files(self._sandbox_key, path) - - async def close(self) -> None: - # Terminate external sandbox AND drop local handle. - await self._manager.terminate(self._sandbox_key, reason="completed") - - async def close_local(self) -> None: - # Drop the local gRPC/TCP connection only; the external sandbox keeps - # running so sibling eval workers and the orchestrator's final - # terminate() can still reach it. e2b's AsyncSandbox.close() does - # exactly this — it closes the stream without sending a kill signal. - await self._sandbox.close() -``` - -The adapter deliberately calls `_sandbox.commands.run` and -`_sandbox.files.{read,write}` directly because `BaseSandboxManager` does -not expose those entry points. `list_files` and `terminate` go through -the manager so its caching/observability stays intact. If -`BaseSandboxManager` grows `run_command` / `read_file` between now and -PR 10, switch to those at that point. - -## Task 2: Make ReActWorker And MiniF2FToolkit Serializable - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Modify: `ergon_builtins/ergon_builtins/toolkits/minif2f.py` - -`ReActWorker.toolkit` round-trips through the `_type` discriminator, so -both the worker AND the toolkit must be Pydantic BaseModels. The toolkit -holds **config**, not live runtime handles — its `tools(...)` method -constructs runtime tool objects lazily at execute time. - -- [ ] **Step 1: Convert `MiniF2FToolkit` to Pydantic BaseModel** - -In `ergon_builtins/ergon_builtins/toolkits/minif2f.py`: - -```python -from pydantic import BaseModel, ConfigDict - - -class MiniF2FToolkit(BaseModel): - """Serializable MiniF2F toolkit config. - - Carries only config (file paths, limits, flags). Runtime tool - handles are built lazily via `tools(sandbox, task)`; they are not - serializable and never round-trip through JSON. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - proof_output_path: str = "/workspace/final_output/proof.lean" - lean_workspace: str = "/workspace/lean" - max_tool_calls: int = 32 - - def tools(self, sandbox, task): - # Lazy import keeps runtime tool construction out of the - # serialization path. The internal module builds AgentTool - # instances bound to the live sandbox. - from ergon_builtins.toolkits._minif2f_tools import build_tools - - return build_tools(self, sandbox=sandbox, task=task) -``` - -Move any runtime tool construction (`AgentTool` instances bound to the -sandbox) into a sibling `_minif2f_tools.py` module. The toolkit -serializes; the tools never do. - -- [ ] **Step 2: Convert `ReActWorker` constructor state to Pydantic fields** - -Make `ReActWorker` inherit `Worker` once `Worker` becomes a Pydantic -model. Fields: - -```python -name: str -model: str | None -system_prompt: str -max_iterations: int = 20 -toolkit: MiniF2FToolkit | None = None -``` - -Runtime-only clients (LLM provider handle, http client) go into -`PrivateAttr`. - -PR 10a / 10b / 10c each extend `ReActWorker.toolkit`'s type union to -include their toolkit, OR — once a third toolkit lands — replace the -union with a `Toolkit` protocol that requires `model_dump()` and -`tools(sandbox, task)`. - -- [ ] **Step 3: Smoke test toolkit round-trip** - -```python -def test_minif2f_toolkit_round_trips_through_json() -> None: - tk = MiniF2FToolkit(max_tool_calls=16) - serialized = tk.model_dump(mode="json") - assert serialized["_type"].endswith(":MiniF2FToolkit") - rebuilt = MiniF2FToolkit.model_validate(serialized) - assert rebuilt.max_tool_calls == 16 -``` - -## Task 3: Convert Benchmark Tasks - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/worker_factory.py` - -- [ ] **Step 1: Replace imports** - -Replace: - -```python -from ergon_core.api.benchmark import Benchmark, BenchmarkRequirements, TaskSpec -``` - -with: - -```python -from ergon_core.api import Benchmark, BenchmarkRequirements, Task -from ergon_builtins.sandboxes import LeanSandbox -from ergon_builtins.toolkits.minif2f import MiniF2FToolkit -``` - -- [ ] **Step 2: Replace `TaskSpec` construction** - -Each task becomes: - -```python -Task[MiniF2FTaskPayload]( - task_slug="prove", - instance_key=instance_key, - description=description, - task_payload=payload, - worker=make_minif2f_worker(), - sandbox=LeanSandbox(), - evaluators=(make_minif2f_rubric(),), -) -``` - -## Task 4: Tests - -**Files:** - -- Modify: `ergon_builtins/tests/unit/` -- Modify: `ergon_core/tests/unit/runtime/test_experiment_definition_service.py` - -- [ ] **Step 1: Add definition JSON assertion** - -Persist a MiniF2F experiment and assert the first task JSON includes: - -```python -assert task_json["worker"]["_type"].endswith(":ReActWorker") -assert task_json["worker"]["toolkit"]["_type"].endswith(":MiniF2FToolkit") -assert task_json["sandbox"]["_type"].endswith(":LeanSandbox") -assert task_json["evaluators"], "evaluators must persist" -# Every evaluator entry must carry a `_type` discriminator so it can -# round-trip through Evaluator.from_definition / Rubric.from_definition. -assert all( - ev.get("_type") for ev in task_json["evaluators"] -), "every evaluator entry must carry a `_type` discriminator" -assert "_legacy" not in task_json, ( - "MiniF2F is now object-bound; the _legacy bridge marker should be absent" -) -``` - -- [ ] **Step 2: Run focused tests** - -```bash -uv run pytest ergon_builtins/tests/unit ergon_core/tests/unit/runtime/test_experiment_definition_service.py -q -``` - -## PR Ledger - -Invariant landed: one builtin proves the v2 authoring vertical. - -Bridge code introduced: manager-backed `LeanSandbox` runtime adapter. - -Bridge code retired (partially): -- MiniF2F tasks now carry `task.worker`/`task.sandbox` inline, so MiniF2F - runs no longer hit the `_legacy_worker_bridge` fallback that PR 5 - Task 4b put on `worker_execute`. The fallback itself stays alive — it - still serves swebench, researchrubrics, and gdpeval until they migrate - in PR 10a/10b/10c. PR 11 deletes the fallback once every benchmark is - on object-bound `Task`. - -Old path still intentionally alive: other builtins, registry, base sandbox -manager, `_legacy_worker_bridge.py`. - -Deletion gate: PR 10 migrates the remaining builtins; PR 11 deletes the -manager bridge AND the legacy-worker fallback. - -Tests added or updated: MiniF2F definition JSON and builtin unit tests. -Add a smoke check that MiniF2F's tasks carry a non-None `worker`/`sandbox` -after `Task.from_definition`, so a future regression that re-emits -TaskSpec snapshots fails loud. - -Modules owned by this PR: MiniF2F and builtin sandbox adapter. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07b-pr-6-5-domain-colocation.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07b-pr-6-5-domain-colocation.md deleted file mode 100644 index 14225c6ee..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07b-pr-6-5-domain-colocation.md +++ /dev/null @@ -1,1402 +0,0 @@ -# PR 6.5 — Domain Colocation + Kill `Experiment` Class - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Two related changes shipped together, structured as **two commits in one PR**: - -1. **(Phase 1: Domain colocation)** Re-organise `ergon_builtins` so the file layout matches the actual coupling cardinalities between `Sandbox`, `Toolkit`, `Worker`, `Evaluator`, `Criterion`, and `Benchmark`. Collapse the misleading top-level `sandboxes/` and `toolkits/` directories into per-benchmark subpackages; leave the genuinely cross-cutting `Worker` classes at the top level; rewrite the v2 RFC's "composability" framing. - -2. **(Phase 2: Kill `Experiment` class)** Delete the `Experiment` class outright. Replace `persist_definition(experiment)` with `persist_benchmark(benchmark, *, name, experiment=None, metadata=None)`. Rename `ExperimentRecord` SQLModel + physical table to `BenchmarkDefinitionRecord`. Add an `experiment: str | None` column for grouping. Delete the CLI authoring route (`ergon experiment define`, `ergon experiment run`). Update all callers / tests / Inngest payloads / dashboard reads. See `docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md` for the full rationale. - -**Why one PR with two commits, not two PRs:** the two pieces share a target architecture and benefit from one-shot cutover (no intermediate state where the per-benchmark layout exists but `Experiment` is still alive). The commits stay clearly labelled so the diff is mentally splittable for review. - -**Architecture (final state after both commits):** - -``` -1:1 with benchmark (lives in benchmarks/<slug>/): - sandbox.py • toolkit.py • prompts.py - workers.py • benchmark.py • rubric.py - criteria/<benchmark-specific criteria> - -N:1 across benchmarks (lives at top level): - workers/baselines/ ← ReActWorker, CoTWorker, ReflexionWorker - sandbox/ ← _manager_backed.py (shared sandbox adapter) - evaluators/criteria/ ← reusable primitives (e.g. LLMJudgeCriterion) - evaluators/rubrics/ ← reusable composites - benchmarks/README.md ← static catalogue (replaces deleted CLI registry) -``` - -Note: no per-benchmark `experiment.py` file. Authoring is Python-only; CLI is observation-only (lifecycle commands move to PR 8). - -**Tech Stack:** File moves, import rewrites, doc updates, SQLModel/table rename, public-API surgery on `ergon_core.api`, CLI command deletion, dashboard contract regeneration (`pnpm run generate:contracts`). Note: the dashboard does **not** use Drizzle; it reads via the REST API + generated Zod contracts from Pydantic JSON Schema (see `ergon-dashboard/scripts/generate-rest-contracts.mjs`). So the dashboard impact of the rename is mostly automated codegen + a typecheck pass — not direct DB queries. - -**Sequencing:** Ships as a follow-up to PR 6, before PR 8 and PR 10a. Locks in both the file layout AND the API shape before the SWEBench / ResearchRubrics / GDPEval verticals replicate the PR 6 pattern (which would otherwise amplify the wrong layout AND build doomed `Experiment(...)` constructors 3× over). PR 8 then ships the slim lifecycle-only CLI on top of the cleaned-up API. - ---- - -## Files - -### Phase 1 (Commit 1: Domain Colocation) - -**Move (PR 6 outputs → benchmark-colocated paths):** - -```text -ergon_builtins/ergon_builtins/sandboxes/lean.py - → ergon_builtins/ergon_builtins/benchmarks/minif2f/sandbox.py -ergon_builtins/ergon_builtins/toolkits/minif2f.py - → ergon_builtins/ergon_builtins/benchmarks/minif2f/toolkit.py -ergon_builtins/ergon_builtins/toolkits/_minif2f_tools.py - → ergon_builtins/ergon_builtins/benchmarks/minif2f/_tools.py -``` - -**Rename:** - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/worker_factory.py - → ergon_builtins/ergon_builtins/benchmarks/minif2f/workers.py -``` - -**Create (Phase 1):** - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/_legacy_workers.py -ergon_builtins/ergon_builtins/sandbox/__init__.py -``` - -**Delete (Phase 1 — now-empty top-level dirs):** - -```text -ergon_builtins/ergon_builtins/sandboxes/__init__.py -ergon_builtins/ergon_builtins/sandboxes/ (directory) -ergon_builtins/ergon_builtins/toolkits/__init__.py -ergon_builtins/ergon_builtins/toolkits/ (directory) -``` - -**Modify (Phase 1 — import path updates + parameterised constructor):** - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/__init__.py -ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py -ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -ergon_builtins/registry_core.py # if it imports MiniF2FReactWorker -ergon_builtins/registry.py # likewise -ergon_builtins/tests/unit/benchmarks/test_minif2f_task_shape.py -ergon_core/tests/unit/runtime/test_experiment_definition_writer.py -ergon_core/tests/unit/runtime/test_experiment_definition_service.py -scripts/check_suppression_budget.py # path comments only -``` - -### Phase 2 (Commit 2: Kill `Experiment` Class) - -**Delete (Phase 2):** - -```text -ergon_core/ergon_core/api/experiment.py # the Experiment class -ergon_core/ergon_core/core/application/experiments/models.py # ExperimentDefineRequest DTO -ergon_cli/ergon_cli/commands/experiment.py # CLI authoring handlers (define/run) -ergon_cli/tests/unit/cli/test_experiment_cli.py # tests for deleted handlers -``` - -(The `ergon_cli/commands/experiment.py` file may be reduced rather than deleted if other commands live there; PR 8 then adds lifecycle commands back to a new `commands/experiment.py` / `commands/run.py`.) - -**Create (Phase 2):** - -```text -ergon_builtins/ergon_builtins/benchmarks/README.md # static benchmark catalogue -``` - -**Rename (Phase 2 — SQLModel + physical table):** - -```text -ergon_core/ergon_core/core/persistence/telemetry/models.py::ExperimentRecord - → BenchmarkDefinitionRecord -# Physical table: rename in the SQLModel definition. No Alembic migration — -# the migration chain is being dropped/regenerated wholesale; no production -# data to preserve. -``` - -**Modify (Phase 2 — API surgery + callsites):** - -```text -ergon_core/ergon_core/api/__init__.py # drop Experiment export, add persist_benchmark -ergon_core/ergon_core/api/persistence.py # rename persist_definition → persist_benchmark; new signature -ergon_core/ergon_core/core/application/experiments/service.py # remove define_benchmark_experiment -ergon_core/ergon_core/core/application/experiments/launch.py # update event payloads -ergon_core/ergon_core/core/application/experiments/definition_writer.py -ergon_core/ergon_core/core/persistence/telemetry/models.py # add experiment: str | None column -# Plus every test that constructs Experiment(...) — find via: -# rg "Experiment\(" ergon_core/tests/ ergon_builtins/tests/ ergon_cli/tests/ tests/ -# Plus every event payload that names "experiment" semantically — find via: -# rg "experiment_id\|experiment=" ergon_core/ ergon_cli/ -# Plus the dashboard contract regeneration: -ergon-dashboard/src/generated/ # regenerated by `pnpm run generate:contracts` -# (No direct file edits in ergon-dashboard/src/ — the Zod schemas are -# produced from backend Pydantic JSON Schema via codegen. Hand-written -# TypeScript files that consume the regenerated types will typecheck-fail -# if naming drifts; fix those manually.) -``` - -### Shared (touched by both phases — finalised in Phase 2) - -```text -docs/architecture/06_builtins.md # updates from both phases -docs/architecture/01_public_api.md # drops Experiment, adds persist_benchmark -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/09-pr-08-cli-composition.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11-pr-10a-swebench.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11b-pr-10b-researchrubrics.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11c-pr-10c-gdpeval.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/12-pr-11-deletion-final-schema.md -docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/00-program.md -``` - -## Current State - -After PR 6, `ergon_builtins/` looks like: - -``` -ergon_builtins/ -├── benchmarks/ -│ └── minif2f/ -│ ├── benchmark.py -│ ├── worker_factory.py ← legacy + v2 factories mixed -│ ├── rubric.py -│ ├── sandbox_manager.py ← v1 (deleted in PR 11) -│ └── ... -├── sandboxes/ ← top-level: looks cross-cutting -│ ├── __init__.py -│ └── lean.py ← but it's 1:1 with minif2f -├── toolkits/ ← top-level: looks cross-cutting -│ ├── __init__.py -│ ├── minif2f.py ← but it's 1:1 with minif2f -│ └── _minif2f_tools.py ← same -├── workers/ -│ └── baselines/ -│ └── react_worker.py ← genuinely N:1 (reused by every benchmark) -└── evaluators/ ← genuinely cross-cutting - ├── criteria/ - └── rubrics/ -``` - -The top-level `sandboxes/` and `toolkits/` directories suggest -cross-benchmark reuse that doesn't exist: `LeanSandbox` is only usable -with `MiniF2FToolkit`; `MiniF2FToolkit` is only usable inside -`LeanSandbox`. Once PR 10a–10c land, every benchmark adds one file to -each top-level dir even though no file is ever imported by anything -outside its own benchmark. - -The genuinely cross-cutting things (`ReActWorker`, the shared -manager-backed runtime adapter `_manager_backed.py` planned in PR 10a) -do belong at the top level. - -## Target State For This PR - -``` -ergon_builtins/ -├── benchmarks/ -│ └── minif2f/ -│ ├── benchmark.py -│ ├── sandbox.py ← was sandboxes/lean.py -│ ├── toolkit.py ← was toolkits/minif2f.py -│ ├── _tools.py ← was toolkits/_minif2f_tools.py -│ ├── workers.py ← was worker_factory.py (v2 factories only) -│ ├── _legacy_workers.py ← legacy MiniF2FReactWorker block, marked -│ │ for PR 11 deletion (clear separation) -│ ├── experiment.py ← (new file: experiment factory; see Task 4) -│ ├── rubric.py -│ ├── sandbox_manager.py ← still v1, still deleted in PR 11 -│ └── criteria/ -├── sandbox/ ← NEW: cross-cutting sandbox-adapter infra -│ └── (PR 10a will add _manager_backed.py here) -├── workers/ -│ └── baselines/ -│ └── react_worker.py ← still here, unchanged -└── evaluators/ ← unchanged -``` - -`sandboxes/` (plural) and `toolkits/` are gone. Every benchmark's -domain-specific bundle (sandbox + toolkit + tools + worker factories + -prompts + rubric + experiment factory) lives in `benchmarks/<slug>/`. -The new top-level `sandbox/` (singular) holds *cross-benchmark* -sandbox infrastructure — currently empty until PR 10a populates it -with `_manager_backed.py`. - -**Why `sandbox/` (singular) instead of `runtime/`:** the dir's -contents are specifically about sandbox-adapter wiring (E2B SDK -adapter, manager-backed runtime adapter). Naming it `sandbox/` -(singular, top-level) signals "infrastructure for the `Sandbox` -abstraction" without the plural's implication of "a catalogue of -sandboxes" that bit us with `sandboxes/`. Distinct from the -`ergon_core.api.sandbox` namespace by virtue of living under -`ergon_builtins/` (no name collision in practice). - -## Why This Layout - -The cardinality matrix (derived from the actual coupling, not the -aspirational "composability" framing): - -| Pair | Cardinality | Lives where | -|------|-------------|-------------| -| Benchmark ↔ Sandbox | 1↔1 (in practice) | `benchmarks/<slug>/sandbox.py` | -| Sandbox ↔ Toolkit | 1↔N possible, 1↔1 in practice | `benchmarks/<slug>/toolkit.py` | -| Toolkit ↔ Worker class | N↔M (loose) | Worker class in `workers/baselines/`, binding in `benchmarks/<slug>/workers.py` | -| Worker class ↔ Benchmark | N↔1 (one ReActWorker, N benchmarks) | Worker class in `workers/baselines/` | -| Evaluator ↔ Benchmark | 1↔1 (rubric); N↔M (reusable criteria) | Rubric in `benchmarks/<slug>/rubric.py`; benchmark-specific criteria in `benchmarks/<slug>/criteria/`; reusable criteria in top-level `evaluators/criteria/` | -| Evaluator ↔ Sandbox | 1↔1 if agentic, 0 otherwise | Same as Evaluator ↔ Benchmark | -| Experiment factory ↔ Benchmark | 1↔N (one factory per strategy ablation) | `benchmarks/<slug>/experiment.py` | -| Experiment registry ↔ Benchmarks | 1↔N (one dict over all builtins) | `ergon_builtins/benchmarks/_registry.py` (added in PR 8) | -| CLI composition helper ↔ Experiment registry | 1↔1 | `ergon_cli/composition/` (lives in CLI package) | - -The test for "where does X live?": - -- 1:1 with a benchmark → `benchmarks/<slug>/` -- N:1 across benchmarks → top-level under its category - -`Sandbox` and `Toolkit` fail the test for cross-cutting (every concrete -class is 1:1 with a benchmark); they move down. `ReActWorker` passes -(one class powers every benchmark); it stays up. The shared -manager-backed sandbox adapter passes (one adapter wraps every -benchmark's `BaseSandboxManager` subclass until PR 11); it gets its own -top-level home in `sandbox/`. - -## Composition Layer - -The composition layer — where worker strategy, benchmark, sandbox, and -evaluator are wired into a runnable `Experiment` — lives in three -places, each at its correct cardinality level: - -``` -ergon_builtins/benchmarks/<slug>/ -├── workers.py ← per-benchmark factories: react_worker(), -│ cot_worker(), reflexion_worker(), … -│ Each binds a Worker class from -│ workers/baselines/ to this benchmark's -│ sandbox + toolkit + prompt. -│ -└── experiment.py ← per-benchmark Experiment factories: - make_react_experiment(args) -> Experiment, - make_cot_experiment(args) -> Experiment, … - Each picks ONE worker factory from workers.py, - constructs the Benchmark instance, attaches - evaluators, returns an Experiment. - -ergon_builtins/benchmarks/_registry.py (added in PR 8) - BUILTIN_EXPERIMENT_FACTORIES: dict[str, Callable] - Maps CLI slug → experiment factory. - -ergon_cli/composition/ - build_experiment(args) — argparse Namespace - in, Experiment out. Looks up slug in - BUILTIN_EXPERIMENT_FACTORIES and dispatches. -``` - -The split is: **what** to compose lives per-benchmark (factories knowing -about the local sandbox/toolkit/prompt); **how to discover** it lives -at the `_registry.py` level; **how to drive it from a CLI** lives in -the CLI package. Each layer is at its correct cardinality and only -imports downward. - -### Making the swap point real: parameterised Benchmark - -The above layering only works if `Benchmark.build_instances()` does -*not* hardcode a specific worker factory. PR 6 today writes: - -```python -# benchmark.py — current -yield Task(..., worker=make_minif2f_worker(), sandbox=LeanSandbox(), ...) -``` - -— which means swapping ReAct → CoT requires editing `benchmark.py`. -That makes `experiment.py` cosmetic. To make it the real swap point, -PR 6.5 parameterises the benchmark: - -```python -# benchmark.py — post-PR-6.5 -class MiniF2FBenchmark(Benchmark): - def __init__( - self, - *, - worker_factory: Callable[[], Worker] = make_minif2f_worker, - sandbox_factory: Callable[[], Sandbox] = LeanSandbox, - ..., - ) -> None: ... - - def build_instances(self): - yield Task( - ..., - worker=self._worker_factory(), - sandbox=self._sandbox_factory(), - ..., - ) -``` - -Then `experiment.py` is genuinely where strategy is chosen: - -```python -def make_react_experiment(args): - return Experiment(benchmark=MiniF2FBenchmark(worker_factory=make_minif2f_react), ...) - -def make_cot_experiment(args): - return Experiment(benchmark=MiniF2FBenchmark(worker_factory=make_minif2f_cot), ...) -``` - -The benchmark module still imports its sandbox (the default value), -which is honest: the sandbox is part of the per-benchmark domain -bundle. The worker, in contrast, is now genuinely swappable from the -outside. - -**This PR does not implement the composition layer.** PR 8 owns the -`_registry.py` + `ergon_cli/composition/` wiring. What PR 6.5 does is: - -1. Move per-benchmark files into `benchmarks/<slug>/` (Tasks 1–3). -2. Create `benchmarks/<slug>/experiment.py` with a stub factory so - the location is locked in before PR 8 lands (Task 4). -3. **Parameterise `MiniF2FBenchmark.__init__`** to accept - `worker_factory` / `sandbox_factory` callables (Task 5), so the - strategy swap is honest and `experiment.py`'s eventual body is - trivial. -4. Update PR 8's plan (Task 10) to reflect the new file paths and - the parameterised-benchmark constructor. - -## Task 1: Move MiniF2F Sandbox - -**Files:** - -- Move: `ergon_builtins/sandboxes/lean.py` → `ergon_builtins/benchmarks/minif2f/sandbox.py` - -- [ ] **Step 1: `git mv` the file** - - ```bash - git mv ergon_builtins/ergon_builtins/sandboxes/lean.py \ - ergon_builtins/ergon_builtins/benchmarks/minif2f/sandbox.py - ``` - -- [ ] **Step 2: Update module docstring** - - Replace "object-bound Lean 4 sandbox for MiniF2F" introduction with a - shorter version that reflects the new colocation; drop the "PR 10 - extracts `_ManagerBackedSandboxRuntime` into - `ergon_builtins/sandboxes/_manager_backed.py`" line and replace with - "PR 10a extracts the shared adapter to `ergon_builtins/sandbox/_manager_backed.py`". - -- [ ] **Step 3: Verify no remaining `sandboxes/lean` references** - - ```bash - rg "from ergon_builtins.sandboxes" ergon_builtins/ ergon_core/ - rg "ergon_builtins\.sandboxes" ergon_builtins/ ergon_core/ - ``` - -## Task 2: Move MiniF2F Toolkit + Tools - -**Files:** - -- Move: `ergon_builtins/toolkits/minif2f.py` → `ergon_builtins/benchmarks/minif2f/toolkit.py` -- Move: `ergon_builtins/toolkits/_minif2f_tools.py` → `ergon_builtins/benchmarks/minif2f/_tools.py` - -- [ ] **Step 1: `git mv` both files** - -- [ ] **Step 2: Update internal imports** - - - `toolkit.py` line 43 (`from ergon_builtins.toolkits._minif2f_tools import build_tools`) - → `from ergon_builtins.benchmarks.minif2f._tools import build_tools` - - `_tools.py` line 22 (`from ergon_builtins.benchmarks.minif2f.constants import ...`) — already correct, no change needed - - `_tools.py` line 25 (`from ergon_builtins.toolkits.minif2f import MiniF2FToolkit` under TYPE_CHECKING) - → `from ergon_builtins.benchmarks.minif2f.toolkit import MiniF2FToolkit` - -- [ ] **Step 3: Re-check the circular-import cycle** - - The PR 6 cycle was: - `toolkits/minif2f.py → toolkits/_minif2f_tools.py → benchmarks.minif2f.constants → benchmarks/minif2f/__init__.py → benchmark.py → worker_factory.py → toolkits/minif2f.py` - - Post-move: - `benchmarks/minif2f/toolkit.py → benchmarks/minif2f/_tools.py → benchmarks/minif2f/constants.py → benchmarks/minif2f/__init__.py → benchmark.py → workers.py → benchmarks/minif2f/toolkit.py` - - Still a cycle. The lazy import inside `MiniF2FToolkit.tools()` is - still required; keep the `# reason:` comment but update the path - description. - -## Task 3: Rename `worker_factory.py` → `workers.py` AND Split Out Legacy - -**Files:** - -- Rename: `ergon_builtins/benchmarks/minif2f/worker_factory.py` → `ergon_builtins/benchmarks/minif2f/workers.py` -- Create: `ergon_builtins/benchmarks/minif2f/_legacy_workers.py` - -**Rationale:** PR 6's `worker_factory.py` mixed v2 factories -(`make_minif2f_worker`, `make_minif2f_rubric`) with the legacy -`MiniF2FReactWorker` class + `_minif2f_run_skill` (kept alive for the -registry-string bridge until PR 11). This task separates the two so -PR 11's deletion is a single-file `rm`, not a multi-block surgery. - -- [ ] **Step 1: `git mv` worker_factory.py → workers.py** - -- [ ] **Step 2: Move the legacy block to `_legacy_workers.py`** - - Cut from `workers.py` and paste into `_legacy_workers.py`: - - `_minif2f_run_skill()` function - - `MiniF2FReactWorker` class - - Imports of `MiniF2FSandboxManager` and `_LegacyMiniF2FToolkit` - - Leave `workers.py` with only the v2 factories - (`make_minif2f_worker`, `make_minif2f_rubric`). - -- [ ] **Step 3: Add a clear deletion marker to `_legacy_workers.py`** - - Top-of-file docstring: - - ```python - """Legacy MiniF2F worker bridge — DELETED IN PR 11. - - This file exists solely so the `"minif2f-react"` registry slug - still resolves for experiments persisted before PR 6. Once PR 11 - retires the legacy registry fallback chain, delete this entire - file along with `sandbox_manager.py`. - - Do NOT import from this file in new code. v2 callers use - `workers.make_minif2f_worker()`. - """ - ``` - -- [ ] **Step 4: Update registry imports** - - Find every reference to `MiniF2FReactWorker` in - `ergon_builtins/registry_core.py` / `ergon_builtins/registry.py`: - - ```bash - rg "MiniF2FReactWorker" ergon_builtins/ - ``` - - Update the import path: - `from ergon_builtins.benchmarks.minif2f.worker_factory import MiniF2FReactWorker` - → `from ergon_builtins.benchmarks.minif2f._legacy_workers import MiniF2FReactWorker` - -- [ ] **Step 5: Update imports in `benchmark.py`** - - `from ergon_builtins.benchmarks.minif2f.worker_factory import (...)` - → `from ergon_builtins.benchmarks.minif2f.workers import (...)` - -- [ ] **Step 6: Update `workers.py` docstring** - - ```python - """MiniF2F worker factories — one per agentic strategy. - - Each factory bundles the MiniF2F sandbox, toolkit, and system - prompt with a chosen worker class (ReActWorker today; CoTWorker / - ReflexionWorker future). Strategies vary independently; the - domain bundle is constant. - - Legacy registry bridge (MiniF2FReactWorker) lives in - `_legacy_workers.py` and is deleted in PR 11. - """ - ``` - -- [ ] **Step 7: Sanity-check no stragglers** - - ```bash - rg "minif2f\.worker_factory" ergon_builtins/ ergon_core/ ergon_cli/ - ``` - - Expect zero hits. - -## Task 4: Parameterise `MiniF2FBenchmark` For Strategy A/B - -**Files:** - -- Modify: `ergon_builtins/benchmarks/minif2f/benchmark.py` - -**Rationale:** PR 6 hardcodes `make_minif2f_worker()` and -`LeanSandbox()` inside `build_instances()`. That makes the -`experiment.py` swap point fictional — swapping worker strategy -today requires editing `benchmark.py`. This task parameterises -the benchmark constructor so the swap point becomes real and the -composition layering described above holds. - -- [ ] **Step 1: Add factory kwargs to `__init__`** - - ```python - from collections.abc import Callable - - from ergon_core.api.sandbox import Sandbox - from ergon_core.api.worker import Worker - - from ergon_builtins.benchmarks.minif2f.sandbox import LeanSandbox - from ergon_builtins.benchmarks.minif2f.workers import ( - make_minif2f_rubric, - make_minif2f_worker, - ) - - - class MiniF2FBenchmark(Benchmark): - def __init__( - self, - *, - # Existing kwargs (name, description, metadata, limit, ...) stay. - worker_factory: Callable[[], Worker] = make_minif2f_worker, - sandbox_factory: Callable[[], Sandbox] = LeanSandbox, - evaluator_factory: Callable[[], "MiniF2FRubric"] = make_minif2f_rubric, - ..., - ) -> None: - super().__init__(...) - self._worker_factory = worker_factory - self._sandbox_factory = sandbox_factory - self._evaluator_factory = evaluator_factory - ... - ``` - -- [ ] **Step 2: Use the factories inside `build_instances`** - - ```python - def build_instances(self) -> Mapping[str, Sequence[Task[MiniF2FTaskPayload]]]: - tasks = [] - for problem in self._load_problems(): - ... - tasks.append( - Task[MiniF2FTaskPayload]( - ..., - worker=self._worker_factory(), - sandbox=self._sandbox_factory(), - evaluators=(self._evaluator_factory(),), - ) - ) - return {"default": tasks} - ``` - -- [ ] **Step 3: Keep behaviour identical for callers that don't pass factories** - - The defaults (`make_minif2f_worker`, `LeanSandbox`, `make_minif2f_rubric`) - reproduce PR 6's behaviour exactly. All existing tests must - continue to pass — confirm by running - `uv run pytest ergon_builtins/tests/unit -q` after this task. - -- [ ] **Step 4: Add a one-line test that the factories swap** - - Append to `ergon_builtins/tests/unit/benchmarks/test_minif2f_task_shape.py`: - - ```python - def test_minif2f_benchmark_accepts_custom_worker_factory() -> None: - """The benchmark uses the worker_factory passed to its constructor.""" - from unittest.mock import MagicMock - - from ergon_builtins.benchmarks.minif2f.benchmark import MiniF2FBenchmark - from ergon_builtins.benchmarks.minif2f.workers import make_minif2f_worker - - sentinel_worker = make_minif2f_worker() - sentinel_worker.name = "sentinel" - factory = MagicMock(return_value=sentinel_worker) - - benchmark = MiniF2FBenchmark(worker_factory=factory, limit=1) - tasks = list(benchmark.build_instances().values())[0] - - assert tasks[0].worker is sentinel_worker - factory.assert_called_once() - ``` - - This is the load-bearing assertion: factories are *called*, not - the class attribute that defaults to them. Without this test, a - future refactor could accidentally revert to hardcoding the - default and the test suite would still pass. - -- [ ] **Step 5: Update `worker_factory` defaults to be a real swap point** - - Sanity-check that calling `MiniF2FBenchmark(worker_factory=lambda: <other_worker>)` produces tasks bound to the new worker, not the default. This is covered by Step 4's test; just verify the assertion is meaningful. - -## Task 5: Update `benchmarks/minif2f/__init__.py` - -**Files:** - -- Modify: `ergon_builtins/benchmarks/minif2f/__init__.py` - -- [ ] **Step 1: Reconsider the eager-export comment** - - The PR 6 comment explains why `MiniF2FBenchmark` cannot be eagerly - re-exported (cycle through `sandboxes/lean.py → MiniF2FSandboxManager`). - Post-move, the cycle is: - `benchmarks/minif2f/__init__.py → benchmark.py → sandbox.py → sandbox_manager.py → __init__.py` - - Still a cycle (sandbox.py still imports MiniF2FSandboxManager until - PR 11). Keep the non-re-export, update the comment to describe the - new (shorter) cycle path, and keep the existing - `TODO(PR 11)` marker. - -## Task 6: Add `sandbox/` Top-Level Dir Stub - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/sandbox/__init__.py` - -- [ ] **Step 1: Empty package init** - - Just a docstring explaining the dir's purpose: - - ```python - """Cross-cutting sandbox infrastructure shared across benchmarks. - - Files in this package implement adapters / utilities that more than - one benchmark's `Sandbox` subclass relies on. Per-benchmark code - (`LeanSandbox`, `SWEBenchSandbox`, etc.) lives in - `ergon_builtins/benchmarks/<slug>/sandbox.py`, NOT here. - - Naming: singular `sandbox/` (this package, cross-cutting infra) - vs. per-benchmark `benchmarks/<slug>/sandbox.py` (single concrete - Sandbox subclass). The deleted `sandboxes/` (plural) directory - conflated the two and is gone. - - PR 10a populates this with `_manager_backed.py` (the shared - BaseSandboxManager → SandboxRuntime adapter). - """ - ``` - -- [ ] **Step 2: Add to test_public_api_boundaries allowlist if needed** - - Check whether any architecture test enforces a closed set of top-level - `ergon_builtins/` packages; if so, add `sandbox` to the allowlist - (and remove `sandboxes` / `toolkits` if present). - - ```bash - rg "ergon_builtins.*(sandbox|workers|sandboxes|toolkits)" ergon_core/tests/unit/architecture/ - ``` - -## Task 7: Update Suppression Budget Comment - -**Files:** - -- Modify: `scripts/check_suppression_budget.py` - -- [ ] **Step 1: Update path references in the comment** - - Change `_minif2f_tools.py build_tools` → `benchmarks/minif2f/_tools.py build_tools` etc. No count change; this is documentation drift. - -## Task 8: Update Architecture Docs - -**Files:** - -- Modify: `docs/architecture/06_builtins.md` -- Modify: `docs/architecture/01_public_api.md` - -- [ ] **Step 1: Update the code map in `06_builtins.md`** - - Replace: - - ``` - | v2 object-bound Sandbox subclasses | `ergon_builtins/sandboxes/` | - | v2 serializable Toolkit configs | `ergon_builtins/toolkits/` | - ``` - - With: - - ``` - | Per-benchmark Sandbox subclass | `ergon_builtins/benchmarks/<slug>/sandbox.py` | - | Per-benchmark Toolkit config | `ergon_builtins/benchmarks/<slug>/toolkit.py` | - | Per-benchmark worker factories | `ergon_builtins/benchmarks/<slug>/workers.py` | - | Per-benchmark experiment factories | `ergon_builtins/benchmarks/<slug>/experiment.py` | - | Per-benchmark legacy (PR 11 deletes) | `ergon_builtins/benchmarks/<slug>/_legacy_workers.py` | - | Cross-cutting sandbox-adapter infra | `ergon_builtins/sandbox/` | - ``` - -- [ ] **Step 2: Add a "Cardinality and colocation" section** - - Insert the cardinality matrix (from "Why This Layout" above) plus - the "1:1 → `benchmarks/<slug>/`; N:1 → top-level" test. - - **`06_builtins.md` is the canonical home for this matrix.** Do - not duplicate it in the RFC docs (`00-readme.md`, - `01-api-surface.md`); those describe authoring intent, not the - internal builtins layout. Cross-references are fine; copies are - not (they drift). - -- [ ] **Step 3: Add an anti-pattern** - - Append to `06_builtins.md` § 6: - - > **Per-benchmark code under top-level `sandboxes/` or `toolkits/`.** - > A `LeanSandbox` is only useful inside MiniF2F; a `MiniF2FToolkit` - > assumes `LeanSandbox`. These belong in `benchmarks/minif2f/`, - > not in cross-cutting top-level dirs that imply reuse that doesn't - > exist. The cardinality test: a file is cross-cutting only if N - > different benchmarks would import it. - -- [ ] **Step 4: Add an anti-pattern (worker side)** - - > **`<Benchmark>ReActWorker(ReActWorker)` subclasses.** Making - > per-benchmark worker subclasses costs N×M classes (N strategies × - > M benchmarks) and forces the agentic-loop logic to live in N - > subclasses that all `super().execute()`. The v2 pattern is: one - > worker class per strategy in `workers/baselines/`, plus a - > per-benchmark factory function in `benchmarks/<slug>/workers.py` - > that binds the strategy to the local sandbox + toolkit + prompt. - > Cost is N + M, not N × M. - -- [ ] **Step 5: Update `01_public_api.md`** - - Find the "add a new benchmark" extension point (added in PR 6) and - replace the file paths. Drop the references to - `ergon_builtins/sandboxes/` and `ergon_builtins/toolkits/`; replace - with `ergon_builtins/benchmarks/<slug>/sandbox.py` and `.../toolkit.py`. - -## Task 9: Update v2 RFC Framing - -**Files:** - -- Modify: `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md` -- Modify: `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md` - -- [ ] **Step 1: Rewrite the "composability" framing in `00-readme.md`** - - Wherever the RFC claims the v2 design lets you compose any - `Sandbox`/`Toolkit`/`Worker` freely, replace with: - - > **Domain colocation + orthogonal worker strategies.** A benchmark - > ships with a coupled triple: its `Sandbox` (runtime container), - > `Toolkit` (LLM-facing tools that assume that container), and - > system prompt. These are co-designed; they are not portable across - > benchmarks. The orthogonal axis is the worker *class*: the same - > `ReActWorker` (or `CoTWorker`, `ReflexionWorker`) is reused for - > every benchmark, parameterised by the per-benchmark triple via a - > factory function. This is what the architecture buys: - > - one place for agentic-loop logic (per worker class), - > - one place for per-benchmark wiring (per benchmark subpackage), - > - serialisable config so tasks round-trip through the - > orchestrator → eval-worker process boundary. - > - > What the design does *not* buy: the ability to use `MiniF2FToolkit` - > inside a `SWEBenchSandbox` or vice versa. Toolkit / sandbox / - > prompt are domain-coupled. - -- [ ] **Step 2: Update `01-api-surface.md`** - - Find any references to `Toolkit` portability or "compose any - worker with any toolkit" and rewrite to match Step 1's framing. - - **Do NOT duplicate the cardinality matrix here.** It lives in - `docs/architecture/06_builtins.md` (Task 7 Step 2) as the - single source of truth. If `01-api-surface.md` needs to reference - the cardinality story, link to `06_builtins.md` rather than - pasting the table. - -## Task 10: Update Later PR Plans (Contradiction Sweep) - -**Files:** - -- Modify: `09-implementation-plan/09-pr-08-cli-composition.md` -- Modify: `09-implementation-plan/11-pr-10a-swebench.md` -- Modify: `09-implementation-plan/11b-pr-10b-researchrubrics.md` -- Modify: `09-implementation-plan/11c-pr-10c-gdpeval.md` -- Modify: `09-implementation-plan/12-pr-11-deletion-final-schema.md` -- Modify: `09-implementation-plan/00-program.md` - -The later PR plans were written assuming PR 6's `sandboxes/<slug>.py` / -`toolkits/<slug>.py` layout. Each one's `Files → Create:` section, -plus any embedded `import` lines in code blocks, must be updated to -the post-PR-6.5 paths. - -**Method: do a thorough scan of each file before editing.** For each -PR plan, before making changes, run: - -```bash -rg "sandboxes/" docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/<file> -rg "toolkits/" docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/<file> -rg "worker_factory" docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/<file> -rg "ergon_builtins\.sandboxes\|ergon_builtins\.toolkits" docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/<file> -``` - -Treat every hit as a candidate edit; update or remove as appropriate. -Do not assume the bullet points below are exhaustive — the sweep is. - -- [ ] **Step 1: PR 8 (CLI composition) path + parameterisation updates** - - In `09-pr-08-cli-composition.md`: - - The plan adds `ergon_builtins/ergon_builtins/benchmarks/_registry.py`. - Confirm it imports `MiniF2FBenchmark` directly from - `benchmarks/minif2f/benchmark.py`, not through any top-level - re-export. - - Any factory functions referenced in example code blocks should use - `benchmarks/minif2f/experiment.py::make_react_experiment` (PR 6.5 - Task 4 creates this file with one factory; PR 8 may add more). - - Drop any references to `worker_factory` (it no longer exists). - - **Update PR 8's example code to use the parameterised benchmark - constructor** (PR 6.5 Task 5): - - ```python - # PR 8 example: experiment factory - def make_react_experiment(args): - benchmark = MiniF2FBenchmark( - worker_factory=make_minif2f_worker, - limit=args.limit, - ) - return Experiment(benchmark=benchmark, ...) - - def make_cot_experiment(args): - benchmark = MiniF2FBenchmark( - worker_factory=make_minif2f_cot_worker, # different strategy - limit=args.limit, - ) - return Experiment(benchmark=benchmark, ...) - ``` - - If PR 8's current draft has experiment factories hardcoding - `MiniF2FBenchmark()` and then ignoring the worker (because the - benchmark builds tasks with `make_minif2f_worker()` baked in), - that's now a contradiction — update it. - -- [ ] **Step 2: PR 10a (SWEBench) path updates** - - In `11-pr-10a-swebench.md`: - - `ergon_builtins/sandboxes/swebench.py` → `ergon_builtins/benchmarks/swebench_verified/sandbox.py` - - `ergon_builtins/sandboxes/_manager_backed.py` → `ergon_builtins/sandbox/_manager_backed.py` - - `ergon_builtins/toolkits/swebench.py` → `ergon_builtins/benchmarks/swebench_verified/toolkit.py` - - `ergon_builtins/toolkits/_swebench_tools.py` (if planned) → `ergon_builtins/benchmarks/swebench_verified/_tools.py` - - `swebench_verified/worker_factory.py` → `swebench_verified/workers.py` - - Mirror PR 6.5's `_legacy_workers.py` split if the SWEBench plan - keeps a legacy worker class during the bridge. - - All imports inside example code blocks: rewrite to match. - -- [ ] **Step 3: PR 10b (ResearchRubrics) path updates** - - Same pattern for `11b-pr-10b-researchrubrics.md`: - - `sandboxes/researchrubrics.py` → `benchmarks/researchrubrics/sandbox.py` - - `toolkits/researchrubrics.py` → `benchmarks/researchrubrics/toolkit.py` - - Imports inside `from ergon_builtins.sandboxes._manager_backed import …` - → `from ergon_builtins.sandbox._manager_backed import …` - - Plus the workers / _legacy_workers split if relevant. - -- [ ] **Step 4: PR 10c (GDPEval) path updates** - - Same pattern for `11c-pr-10c-gdpeval.md`: - - `sandboxes/gdpeval.py` → `benchmarks/gdpeval/sandbox.py` - - `toolkits/gdpeval.py` → `benchmarks/gdpeval/toolkit.py` - - Same import / workers / _legacy_workers updates as above. - -- [ ] **Step 5: PR 11 (deletion) — thorough scan** - - PR 11 is the most likely to contain stale path assumptions because - it lists files to delete and symbols to remove. Treat this as the - most important sweep target. - - In `12-pr-11-deletion-final-schema.md`: - - a. **Files To Delete** list: - - Confirm `benchmarks/<slug>/sandbox_manager.py` paths are - present and correct (per-benchmark managers). - - REMOVE any entry that lists `ergon_builtins/sandboxes/` or - `ergon_builtins/toolkits/` as a deletion target — those dirs - were already deleted by PR 6.5. - - REMOVE any entry that lists - `ergon_builtins/sandbox/_manager_backed.py` as a deletion - target (the adapter stays; only its internals change to call - E2B directly). If PR 11's plan currently says to delete the - adapter, it's a contradiction — flag for resolution. - - If PR 11 deletes `MiniF2FReactWorker`, the path is now - `benchmarks/minif2f/_legacy_workers.py` (whole file goes), - not `benchmarks/minif2f/worker_factory.py`. Same for - SWEBench's `SWEBenchReactWorker` (or whatever it's called) - if it gets a `_legacy_workers.py` in PR 10a. - - b. **Symbol deletion table** (if any): - - `_minif2f_run_skill` moves from `worker_factory.py` to - `_legacy_workers.py`; the deletion target updates. - - c. **PR 11 sandbox-rewrite plan**: - - `LeanSandbox.provision()` and `_bind_runtime()` are rewritten - to call E2B directly (drop `MiniF2FSandboxManager` calls). - Confirm PR 11's plan reflects this. If it currently says - "delete `LeanSandbox`", that's wrong — `LeanSandbox` stays, - only its body changes. - - d. **Imports in PR 11's example code blocks**: - - Any `from ergon_builtins.sandboxes…` → `from ergon_builtins.benchmarks.<slug>.sandbox…` - - Any `from ergon_builtins.toolkits…` → `from ergon_builtins.benchmarks.<slug>.toolkit…` - -- [ ] **Step 6: 00-program.md bridge table + ledger updates** - - - Bridge row "Sandbox subclasses beside BaseSandboxManager" doesn't - reference paths. No path change needed. - - Add a short note in the "Per-PR ledger updates" section calling - out PR 6.5's file moves so the v2 transition ledger test - (`test_v2_transition_ledger.py`) doesn't get surprised when - `sandboxes/` disappears. - - Add PR 6.5 to the program timeline / ordering diagram if one - exists. - -## Task 11: Update PR 6 Inline TODO Markers - -**Files:** - -- Modify: `ergon_builtins/benchmarks/minif2f/sandbox.py` (post-move) -- Modify: `ergon_builtins/benchmarks/minif2f/toolkit.py` (post-move) -- Modify: `ergon_builtins/workers/baselines/react_worker.py` - -- [ ] **Step 1: Update path references in `TODO(PR 10a/11)` comments** - - The TODOs in PR 6's output reference paths like - `ergon_builtins/sandboxes/_manager_backed.py` (for PR 10a). Update - to `ergon_builtins/sandbox/_manager_backed.py` (singular). - - Specifically the docstring of `LeanSandbox` (now at - `benchmarks/minif2f/sandbox.py`) and the `TODO(PR 10a)` comment - above `_ManagerBackedSandboxRuntime`. - -## Task 12: Update Test File Imports - -**Files:** - -- Modify: `ergon_builtins/tests/unit/benchmarks/test_minif2f_task_shape.py` -- Modify: `ergon_core/tests/unit/runtime/test_experiment_definition_writer.py` -- Modify: `ergon_core/tests/unit/runtime/test_experiment_definition_service.py` - -- [ ] **Step 1: Rewrite imports** - - - `from ergon_builtins.sandboxes.lean import LeanSandbox` - → `from ergon_builtins.benchmarks.minif2f.sandbox import LeanSandbox` - - `from ergon_builtins.toolkits.minif2f import MiniF2FToolkit` - → `from ergon_builtins.benchmarks.minif2f.toolkit import MiniF2FToolkit` - - `from ergon_builtins.benchmarks.minif2f.worker_factory import (make_minif2f_rubric, make_minif2f_worker)` - → `from ergon_builtins.benchmarks.minif2f.workers import (make_minif2f_rubric, make_minif2f_worker)` - -- [ ] **Step 2: Run focused tests** - - ```bash - uv run pytest ergon_builtins/tests/unit ergon_core/tests/unit/runtime/test_experiment_definition_service.py ergon_core/tests/unit/runtime/test_experiment_definition_writer.py -q - ``` - - Expect: all previously-passing tests still pass. - -## Task 13: Update ReActWorker Import - -**Files:** - -- Modify: `ergon_builtins/workers/baselines/react_worker.py` - -- [ ] **Step 1: Rewrite the import** - - `from ergon_builtins.toolkits.minif2f import MiniF2FToolkit` - → `from ergon_builtins.benchmarks.minif2f.toolkit import MiniF2FToolkit` - - Note: this import is the entry point to the dependency that PR 11 - replaces with a `Toolkit` protocol (see existing TODO comment on the - `toolkit:` field). The import path change is mechanical; the TODO - stays. - -## Task 14: Phase 1 Check Suite - -Phase 1 (file moves + parameterised benchmark) is now complete. Verify before committing. - -- [ ] **Step 1: Run lint + format + type-check + slopcop** - - ```bash - pnpm run check:be - ``` - - All checks must pass. - -- [ ] **Step 2: Run fast test suite** - - ```bash - pnpm run test:be:fast - ``` - - Expect: same pass count as before PR 6.5 Phase 1 (no behaviour change yet — `Experiment` class still alive). - -- [ ] **Step 3: Search for stragglers** - - ```bash - rg "ergon_builtins\.sandboxes" . - rg "ergon_builtins\.toolkits" . - rg "minif2f\.worker_factory" . - ``` - - Any hit outside of `docs/rfcs/accepted/` (historical) is a bug. - -## Task 15: Phase 1 Commit - -```bash -git add -A -git commit -m "PR 6.5 (1/2): domain colocation in ergon_builtins - -- Move per-benchmark sandbox/toolkit/tools into benchmarks/minif2f/ -- Rename worker_factory.py → workers.py; split legacy into _legacy_workers.py -- Parameterise MiniF2FBenchmark(worker_factory=..., sandbox_factory=...) -- Add top-level sandbox/ stub for PR 10a's shared adapter -- Update architecture docs, v2 RFC framing, suppression budget -- No behaviour change; all tests still pass" -``` - -Phase 2 begins below. Do NOT mix Phase 2 changes into this commit. - ---- - -## Task 16: (Phase 2 starts) Hard-Delete `Experiment` Class - -**Files:** - -- Delete: `ergon_core/ergon_core/api/experiment.py` -- Modify: `ergon_core/ergon_core/api/__init__.py` (drop `Experiment` export) - -- [ ] **Step 1: Inventory `Experiment` callsites** - - ```bash - rg "from ergon_core\.api.* import .*Experiment\b" . - rg "Experiment\(" --type py - rg "Experiment\b" ergon_core/ ergon_cli/ ergon_builtins/ tests/ | grep -v _test_data | wc -l - ``` - - Expect ~20-30 hits across tests + the application services + CLI handlers. Note where each comes from; each one needs an edit in subsequent tasks. - -- [ ] **Step 2: Delete the class file** - - ```bash - rm ergon_core/ergon_core/api/experiment.py - ``` - -- [ ] **Step 3: Drop the export** - - Edit `ergon_core/ergon_core/api/__init__.py`: - - Remove `from ergon_core.api.experiment import Experiment` - - Remove `"Experiment"` from `__all__` - - Add `from ergon_core.api.persistence import persist_benchmark` (Task 17 creates this) - - Project will not import-clean until Task 17 lands. Acceptable — these two tasks must land in the same commit. - -## Task 17: Rename `persist_definition` → `persist_benchmark` (Signature Change) - -**Files:** - -- Modify: `ergon_core/ergon_core/api/persistence.py` (or wherever the existing `persist_definition` lives — find via `rg "def persist_definition"`) -- Modify: every callsite found in `rg "persist_definition\("` - -- [ ] **Step 1: Locate the current function** - - ```bash - rg "def persist_definition" --type py - ``` - - Likely `ergon_core/ergon_core/core/application/experiments/definition_writer.py`. - -- [ ] **Step 2: Rename + change signature** - - Old: - ```python - def persist_definition(experiment: Experiment) -> DefinitionHandle: - ... - ``` - - New: - ```python - def persist_benchmark( - benchmark: Benchmark, - *, - name: str, - experiment: str | None = None, - metadata: dict[str, Any] | None = None, - ) -> DefinitionHandle: - """Persist a configured Benchmark as a definition row. - - ``experiment`` is an optional string tag grouping related - definitions (e.g. an ablation study). It is NOT a class — see - docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md - for the rationale. - """ - ... - ``` - - Body: pull `benchmark`, `name`, `metadata` directly from kwargs (no `experiment.benchmark` / `experiment.name` indirection). Write the new `experiment: str | None` column on the persisted row (Task 18 adds the column). - -- [ ] **Step 3: Update every callsite** - - Old shape (`define_benchmark_experiment` / direct `persist_definition(experiment)`): - ```python - experiment = Experiment(benchmark=b, name="foo", metadata={...}) - handle = persist_definition(experiment) - ``` - - New shape: - ```python - handle = persist_benchmark(b, name="foo", metadata={...}) - ``` - - Callsites to update (non-exhaustive, use `rg` to find all): - - `ergon_core/ergon_core/core/application/experiments/service.py::define_benchmark_experiment` — delete; replace with direct `persist_benchmark` calls at the new CLI / handler boundary (most are removed in Task 21). - - Any integration test that round-trips through persist + load. - - Any fixture in `ergon_core/tests/conftest.py` or `ergon_builtins/tests/conftest.py`. - -- [ ] **Step 4: Confirm no stragglers** - - ```bash - rg "persist_definition\b" . - rg "Experiment\(" . - ``` - - Expect zero hits in production code. Test fixtures that still reference these are tracked in Task 21 for cleanup. - -## Task 18: Rename `ExperimentRecord` SQLModel + Physical Table - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Modify: every repository / query that names `ExperimentRecord` - -- [ ] **Step 1: Rename the SQLModel class AND the physical table** - - In `models.py`: - - ```python - class BenchmarkDefinitionRecord(SQLModel, table=True): - __tablename__ = "benchmark_definitions" # was "experiments" - ... - experiment: str | None = Field(default=None, index=True) # NEW - ``` - - Both the Python class name and the physical table name change. There's no production data to preserve and the Alembic chain is being dropped/regenerated wholesale (see PR 11), so no migration dance is required. - -- [ ] **Step 2: Add an explicit ledger entry** - - In the `BenchmarkDefinitionRecord` class docstring: - - ```python - """Persisted definition row for one configured Benchmark. - - Renamed from ``ExperimentRecord`` in PR 6.5 Phase 2 (kill-Experiment - refactor). Table also renamed: ``experiments`` → ``benchmark_definitions``. - The ``experiment: str | None`` column is the user-facing grouping tag - (e.g. ``experiment="strategy-ablation-2026-05-14"``) — it is a label, not - a foreign key. - """ - ``` - -- [ ] **Step 3: Update all reads** - - ```bash - rg "ExperimentRecord\b" . - ``` - - Every hit gets rewritten to `BenchmarkDefinitionRecord`. Likely files: - - Every `*_repository.py` and `*_repositories.py` in `ergon_core/core/persistence/` - - `ergon_core/core/application/experiments/` query helpers - - Inngest function payload classes that reference the table indirectly - - The dashboard's generated contract schemas (regenerated via `pnpm run generate:contracts` — separate Task 21) - -## Task 19: Delete CLI Authoring Commands - -**Files:** - -- Delete: `ergon_cli/ergon_cli/commands/experiment.py::handle_experiment_define` -- Delete: `ergon_cli/ergon_cli/commands/experiment.py::handle_experiment_run` -- Delete: argparse subparsers for `experiment define` / `experiment run` -- Delete: `ergon_cli/tests/unit/cli/test_experiment_cli.py` (or the affected test cases) - -- [ ] **Step 1: Locate the CLI surface** - - ```bash - rg "experiment.define\|experiment.run\|handle_experiment_define\|handle_experiment_run" ergon_cli/ - ``` - -- [ ] **Step 2: Delete the handlers + argparse plumbing** - - Remove the subparser registrations from wherever the argparse tree is built (likely `ergon_cli/__main__.py` or `ergon_cli/cli.py`). The `experiment` top-level command may continue to exist for the lifecycle subcommands `show` / `list` added in PR 8 — for now, drop just the `define` and `run` subcommands. - -- [ ] **Step 3: Delete the corresponding tests** - - ```bash - rg "handle_experiment_define\|handle_experiment_run" ergon_cli/tests/ - ``` - - Remove every test case referencing these. Run `uv run pytest ergon_cli/tests/unit -q` to confirm the CLI tests still pass (other commands unaffected). - -- [ ] **Step 4: Document the hard break** - - Add an entry to the repo CHANGELOG (or wherever user-facing breaking changes go) noting that `ergon experiment define` / `ergon experiment run` are gone. Direct users to the Python API (`persist_benchmark` + `launch_run`). See the example `kick_off.py` script in `docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md`. - -## Task 20: Add `benchmarks/README.md` Catalogue - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/benchmarks/README.md` - -The CLI no longer dispatches benchmarks, so the discoverability surface is documentation. This README is the catalogue. - -- [ ] **Step 1: Write the README** - - ```markdown - # Builtin Benchmarks - - Each subdirectory is one benchmark. Import from Python; there is no - CLI authoring path. - - | Benchmark | Module | Worker factories | Default sandbox | - |---|---|---|---| - | MiniF2F | `ergon_builtins.benchmarks.minif2f` | `make_minif2f_worker` (ReAct) | `LeanSandbox` | - - Adding a new benchmark = a new subdirectory. Update this table in - the same PR. - - ## Authoring example - - See `docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md` - for the full Python authoring example. Minimal: - - from ergon_builtins.benchmarks.minif2f import MiniF2FBenchmark, make_minif2f_worker - from ergon_core.api import persist_benchmark, launch_run - - benchmark = MiniF2FBenchmark(worker_factory=make_minif2f_worker, limit=10) - handle = persist_benchmark(benchmark, name="minif2f-react", experiment="ablation-2026-05-15") - await launch_run(handle.definition_id) - ``` - -- [ ] **Step 2: Add an architecture-doc cross-reference** - - In `docs/architecture/06_builtins.md`, add a "Discovery" sub-section pointing at this README as the canonical catalogue. See Task 8 (already covers other 06_builtins.md edits; add this one as a step there or as a Phase 2 addendum). - -## Task 21: Update All Remaining Callsites (Tests, Inngest, Dashboard) - -**Files:** - -- Modify: every test file constructing `Experiment(...)` (~20 files; identify via `rg`) -- Modify: every Inngest event payload naming `experiment` semantically -- Regenerate: dashboard contract schemas (`pnpm run generate:contracts`); hand-fix any consumer TypeScript that typecheck-fails -- Modify: any fixture or conftest that yields `Experiment` instances - -- [ ] **Step 1: Tests** - - ```bash - rg "Experiment\(" --type py ergon_core/tests/ ergon_builtins/tests/ ergon_cli/tests/ tests/ - ``` - - Rewrite each from `Experiment(benchmark=b, name="x", metadata={})` → either: - - direct `persist_benchmark(b, name="x", metadata={})` if the test was exercising persist; - - or just drop the `Experiment` wrapper if the test was using it as a struct; - - update assertions that read `experiment.benchmark` / `experiment.name` to read the kwarg directly. - -- [ ] **Step 2: Inngest event payloads** - - ```bash - rg "experiment_id\b\|experiment\s*:\s*Experiment\|experiment\s*:\s*\"" ergon_core/ - ``` - - Likely sites: `core/application/experiments/launch.py` event payloads, any `*_inngest.py` handlers. Rewrite to use `definition_id` for the runtime identity (it always was the actual key; the name was just misleading) and `experiment: str | None` for the tag. - -- [ ] **Step 3: Dashboard TypeScript** - - ```bash - # Find consumer TypeScript that references the renamed types: - rg "ExperimentRecord\|experiment_record\|ExperimentDefineRequest" ergon-dashboard/src/ --type ts --type tsx | grep -v generated - ``` - - Workflow: (1) run `pnpm run generate:contracts` from `ergon-dashboard/` — this regenerates the Zod schemas from the backend's renamed Pydantic models, picking up `BenchmarkDefinitionRecord` and the new `experiment: str | None` field automatically. (2) Run `pnpm run typecheck`; for each consumer file that fails, rename the referenced type / field by hand. (3) Update any UI surface labeling — the `experiment` column is the optional grouping tag; the `name` column is the per-definition human label. Both are user-facing. - -- [ ] **Step 4: Run full test suite** - - ```bash - pnpm run test:be:fast - pnpm run check:fe # dashboard typecheck + lint - ``` - - Both must pass. If frontend has tests, run those too. - -## Task 22: Finalise Architecture Docs + RFC Framing for Phase 2 - -**Files:** - -- Modify: `docs/architecture/01_public_api.md` -- Modify: `docs/architecture/06_builtins.md` -- Modify: `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md` -- Modify: `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/01-api-surface.md` - -(Some of these may have been partly updated in Task 8 / Task 9 for Phase 1's framing — finalise here for the post-Experiment-kill state.) - -- [ ] **Step 1: `01_public_api.md`** — drop the `Experiment` type from the type list. Add `persist_benchmark(benchmark, *, name, experiment=None, metadata=None) -> DefinitionHandle` as the canonical authoring API. Note: the word "experiment" survives only as a `str | None` field, not a class. - -- [ ] **Step 2: `06_builtins.md`** — add the Discovery sub-section pointing at `benchmarks/README.md`. Note that the CLI is observation-only after PR 6.5. - -- [ ] **Step 3: `00-readme.md`** — rewrite the "experiment" terminology section to match the brainstorm doc. The class is gone; the word is a label. Cross-reference the brainstorm doc for the longer rationale. - -- [ ] **Step 4: `01-api-surface.md`** — drop `Experiment` from the public API type list. Add `persist_benchmark`. Add a short note on `experiment: str | None` semantics (tag, not class). - -## Task 23: Phase 2 Check Suite - -- [ ] **Step 1: Full backend checks** - - ```bash - pnpm run check:be - ``` - - All green. - -- [ ] **Step 2: Full backend tests** - - ```bash - pnpm run test:be:fast - ``` - - All green. Expect a slight test-count change (deleted CLI authoring tests) but no failures. - -- [ ] **Step 3: Frontend checks (dashboard)** - - ```bash - pnpm run check:fe - ``` - - All green. - -- [ ] **Step 4: Verify the Experiment class is truly gone** - - ```bash - rg "class Experiment\b" . # zero hits in code (history/docs OK) - rg "from ergon_core.*import.*Experiment\b" . # zero hits - rg "ExperimentRecord\b" . # zero hits in code - rg "ExperimentDefineRequest\b" . # zero hits - ``` - - Any hit outside of `docs/superpowers/brainstorms/` or `docs/rfcs/active/2026-05-11-.../` is a bug. - -## Task 24: Phase 2 Commit - -```bash -git add -A -git commit -m "PR 6.5 (2/2): kill Experiment class; persist_benchmark + experiment column - -- Hard delete ergon_core.api.experiment.Experiment (no alias) -- Rename persist_definition(experiment) → persist_benchmark(benchmark, *, name, experiment=None, ...) -- Rename ExperimentRecord SQLModel + 'experiments' table → BenchmarkDefinitionRecord / 'benchmark_definitions' -- Add experiment: str | None column for grouping related definitions -- Delete CLI authoring commands (ergon experiment define / run) — clean break -- Add ergon_builtins/benchmarks/README.md as catalogue (replaces deleted CLI registry) -- Update all tests, Inngest payloads, dashboard TypeScript -- See docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md for rationale" -``` - -PR is now ready to push. Two commits, one PR: file moves + Experiment kill. - -## Verification - -After both commits land: - -- All `pnpm run check:be` steps green. -- All `pnpm run test:be:fast` tests pass. -- All `pnpm run check:fe` checks pass. -- `rg "ergon_builtins\.sandboxes"` zero hits in code. -- `rg "ergon_builtins\.toolkits"` zero hits in code. -- `rg "class Experiment\b"` zero hits in code. -- `rg "ExperimentRecord\b"` zero hits in code. -- `rg "persist_definition\b"` zero hits in code. -- The architecture docs (`06_builtins.md`, `01_public_api.md`) describe the new framing. -- The cardinality matrix appears in `06_builtins.md`. -- `ergon_builtins/benchmarks/README.md` exists and lists the builtin benchmarks. - -## What This PR Is NOT - -- Not a worker-subclass-per-benchmark rewrite. The point is the opposite: lock in the "factory function per strategy, not subclass per benchmark" pattern before PR 10a/10b/10c replicate it. -- Not a `Sandbox` / `Toolkit` API redesign. The Pydantic-bound model + `_type` discriminator pattern from PR 5 stands. -- Not a CLI rewrite. Lifecycle commands (`run status`, `run cancel`, etc.) are added in PR 8 — *not* in this PR. PR 6.5 only deletes the CLI authoring route. -- Not a behaviour change in `Benchmark.build_instances()`. PR 5/6's design (inline worker/sandbox/evaluators on each Task) stands. - -## Risks - -- **Import cycles re-emerge differently.** The PR 6 cycle (lazy import in `MiniF2FToolkit.tools()`) survives Phase 1's move but the path description in the `# reason:` comment needs updating. Task 2 Step 3 covers this; double-check post-move. -- **Test discovery surprises.** Pytest discovery rooted at `ergon_builtins/tests/unit` finds tests by directory; moved files shouldn't affect discovery. Confirm with `pytest --collect-only` if anything looks off. -- **Registry references survive.** Search for any `worker_factory` string in `registry_core.py` / `registry.py`; PR 6 didn't remove the legacy registry binding for `"minif2f-react"`, so the file is still referenced by string. Task 3 Step 4 covers this — if it imports `MiniF2FReactWorker` from `worker_factory`, the rename needs to update that import too. -- **Dashboard contract drift.** The frontend reads via the REST API + Zod schemas generated from Pydantic JSON Schema (no Drizzle). Phase 2 Task 21 Step 3 covers regenerating contracts and fixing typecheck failures. Risk: forgetting to run `pnpm run generate:contracts` after backend renames, leaving the dashboard typing against stale schemas. *Mitigation: Task 21 Step 3 explicitly runs the regeneration; Task 23 runs `pnpm run check:fe`.* -- **Inngest event payload drift.** Event payloads that named `experiment_id` semantically need rewriting to `definition_id`. Phase 2 Task 21 Step 2 covers this — risk is missing one and silently breaking a workflow. *Mitigation: run E2E tests if available.* -- **Phase 1 and Phase 2 commits must land together.** Phase 1 alone leaves `Experiment` alive but PR 8's CLI dispatch references the now-renamed file paths — landing only Phase 1 creates a broken intermediate state. Don't split the PR. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/08-pr-07-persistence-collapse.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/08-pr-07-persistence-collapse.md deleted file mode 100644 index 9d49c3ac9..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/08-pr-07-persistence-collapse.md +++ /dev/null @@ -1,491 +0,0 @@ -# PR 7 — Persistence Collapse Behind Bridges - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Make new definitions and runs work without -`BenchmarkDefinitionRecord` (PR 6.5's renamed `ExperimentRecord`), while -old read paths remain bridged until final deletion. - -**Architecture:** Add definition metadata columns and canonical launch by -`definition_id`. Keep `BenchmarkDefinitionRecord` importable for old -read models until PR 11. - -**Tech Stack:** SQLModel, Alembic additive migration, read-model tests. - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/core/persistence/definitions/models.py -ergon_core/ergon_core/core/persistence/telemetry/models.py -ergon_core/ergon_core/core/application/experiments/definition_writer.py -ergon_core/ergon_core/core/application/experiments/launch.py -ergon_core/ergon_core/core/application/experiments/service.py -ergon_core/ergon_core/core/application/read_models/experiments.py -ergon_core/ergon_core/core/application/read_models/runs.py -ergon_core/ergon_core/core/application/read_models/cohorts.py -ergon_core/tests/unit/runtime/test_experiment_launch_service.py -ergon_core/tests/unit/runtime/test_experiment_read_service.py -ergon_core/tests/unit/state/test_type_invariants.py -``` - -**Create:** - -```text -ergon_core/migrations/versions/aabbccdd0005_definition_metadata_and_launch.py -ergon_core/ergon_core/core/application/experiments/errors.py -``` - -(Migration id `aabbccdd0005` continues the existing numbering sequence — -`aabbccdd0001` (run-graph task_json), `aabbccdd0002` (worker_output_json), -`aabbccdd0003` (definition task_json), `aabbccdd0004` (PR 6.5 -`add_experiment_tag`, already shipped on the public PR 6.5 branch / PR -#58 CI). PR 7 takes `aabbccdd0005` because PR 6.5 pushed `0004` first — -migration ids are monotonic by push order. Downstream plans (PR 10a/b/c, -PR 11) need to claim `aabbccdd0006` as the next free id.) - -## Current State - -`BenchmarkDefinitionRecord` (formerly `ExperimentRecord` pre-PR-6.5) -carries user-facing metadata and run launch loads by `experiment_id`. -`ExperimentDefinition` carries only benchmark type, metadata JSON, and -created timestamp. - -## Target State For This PR - -New path: - -```python -definition = ExperimentDefinition( - id=definition_id, - name=benchmark.name or benchmark_type, - description=benchmark.description, - metadata_json=dict(benchmark.metadata), - created_by=benchmark.created_by, -) -run = launch_run(definition_id) -``` - -Old `BenchmarkDefinitionRecord` (renamed from `ExperimentRecord` in PR -6.5) remains for old read models. - -**Reconciliation note (post-PR-6.5).** This plan was originally written -when the `Experiment` wrapper class was the source of identity. PR 6.5 -deleted that wrapper — identity now lives on the `Benchmark` instance. -Throughout this document, references to ``experiment.name`` / -``experiment.description`` / ``experiment.created_by`` / -``experiment.metadata`` resolve to the matching ``benchmark.*`` fields. -References to ``ExperimentRecord`` resolve to ``BenchmarkDefinitionRecord``. -References to ``ExperimentService.run_experiment`` resolve to the -module-level ``run_experiment`` function in -``application/experiments/service.py``. - -## Task 1: Add Definition Metadata - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/definitions/models.py` -- Create: migration file - -- [ ] **Step 1: Add fields** - -```python -name: str = Field(index=True) -description: str | None = None -created_by: str | None = None -``` - -Keep `metadata_json` as the JSONB/free-form metadata field. - -- [ ] **Step 2: Add migration** - -```python -def upgrade() -> None: - op.add_column("experiment_definitions", sa.Column("name", sa.Text(), nullable=True)) - op.add_column("experiment_definitions", sa.Column("description", sa.Text(), nullable=True)) - op.add_column("experiment_definitions", sa.Column("created_by", sa.Text(), nullable=True)) - op.create_index("ix_experiment_definitions_name", "experiment_definitions", ["name"]) - op.execute("UPDATE experiment_definitions SET name = benchmark_type WHERE name IS NULL") - op.alter_column("experiment_definitions", "name", nullable=False) -``` - -Downgrade drops the index and columns. - -## Task 2: Persist Metadata - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/experiments/definition_writer.py` - -This task is the **denormalization site** for `02-persistence-layer.md` -§3's indexed-column rule. `name` and `description` round-trip through -`experiment_json` AND are written into dedicated indexed columns so the -dashboard can `SELECT id FROM experiment_definitions WHERE name = ...` -without JSON path queries. The extraction happens here, inline, at -write time — there is **no** `Experiment.metadata_columns()` helper or -`model_validator` that produces the column dict, because: - -1. The writer is the only caller; abstracting it adds indirection - without a second consumer. -2. The column set is short (three fields); a helper would barely save - code. -3. Keeping the extraction inline keeps the SQL writer's full shape - visible in one place, which matters for the v1-audit-style review - that asks "what columns get populated when". - -If a second caller appears later (e.g. an admin tool that wants to -reproduce the denormalization for back-fills), promote the inline block -to a `_metadata_columns(experiment)` private helper in the same module -at that point — not preemptively. - -- [ ] **Step 1: Add `created_by` to `Benchmark.__init__`** - -PR 6.5 made `Benchmark` the source of identity (`name`, `description`, -`metadata` are already first-class kwargs on ``Benchmark.__init__``). -Add ``created_by: str | None = None`` to the constructor and store it -as ``self.created_by``. Symmetric with the existing identity fields; -no separate wrapper class involved. - -- [ ] **Step 2: Change definition row construction** - -Replace: - -```python -definition_row = ExperimentDefinition( - id=definition_id, - benchmark_type=benchmark_type, - metadata_json=resolved_metadata, - created_at=now, -) -``` - -with: - -```python -definition_row = ExperimentDefinition( - id=definition_id, - benchmark_type=benchmark_type, - name=benchmark.name if benchmark.name is not None else benchmark_type, - description=benchmark.description, - created_by=benchmark.created_by, - metadata_json=resolved_metadata, - created_at=now, -) -``` - -The `if ... is not None` form (rather than `benchmark.name or -benchmark_type`) is deliberate: `or` would also fall through on the -empty string, which is a valid (if unusual) author choice; `is not -None` only falls through when the author left the field unset. - -(Today ``benchmark.name`` is always a non-empty string because -``Benchmark.__init__`` defaults to ``self.__class__.__name__``. Once a -benchmark subclass overrides `__init__` to leave it unset, the -``is not None`` guard becomes load-bearing — keep it.) - -## Task 3: Add Canonical Launch By Definition - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/experiments/launch.py` - -- [ ] **Step 1: Add public function** - -```python -async def launch_run( - definition_id: UUID, - *, - metadata: Mapping[str, Any] | None = None, - emit_workflow_started: WorkflowStartedEmitter | None = None, -) -> ExperimentRunResult: - emitter = emit_workflow_started or _emit_workflow_started - with get_session() as session: - definition = session.get(ExperimentDefinition, definition_id) - if definition is None: - raise ValueError(f"ExperimentDefinition {definition_id} not found") - run = create_run( - DefinitionHandle(definition_id=definition.id, benchmark_type=definition.benchmark_type), - experiment_id=None, - workflow_definition_id=definition.id, - instance_key=None, - worker_team_json={}, - evaluator_slug=None, - model_target=None, - sandbox_slug=None, - dependency_extras_json={}, - assignment_json=dict(metadata or {}), - seed=None, - ) - await emitter(run.id, definition_id) - return ExperimentRunResult( - experiment_id=definition_id, - run_ids=[run.id], - workflow_definition_ids=[definition_id], - ) -``` - -The `create_run` call still accepts old telemetry fields. PR 11 narrows it. - -- [ ] **Step 2: Make the module-level `run_experiment` delegate** - -PR 6.5's cleanup collapsed `ExperimentService` to the module-level -``run_experiment(request, *, workflow_definition_factory=None, -emit_workflow_started=None)`` function in -``application/experiments/service.py``. If the request field is still -named ``experiment_id``, detect whether it is an ``ExperimentDefinition`` -first (call ``launch_run(definition_id)`` from Task 3 Step 1). Fall -back to ``BenchmarkDefinitionRecord`` lookup only if no definition -exists. - -## Task 4: Read Models Prefer Definitions - -**Files:** - -- Modify read model files listed above - -- [ ] **Step 1: Change list/show queries** - -Queries should select `ExperimentDefinition` first and map: - -```python -experiment_id = definition.id -name = definition.name -description = definition.description -benchmark_type = definition.benchmark_type -metadata = definition.parsed_metadata() -``` - -Keep a `BenchmarkDefinitionRecord` fallback method named -`_legacy_benchmark_definition_record_detail` for old rows that -predate the new columns. - -## Task 5: Tests - -**Files:** - -- Modify: `ergon_core/tests/unit/runtime/test_experiment_launch_service.py` -- Modify: `ergon_core/tests/unit/runtime/test_experiment_read_service.py` - -- [ ] **Step 1: Launch test** - -```python -@pytest.mark.asyncio -async def test_launch_run_accepts_definition_id_without_experiment_record(session): - definition = ExperimentDefinition( - benchmark_type="mini", - name="mini", - metadata_json={}, - ) - session.add(definition) - session.commit() - - result = await launch_run(definition.id, emit_workflow_started=AsyncMock()) - - assert result.workflow_definition_ids == [definition.id] - assert result.run_ids -``` - -- [ ] **Step 2: Read model test** - -```python -from sqlmodel import select - -from ergon_core.core.application.read_models.experiments import ( - ExperimentReadService, -) -from ergon_core.core.persistence.definitions.models import ExperimentDefinition -from ergon_core.core.persistence.telemetry.models import BenchmarkDefinitionRecord - - -@pytest.mark.asyncio -async def test_read_service_returns_definition_metadata_without_benchmark_definition_record( - session, -): - definition = ExperimentDefinition( - benchmark_type="mini", - name="mini-experiment", - description="smoke for read model", - metadata_json={"created_by": "test"}, - ) - session.add(definition) - session.commit() - - # Sanity-check setup: no BenchmarkDefinitionRecord exists for this id. - assert ( - session.exec( - select(BenchmarkDefinitionRecord).where( - BenchmarkDefinitionRecord.id == definition.id - ) - ).first() - is None - ) - - detail = await ExperimentReadService().get_experiment(definition.id) - - assert detail is not None - assert detail.experiment_id == definition.id - assert detail.name == "mini-experiment" - assert detail.description == "smoke for read model" - assert detail.benchmark_type == "mini" - assert detail.metadata.get("created_by") == "test" - - -@pytest.mark.asyncio -async def test_read_service_falls_back_to_benchmark_definition_record_for_legacy_rows( - session, -): - """Old rows with a BenchmarkDefinitionRecord but no ExperimentDefinition - name still resolve via the legacy path until PR 11 deletes the legacy - table.""" - - legacy = BenchmarkDefinitionRecord(name="legacy-only", benchmark_type="mini") - session.add(legacy) - session.commit() - - detail = await ExperimentReadService().get_experiment(legacy.id) - assert detail is not None - assert detail.name == "legacy-only" -``` - -- [ ] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_experiment_launch_service.py \ - ergon_core/tests/unit/runtime/test_experiment_read_service.py \ - ergon_core/tests/unit/state/test_type_invariants.py -q -``` - -## Task 6: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` -- Create: `ergon_core/ergon_core/core/application/experiments/errors.py` -- Modify: `ergon_core/ergon_core/core/application/experiments/repository.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` - -PR 7 lands `persist_definition` writing only the collapsed definition -table; the corresponding smoketest case flips green. PR 7 is also the -natural home for adding `experiments/errors.py` since this PR is -already heavily editing the experiments package. - -- [ ] **Step 1: Remove the xfail on `test_persist_definition_writes_only_intended_tables`** - -In `test_walkthrough_smoketest.py`, delete the -`@pytest.mark.xfail(reason="PR 7: ...")` decorator. The test body -already asserts the correct invariant (one `ExperimentDefinition` row, -N `ExperimentDefinitionTask` rows) and now passes against the collapsed -schema this PR delivers. - -- [ ] **Step 2: Add `experiments/errors.py`** - -```python -"""Errors raised by the experiments domain. - -Typed exceptions so callers can `except DefinitionNotFoundError:` -specifically rather than catching generic `ValueError` and string- -matching the message. See 07-test-strategy.md § Repository layer -standard rule 8. -""" - -from uuid import UUID - - -class ExperimentDomainError(Exception): - """Base for all experiments-domain errors.""" - - -class DefinitionNotFoundError(ExperimentDomainError): - """Lookup failed for an `ExperimentDefinition` row.""" - - def __init__(self, definition_id: UUID) -> None: - super().__init__(f"ExperimentDefinition {definition_id} not found") - self.definition_id = definition_id - - -class DefinitionTaskNotFoundError(ExperimentDomainError): - """Lookup failed for an `ExperimentDefinitionTask` row.""" - - def __init__(self, task_id: UUID) -> None: - super().__init__(f"ExperimentDefinitionTask {task_id} not found") - self.task_id = task_id - - -class DefinitionInstanceNotFoundError(ExperimentDomainError): - """Lookup failed for an `ExperimentDefinitionInstance` row.""" - - def __init__(self, instance_id: UUID) -> None: - super().__init__( - f"ExperimentDefinitionInstance {instance_id} not found" - ) - self.instance_id = instance_id -``` - -- [ ] **Step 3: Replace `ValueError` raises in the repository** - -In `experiments/repository.py`, replace: - -```python -raise ValueError(f"ExperimentDefinitionTask {task_id} not found") -raise ValueError(f"ExperimentDefinitionInstance {task.instance_id} not found") -``` - -with: - -```python -raise DefinitionTaskNotFoundError(task_id) -raise DefinitionInstanceNotFoundError(task.instance_id) -``` - -Update imports accordingly. Run `rg "ExperimentDefinitionTask.*not found"` -to confirm no other inline raise sites remain. - -- [ ] **Step 4: Remove the xfail entry** - -In `test_repository_companion_files.py`, delete: - -```python -("test_package_has_errors_py_if_it_raises", - "ergon_core/ergon_core/core/application/experiments"): - "PR 7: add experiments/errors.py with typed DefinitionNotFoundError " - "and replace ValueError raises", -``` - -- [ ] **Step 5: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py \ - ergon_core/tests/unit/architecture/test_repository_companion_files.py -q -``` - -Expected: the PR 7 cases PASS; remaining cases still XFAIL. - -## PR Ledger - -Invariant landed: new definitions can launch without `BenchmarkDefinitionRecord` -rows (the table formerly known as `ExperimentRecord` pre-PR-6.5). - -Bridge code introduced: `BenchmarkDefinitionRecord` fallback read/launch -path; ``run_experiment`` detects which side of the bridge to take. - -Old path still intentionally alive: `BenchmarkDefinitionRecord`, old run -telemetry fields. - -Deletion gate: PR 11 deletes `BenchmarkDefinitionRecord` and narrows -`create_run`. - -Tests added or updated: launch-by-definition and read-model tests. - -Modules owned by this PR: definition metadata, launch, read models. - -Files added: `experiments/errors.py` (typed exceptions), -`migrations/versions/aabbccdd0005_definition_metadata_and_launch.py`. - -Identity source assumed: `Benchmark.{name, description, created_by, metadata}` -(PR 6.5 deleted the `Experiment` wrapper that previously owned these). diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/09-pr-08-cli-composition.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/09-pr-08-cli-composition.md deleted file mode 100644 index 1fcd3d70e..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/09-pr-08-cli-composition.md +++ /dev/null @@ -1,487 +0,0 @@ -# PR 8 — Lifecycle CLI Cleanup - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Round out the lifecycle / observation commands the CLI keeps after PR 6.5 killed the authoring route. No new abstractions. No factory dispatch. No per-benchmark CLI registration burden. - -**Context:** PR 6.5 made the call (and recorded the rationale in `docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md`) that the CLI has **exactly one role**: lifecycle and observation of persisted state. Authoring is Python-only. PR 6.5 deleted the `ergon experiment define` / `ergon experiment run` *authoring* commands — but kept the *observation* commands (`experiment show <UUID>`, `experiment list`, `run list`, `run cancel`) that operate on persisted state. PR 8 fills in the remaining gaps: a single-run status command, an experiment-tag-based filter on `run list`, and tag-grouping commands that surface `BenchmarkDefinitionRecord.experiment` (which is otherwise unobservable from the CLI today). - -**Scope (much smaller than the original plan):** three new commands and one new flag. Each is a thin wrapper around an existing repository read. No DTOs, no factory registries, no new abstractions. - -**Tech Stack:** argparse CLI handlers, existing repositories (`DefinitionRepository`, direct `RunRecord` reads, `ExperimentReadService`), pytest CLI tests. - ---- - -## Files - -**Create:** - -```text -ergon_cli/tests/unit/cli/test_run_cli.py # tests for the new commands -``` - -**Modify:** - -```text -ergon_cli/ergon_cli/commands/run.py # add `status`, --experiment filter -ergon_cli/ergon_cli/commands/experiment.py # add `tags`, `by-tag` -ergon_cli/ergon_cli/main.py # register new subparsers -ergon_cli/tests/unit/cli/test_experiment_cli.py # tests for new commands -ergon_core/ergon_core/core/application/experiments/repository.py # add tag helpers -ergon_core/ergon_core/core/application/workflows/runs.py # add list helpers if missing -``` - -## Current State (after PR 6.5 + PR 7) - -After PR 7 lands (one layer up from this PR): - -- The public `Experiment` wrapper class is gone (PR 6.5). -- `persist_benchmark(benchmark) -> DefinitionHandle` is the authoring API - (module-level function in `ergon_core.api`). Identity fields (``name``, - ``description``, ``metadata``, ``created_by``) are read off the - ``Benchmark`` instance directly — no kwargs to `persist_benchmark`. -- `BenchmarkDefinitionRecord` is the persisted legacy row (table - ``experiments``), with an `experiment: str | None` column. Multiple - records sharing the same `experiment` tag belong to the same logical - experiment. No `Benchmark(experiment=...)` constructor kwarg exists - today; tagging is currently a write-side-only column populated by the - cohort / test harness. -- `ExperimentDefinition` is the canonical v2 row (table - ``experiment_definitions``) with `name`/`description`/`created_by` - columns added in PR 7. -- The CLI **already has** these observation commands (kept by PR 6.5): - - `ergon experiment show <UUID>` — full detail via `ExperimentReadService.get_experiment` - - `ergon experiment list --limit N` — list summaries via `ExperimentReadService.list_experiments` - - `ergon run list --limit N --status S` — direct `RunRecord` query - - `ergon run cancel <UUID>` — wraps `workflows.runs.cancel_run` (sync) -- The CLI **does not yet have**: - - A way to inspect a single run by id (status snapshot). - - A way to filter `run list` by the experiment-tag column. - - A way to discover or browse the experiment-tag namespace from the CLI. - -PR 8 fills exactly those three gaps and nothing else. - -## Target State For This PR - -```bash -# Run lifecycle (additions only — existing commands unchanged) -ergon run status <run-id> # NEW — show single run status -ergon run list [--status=S] [--experiment=<tag>] [--limit=N] # EXTENDED — adds --experiment - -# Experiment-tag observation (additive — existing UUID-based commands unchanged) -ergon experiment tags # NEW — list distinct experiment-tag strings -ergon experiment by-tag <tag> # NEW — list definitions in a tag, with latest run -``` - -That's the full new CLI surface. All read-only against persisted state. Existing commands — `experiment show <UUID>`, `experiment list`, `run list` (without --experiment), `run cancel` — keep working exactly as they do today. - -## Task 1: Add Repository Helpers - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/experiments/repository.py` -- Modify: `ergon_core/ergon_core/core/application/workflows/runs.py` - -The new CLI handlers need a few thin reads. Add them next to the existing repository code, not as standalone abstractions. - -- [ ] **Step 1: `DefinitionRepository.list_by_experiment_tag`** - - In `core/application/experiments/repository.py`, add a method that takes a session and an experiment tag string, and returns the matching `BenchmarkDefinitionRecord` rows. - - ```python - def list_by_experiment_tag( - self, - session: Session, - tag: str, - ) -> list[BenchmarkDefinitionRecord]: - """List ``BenchmarkDefinitionRecord`` rows tagged with ``tag``. - - The ``experiment`` column groups records into a named logical - experiment; this helper is the read side of that grouping. - """ - stmt = select(BenchmarkDefinitionRecord).where( - BenchmarkDefinitionRecord.experiment == tag, - ) - return list(session.exec(stmt).all()) - ``` - -- [ ] **Step 2: `DefinitionRepository.distinct_experiment_tags`** - - ```python - def distinct_experiment_tags(self, session: Session) -> list[str]: - """Distinct non-null ``experiment`` tag values across all records.""" - stmt = ( - select(BenchmarkDefinitionRecord.experiment) - .where(BenchmarkDefinitionRecord.experiment.is_not(None)) - .distinct() - ) - return [row for row in session.exec(stmt).all() if row is not None] - ``` - -- [ ] **Step 3: `latest_run_for_definition` helper** - - In `core/application/workflows/runs.py`, add a module-level helper (matches the style of the existing `cancel_run` / `create_run` functions in that file): - - ```python - def latest_run_for_definition(definition_id: UUID) -> RunRecord | None: - """Most-recent ``RunRecord`` for a given workflow definition, or None.""" - with get_session() as session: - stmt = ( - select(RunRecord) - .where(RunRecord.workflow_definition_id == definition_id) - .order_by(RunRecord.created_at.desc()) - .limit(1) - ) - return session.exec(stmt).first() - ``` - - Note: this uses ``workflow_definition_id`` (the ``ExperimentDefinition`` FK), not ``experiment_id``. The ``experiment_id`` column on ``RunRecord`` still exists but is being narrowed in PR 11. - -## Task 2: Add `ergon run status` Command - -**Files:** - -- Modify: `ergon_cli/ergon_cli/commands/run.py` - -The existing `run.py` already has `list_runs` and `cancel_run` handlers and a `handle_run` dispatcher. Add a third action. - -- [ ] **Step 1: `status_run` handler** - - ```python - def status_run(args: Namespace) -> int: - ensure_db() - try: - run_id = UUID(args.run_id) - except ValueError: - print(f"Invalid UUID: {args.run_id}") - return 1 - - with get_session() as session: - run = session.get(RunRecord, run_id) - if run is None: - print(f"No run found with id {args.run_id}") - return 1 - - print(f"run_id: {run.id}") - print(f"status: {run.status}") - print(f"benchmark_type: {run.benchmark_type}") - print(f"workflow_definition_id: {run.workflow_definition_id}") - print(f"instance_key: {run.instance_key}") - if run.evaluator_slug is not None: - print(f"evaluator: {run.evaluator_slug}") - if run.model_target is not None: - print(f"model: {run.model_target}") - created = run.created_at.strftime("%Y-%m-%d %H:%M:%S") if run.created_at else "-" - print(f"created_at: {created}") - if run.started_at: - print(f"started_at: {run.started_at.strftime('%Y-%m-%d %H:%M:%S')}") - if run.completed_at: - print(f"completed_at: {run.completed_at.strftime('%Y-%m-%d %H:%M:%S')}") - if run.error_message: - print(f"error: {run.error_message}") - return 0 - ``` - -- [ ] **Step 2: Dispatch in `handle_run`** - - Update the existing `handle_run` dispatcher to route `status`: - - ```python - def handle_run(args: Namespace) -> int: - if args.run_action == "list": - return list_runs(args) - elif args.run_action == "cancel": - return cancel_run(args) - elif args.run_action == "status": - return status_run(args) - else: - print("Usage: ergon run {list|status|cancel}") - return 1 - ``` - -## Task 3: Add `--experiment` Filter to `ergon run list` - -**Files:** - -- Modify: `ergon_cli/ergon_cli/commands/run.py` - -The existing `list_runs` filters by `--status` and `--limit`. Add an optional `--experiment=<tag>` filter that joins through `BenchmarkDefinitionRecord.experiment`. - -- [ ] **Step 1: Extend `list_runs`** - - In `list_runs`, after the status filter, add the experiment-tag join: - - ```python - if args.experiment: - stmt = stmt.join( - BenchmarkDefinitionRecord, - RunRecord.experiment_id == BenchmarkDefinitionRecord.id, - ).where(BenchmarkDefinitionRecord.experiment == args.experiment) - ``` - - The join is via `RunRecord.experiment_id` → `BenchmarkDefinitionRecord.id`. This is the legacy FK from PR 6.5; PR 11 will narrow it. Once narrowed, this filter will need to route through `ExperimentDefinition` / a tagging story — flag this in the PR 11 plan. - - Add the import at the top of the file: - ```python - from ergon_core.core.persistence.telemetry.models import BenchmarkDefinitionRecord, RunRecord - ``` - -- [ ] **Step 2: Update empty-result message** - - When no rows match, mention the filters that were in play: - - ```python - if not runs: - parts = ["No runs found"] - if args.status: - parts.append(f"with status={args.status!r}") - if args.experiment: - parts.append(f"for experiment={args.experiment!r}") - print(" ".join(parts)) - return 0 - ``` - -## Task 4: Add `ergon experiment tags` + `experiment by-tag` Commands - -**Files:** - -- Modify: `ergon_cli/ergon_cli/commands/experiment.py` - -The existing `experiment.py` already has `handle_experiment_show` and `handle_experiment_list` (both UUID-based, both kept). Add two new tag-namespace handlers. - -- [ ] **Step 1: `handle_experiment_tags`** - - ```python - def handle_experiment_tags(args: Namespace) -> int: - _ensure_cli_logging() - with get_session() as session: - tags = DefinitionRepository().distinct_experiment_tags(session) - if not tags: - logger.info( - "No experiment tags yet. Tag definitions by setting " - "`experiment` on the underlying record (cohort harness)." - ) - return 0 - for tag in tags: - logger.info("%s", tag) - return 0 - ``` - -- [ ] **Step 2: `handle_experiment_by_tag`** - - ```python - def handle_experiment_by_tag(args: Namespace) -> int: - _ensure_cli_logging() - with get_session() as session: - records = DefinitionRepository().list_by_experiment_tag( - session, args.tag, - ) - if not records: - logger.info("No definitions tagged with experiment=%r", args.tag) - return 0 - logger.info("DEFINITION_ID\tNAME\tBENCHMARK\tSTATUS\tLATEST_RUN_STATUS") - for record in records: - latest = latest_run_for_definition(record.id) - latest_status = latest.status if latest else "no runs" - logger.info( - "%s\t%s\t%s\t%s\t%s", - record.id, - record.name, - record.benchmark_type, - record.status, - latest_status, - ) - return 0 - ``` - - Imports at the top of the file: - - ```python - from ergon_core.core.application.experiments.repository import DefinitionRepository - from ergon_core.core.application.workflows.runs import latest_run_for_definition - from ergon_core.core.persistence.shared.db import get_session - ``` - -- [ ] **Step 3: Dispatch in `handle_experiment`** - - Update the existing `handle_experiment` dispatcher: - - ```python - async def handle_experiment(args: Namespace) -> int: - _ensure_cli_logging() - if args.experiment_action == "show": - return handle_experiment_show(args) - if args.experiment_action == "list": - return handle_experiment_list(args) - if args.experiment_action == "tags": - return handle_experiment_tags(args) - if args.experiment_action == "by-tag": - return handle_experiment_by_tag(args) - logger.error("Usage: ergon experiment {show|list|tags|by-tag}") - return 1 - ``` - -## Task 5: Register Subparsers + Tests - -**Files:** - -- Modify: `ergon_cli/ergon_cli/main.py` -- Create: `ergon_cli/tests/unit/cli/test_run_cli.py` -- Modify: `ergon_cli/tests/unit/cli/test_experiment_cli.py` - -- [ ] **Step 1: Register new `run` subcommand parsers in `main.py`** - - Find the existing `run` subparser block and add `status` and the `--experiment` flag on `list`: - - ```python - run_status_parser = run_sub.add_parser("status", help="Show status of one run") - run_status_parser.add_argument("run_id", help="Run ID (UUID)") - - # Extend the existing run_list_parser - run_list_parser.add_argument( - "--experiment", - default=None, - help="Filter by experiment tag (BenchmarkDefinitionRecord.experiment)", - ) - ``` - -- [ ] **Step 2: Register new `experiment` subcommand parsers in `main.py`** - - Find the existing `experiment` subparser block and add `tags` and `by-tag`: - - ```python - experiment_sub.add_parser("tags", help="List distinct experiment tags") - experiment_by_tag_parser = experiment_sub.add_parser( - "by-tag", help="List definitions for an experiment tag" - ) - experiment_by_tag_parser.add_argument("tag", help="Experiment tag") - ``` - -- [ ] **Step 3: Create `tests/unit/cli/test_run_cli.py`** - - Cover four cases. Use the same pattern as `test_experiment_cli.py` (monkeypatched fakes, `Namespace` direct calls, `capsys`): - - - `test_run_status_prints_status_fields` — fake a `RunRecord` returned by `session.get`, assert key fields land in stdout. - - `test_run_status_reports_invalid_uuid` — pass a non-UUID, assert exit code 1 and helpful message. - - `test_run_status_reports_missing_run` — fake `session.get` returning `None`, assert exit code 1 and helpful message. - - `test_run_list_filters_by_experiment` — wire a fake session whose `select(...).where(...).join(...).where(...).limit(...)` chain captures the call shape; assert the filter is applied. (Alternatively: integration test against an in-memory SQLite session if the project already has a helper for that.) - -- [ ] **Step 4: Extend `test_experiment_cli.py`** - - Add three cases: - - - `test_experiment_tags_lists_distinct_tags` — monkeypatch `DefinitionRepository` with `distinct_experiment_tags` returning a list; assert tags appear in `caplog.text`. - - `test_experiment_tags_handles_empty` — monkeypatch returns `[]`; assert helpful empty-state message. - - `test_experiment_by_tag_lists_definitions_with_latest_run_status` — monkeypatch `DefinitionRepository.list_by_experiment_tag` and `latest_run_for_definition` to return fakes; assert each definition and its latest run status appear. - - Also extend the existing parser test: - - ```python - def test_experiment_subcommands_are_registered_in_main_parser() -> None: - parser = build_parser() - tags_args = parser.parse_args(["experiment", "tags"]) - by_tag_args = parser.parse_args(["experiment", "by-tag", "alpha"]) - assert tags_args.experiment_action == "tags" - assert by_tag_args.experiment_action == "by-tag" - assert by_tag_args.tag == "alpha" - ``` - -- [ ] **Step 5: Run focused tests** - - ```bash - uv run pytest ergon_cli/tests/unit/cli -q - ``` - - All green. - -## Task 6: Documentation - -**Files:** - -- Modify: `docs/architecture/06_builtins.md` (the "Discovery" section added in PR 6.5) -- Modify: `ergon_builtins/ergon_builtins/benchmarks/README.md` (if present — cross-link to the new CLI commands) -- Modify: top-level `README.md` (if it documents CLI commands) - -- [ ] **Step 1: Update the discovery section** - - In `06_builtins.md`, expand the observation section to mention: - - > After kicking off a run from Python, observe it via the CLI: - > - `ergon run status <run-id>` — current state of one run - > - `ergon run list [--status=S] [--experiment=<tag>]` — list runs, optionally filtered - > - `ergon experiment show <UUID>` — full experiment detail (UUID-based) - > - `ergon experiment list` — list recent experiments - > - `ergon experiment tags` — list distinct experiment-tag strings - > - `ergon experiment by-tag <tag>` — list definitions sharing a tag, with latest run status - -- [ ] **Step 2: Update the catalogue README** (if it exists) - - Mirror the same six-bullet block. If the file doesn't exist, skip — don't create one. - -- [ ] **Step 3: Update top-level README** - - If the repo's `README.md` documents CLI commands, add the new ones; remove any references to deleted commands (PR 6.5 should have already cleaned those up). - -## Task 7: Full Check Suite + Commit - -- [ ] **Step 1: Backend checks** - - ```bash - pnpm run check:be - ``` - -- [ ] **Step 2: Backend tests** - - ```bash - pnpm run test:be:fast - ``` - -- [ ] **Step 3: Commit** - - ```bash - git add -A - git commit -m "PR 8: lifecycle CLI commands (run status, --experiment filter, experiment tags/by-tag) - - - Add ergon run status <run-id> - - Add --experiment filter to ergon run list - - Add ergon experiment tags + experiment by-tag <tag> - - Add repository helpers (distinct_experiment_tags, list_by_experiment_tag, - latest_run_for_definition) - - Documentation updates pointing users at the new observation commands - - No authoring path on the CLI; Python remains the only way to start a run." - ``` - -## Verification - -- All `pnpm run check:be` steps green. -- All `pnpm run test:be:fast` tests pass (including new CLI tests). -- `ergon run status <run-id>` returns expected status for a known run. -- `ergon experiment tags` returns the set of distinct tag strings. -- `ergon experiment by-tag <tag>` returns definitions tagged with that string and their latest-run status. - -## What This PR Is NOT - -- **Not an authoring path.** No `ergon experiment define`, no `ergon run <benchmark>`, no benchmark / worker registry dicts. Authoring is Python-only (PR 6.5 made this decision). -- **Not a new abstraction layer.** These commands are thin wrappers around existing repositories — no DTOs, no service classes, no factory registries. -- **Not a replacement of `experiment show <UUID>` / `experiment list`.** Those PR-6.5 commands stay. The new `tags` / `by-tag` commands are *additive* — they surface the experiment-tag namespace (`BenchmarkDefinitionRecord.experiment`) which the UUID-based commands don't expose. -- **Not a dashboard replacement.** CLI commands are for quick terminal-driven observation. Rich UI lives in the dashboard. -- **Not a change to `persist_benchmark` or `launch_run`.** Those are stable after PR 6.5 / 7. -- **Not a `Benchmark(experiment=...)` constructor kwarg.** Tagging is still write-side-only today; PR 8 only adds *read* commands. A constructor kwarg can be added later if needed. - -## Risks - -- **`RunRecord.experiment_id` is being narrowed in PR 11.** Today the `run list --experiment=<tag>` filter joins via `RunRecord.experiment_id → BenchmarkDefinitionRecord.id`. PR 11 will narrow that FK; the filter logic will need to follow. *Mitigation:* call this out in the PR 11 plan as a CLI follow-up. -- **Tag concept is half-wired.** The `experiment` column is populated by the cohort / test harness today but not by the public authoring API. Until a `Benchmark(experiment=...)` kwarg lands, users coming through `persist_benchmark` won't have anything to query. *Mitigation:* empty-state messages on the new commands point at the cohort harness as the only path to a tag today. -- **`cancel_run` is sync, but `handle_experiment` is async.** The CLI's argparse dispatch already handles a mix of sync and async handlers (see `main.py`). No change needed. - -## PR Ledger - -- **Invariant landed:** the CLI is observation-only; authoring is Python-only. -- **Bridge code introduced:** none. -- **Old paths still alive:** `RunRecord.experiment_id` (gated for PR 11); `BenchmarkDefinitionRecord` table (gated for PR 11). -- **Deletion gate:** none new from this PR. -- **Tests added or updated:** `test_run_cli.py` (new), `test_experiment_cli.py` (extended). -- **Modules owned by this PR:** `ergon_cli/commands/run.py` (extended), `ergon_cli/commands/experiment.py` (extended), repository helpers in `ergon_core/core/application/experiments/repository.py` and `ergon_core/core/application/workflows/runs.py`. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/10-pr-09-dynamic-subtasks.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/10-pr-09-dynamic-subtasks.md deleted file mode 100644 index 1833aca53..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/10-pr-09-dynamic-subtasks.md +++ /dev/null @@ -1,603 +0,0 @@ -# PR 9 — Dynamic Subtasks Are Graph-Native - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Make `WorkerContext.spawn_task` write only `run_graph_nodes` with -`is_dynamic=true` and task JSON; no synthetic definition rows. - -**Architecture:** Dynamic and static tasks share `graph_repo.node`. The only -difference is row origin. - -**Tech Stack:** WorkerContext facade, task management service, graph repo, -pytest containment tests. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. They are NOT new tasks layered on top — they are corrections that -must be applied to the task descriptions below before implementation starts. - -1. **Unfreeze `WorkerContext`.** Current code has - `model_config = {"frozen": True}` on `WorkerContext(BaseModel)`. The RFC - (`03-runtime.md` lines 299–313, 457–501) specifies a NON-frozen `BaseModel` - with `PrivateAttr` fields `_task_mgmt`, `_task_inspect`, `_resource_repo`, - plus a `_for_job` classmethod that injects services via - `object.__setattr__`. Task 2's `WorkerContext` facade must first remove the - `frozen` config and add the `PrivateAttr` declarations before the facade - methods (`spawn_task`, `_assert_descendant`) can be added. -2. **Drop the `Task.created_by` assumption.** PR 7 added `created_by` to - `Benchmark`, NOT to `Task`. If dynamic spawns need spawner identity, - record it on the graph node or audit metadata. Audit any plan section - that refers to `Task.created_by` and reroute. -3. **Add the missing graph + inspection methods.** - `WorkflowGraphRepository.descendants_by_parent(...)` and - `TaskInspectionService.descendant_ids(...)` do NOT exist yet — they need - to be added as part of this PR (the SQL CTE in Task 2b is correct; - just needs to be implemented rather than assumed-existing). -4. **Reconcile `add_subtask` signature drift.** Current - `TaskManagementService.add_subtask(session, command: AddSubtaskCommand)` - uses a command-object pattern. Task 3 must either extend the command to - carry a full `Task` instance OR add a thin wrapper on `WorkerContext` - that builds the command from `(task, depends_on)` and calls the service. - Pick one and update Task 3 accordingly. -5. **Create `SpawnedTaskHandle` and `ContainmentViolation` as part of this PR.** - Task 1 already covers `SpawnedTaskHandle`; ensure `ContainmentViolation` - is added to `ergon_core/api/errors.py` (Task 2 Step 3's containment check - raises it). - ---- - -## Files - -**Modify:** - -```text -ergon_core/ergon_core/api/worker/context.py -ergon_core/ergon_core/api/worker/results.py -ergon_core/ergon_core/core/application/tasks/management.py -ergon_core/ergon_core/core/application/tasks/inspection.py -ergon_core/ergon_core/core/application/graph/repository.py -ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py -ergon_core/tests/unit/runtime/test_worker_context_containment.py -``` - -## Current State - -Dynamic graph nodes exist, but runtime identity still mixes `node_id` and -`definition_task_id`. `WorkerContext` is mostly identity fields and does not -own the curated facade. - -## Target State For This PR - -```python -handle = await context.spawn_task( - Task( - task_slug="child", - instance_key="sample-1", - description="Research one child question.", - worker=child_worker, - sandbox=child_sandbox, - evaluators=(), - ) -) -``` - -inserts: - -```python -RunGraphNode( - run_id=context.run_id, - task_id=new_task_id, - parent_task_id=context.task_id, - task_json=task.model_dump(mode="json"), - is_dynamic=True, -) -``` - -In the current schema this maps to `id=new_task_id` and -`parent_node_id=context.node_id`; PR 11 renames columns. - -## Task 1: Add Spawned Handle - -**Files:** - -- Modify: `ergon_core/ergon_core/api/worker/results.py` - -- [ ] **Step 1: Add model** - -```python -class SpawnedTaskHandle(BaseModel): - model_config = {"frozen": True} - - task_id: UUID - - async def wait(self) -> None: - raise NotImplementedError("await_completion is deferred in v2") -``` - -Export it from `api/worker/__init__.py` and `api/__init__.py`. - -## Task 2: Add WorkerContext Facade - -**Files:** - -- Modify: `ergon_core/ergon_core/api/worker/context.py` - -- [ ] **Step 1: Add private services and constructor** - -```python -class WorkerContext(BaseModel): - run_id: UUID - task_id: UUID - execution_id: UUID - definition_id: UUID - node_id: UUID | None = None - _task_mgmt: TaskManagementService = PrivateAttr() - _task_inspect: TaskInspectionService = PrivateAttr() - - @classmethod - def _for_job(cls, *, run_id, task_id, execution_id, definition_id, node_id, task_mgmt, task_inspect, resource_repo): - instance = cls(run_id=run_id, task_id=task_id, execution_id=execution_id, definition_id=definition_id, node_id=node_id) - object.__setattr__(instance, "_task_mgmt", task_mgmt) - object.__setattr__(instance, "_task_inspect", task_inspect) - object.__setattr__(instance, "_resource_repo", resource_repo) - return instance -``` - -- [ ] **Step 2: Add containment exception** - -In `ergon_core/ergon_core/api/errors.py` add: - -```python -class ContainmentViolation(RuntimeError): - """Raised when a worker tries to act on a task it does not own.""" - - def __init__(self, *, parent_task_id: UUID, target_task_id: UUID) -> None: - super().__init__( - f"Task {target_task_id} is not a descendant of {parent_task_id}; " - "WorkerContext can only mutate tasks it spawned or their descendants." - ) - self.parent_task_id = parent_task_id - self.target_task_id = target_task_id -``` - -Export it from `ergon_core.api`. - -- [ ] **Step 3: Add facade methods** - -In `WorkerContext`: - -```python -async def spawn_task( - self, - task: Task, - *, - depends_on: tuple[UUID, ...] = (), -) -> SpawnedTaskHandle: - return await self._task_mgmt.add_subtask( - run_id=self.run_id, - parent_task_id=self.task_id, - task=task, - depends_on=depends_on, - ) - -async def cancel_task(self, task_id: UUID, *, reason: str = "") -> None: - await self._assert_descendant(task_id) - await self._task_mgmt.cancel_task( - run_id=self.run_id, - task_id=task_id, - reason=reason, - ) - -async def refine_task(self, task_id: UUID, *, description: str) -> None: - await self._assert_descendant(task_id) - await self._task_mgmt.refine_task( - run_id=self.run_id, - task_id=task_id, - description=description, - ) - -async def restart_task(self, task_id: UUID) -> SpawnedTaskHandle: - await self._assert_descendant(task_id) - return await self._task_mgmt.restart_task( - run_id=self.run_id, - task_id=task_id, - ) - -async def subtasks(self) -> tuple[Task, ...]: - return await self._task_inspect.children( - run_id=self.run_id, - parent_task_id=self.task_id, - ) - -async def descendants(self) -> tuple[Task, ...]: - return await self._task_inspect.descendants( - run_id=self.run_id, - root_task_id=self.task_id, - ) - -async def get_task(self, task_id: UUID) -> Task: - await self._assert_descendant(task_id) - return await self._task_inspect.get( - run_id=self.run_id, - task_id=task_id, - ) - -async def _assert_descendant(self, task_id: UUID) -> None: - """Raise ContainmentViolation if task_id is not self.task_id or a descendant.""" - - if task_id == self.task_id: - return - descendant_ids = await self._task_inspect.descendant_ids( - run_id=self.run_id, - root_task_id=self.task_id, - ) - if task_id not in descendant_ids: - raise ContainmentViolation( - parent_task_id=self.task_id, - target_task_id=task_id, - ) -``` - -The implementation deliberately re-reads descendants on every call rather -than caching them — `TaskInspectionService.descendant_ids` already memoizes -within a single Inngest step and a worker that spawns mid-step needs the -fresh set. `_assert_descendant` lives on `WorkerContext`, not on the service, -because containment is a facade-level rule. - -## Task 2b: Add `descendant_ids` To Inspection Service - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/tasks/inspection.py` - -- [ ] **Step 1: Add lookup** - -```python -async def descendant_ids( - self, - *, - run_id: UUID, - root_task_id: UUID, -) -> frozenset[UUID]: - """Return all task_ids reachable as children/grandchildren of root_task_id.""" - - async with self._session() as session: - rows = await self._graph_repo.descendants_by_parent( - session, - run_id=run_id, - root_task_id=root_task_id, - ) - return frozenset(row.task_id for row in rows) -``` - -Add the supporting `WorkflowGraphRepository.descendants_by_parent`. The -CTE is recursive on `parent_node_id` during the transition and renames to -`parent_task_id` after PR 11. Both SQLite and Postgres accept the -`WITH RECURSIVE` form below. - -```python -from collections.abc import Sequence -from uuid import UUID - -from sqlalchemy import text - -from ergon_core.core.persistence.graph.models import RunGraphNode - - -class WorkflowGraphRepository: - ... - - async def descendants_by_parent( - self, - session, - *, - run_id: UUID, - root_task_id: UUID, - ) -> Sequence[RunGraphNode]: - """Return all RunGraphNode rows transitively reachable from - root_task_id via parent_node_id, NOT including the root itself. - """ - - # During the transition the canonical identity column is `id` - # (with `parent_node_id` pointing at parent.id). PR 11 renames to - # `task_id` / `parent_task_id`; update the column references in - # the same commit that does the rename. - cte_sql = text( - """ - WITH RECURSIVE descendants AS ( - SELECT id, parent_node_id, run_id - FROM run_graph_nodes - WHERE run_id = :run_id - AND parent_node_id = :root_task_id - - UNION ALL - - SELECT child.id, child.parent_node_id, child.run_id - FROM run_graph_nodes AS child - JOIN descendants ON child.parent_node_id = descendants.id - WHERE child.run_id = :run_id - ) - SELECT id FROM descendants - """ - ) - result = session.exec( - cte_sql.bindparams(run_id=str(run_id), root_task_id=str(root_task_id)) - ).all() - descendant_ids = [row[0] for row in result] - if not descendant_ids: - return () - - return session.exec( - select(RunGraphNode).where(RunGraphNode.id.in_(descendant_ids)) - ).all() -``` - -SQLite needs the foreign-key parameters as strings (`str(run_id)`) because -the `sqlite3` driver does not auto-cast `UUID`. Postgres accepts either. - -The recursion bound is implicit (the run graph is a DAG built only by -`spawn_task`; cycles are prevented at insertion time by -`add_node`'s parent-resolution check). If a stress test later needs an -explicit depth cap, add `WHERE level < :max_depth` to the recursive arm. - -## Task 3: Management Service Inserts Dynamic Task JSON - -**Files:** - -- Modify: `ergon_core/ergon_core/core/application/tasks/management.py` - -- [ ] **Step 1: Add object-bound overload** - -```python -async def add_subtask( - self, - *, - run_id: UUID, - parent_task_id: UUID, - task: Task, - depends_on: tuple[UUID, ...] = (), -) -> SpawnedTaskHandle: - new_task_id = uuid4() - async with self._session() as session: - parent = self._graph_repo.node(session, run_id=run_id, task_id=parent_task_id) - node = await self._graph_repo.add_node( - session, - run_id, - task_slug=task.task_slug, - instance_key=task.instance_key, - description=task.description, - status=graph_status.PENDING, - parent_node_id=parent.node_id, - level=parent.level + 1, - task_json=task.model_dump(mode="json"), - is_dynamic=True, - meta=MutationMeta(actor="worker-context", reason="spawn_task"), - ) - for dep in depends_on: - dep_node = self._graph_repo.node(session, run_id=run_id, task_id=dep) - await self._graph_repo.add_edge( - session, - run_id, - source_node_id=dep_node.node_id, - target_node_id=node.id, - status=graph_status.PENDING, - meta=MutationMeta(actor="worker-context", reason="spawn dependency"), - ) - session.commit() - return SpawnedTaskHandle(task_id=new_task_id) -``` - -Adjust field names to the actual DTO returned by `add_node`; the invariant -is that no definition table is written. - -## Task 4: Tests - -**Files:** - -- Create: `ergon_core/tests/unit/runtime/test_worker_context_containment.py` -- Modify: `ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py` - -- [ ] **Step 1: Dynamic spawn writes no definition row** - -```python -from sqlmodel import select, func - -from ergon_core.core.persistence.definitions.models import ExperimentDefinitionTask -from ergon_core.core.persistence.graph.models import RunGraphNode -from ergon_core.api.benchmark.task import Task -from tests.unit.runtime._test_workers import EchoWorker, EchoSandbox - - -@pytest.mark.asyncio -async def test_spawn_task_does_not_write_definition_task_row( - session, - worker_context_factory, - run_graph_factory, -): - graph = run_graph_factory(nodes=[("root", None)]) - context = worker_context_factory( - run_id=graph.run_id, - task_id=graph.task_id("root"), - ) - - before = session.exec( - select(func.count()).select_from(ExperimentDefinitionTask) - ).one() - - handle = await context.spawn_task( - Task( - task_slug="child", - instance_key="sample-1", - description="spawned child", - worker=EchoWorker(name="echo", model=None), - sandbox=EchoSandbox(), - evaluators=(), - ) - ) - - after = session.exec( - select(func.count()).select_from(ExperimentDefinitionTask) - ).one() - assert after == before, "dynamic spawn must not create a definition row" - - # Spawned node exists in run_graph_nodes with is_dynamic=true. - row = session.exec( - select(RunGraphNode).where(RunGraphNode.id == handle.task_id) - ).one() - assert row.is_dynamic is True - assert row.task_json["task_slug"] == "child" -``` - -- [ ] **Step 2: Same runtime path loads spawned tasks** - -```python -@pytest.mark.asyncio -async def test_spawned_task_inflates_through_graph_repo_node( - session, - worker_context_factory, - run_graph_factory, - graph_repo, -): - graph = run_graph_factory(nodes=[("root", None)]) - context = worker_context_factory( - run_id=graph.run_id, - task_id=graph.task_id("root"), - ) - - handle = await context.spawn_task( - Task( - task_slug="child", - instance_key="sample-1", - description="spawned child", - worker=EchoWorker(name="echo", model=None), - sandbox=EchoSandbox(), - evaluators=(), - ) - ) - - view = graph_repo.node( - session, - run_id=graph.run_id, - task_id=handle.task_id, - ) - - assert view.task.task_slug == "child" - assert view.task.task_id == handle.task_id - assert view.is_dynamic is True - # Object-bound reconstruction works for spawned snapshots too: - assert isinstance(view.task.worker, EchoWorker) - assert isinstance(view.task.sandbox, EchoSandbox) -``` - -- [ ] **Step 3: Containment** - -```python -@pytest.mark.asyncio -async def test_worker_context_cancel_raises_on_non_descendant( - worker_context_factory, - run_graph_factory, -): - graph = run_graph_factory( - nodes=[ - ("root", None), - ("sibling", None), # peer of root, NOT a child - ("child", "root"), - ] - ) - context = worker_context_factory( - run_id=graph.run_id, - task_id=graph.task_id("root"), - ) - # Spawned descendants are fine: - await context.cancel_task(graph.task_id("child")) - - # A sibling task is outside the containment boundary: - with pytest.raises(ContainmentViolation) as excinfo: - await context.cancel_task(graph.task_id("sibling")) - - assert excinfo.value.parent_task_id == graph.task_id("root") - assert excinfo.value.target_task_id == graph.task_id("sibling") -``` - -The `run_graph_factory` fixture is the same one used by the dynamic-spawn -test in Step 2; if it does not yet support a `nodes=[(slug, parent_slug)]` -form, extend it in the same commit. - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py \ - ergon_core/tests/unit/runtime/test_worker_context_containment.py -q -``` - -## Task 5: Flip XFails Landed By This PR - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` -- Modify: `ergon_core/tests/unit/architecture/test_dead_path_audit.py` -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` -- Modify: `ergon_core/tests/unit/runtime/test_identity_invariants.py` - -PR 9 closes Δ.3 — dynamic subtasks are graph-native. Four ledger entries -flip: - -- [ ] **Step 1: Remove `materialize_dynamic_subtask_definition_is_gone` from `_XFAIL_BY_NAME`** - -In `test_v2_final_state_ledger.py`, delete: - -```python -"materialize_dynamic_subtask_definition_is_gone": "PR 9 makes dynamic subtasks graph-native", -``` - -- [ ] **Step 2: Remove the dead-path entry** - -In `test_dead_path_audit.py`, delete: - -```python -"materialize_dynamic_subtask_definition": "PR 9: graph-native dynamic spawn", -``` - -- [ ] **Step 3: Flip the smoketest and identity cases** - -In `test_walkthrough_smoketest.py`, remove the xfail decorator on -`test_dynamic_spawn_writes_only_to_run_graph_nodes` and implement the -real body: drive `WorkerContext.spawn_task` from a parent task context, -then assert the spawned `task_id` has zero rows in -`experiment_definition_tasks` and exactly one row in `run_graph_nodes` -with `is_dynamic=True`. - -In `test_identity_invariants.py`, remove the xfail on -`test_dynamic_task_id_has_no_definition_row` and implement the real body. - -- [ ] **Step 4: Run the ledgers** - -```bash -uv run pytest \ - ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py \ - ergon_core/tests/unit/architecture/test_dead_path_audit.py \ - ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py \ - ergon_core/tests/unit/runtime/test_identity_invariants.py -q -``` - -Expected: four more cases PASS; PR 11 entries still XFAIL. - -## PR Ledger - -Invariant landed: dynamic tasks are graph-native. - -Bridge code introduced: `node_id` remains in WorkerContext during schema -transition. - -Old path still intentionally alive: runtime identity fields until PR 11. - -Deletion gate: PR 11 removes `node_id` from context and graph DTOs. - -Tests added or updated: dynamic spawn and containment tests. - -Modules owned by this PR: WorkerContext and task management. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11-pr-10a-swebench.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11-pr-10a-swebench.md deleted file mode 100644 index e7d63ed20..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11-pr-10a-swebench.md +++ /dev/null @@ -1,586 +0,0 @@ -# PR 10a — SWEBench Verified Vertical - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** SWEBench tasks construct object-bound `Task` with -`SWEBenchSandbox` and a serializable solver worker. - -**Architecture:** Repeat the MiniF2F vertical pattern (PR 6) for -SWEBench. Land as a standalone PR so the conversion fits the 2.5k -non-generated-changed-lines budget and reviewers can focus on one -benchmark at a time. - -**Tech Stack:** Builtin benchmark modules, sandbox adapters, Pydantic -worker config, pytest. - -This is the **first of three independent sub-PRs** that complete the -builtins migration. PR 10b (ResearchRubrics) and PR 10c (GDPEval) follow -the same template. Each can ship independently — there are no -sequential dependencies between the three, only the shared post-merge -cleanup that PR 10c executes. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. Most are confirmations that the existing tasks scope work correctly; -one is a real prerequisite call-out. - -1. **Confirm shared adapter extraction is in scope.** Task 1 calls for moving - the `_ManagerBackedSandboxRuntime` adapter (currently inlined in - `ergon_builtins/benchmarks/minif2f/sandbox.py`) into a shared module at - `ergon_builtins/sandbox/_manager_backed.py`. The audit confirmed the - shared module does NOT exist yet, so the extraction is real PR 10a work - (not assumed-already-done). PRs 10b and 10c import from the shared module - on the assumption that PR 10a lands first. -2. **Sandbox subdirectory rename.** Before creating - `ergon_builtins/benchmarks/swebench_verified/sandbox.py`, the existing - `swebench_verified/sandbox/` directory must be renamed (e.g., - `git mv swebench_verified/sandbox swebench_verified/sandbox_template`) to - avoid module shadowing. Update all `sandbox.utils` imports to - `sandbox_template.utils`. Call this out as a prerequisite step in Task 1. -3. **Worker factory pattern.** Plan renames - `swebench_verified/worker_factory.py` → `workers.py` and converts the - `SWEBenchReactWorker` class to a `make_swebench_worker()` factory - function. Audit confirmed current code is class-based — this is real - PR 10a work. -4. **Toolkit Pydantic conversion.** Plan converts `SWEBenchToolkit` from a - regular class (`__init__(sandbox, workdir)`, `get_tools()`) into a - Pydantic `BaseModel` with a `tools(sandbox, task)` method. Audit - confirmed current code is regular-class — this is real PR 10a work. - ---- - -## Common Conversion Recipe - -Every vertical sub-PR (10a, 10b, 10c) follows the same template. PR 6.5 fixed the file layout + killed the `Experiment` class, so this recipe assumes the post-PR-6.5 world: - -1. Create `Sandbox` subclass at **`ergon_builtins/ergon_builtins/benchmarks/<slug>/sandbox.py`** (per-benchmark, NOT under `sandboxes/` — PR 6.5 deleted that top-level dir). -2. Move sandbox provisioning behavior into the subclass's `provision()` body, attaching `_runtime` via `ManagerBackedSandboxRuntime` from `ergon_builtins/sandbox/_manager_backed.py` (singular, top-level — created by PR 10a; reused by PR 10b/10c). -3. Add `_bind_runtime(sandbox_id)` so eval workers can attach to an already-running sandbox. -4. Move reusable toolkit construction into a serializable Pydantic object at **`ergon_builtins/ergon_builtins/benchmarks/<slug>/toolkit.py`** (per-benchmark, NOT under `toolkits/`). -5. Add **`benchmarks/<slug>/workers.py`** (renamed from `worker_factory.py`) with factory functions like `make_<slug>_worker()` returning a concrete `Worker` instance. Parameterise the benchmark constructor (`worker_factory=...`, `sandbox_factory=...`) following the PR 6.5 MiniF2F pattern. -6. Convert `benchmark.py` to return `Task` (not `TaskSpec`). -7. Add a unit test that persists the benchmark and asserts the stored task JSON has `_type` entries for `worker`, `sandbox`, and every `evaluators[i]`. -8. Add one line to **`ergon_builtins/ergon_builtins/benchmarks/README.md`** (the catalogue PR 6.5 added) listing this benchmark and its worker factories. -9. Leave the old `sandbox_manager.py` and registry registrations in place; PR 11 deletes them. - -**No CLI factory registration step.** PR 6.5 deleted `BUILTIN_EXPERIMENT_FACTORIES` and the entire CLI authoring route. Authoring is Python-only — users import the benchmark class directly from `ergon_builtins.benchmarks.<slug>` and call `persist_benchmark(...)`. The CLI observes via `ergon experiment show` / `ergon run status` (added in PR 8). - -The `ManagerBackedSandboxRuntime` adapter is shared infrastructure — PR 10a creates it at `ergon_builtins/sandbox/_manager_backed.py`; PR 10b and PR 10c import from there. - -## Files - -**Create:** - -```text -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox.py # was sandboxes/swebench.py -ergon_builtins/ergon_builtins/sandbox/_manager_backed.py # singular top-level; was sandboxes/_manager_backed.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py # was toolkits/swebench.py -ergon_builtins/tests/unit/test_swebench_v2_definition.py -``` - -**Rename:** - -```text -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/worker_factory.py - → ergon_builtins/ergon_builtins/benchmarks/swebench_verified/workers.py -``` - -**Modify:** - -```text -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/rubric.py -ergon_builtins/ergon_builtins/benchmarks/README.md # add SWEBench row -ergon_core/tests/unit/runtime/test_definition_writer.py -tests/fixtures/smoke_components/benchmarks.py # migrate SweBenchSmokeBenchmark in lockstep -``` - -**Note: no `ergon_cli/ergon_cli/commands/_registry.py` edit.** PR 6.5 deleted `BUILTIN_EXPERIMENT_FACTORIES`; there is no CLI registry to add an entry to. The benchmark is discoverable via the `benchmarks/README.md` catalogue and importable from Python. - -## Task 1: Add `SWEBenchSandbox` - -- [ ] **Step 1: Extract the shared `ManagerBackedSandboxRuntime`** - -Create `ergon_builtins/ergon_builtins/sandbox/_manager_backed.py` (PR 6.5 created the empty `sandbox/` package dir specifically for this file) so PR 10b and PR 10c can reuse it: - -```python -"""Adapter that lets a Sandbox subclass delegate to a legacy -*SandboxManager. - -PR 10a creates this; PR 10b and PR 10c import it. PR 11 deletes the -managers it wraps (after the SandboxRuntime protocol becomes the only -contract). -""" - -from typing import Any - - -class ManagerBackedSandboxRuntime: - """Adapter that lets a sandbox subclass delegate to a legacy manager.""" - - def __init__(self, *, manager: Any, sandbox: Any) -> None: - self._manager = manager - self._sandbox = sandbox - self.sandbox_id: str = sandbox.sandbox_id - - async def run_command(self, cmd, *, timeout=None): - return await self._manager.run_command( - self._sandbox.task_id, cmd, timeout=timeout - ) - - async def write_file(self, path: str, content: bytes) -> None: - await self._manager.upload_file(self._sandbox.task_id, path, content) - - async def read_file(self, path: str) -> bytes: - return await self._manager.read_file(self._sandbox.task_id, path) - - async def list_files(self, path: str) -> list[str]: - return await self._manager.list_files(self._sandbox.task_id, path) - - async def close(self) -> None: - # Terminate the external sandbox AND drop local resources. - await self._manager.terminate(self._sandbox.task_id) - - async def close_local(self) -> None: - # Drop local handles only; leave the external sandbox alive so - # the orchestrator's release can be the sole terminator. - await self._manager.close_local(self._sandbox.task_id) -``` - -If a manager's API differs (e.g. `delete` instead of `terminate`, no -`close_local`), fix it at the manager layer rather than diverging the -adapter — every `*SandboxManager` ends up with the same five -operations. - -PR 6.5's `LeanSandbox` at `ergon_builtins/benchmarks/minif2f/sandbox.py` (which inlined this adapter) must be updated in this PR to import from the shared location: - -```python -# Before (PR 6.5): the adapter was inlined inside benchmarks/minif2f/sandbox.py -# After (PR 10a): import from the shared module -from ergon_builtins.sandbox._manager_backed import ManagerBackedSandboxRuntime -``` - -The edit is mechanical; include it as part of Step 1. PR 6.5's `# TODO(PR 10a):` markers on the inlined adapter point at this exact change. - -- [ ] **Step 2: Resolve the `sandbox/` directory name conflict** - -SWEBench currently has both `benchmarks/swebench_verified/sandbox_manager.py` (the manager) AND a `benchmarks/swebench_verified/sandbox/` *directory* containing `Dockerfile`, `e2b.toml.template`, and `utils.py`. Creating a new `sandbox.py` file at the same level would shadow that directory in Python's module resolution. - -Rename the directory first: - -```bash -git mv ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_template -``` - -Update any `from ergon_builtins.benchmarks.swebench_verified.sandbox.utils import ...` imports to `sandbox_template.utils`. - -- [ ] **Step 3: Create the SWEBench subclass** - -`ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox.py`: - -```python -from uuid import uuid4 - -from ergon_core.api.sandbox import Sandbox -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) -from ergon_builtins.sandbox._manager_backed import ( - ManagerBackedSandboxRuntime, -) - - -class SWEBenchSandbox(Sandbox): - """E2B-backed sandbox for SWEBench verified instances.""" - - image_tag: str = "ergon-swebench-v1" - repo_url: str | None = None - base_commit: str | None = None - requires_network: bool = True - - async def provision(self) -> None: - manager = SWEBenchSandboxManager( - image_tag=self.image_tag, - repo_url=self.repo_url, - base_commit=self.base_commit, - ) - sandbox = await manager.create(task_id=uuid4(), envs=self.env) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) - - async def _bind_runtime(self, sandbox_id: str) -> None: - # Eval-side attach: reconnect to an already-running sandbox. - manager = SWEBenchSandboxManager( - image_tag=self.image_tag, - repo_url=self.repo_url, - base_commit=self.base_commit, - ) - sandbox = await manager.connect(sandbox_id=sandbox_id) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) -``` - -If `SWEBenchSandboxManager` does not expose `connect(sandbox_id=...)`, -add the method at the manager layer — every benchmark's manager must -support reconnect-by-id for the synchronous-fanout eval path to work. - -## Task 2: Move Toolkit Construction - -- [ ] **Step 1: Move toolkit into the per-benchmark subpackage** - -If a `toolkit.py` already exists under `swebench_verified/`, that's the target path — no move needed. If toolkit logic lives elsewhere, move it: - -```bash -# Example if toolkit logic is currently in a sibling location: -git mv <current_path>/toolkit.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py -``` - -**Do NOT create `ergon_builtins/toolkits/`** — PR 6.5 explicitly deleted that top-level dir as a misleading cross-cutting namespace. Per-benchmark toolkits live alongside their benchmark. - -- [ ] **Step 2: Convert toolkit to Pydantic BaseModel** - -`SWEBenchToolkit` must be a `BaseModel` so `Worker.toolkit` round-trips through `_type` discrimination. The toolkit holds **config**, not runtime handles: - -```python -from pydantic import BaseModel, ConfigDict - - -class SWEBenchToolkit(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - repo_root: str = "/workspace/repo" - patch_output_path: str = "/workspace/final_output/patch.diff" - max_tool_calls: int = 32 - - def tools(self, sandbox, task): - # Lazy import keeps runtime tool construction out of the - # serialization path. PR 6.5's MiniF2FToolkit followed the - # same pattern; reference its `# reason:` comment for the - # circular-import rationale. - from ergon_builtins.benchmarks.swebench_verified._tools import build_tools - - return build_tools(self, sandbox=sandbox, task=task) -``` - -Move runtime tool construction into a sibling `_tools.py` module (under `benchmarks/swebench_verified/`); the toolkit serializes; the tools do not. - -- [ ] **Step 3: Update all importers** - -```bash -rg "from ergon_builtins.toolkits\b" ergon_builtins ergon_core # should be zero hits — PR 6.5 deleted the dir -rg "from ergon_builtins.benchmarks.swebench_verified.toolkit import" \ - ergon_builtins ergon_core -``` - -The first command exists to fail loudly if anyone still references the deleted top-level `toolkits/`. The second is what should resolve cleanly. - -## Task 3: Convert Worker Factory - -- [ ] **Step 1: Replace registry-driven factory with direct constructor** - -In `ergon_builtins/benchmarks/swebench_verified/workers.py` (renamed from `worker_factory.py`): - -```python -from ergon_builtins.benchmarks.swebench_verified.toolkit import SWEBenchToolkit -from ergon_builtins.workers.baselines.react_worker import ReActWorker - - -def make_swebench_worker( - *, - model: str = "openai:gpt-4o-mini", - max_iterations: int = 24, -) -> ReActWorker: - return ReActWorker( - name="swebench-solver", - model=model, - system_prompt=_SYSTEM_PROMPT, - max_iterations=max_iterations, - toolkit=SWEBenchToolkit(), - ) -``` - -`ReActWorker.toolkit` field gains `SWEBenchToolkit` in its union (PR 6.5's `# TODO(PR 10a/10b/10c):` comment on the field flagged this). Update the union in the same commit. PR 11 collapses the union into a `Toolkit` protocol once 3+ toolkits exist. - -**Also: parameterise `SWEBenchVerifiedBenchmark.__init__`** to accept `worker_factory` / `sandbox_factory` / `evaluator_factory` kwargs with defaults — mirror the PR 6.5 MiniF2F pattern. This is what makes the worker swap point real for downstream `experiment.py` / Python authoring users. - -## Task 4: Convert Benchmark Tasks - -- [ ] **Step 1: Replace imports** - -In `benchmark.py`: - -```python -# Delete: -from ergon_core.api.benchmark import Benchmark, BenchmarkRequirements, TaskSpec - -# Add: -from ergon_core.api import Benchmark, BenchmarkRequirements, Task -from ergon_builtins.benchmarks.swebench_verified.sandbox import SWEBenchSandbox -from ergon_builtins.benchmarks.swebench_verified.workers import ( - make_swebench_worker, -) -from ergon_builtins.benchmarks.swebench_verified.rubric import ( - make_swebench_rubric, -) -``` - -- [ ] **Step 2: Replace `TaskSpec` construction** - -Every `TaskSpec(...)` call inside `build_instances` becomes: - -```python -Task[SWEBenchTaskPayload]( - task_slug="swebench-instance", - instance_key=instance.instance_id, - description=instance.problem_statement, - task_payload=SWEBenchTaskPayload( - instance_id=instance.instance_id, - repo=instance.repo, - base_commit=instance.base_commit, - ), - worker=make_swebench_worker(), - sandbox=SWEBenchSandbox( - repo_url=f"https://github.com/{instance.repo}", - base_commit=instance.base_commit, - ), - evaluators=(make_swebench_rubric(),), -) -``` - -## Task 5: Add SWEBench To The Benchmarks Catalogue - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/README.md` (created in PR 6.5 Task 20) - -PR 6.5 killed `BUILTIN_EXPERIMENT_FACTORIES` and the entire CLI authoring route. There is **no CLI registry to update**. Discovery is documentation-only. - -- [ ] **Step 1: Add one row to the catalogue** - -Open `ergon_builtins/ergon_builtins/benchmarks/README.md` and add a row for SWEBench under the existing table: - -```markdown -| SWEBench Verified | `ergon_builtins.benchmarks.swebench_verified` | `make_swebench_worker` | `SWEBenchSandbox` | -``` - -- [ ] **Step 2: Document the Python authoring example for SWEBench** - -If the README has per-benchmark example snippets, add one for SWEBench mirroring the MiniF2F example PR 6.5 added: - -```python -from ergon_builtins.benchmarks.swebench_verified import ( - SWEBenchVerifiedBenchmark, - make_swebench_worker, -) -from ergon_core.api import persist_benchmark, launch_run - -benchmark = SWEBenchVerifiedBenchmark( - name="swebench-react", - metadata={"experiment": "swebench-eval-2026"}, - worker_factory=make_swebench_worker, - limit=10, -) -handle = persist_benchmark(benchmark) -await launch_run(handle.definition_id) -``` - -That's the entire "register the benchmark" workflow. Users discover it via the README; they author runs via Python. - -## Task 5.5: Migrate The Matching Smoke Fixture - -**Files:** - -- Modify: `tests/fixtures/smoke_components/benchmarks.py` — `SweBenchSmokeBenchmark` -- Modify (if needed): `tests/fixtures/smoke_components/criteria/smoke_rubrics.py` — `SweBenchSmokeRubric` - -PR 5's retirement of `_evaluator_bridge` + PR 6's object-bound migration -created an asymmetry: the **production** SWEBench benchmark migrates to -`Task` here, but the **smoke fixture** at -`tests/fixtures/smoke_components/benchmarks.py` still uses `TaskSpec`. -Both must move together or PR 11 cannot delete `TaskSpec`. - -- [ ] **Step 1: Add a concrete `SweBenchSmokeTask(Task[...])` subclass** - -Mirrors the named-subclass pattern from PR 6 minif2f. Avoids the -parameterized-generic ``Task[X]`` discriminator that `import_component` -cannot resolve via ``getattr(module, "Task[X]")``. - -- [ ] **Step 2: Override `build_instances` to return `Task`** - -```python -class SweBenchSmokeTask(Task[SWEBenchTaskPayload]): - ... - -class SweBenchSmokeBenchmark(_SingleTaskSmokeBenchmark): - ... - - def build_instances(self) -> Mapping[str, Sequence[Task[SWEBenchTaskPayload]]]: - payload = SWEBenchTaskPayload.model_validate(self.task_payload) - task = SweBenchSmokeTask( - task_slug=self.task_slug, - instance_key="default", - description=self.task_description, - evaluator_binding_keys=("default", "post-root"), - task_payload=payload, - evaluators=( - SweBenchSmokeRubric(name="default"), - SmokePostRootTimingRubric(name="post-root"), - ), - ) - return {"default": [task]} -``` - -- [ ] **Step 3: Migrate `SweBenchSmokeRubric` to pure Pydantic** - -The smoke rubric currently has a custom `__init__(self, *, name, metadata=None)` -that's incompatible with Pydantic's `model_validate` (used by -`Evaluator.from_definition`). Replace with the -`Field(default_factory=tuple, exclude=True)` + `@model_validator(mode="after")` -pattern (see `ergon_builtins/benchmarks/minif2f/rubric.py` for the -exemplar). PR 6 did this for `MiniF2FSmokeRubric`; this step does the -same for the swebench counterpart. - -## Task 6: Tests - -- [ ] **Step 1: Definition JSON test** - -Create `ergon_builtins/tests/unit/test_swebench_v2_definition.py`: - -```python -import pytest -from ergon_core.api import persist_benchmark -from ergon_builtins.benchmarks.swebench_verified.benchmark import ( - SWEBenchVerifiedBenchmark, -) - - -@pytest.mark.asyncio -async def test_swebench_persists_object_bound_task_json(session_factory): - benchmark = SWEBenchVerifiedBenchmark( - name="swebench-smoke", - metadata={"author": "test"}, - limit=1, - ) - - handle = persist_benchmark(benchmark) - with session_factory() as session: - rows = session.exec( - "SELECT task_json FROM experiment_definition_tasks " - "WHERE definition_id = :d", - {"d": handle.definition_id}, - ).all() - assert rows, "expected at least one persisted task" - task_json = rows[0][0] - assert task_json["worker"]["_type"].endswith(":ReActWorker") - assert task_json["sandbox"]["_type"].endswith(":SWEBenchSandbox") - assert task_json["evaluators"], "evaluators must persist" - assert all( - ev.get("_type") for ev in task_json["evaluators"] - ), "every evaluator entry must carry a `_type` discriminator" - assert "_legacy" not in task_json -``` - -- [ ] **Step 2: Reconstruction test** - -```python -from uuid import uuid4 - -import pytest -from ergon_core.api.benchmark.task import Task - - -@pytest.mark.asyncio -async def test_swebench_task_json_round_trips_through_from_definition(): - benchmark = SWEBenchVerifiedBenchmark(limit=1) - task = next(iter(benchmark.build_instances().values()))[0] - task_json = task.model_dump(mode="json") - - rebuilt = await Task.from_definition(task_json, task_id=uuid4()) - - assert rebuilt.worker is not None - assert rebuilt.sandbox is not None - assert isinstance(rebuilt.sandbox, type(task.sandbox)) -``` - -- [ ] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_builtins/tests/unit/test_swebench_v2_definition.py \ - ergon_core/tests/unit/runtime/test_definition_writer.py -q -``` - -Expected: pass; persisted JSON includes object-bound `_type`s for -worker, sandbox, and every evaluator entry. - -## Task 7: Commit - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox.py \ - ergon_builtins/ergon_builtins/sandbox/_manager_backed.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/_tools.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/workers.py \ - ergon_builtins/ergon_builtins/benchmarks/minif2f/sandbox.py \ - ergon_builtins/ergon_builtins/benchmarks/README.md \ - ergon_builtins/tests/unit/test_swebench_v2_definition.py -git commit -m "feat(builtins): convert SWEBench to object-bound Task (PR 10a)" -``` - -Note: also includes `benchmarks/minif2f/sandbox.py` because PR 10a updates the MiniF2F sandbox to import the now-shared `ManagerBackedSandboxRuntime` from `sandbox/_manager_backed.py`. - -## PR Ledger - -Invariant landed: SWEBench builtin constructs object-bound task graphs; -shared `ManagerBackedSandboxRuntime` adapter exists for PR 10b/10c. - -Bridge code introduced: `SWEBenchSandbox` wraps the legacy -`SWEBenchSandboxManager` via `ManagerBackedSandboxRuntime`; toolkit is -Pydantic-serializable. - -Bridge code retired (partially): -- SWEBench tasks now carry `task.worker`/`task.sandbox` inline, so - SWEBench runs no longer hit the `_legacy_worker_bridge` fallback that - PR 5 Task 4b put on `worker_execute`. -- SWEBench tasks also carry `task.evaluators` inline, so SWEBench runs - no longer hit the symmetric `_legacy_evaluator_bridge` fallback that - PR 5 (restored post-cleanup) put on `evaluate_task_run`. -- Both fallbacks stay alive — they still serve researchrubrics, gdpeval, - and the matching smoke fixtures until they migrate in PR 10b/10c. - PR 11 deletes both bridges once every benchmark is on object-bound - `Task`. - -Old path still intentionally alive: `swebench_verified/sandbox_manager.py`, -registry registrations in `ergon_builtins/registry*.py`, -`_legacy_worker_bridge.py`, and `_legacy_evaluator_bridge.py` (the last -two are still required by researchrubrics, gdpeval, and any unmigrated -smoke fixtures). - -Migrations: this PR adds **no Alembic migration** (the SWEBench changes -are code-only). The next free migration id is `aabbccdd0006` (PR 6.5 -took `aabbccdd0004` for `add_experiment_tag`; PR 7 took `aabbccdd0005` -for `definition_metadata_and_launch`); reserve it for PR 10b if/when -needed. - -Deletion gate: PR 11 deletes the manager file and registry registrations. -PR 10c's cross-cutting cleanup verifies migrated benchmarks no longer -import `ComponentRegistry`. - -Tests added or updated: SWEBench definition JSON + reconstruction. - -Modules owned by this PR: `swebench_verified/`, the shared sandbox -adapter, and the SWEBench toolkit. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11b-pr-10b-researchrubrics.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11b-pr-10b-researchrubrics.md deleted file mode 100644 index df72ff03c..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11b-pr-10b-researchrubrics.md +++ /dev/null @@ -1,453 +0,0 @@ -# PR 10b — ResearchRubrics Vertical - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** ResearchRubrics tasks construct object-bound `Task` with -`ResearchE2BSandbox` and a serializable researcher worker. - -**Architecture:** Repeat the MiniF2F + SWEBench vertical pattern for -ResearchRubrics. Imports the shared `ManagerBackedSandboxRuntime` -adapter PR 10a created. Adds one wrinkle: the rubric carries a -`JudgeCriterion` whose `judge_model` field must persist, so the -criterion itself becomes a Pydantic subclass. - -**Tech Stack:** Builtin benchmark modules, sandbox adapters, Pydantic -worker config, pytest. - -PR 10a and PR 10b have no sequential dependency; either can land first. -PR 10c's cleanup is the only place ordering matters. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. The first one is **architectural and must land before Task 4** — -either as a Task 0 inside this PR, or as a separate "Criterion micro-PR" -that lands before PR 10b. The second is a real plan addition. - -1. **Convert `Criterion` base class from pure ABC to `BaseModel + ABC`.** - The RFC (`01-api-surface.md` lines 1030–1082) specifies - `class Criterion(BaseModel, ABC)` with `type_slug: ClassVar[str]`, - `required_packages: ClassVar[list[str]] = []`, and a `from_definition` - classmethod. Current code (`ergon_core/api/criterion/criterion.py`) has - pure `class Criterion(ABC)`. This conversion touches **every existing - `Criterion` subclass** across `ergon_core` and `ergon_builtins` — - subclasses currently pass identity fields via `__init__` and will need - to be converted to Pydantic field declarations. Task 4's - `JudgeCriterion(Criterion)` Pydantic conversion ONLY works once the base - class is `BaseModel`. Treat this as a prerequisite — add a **Task 0: - Criterion base class** at the start of this plan that does the base - conversion + subclass sweep, or extract it into a "PR 10b-prep" micro-PR - that lands first. - -2. **Add factory kwargs to `ResearchRubricsBenchmark.__init__`.** Current - constructor takes only `limit`/`name`/`description`/`metadata`. Plan - requires `worker_factory`, `sandbox_factory`, and `evaluator_factory` - kwargs (mirroring the MiniF2F pattern). Add these as part of Task 5 - (Convert Benchmark Tasks). - ---- - -## Common Conversion Recipe - -See [`11-pr-10a-swebench.md`](11-pr-10a-swebench.md) § Common -Conversion Recipe. The 8-step template is identical for every -vertical; this PR adds one extra step (re-home judge criterion). - -## Files - -**Create:** - -```text -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/sandbox.py # was sandboxes/research_e2b.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/toolkit.py # was toolkits/research_rubrics.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/_tools.py # runtime tool construction -ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py -``` - -**Rename:** - -```text -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/worker_factory.py - → ergon_builtins/ergon_builtins/benchmarks/researchrubrics/workers.py -``` - -**Modify:** - -```text -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/criteria.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py -ergon_builtins/ergon_builtins/benchmarks/README.md # add ResearchRubrics row -``` - -**Note: no `ergon_cli/_registry.py` edit.** PR 6.5 deleted `BUILTIN_EXPERIMENT_FACTORIES`. Discovery is via the README catalogue; authoring is Python-only. - -## Task 1: Add `ResearchE2BSandbox` - -- [ ] **Step 1: Create the subclass** - -`ergon_builtins/ergon_builtins/benchmarks/researchrubrics/sandbox.py`: - -```python -from uuid import uuid4 - -from ergon_core.api.sandbox import Sandbox -from ergon_builtins.benchmarks.researchrubrics.sandbox_manager import ( - ResearchRubricsSandboxManager, -) -from ergon_builtins.sandbox._manager_backed import ( - ManagerBackedSandboxRuntime, -) - - -class ResearchE2BSandbox(Sandbox): - template_id: str = "ergon-research-v1" - requires_network: bool = True - research_data_dir: str = "/workspace/research_data" - - async def provision(self) -> None: - manager = ResearchRubricsSandboxManager(template_id=self.template_id) - sandbox = await manager.create(task_id=uuid4(), envs=self.env) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) - - async def _bind_runtime(self, sandbox_id: str) -> None: - # Eval-side attach: reconnect to an already-running sandbox so - # evaluate_task_run can call task.sandbox.run_command(...) on the - # same external sandbox worker_execute provisioned. - manager = ResearchRubricsSandboxManager(template_id=self.template_id) - sandbox = await manager.connect(sandbox_id=sandbox_id) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) -``` - -If `ResearchRubricsSandboxManager` lacks `connect(sandbox_id=...)`, -add it at the manager layer — the synchronous-fanout eval path -requires reconnect-by-id on every builtin sandbox. - -The shared `ManagerBackedSandboxRuntime` is the one PR 10a landed at `ergon_builtins/sandbox/_manager_backed.py` (singular `sandbox/`, top-level — PR 6.5 created the empty package). If PR 10b lands first (rather than after 10a), create the adapter as Step 0 — see PR 10a's Task 1 Step 1 for the exact body. - -## Task 2: Move Toolkit Helpers - -- [ ] **Step 1: Move toolkit types** - -```bash -git mv ergon_builtins/ergon_builtins/benchmarks/researchrubrics/toolkit_types.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/toolkit.py -``` - -**Do NOT create `ergon_builtins/toolkits/`** — PR 6.5 deleted that top-level dir. Toolkit lives alongside its benchmark. - -- [ ] **Step 2: Convert to Pydantic** - -Replace any dataclasses / namedtuples with a Pydantic BaseModel: - -```python -from pydantic import BaseModel, ConfigDict - - -class ResearchRubricsToolkit(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - judge_model: str = "openai:gpt-4o" - max_search_calls: int = 12 - enable_web_browse: bool = True - - def tools(self, sandbox, task): - from ergon_builtins.benchmarks.researchrubrics._tools import build_tools - - return build_tools(self, sandbox=sandbox, task=task) -``` - -Move runtime tool construction into `benchmarks/researchrubrics/_tools.py`; the toolkit holds config only. - -- [ ] **Step 3: Update importers** - -```bash -rg "from ergon_builtins.benchmarks.researchrubrics.toolkit_types import" \ - ergon_builtins ergon_core -rg "from ergon_builtins.toolkits" \ # should be zero hits — PR 6.5 deleted the dir - ergon_builtins ergon_core -``` - -Replace with `from ergon_builtins.benchmarks.researchrubrics.toolkit import ResearchRubricsToolkit`. - -## Task 3: Convert Worker Factory - -- [ ] **Step 1: Replace registry call with constructor** - -In `researchrubrics/workers.py` (renamed from `worker_factory.py` — mirror PR 6.5's MiniF2F rename): - -```python -from ergon_builtins.benchmarks.researchrubrics.toolkit import ResearchRubricsToolkit -from ergon_builtins.workers.baselines.react_worker import ReActWorker - - -def make_research_worker( - *, - model: str = "openai:gpt-4o-mini", - max_iterations: int = 16, -) -> ReActWorker: - return ReActWorker( - name="research-runner", - model=model, - system_prompt=_RESEARCH_SYSTEM_PROMPT, - max_iterations=max_iterations, - toolkit=ResearchRubricsToolkit(), - ) -``` - -**Also: parameterise `ResearchRubricsBenchmark.__init__`** to accept `worker_factory` / `sandbox_factory` kwargs with defaults — mirror the PR 6.5 MiniF2F pattern. This makes the worker swap point real for Python authoring. - -## Task 4: Re-Home Judge Criterion (Pydantic conversion) - -ResearchRubrics rubrics carry a `JudgeCriterion` that today uses the -registry to look up its judge model. After PR 10b, the rubric must -persist alongside the worker config — the judge model field flows -through `_type`-discriminated JSON. - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py` - -- [ ] **Step 1: Convert `JudgeCriterion` to a Pydantic `Criterion` subclass** - -```python -from ergon_core.api.benchmark.task import Task -from ergon_core.api.criterion.criterion import Criterion -from ergon_core.api.rubric.evaluator import CriterionResult -from ergon_core.api.worker.results import WorkerOutput - - -class JudgeCriterion(Criterion): - """LLM-judge criterion that scores worker_output against a rubric.""" - - judge_model: str = "openai:gpt-4o" - rubric_text: str - - async def evaluate( - self, - task: Task, - worker_output: WorkerOutput, - ) -> CriterionResult: - # Per 01-api-surface.md, Criterion.evaluate takes - # (task, worker_output) — the sandbox lives on task.sandbox. - ... -``` - -`judge_model` and `rubric_text` are persistent fields — they round-trip -through `_type` JSON. The body of `evaluate` calls the judge model with -`task.description`, `worker_output.final_text`, and the rubric text. - -## Task 5: Convert Benchmark Tasks - -- [ ] **Step 1: Replace imports** - -In `researchrubrics/benchmark.py`: - -```python -# Delete: -from ergon_core.api.benchmark import Benchmark, BenchmarkRequirements, TaskSpec - -# Add: -from ergon_core.api import Benchmark, BenchmarkRequirements, Task -from ergon_builtins.benchmarks.researchrubrics.sandbox import ResearchE2BSandbox -from ergon_builtins.benchmarks.researchrubrics.workers import ( - make_research_worker, -) -from ergon_builtins.benchmarks.researchrubrics.rubric import ( - make_research_rubric_for_instance, -) -``` - -- [ ] **Step 2: Replace `TaskSpec` construction** - -Per-instance task becomes: - -```python -Task[ResearchRubricsTaskPayload]( - task_slug="research-rubric", - instance_key=instance.instance_id, - description=instance.prompt, - task_payload=ResearchRubricsTaskPayload( - instance_id=instance.instance_id, - question=instance.prompt, - ground_truth_rubric=instance.rubric, - ), - worker=make_research_worker(), - sandbox=ResearchE2BSandbox(), - evaluators=(make_research_rubric_for_instance(instance),), -) -``` - -## Task 6: Add ResearchRubrics To The Benchmarks Catalogue - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/README.md` (created in PR 6.5 Task 20) - -PR 6.5 killed `BUILTIN_EXPERIMENT_FACTORIES` and the entire CLI authoring route. No CLI registry entry needed. - -- [ ] **Step 1: Add one row to the catalogue** - -```markdown -| ResearchRubrics | `ergon_builtins.benchmarks.researchrubrics` | `make_research_worker` | `ResearchE2BSandbox` | -``` - -- [ ] **Step 2: Add a Python authoring example** - -```python -from ergon_builtins.benchmarks.researchrubrics import ( - ResearchRubricsBenchmark, - make_research_worker, -) -from ergon_core.api import persist_benchmark, launch_run - -benchmark = ResearchRubricsBenchmark( - name="research-react", - metadata={"experiment": "rr-eval-2026"}, - worker_factory=make_research_worker, - limit=10, -) -handle = persist_benchmark(benchmark) -await launch_run(handle.definition_id) -``` - -## Task 6.5: Migrate The Matching Smoke Fixture - -**Files:** - -- Modify: `tests/fixtures/smoke_components/benchmarks.py` — `ResearchRubricsSmokeBenchmark` -- Modify: `tests/fixtures/smoke_components/criteria/smoke_rubrics.py` — `ResearchRubricsSmokeRubric` - -Symmetric with PR 10a's smoke-fixture migration. The production -ResearchRubrics benchmark migrates to `Task` here, but the smoke -fixture at `tests/fixtures/smoke_components/benchmarks.py` still uses -`TaskSpec`. Migrate it in lockstep so `_legacy_evaluator_bridge` and -`_legacy_worker_bridge` get one step closer to deletable. - -- [ ] **Step 1: Add `ResearchRubricsSmokeTask(Task[...])` concrete subclass** -- [ ] **Step 2: Override `build_instances` to return that Task with `evaluators=(...)`** (mirror PR 6 minif2f + PR 10a swebench) -- [ ] **Step 3: Migrate `ResearchRubricsSmokeRubric` to pure Pydantic** (drop custom `__init__`; use `model_validator(mode="after")` to build criteria) - -## Task 7: Tests - -- [ ] **Step 1: Definition JSON test** - -Create `ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py`: - -```python -import pytest -from ergon_core.api import persist_benchmark -from ergon_builtins.benchmarks.researchrubrics.benchmark import ( - ResearchRubricsBenchmark, -) - - -@pytest.mark.asyncio -async def test_research_rubrics_persists_object_bound_task_json(session_factory): - benchmark = ResearchRubricsBenchmark( - name="research-smoke", - metadata={"author": "test"}, - limit=1, - ) - - handle = persist_benchmark(benchmark) - with session_factory() as session: - rows = session.exec( - "SELECT task_json FROM experiment_definition_tasks " - "WHERE definition_id = :d", - {"d": handle.definition_id}, - ).all() - assert rows - task_json = rows[0][0] - assert task_json["sandbox"]["_type"].endswith(":ResearchE2BSandbox") - assert task_json["evaluators"], "evaluators must persist" - assert all( - ev.get("_type") for ev in task_json["evaluators"] - ), "every evaluator entry must carry a `_type` discriminator" - assert task_json["evaluators"][0]["_type"].endswith(":Rubric") -``` - -- [ ] **Step 2: Judge model round-trips** - -```python -def test_research_rubric_judge_model_is_persisted(): - benchmark = ResearchRubricsBenchmark(limit=1) - task = next(iter(benchmark.build_instances().values()))[0] - serialized = task.model_dump(mode="json") - rubric_json = serialized["evaluators"][0] - judges = [ - c - for c in rubric_json["criteria"] - if c.get("_type", "").endswith(":JudgeCriterion") - ] - assert judges, "rubric must contain at least one JudgeCriterion" - assert judges[0]["judge_model"], "judge_model must round-trip in JSON" -``` - -- [ ] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py -q -``` - -## Task 8: Commit - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/researchrubrics/ \ - ergon_builtins/ergon_builtins/benchmarks/README.md \ - ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py -git commit -m "feat(builtins): convert ResearchRubrics to object-bound Task (PR 10b)" -``` - -## PR Ledger - -Invariant landed: ResearchRubrics builtin constructs object-bound task -graphs; `JudgeCriterion` is a Pydantic-persistent criterion subclass. - -Bridge code introduced: `ResearchE2BSandbox` wraps the legacy manager -via `ManagerBackedSandboxRuntime` (the shared adapter PR 10a created). - -Bridge code retired (partially): -- ResearchRubrics tasks now carry `task.worker`/`task.sandbox` inline, - so ResearchRubrics runs no longer hit the `_legacy_worker_bridge` - fallback on `worker_execute`. -- ResearchRubrics tasks also carry `task.evaluators` inline, so they no - longer hit the symmetric `_legacy_evaluator_bridge` fallback on - `evaluate_task_run`. (Particularly relevant here since ResearchRubrics - is judge-driven and lives on the eval side.) -- Both fallbacks stay alive — they still serve gdpeval and any - unmigrated smoke fixtures until PR 10c migrates them. PR 11 deletes - both bridges once every benchmark is on object-bound `Task`. - -Old path still intentionally alive: `researchrubrics/sandbox_manager.py`, -registry registrations, `_legacy_worker_bridge.py`, and -`_legacy_evaluator_bridge.py` (the last two still required by gdpeval). - -Migrations: this PR adds **no Alembic migration** (the change is -code-only). If a migration is needed, it claims `aabbccdd0006` (PR 6.5 -took `aabbccdd0004` for `add_experiment_tag`; PR 7 took `aabbccdd0005` -for `definition_metadata_and_launch`; PR 10a does not add one). - -Deletion gate: PR 11 deletes the manager file and registry registrations. -PR 10c's cross-cutting cleanup verifies migrated benchmarks no longer -import `ComponentRegistry`. - -Tests added or updated: ResearchRubrics definition JSON + judge model -round-trip. - -Modules owned by this PR: `researchrubrics/`, the ResearchRubrics -toolkit, and `JudgeCriterion`. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11c-pr-10c-gdpeval.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11c-pr-10c-gdpeval.md deleted file mode 100644 index fac36cecc..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/11c-pr-10c-gdpeval.md +++ /dev/null @@ -1,486 +0,0 @@ -# PR 10c — GDPEval Vertical + Builtins Cleanup - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps -> use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** GDPEval tasks construct object-bound `Task` with -`GDPEvalSandbox` and a serializable evaluator worker. Additionally, the -final vertical runs the cross-cutting cleanup: a registry-import guard -that verifies every migrated benchmark has stopped depending on -`ComponentRegistry`. - -**Architecture:** Repeat the MiniF2F + SWEBench + ResearchRubrics -vertical pattern for GDPEval. Add the post-migration cleanup that -PRs 6 / 10a / 10b together enable. - -**Tech Stack:** Builtin benchmark modules, sandbox adapters, Pydantic -worker config, pytest, textual architecture guards. - -PR 10c **must merge last** of the three sub-PRs — its cleanup -asserts every migrated benchmark (minif2f, swebench_verified, -researchrubrics, gdpeval) is `ComponentRegistry`-free, which is only -true once 10a, 10b, and 10c are all in. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. Each is a confirmation that an existing plan task scopes the work -correctly — no architectural surprises. - -1. **`make_gdpeval_worker()` factory pattern.** Plan converts - `GDPEvalReactWorker` class (in `worker_factory.py`) into a - `make_gdpeval_worker()` factory function (in `workers.py`), mirroring the - MiniF2F pattern. Audit confirmed current code is class-based — this is - real PR 10c work. -2. **Toolkit Pydantic conversion.** Plan converts `GDPEvalToolkit` from a - regular class (`__init__(sandbox_manager, task_id, run_id)`) into a - Pydantic `BaseModel`. Audit confirmed current code is regular-class with - non-serializable constructor params — this is real PR 10c work. -3. **`GDPEvalSandbox(Sandbox)` subclass creation.** Audit confirmed the - subclass does NOT exist today (only `GDPEvalSandboxManager` exists). - Task 1 already covers this. -4. **`GDPEvalBenchmark.__init__` factory kwargs.** Current constructor takes - only dataset/split/limit. Add `worker_factory`, `sandbox_factory`, - `evaluator_factory` kwargs as part of Task 4 (Convert Benchmark Tasks). -5. **Smoke fixture row addition.** Audit confirmed GDPEval is missing from - `tests/fixtures/smoke_components/benchmarks.py`. Ensure this PR adds a - `GDPEvalSmokeTask` entry using the object-bound `Task[...]` shape. - ---- - -## Common Conversion Recipe - -See [`11-pr-10a-swebench.md`](11-pr-10a-swebench.md) § Common -Conversion Recipe. - -## Files - -**Create:** - -```text -ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox.py # was sandboxes/gdpeval.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/toolkit.py # was toolkits/gdpeval.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/_tools.py # runtime tool construction -ergon_builtins/tests/unit/test_gdpeval_v2_definition.py -ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -``` - -**Rename:** - -```text -ergon_builtins/ergon_builtins/benchmarks/gdpeval/worker_factory.py - → ergon_builtins/ergon_builtins/benchmarks/gdpeval/workers.py -``` - -**Modify:** - -```text -ergon_builtins/ergon_builtins/benchmarks/gdpeval/benchmark.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/rubric.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria.py -ergon_builtins/ergon_builtins/registry.py -ergon_builtins/ergon_builtins/registry_core.py -ergon_builtins/ergon_builtins/registry_data.py -ergon_builtins/ergon_builtins/benchmarks/README.md # add GDPEval row -``` - -**Note: no `ergon_cli/_registry.py` edit.** PR 6.5 deleted `BUILTIN_EXPERIMENT_FACTORIES`. - -## Task 1: Add `GDPEvalSandbox` - -GDPEval today has `sandbox.py` (the manager) and `sandbox_utils.py`. The existing `sandbox.py` conflicts with the new `Sandbox` subclass file we want to create. Rename the manager first to match the convention used by MiniF2F / SWEBench / ResearchRubrics: - -- [ ] **Step 0: Rename existing `sandbox.py` → `sandbox_manager.py`** - -```bash -git mv ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox.py \ - ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox_manager.py -``` - -Update every import of `GDPEvalSandboxManager` to use the new path: - -```bash -rg "from ergon_builtins.benchmarks.gdpeval.sandbox import" ergon_builtins ergon_core -``` - -Replace with `from ergon_builtins.benchmarks.gdpeval.sandbox_manager import GDPEvalSandboxManager`. - -- [ ] **Step 1: Create subclass** - -`ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox.py` (now free after Step 0): - -```python -from uuid import uuid4 - -from ergon_core.api.sandbox import Sandbox -from ergon_builtins.benchmarks.gdpeval.sandbox_manager import ( - GDPEvalSandboxManager, -) -from ergon_builtins.sandbox._manager_backed import ( - ManagerBackedSandboxRuntime, -) - - -class GDPEvalSandbox(Sandbox): - template_id: str = "ergon-gdpeval-v1" - requires_network: bool = False - workspace_dir: str = "/workspace/gdpeval" - - async def provision(self) -> None: - manager = GDPEvalSandboxManager(template_id=self.template_id) - sandbox = await manager.create(task_id=uuid4(), envs=self.env) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) - - async def _bind_runtime(self, sandbox_id: str) -> None: - manager = GDPEvalSandboxManager(template_id=self.template_id) - sandbox = await manager.connect(sandbox_id=sandbox_id) - object.__setattr__( - self, - "_runtime", - ManagerBackedSandboxRuntime(manager=manager, sandbox=sandbox), - ) -``` - -If `GDPEvalSandboxManager` does not expose -`create / run_command / upload_file / read_file / list_files / terminate / connect`, -add thin adapter methods at the manager — do not reshape the -`SandboxRuntime` protocol. - -## Task 2: Move Toolkit - -- [ ] **Step 1: Convert existing `toolkit.py` to Pydantic in place** - -`toolkit.py` already lives at `benchmarks/gdpeval/toolkit.py` — no move needed. Do not create `ergon_builtins/toolkits/gdpeval.py` (PR 6.5 deleted the top-level `toolkits/` dir). - -- [ ] **Step 2: Convert to Pydantic** - -```python -from pydantic import BaseModel, ConfigDict - - -class GDPEvalToolkit(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - output_path: str = "/workspace/gdpeval/output.json" - allow_shell: bool = False - - def tools(self, sandbox, task): - from ergon_builtins.benchmarks.gdpeval._tools import build_tools - - return build_tools(self, sandbox=sandbox, task=task) -``` - -Move runtime tool construction into `benchmarks/gdpeval/_tools.py`. - -## Task 3: Convert Worker Factory - -- [ ] **Step 1: Replace registry call** - -In `gdpeval/workers.py` (renamed from `worker_factory.py`): - -```python -from ergon_builtins.benchmarks.gdpeval.toolkit import GDPEvalToolkit -from ergon_builtins.workers.baselines.react_worker import ReActWorker - - -def make_gdpeval_worker( - *, - model: str = "openai:gpt-4o-mini", - max_iterations: int = 12, -) -> ReActWorker: - return ReActWorker( - name="gdpeval-runner", - model=model, - system_prompt=_GDPEVAL_SYSTEM_PROMPT, - max_iterations=max_iterations, - toolkit=GDPEvalToolkit(), - ) -``` - -**Also: parameterise `GDPEvalBenchmark.__init__`** to accept `worker_factory` / `sandbox_factory` kwargs with defaults — mirror PR 6.5's MiniF2F pattern. - -## Task 4: Convert Benchmark Tasks - -- [ ] **Step 1: Replace imports** - -```python -# Delete: -from ergon_core.api.benchmark import Benchmark, BenchmarkRequirements, TaskSpec - -# Add: -from ergon_core.api import Benchmark, BenchmarkRequirements, Task -from ergon_builtins.benchmarks.gdpeval.sandbox import GDPEvalSandbox -from ergon_builtins.benchmarks.gdpeval.workers import ( - make_gdpeval_worker, -) -from ergon_builtins.benchmarks.gdpeval.rubric import make_gdpeval_rubric -``` - -- [ ] **Step 2: Replace `TaskSpec` construction** - -```python -Task[GDPEvalTaskPayload]( - task_slug="gdpeval-instance", - instance_key=instance.instance_id, - description=instance.prompt, - task_payload=GDPEvalTaskPayload( - instance_id=instance.instance_id, - question=instance.prompt, - expected=instance.expected, - ), - worker=make_gdpeval_worker(), - sandbox=GDPEvalSandbox(), - evaluators=(make_gdpeval_rubric(),), -) -``` - -## Task 5: Add GDPEval To The Benchmarks Catalogue - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/README.md` (created in PR 6.5 Task 20) - -PR 6.5 killed `BUILTIN_EXPERIMENT_FACTORIES` and the CLI authoring route. No CLI registry entry needed. - -- [ ] **Step 1: Add one row to the catalogue** - -```markdown -| GDPEval | `ergon_builtins.benchmarks.gdpeval` | `make_gdpeval_worker` | `GDPEvalSandbox` | -``` - -- [ ] **Step 2: Add a Python authoring example** - -```python -from ergon_builtins.benchmarks.gdpeval import ( - GDPEvalBenchmark, - make_gdpeval_worker, -) -from ergon_core.api import persist_benchmark, launch_run - -benchmark = GDPEvalBenchmark( - name="gdpeval-react", - metadata={"experiment": "gdp-eval-2026"}, - worker_factory=make_gdpeval_worker, - limit=10, -) -handle = persist_benchmark(benchmark) -await launch_run(handle.definition_id) -``` - -## Task 5.5: Migrate The Matching Smoke Fixture (closes the gate) - -**Files:** - -- Modify: `tests/fixtures/smoke_components/benchmarks.py` — `GDPEvalSmokeBenchmark` (or whichever subclass exists; add one if not) - -PR 6 (minif2f), PR 10a (swebench), PR 10b (researchrubrics) each -migrated their matching smoke-fixture row. PR 10c does the last one. -After this step, **`tests/fixtures/smoke_components/benchmarks.py` -imports `Task` (not `TaskSpec`) for every benchmark** — which is the -prerequisite PR 11 needs to delete `TaskSpec` outright. - -- [ ] **Step 1: Add a concrete `GDPEvalSmokeTask(Task[...])` subclass** -- [ ] **Step 2: Override `build_instances` to return that Task with `evaluators=(...)`** (mirror the prior three smoke-fixture migrations) -- [ ] **Step 3: Sweep the file for any remaining `TaskSpec` references** — after this PR, the import line should drop `TaskSpec` entirely. -- [ ] **Step 4: Update `_SingleTaskSmokeBenchmark` base class** — once every subclass overrides `build_instances`, the base's `TaskSpec`-shaped default can be deleted. Either delete the base method (preferred — each subclass owns its build) or convert it to raise `NotImplementedError`. - -## Task 6: Tests - -- [ ] **Step 1: Definition JSON test** - -Create `ergon_builtins/tests/unit/test_gdpeval_v2_definition.py`: - -```python -import pytest -from ergon_core.api import persist_benchmark -from ergon_builtins.benchmarks.gdpeval.benchmark import GDPEvalBenchmark - - -@pytest.mark.asyncio -async def test_gdpeval_persists_object_bound_task_json(session_factory): - benchmark = GDPEvalBenchmark( - name="gdpeval-smoke", - metadata={"author": "test"}, - limit=1, - ) - - handle = persist_benchmark(benchmark) - with session_factory() as session: - rows = session.exec( - "SELECT task_json FROM experiment_definition_tasks " - "WHERE definition_id = :d", - {"d": handle.definition_id}, - ).all() - assert rows - task_json = rows[0][0] - assert task_json["worker"]["_type"].endswith(":ReActWorker") - assert task_json["sandbox"]["_type"].endswith(":GDPEvalSandbox") - assert task_json["evaluators"], "evaluators must persist" - assert all( - ev.get("_type") for ev in task_json["evaluators"] - ), "every evaluator entry must carry a `_type` discriminator" -``` - -- [ ] **Step 2: Reconstruction test** - -```python -from uuid import uuid4 - -import pytest -from ergon_core.api.benchmark.task import Task - - -@pytest.mark.asyncio -async def test_gdpeval_task_json_round_trips_through_from_definition(): - benchmark = GDPEvalBenchmark(limit=1) - task = next(iter(benchmark.build_instances().values()))[0] - task_json = task.model_dump(mode="json") - - rebuilt = await Task.from_definition(task_json, task_id=uuid4()) - - assert isinstance(rebuilt.sandbox, type(task.sandbox)) - assert rebuilt.worker.toolkit is not None -``` - -- [ ] **Step 3: Run focused tests** - -```bash -uv run pytest ergon_builtins/tests/unit/test_gdpeval_v2_definition.py -q -``` - -## Task 7: Cross-Cutting Cleanup — Registry Import Shrink - -After 10a / 10b / 10c land, no migrated benchmark module should import -`ComponentRegistry`. PR 10c verifies this with a textual guard test and -trims the registry modules' loader to only register the parts that -still have non-builtin callers (model backends, training pipelines). - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/registry.py` -- Modify: `ergon_builtins/ergon_builtins/registry_core.py` -- Modify: `ergon_builtins/ergon_builtins/registry_data.py` -- Create: `ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py` - -- [ ] **Step 1: Remove migrated-benchmark imports from registry files** - -In `registry.py`, `registry_core.py`, `registry_data.py`, delete the -import blocks and registration lines for each migrated benchmark. -Remaining registrations should only be model-backend or training-side -entries; if those don't exist, the file becomes a thin stub. Leave the -files importable until PR 11 — worktrees that haven't rebased still -need to import them. - -- [ ] **Step 2: Add the no-registry guard** - -Create `ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py`: - -```python -"""After PR 10a/10b/10c land, no migrated benchmark module should -import `ComponentRegistry` — object-bound benchmarks construct Tasks -directly and don't go through registry resolution.""" - -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[4] - -_MIGRATED_BENCHMARKS = ( - "minif2f", - "swebench_verified", - "researchrubrics", - "gdpeval", -) - - -@pytest.mark.parametrize("slug", _MIGRATED_BENCHMARKS) -def test_object_bound_benchmark_does_not_import_component_registry( - slug: str, -) -> None: - pkg = ROOT / "ergon_builtins" / "ergon_builtins" / "benchmarks" / slug - offenders: list[str] = [] - for path in pkg.rglob("*.py"): - text = path.read_text() - if "ComponentRegistry" in text: - offenders.append(str(path.relative_to(ROOT))) - assert offenders == [], ( - f"{slug} still depends on ComponentRegistry; PR 10 was supposed " - f"to remove that dependency. Offenders: {offenders}" - ) -``` - -- [ ] **Step 3: Run the full builtins test sweep** - -```bash -uv run pytest ergon_builtins/tests/unit -q -uv run pytest ergon_core/tests/unit/runtime/test_experiment_definition_service.py -q -uv run pytest ergon_core/tests/unit/architecture -q -``` - -Expected: all green; no `TaskSpec`, `ComponentRegistry`, or -`BaseSandboxManager` references inside object-bound benchmark modules. - -## Task 8: Commit - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/gdpeval/ \ - ergon_builtins/ergon_builtins/benchmarks/README.md \ - ergon_builtins/ergon_builtins/registry*.py \ - ergon_builtins/tests/unit/test_gdpeval_v2_definition.py \ - ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -git commit -m "feat(builtins): convert GDPEval + close out builtins migration (PR 10c)" -``` - -## PR Ledger - -Invariant landed: GDPEval builtin constructs object-bound task graphs; -every migrated benchmark is `ComponentRegistry`-free; registry modules -are slimmed to model-backend registrations only. - -Bridge code introduced: `GDPEvalSandbox` wraps the legacy manager via -`ManagerBackedSandboxRuntime`. - -Bridge code retired (fully — after this PR's smoke-fixture migration): -- GDPEval tasks now carry `task.worker`/`task.sandbox`/`task.evaluators` - inline, so GDPEval runs no longer hit either the - `_legacy_worker_bridge` fallback on `worker_execute` or the symmetric - `_legacy_evaluator_bridge` fallback on `evaluate_task_run`. -- After PR 10c + the smoke-fixture migration in Task 6.5, **no - benchmark — production or smoke fixture — still produces `TaskSpec`**. - The "must support" sets for both `_legacy_worker_bridge` and - `_legacy_evaluator_bridge` are empty. -- The two bridge files are not deleted here; PR 11 owns the `git rm` - plus the `if worker is None:` and `if not task.evaluators:` branch - removals in `worker_execute.py` and `evaluate_task_run.py`. - -Old path still intentionally alive: `gdpeval/sandbox.py`, -`gdpeval/sandbox_utils.py`, slimmed `registry*.py` modules, -`BaseSandboxManager`, all per-benchmark `sandbox_manager.py` files (kept -until PR 11 deletes them in one sweep), `_legacy_worker_bridge.py`, -and `_legacy_evaluator_bridge.py` (both unreachable from any benchmark -after this PR, but the files stay until PR 11 removes them together -with the matching fallback branches). - -Migrations: this PR adds **no Alembic migration** (code-only changes). -Next free migration id is `aabbccdd0006` (PR 6.5 took `aabbccdd0004` -for `add_experiment_tag`; PR 7 took `aabbccdd0005` for -`definition_metadata_and_launch`; PR 10a / 10b do not add migrations). - -Deletion gate: PR 11 deletes the registry modules, per-benchmark -sandbox managers, `_legacy_worker_bridge.py`, and -`_legacy_evaluator_bridge.py`. - -Tests added or updated: GDPEval definition JSON + reconstruction + -no-registry architecture guard across all four migrated benchmarks. - -Modules owned by this PR: `gdpeval/`, the GDPEval toolkit, the -registry import shrink, and the cross-cutting no-registry guard. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/12-pr-11-deletion-final-schema.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/12-pr-11-deletion-final-schema.md deleted file mode 100644 index 7b419784b..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/12-pr-11-deletion-final-schema.md +++ /dev/null @@ -1,1196 +0,0 @@ -# PR 11 — Deletion And Final Schema - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Delete all transition bridges and replace additive migrations with -one final v2 initial schema. - -**Architecture:** This is cleanup only. No new behavior lands here except -guards that prevent deleted symbols from returning. - -**Tech Stack:** source deletion, SQLModel final schema, Alembic reset, -architecture tests. - -> **Note: PR 6.5 (and the post-PR-6.5 cleanup) already deleted several -> things this plan used to claim.** Anything in the list below is **not** -> in PR 11's deletion list anymore: -> - The public `Experiment` class (`ergon_core.api.experiment.Experiment`) — already gone (PR 6.5). -> - `ExperimentRecord` SQLModel + `experiments` table — already renamed -> to `BenchmarkDefinitionRecord` / `benchmark_definitions` (PR 6.5). -> **PR 11 KEEPS `BenchmarkDefinitionRecord`** — the rename made it the -> canonical v2 telemetry row for unstarted/launched experiments. -> - `persist_definition` top-level function — already renamed to -> `persist_benchmark` (PR 6.5). -> - `ExperimentDefineRequest`, `define_benchmark_experiment`, -> `BUILTIN_EXPERIMENT_FACTORIES` — already deleted (PR 6.5 Phase 2). -> - CLI authoring commands (`ergon experiment define`, `ergon experiment run`) — already deleted (PR 6.5). -> - Top-level `ergon_builtins/sandboxes/` and `toolkits/` dirs — already deleted (PR 6.5). -> - `ExperimentService` class — already deleted (post-PR-6.5 cleanup); replaced by module-level `run_experiment` in `application/experiments/service.py`. -> - Unused `name`/`description`/`created_by` kwargs on `persist_benchmark` — already dropped (post-PR-6.5 cleanup). -> -> What PR 11 still owns: -> - The **domain** `Experiment` class (different class — see below), -> `TaskSpec`, the per-benchmark `sandbox_manager.py` files, the -> per-benchmark `_legacy_workers.py` files (created by PR 6.5 / 10a / -> 10b / 10c specifically as PR 11 deletion targets), the legacy -> worker fallback chain. -> - The **symmetric** legacy evaluator fallback: `_legacy_evaluator_bridge.py` -> was restored after PR 5's premature retirement; PR 11 deletes it -> alongside `_legacy_worker_bridge.py` (same deletion gate: every -> benchmark — production + smoke fixture — on object-bound `Task`). -> - The PR 1 task-snapshot bridge helpers: `_definition_task_snapshot` -> and `_dynamic_task_snapshot` in `core/application/graph/repository.py` -> (docstrings already mark them for PR 11 deletion), plus the -> `task_json=task.task_json or _definition_task_snapshot(...)` fallback -> in `initialize_from_definition`. -> - The v1 `_ExperimentDefinitionWriter` class in `definition_writer.py` -> (docstring already marks it for PR 11 deletion). -> - The `terminate_sandbox_by_id` helper — but PR 4 moved its caller out -> of `execute_task.py` into a sibling Inngest function at -> `core/application/jobs/sandbox_cleanup.py` (triggered by -> `task/completed` / `task/failed`). PR 11 either keeps the sandbox -> cleanup function (if external sandboxes are still in scope) or -> deletes it alongside the legacy bridges. -> - The final schema/identity collapse. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. Most are deletions-from-the-deletion-list (items the plan still -claims to delete but that are already gone) plus xfail name sync. - -1. **Drop `Worker.from_buffer` from the deletion list.** Already deleted in - an earlier PR. Any task referencing it must be removed. -2. **Drop the `Worker.validate` → `validate_runtime_deps` rename task.** - Already done in PR 5. If the plan still calls for renaming it, remove - that task. -3. **Sync xfail symbol names in `test_dead_path_audit.py`.** Plan's xfail - flip checklist references `_worker_from_payload_bridge`, - `_legacy_worker_bridge`, `_legacy_evaluator_bridge` (etc.) — actual - xfails in the test file use different names (e.g., - `legacy_worker_from_payload`, `_prepare_legacy_graph_native`, - `_prepare_legacy_definition`, `execute_task`, `sandbox_setup`, - `persist_outputs`). Before flipping any xfail to a hard assertion, the - plan must be updated to match the actual symbol names in the test file - AT THE TIME PR 11 IS PICKED UP (the test xfail set may evolve as PRs 9 - and 10a/b/c land). -4. **Cross-check `saved_specs` deletion target.** Plan still lists it; audit - confirmed it still exists in `ergon_core/core/persistence/saved_specs/`. - Verify after PRs 9 and 10a/b/c whether anyone still references it; if - not, this deletion is still valid. -5. **Natural sequencing reminder.** PR 11 is the deletion gate. It is only - safe to run when every benchmark (production + smoke fixture) emits - object-bound `Task[...]` — i.e., after PRs 10a, 10b, and 10c land. Don't - start PR 11 implementation before confirming all three are merged. - ---- - -## Files To Delete - -```text -ergon_core/ergon_core/api/registry.py -ergon_core/ergon_core/core/domain/experiments/ # whole package: Experiment, WorkerSpec, validation -ergon_core/ergon_core/core/application/components/catalog.py # ComponentCatalogService (audit first) -ergon_core/ergon_core/core/application/components/ # whole package once catalog.py is gone -ergon_core/ergon_core/core/application/experiments/repository.py # DefinitionRepository (after prepare_run inlines its one remaining caller) -ergon_core/ergon_core/core/persistence/saved_specs/ -ergon_core/ergon_core/core/application/evaluation/executors.py -ergon_core/ergon_core/core/application/evaluation/inngest_executor.py -ergon_core/ergon_core/core/application/jobs/check_evaluators.py -ergon_core/ergon_core/core/application/jobs/_legacy_worker_bridge.py -ergon_core/ergon_core/core/application/jobs/_legacy_evaluator_bridge.py -ergon_builtins/ergon_builtins/registry.py -ergon_builtins/ergon_builtins/registry_core.py -ergon_builtins/ergon_builtins/registry_data.py -``` - -**Note (post-reconciliation):** earlier drafts of this plan listed -`execute_task.py`, `sandbox_setup.py`, and `persist_outputs.py` as -deletion targets on the theory that PR 4 would collapse them into -`worker_execute`'s body. **That collapse did not happen.** PR 4 kept -the four-function orchestration (`execute_task` → `sandbox_setup` → -`worker_execute` → `persist_outputs` + per-evaluator `evaluate_task_run` -fanout) and added `sandbox_cleanup` as a sibling triggered by -`task/completed` / `task/failed`. Those four files are part of the -final v2 shape, not deletion targets. - -The two legacy bridge files (`_legacy_worker_bridge.py`, -`_legacy_evaluator_bridge.py`) ARE deletion targets once every benchmark -(production + smoke fixture) migrates to object-bound `Task`. The -deletion gate is "no benchmark still produces `TaskSpec`" — PR 6 -migrated minif2f, PR 10a/b/c migrate swebench/researchrubrics/gdpeval -plus their matching smoke fixtures. PR 11 verifies the call sets are -empty, then `git rm`s both files and deletes the matching -`if worker is None:` / `if not task.evaluators:` branches in -`worker_execute.py` / `evaluate_task_run.py`. - -**Audit-before-delete:** - -- `ComponentCatalogService` and the `components/` package — PR 5 forbids - `worker_execute` from importing it; after PR 10c every builtin uses - object-bound `Task`. Confirm zero production callers remain before the - `git rm`. If a dashboard or CLI listing still uses it, decide whether - to migrate or keep the package. -- `DefinitionRepository` — PR 3 removes its `worker_execute` caller; PR 4 - removes its `evaluate_task_run` caller. The remaining call site - (`prepare_run` copying definition→run-tier) should inline the one - method it needs and delete the repository class. - -**Do NOT delete:** - -- `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` — - reshaped by PR 4 to the thin id-only payload + `graph_repo.node` - loader. The file, the Inngest function, the slug, and the - registration all survive. Only the v1 body was removed (in PR 4), - not the file itself. - -Delete old sandbox manager files only after each benchmark has a typed sandbox subclass: - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/sandbox_manager.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager_support.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/sandbox_manager.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox_manager.py # PR 10c renamed from sandbox.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox_utils.py -``` - -Delete per-benchmark `_legacy_workers.py` files (created by PR 6.5 / 10a / 10b / 10c specifically as PR 11 deletion targets — they hold the legacy worker classes that the v1 registry strings still resolve to): - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/_legacy_workers.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/_legacy_workers.py # if PR 10a created one -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/_legacy_workers.py # if PR 10b created one -ergon_builtins/ergon_builtins/benchmarks/gdpeval/_legacy_workers.py # if PR 10c created one -``` - -(Some verticals may not have a `_legacy_workers.py` if their legacy worker block was already minimal. Check before `git rm`.) - -## Required Code Deletions - -**Public API and v1 abstractions:** - -- Remove `TaskSpec` from `ergon_core/ergon_core/api/benchmark/task.py`. -- Remove `Worker.from_buffer` (replaced by nothing — had no callers). -- Remove `Worker.validate()`. PR 5 renamed it to `validate_runtime_deps`; - grep-confirm the old name is gone from these callsites: - `ergon_core.api.rubric.rubric.Rubric.validate`, - `ergon_core.core.domain.experiments.validation`, - `ergon_core.core.application.experiments.definition_writer.persist_benchmark` # renamed by PR 6.5, - `ergon_core.core.application.experiments.launch.launch_run`, - `ergon_builtins.benchmarks.gdpeval.rubric`. -- Remove the **domain** `Experiment` (`ergon_core.core.domain.experiments.Experiment`). PR 6.5 deleted the public `Experiment` class; PR 11 deletes the remaining domain-layer class. After PR 11, **there is no `Experiment` class anywhere** — the word survives only as a `str | None` column on `BenchmarkDefinitionRecord` (the "experiment tag" introduced by PR 6.5). -- Remove `EvaluateTaskRunRequest` (the v1 multi-field payload). The - replacement is `TaskEvaluateRequest` (id-only), already in use since - PR 4. -- Remove `_task_to_definition_json` support for `TaskSpec` (the function - itself is the `_legacy` branch; PR 11 deletes the function). -- Remove `_definition_task_snapshot` and `_dynamic_task_snapshot` from - `core/application/graph/repository.py` (PR 1 bridge helpers; their - docstrings already mark them as PR 11 deletion targets). Then narrow - `initialize_from_definition` from - `task_json=task.task_json or _definition_task_snapshot(...)` to just - `task_json=task.task_json`. -- Remove the v1 `_ExperimentDefinitionWriter` class from - `definition_writer.py` (its docstring already marks it for PR 11 - deletion). The class is the leftover v1 launch path; the canonical - v2 path is the module-level `persist_benchmark` function. -- Remove `terminate_sandbox_by_id`. **PR 4 moved this out of - `execute_task.py`'s deleted `try/finally`** — it now lives in the - sibling Inngest job at - `ergon_core/core/application/jobs/sandbox_cleanup.py` (plus the - matching handler at - `core/infrastructure/inngest/handlers/sandbox_cleanup.py`). If the - final v2 still terminates external sandboxes after each task, keep - the `sandbox_cleanup.py` job and only delete the helper if a more - direct API replaces it; otherwise delete the helper, the job, AND - the handler. - -**Runtime identity and DTOs:** - -- Remove `node_id` as runtime identity from event payloads and DTOs. -- Remove `PreparedTaskExecution.node_id`, `.definition_task_id`, - `.worker_type`, `.assigned_worker_slug`, `.model_target`. PR 3 carried - these as a bridge; after PR 5 makes `task.worker` canonical and PR 11 - collapses identity, they're dead. The DTO simplifies to - `run_id, definition_id, task_id, task_slug, task_description, benchmark_type, execution_id`. -- Remove `_prepare_legacy_definition` from `TaskExecutionService`. -- Retire the legacy worker fallback chain. PR 5 retired the *in-body* - `_worker_from_payload_bridge` but kept a narrow legacy fallback at - `core/application/jobs/_legacy_worker_bridge.py` for unmigrated - benchmarks. PR 6 / PR 10a / PR 10b / PR 10c migrate the four builtins - (minif2f, swebench, researchrubrics, gdpeval) **plus the matching - smoke fixtures in `tests/fixtures/smoke_components/benchmarks.py`** — - after PR 10c, no benchmark (production or fixture) still produces - `TaskSpec`. PR 11 performs the final deletion (see Task 1.5 below). - The grep for `_worker_from_payload_bridge` must come back empty. -- Retire the **symmetric** legacy evaluator fallback chain. Post-PR-5 - cleanup restored `core/application/jobs/_legacy_evaluator_bridge.py` - (which PR 5 had prematurely deleted) as the eval-side counterpart to - `_legacy_worker_bridge.py`. Same deletion gate, same migration - sequence: PR 6 / 10a / 10b / 10c remove each benchmark from the call - set. PR 11 `git rm`s the file and deletes the - `if not task.evaluators:` fallback branch in - `evaluate_task_run.py`. Add the eval-side `git rm` to Task 1.5 - alongside the worker-side `git rm`. - -**Schema (run-tier collapse — composite PK `(run_id, task_id)`):** - -- Drop `run_graph_nodes.id` column. New composite PK is - `(run_id, task_id)` per `02-persistence-layer.md` §2. -- Drop `run_graph_nodes.definition_task_id`. Identity is `task_id` only. -- Drop `run_graph_nodes.parent_node_id`. Rename to `parent_task_id` - (per PR 9 § "In the current schema this maps to ... PR 11 renames - columns"). -- Rename `run_graph_edges.source_node_id` → `source_task_id` and - `target_node_id` → `target_task_id`. -- Drop `run_graph_edges.definition_dependency_id` (mirror of - `definition_task_id` for edges). -- Drop `RunTaskExecution.node_id`; rename to `task_id`. Drop - `RunTaskExecution.definition_task_id`. -- Drop the matching columns on `RunTaskEvaluation` - (`node_id`, `definition_task_id`). -- ~~Remove `ExperimentRecord` from telemetry models~~ — **already done by PR 6.5** (renamed to `BenchmarkDefinitionRecord`). Task 2's schema reset uses the renamed model. - -**Inngest events:** - -- Audit whether `task/completed` still has a real consumer after PR 4. - Current consumer chain in tree: - `ergon_core.core.application.jobs.propagate_execution` listens to - `task/completed` for graph propagation — that's still load-bearing in - v2, so the event survives. The audit is to grep for stale listeners - and confirm `propagate_execution` is the only one. - -## Final Schema Shape - -`RunGraphNode` final identity: - -```python -class RunGraphNode(SQLModel, table=True): - __tablename__ = "run_graph_nodes" - - run_id: UUID = Field(foreign_key="runs.id", primary_key=True) - task_id: UUID = Field(primary_key=True) - parent_task_id: UUID | None = Field(default=None, index=True) - # NOT NULL with no default — task_json is the single source of truth - # for what to execute. An insert without a snapshot is a programming - # error; the DB rejects it. The PR 1 additive migration had a - # `server_default='{}'` only for the additive backfill; PR 11 drops - # it because every legitimate writer (definition copy in PR 1, - # object-bound model_dump in PR 5, dynamic spawn in PR 9) sets it - # explicitly. - task_json: dict = Field(sa_column=Column(JSON, nullable=False)) - status: str = Field(index=True) - level: int = 0 - is_dynamic: bool = False - last_error: str | None = None - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - updated_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) -``` - -`BenchmarkDefinitionRecord` final metadata (renamed from `ExperimentDefinition` by PR 6.5; PR 11 finalises field shape): - -```python -class BenchmarkDefinitionRecord(SQLModel, table=True): - __tablename__ = "benchmark_definitions" # renamed by PR 6.5 - - id: UUID = Field(default_factory=uuid4, primary_key=True) - name: str = Field(index=True) - description: str | None = None - benchmark_type: str = Field(index=True) - benchmark_json: dict = Field(default_factory=dict, sa_column=Column(JSON)) - experiment: str | None = Field(default=None, index=True) # PR 6.5 added — the experiment-tag column - metadata_json: dict = Field(default_factory=dict, sa_column=Column(JSON)) - created_by: str | None = None - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) -``` - -Notes for the schema reset: -- The `experiment_json` field from the v1 shape is gone — there is no `Experiment` class to persist. All metadata lives in `metadata_json` or directly in `benchmark_json`. -- The `experiment: str | None` column is PR 6.5's tag for grouping related definitions (e.g. ablation studies). It is **not** a foreign key — just an indexed string column. - -## Task 1: Delete Symbols - -- [ ] Remove files listed above with `git rm`. -- [ ] Remove exports from `__init__.py` files. -- [ ] Run `rg` for each final deleted symbol and remove remaining production - hits. - -Commands: - -```bash -rg "TaskSpec|WorkerSpec|ComponentRegistry|saved_specs|ExperimentRecord|definition_task_id|EvaluateTaskRunRequest|CriterionExecutor|from_buffer|terminate_sandbox_by_id" ergon_core ergon_builtins ergon_cli -``` - -Note: `evaluate_task_run` is intentionally **not** in this grep — the -function name survives per Δ.4. Run a second grep to confirm -`evaluate_task_run` IS still present in production code: - -```bash -rg "evaluate_task_run" ergon_core ergon_builtins ergon_cli -``` - -Expected: hits in `jobs/evaluate_task_run.py` (the reshaped body), -`infrastructure/inngest/registry.py` (the registration), and the -walkthrough/regression tests. - -Expected: only docs and deleted-symbol tests contain hits. - -## Task 1.3: Shrink `CriterionContext` To A Pure Data Carrier - -**Deferred from PR 5.** The PR 5 plan specified removing the 12 runtime -proxy methods from `CriterionContext` alongside the `_runtime` PrivateAttr -and `sandbox_id` field. This was not done — criteria in PRs 6, 10a, 10b, -10c still call `context.run_command(...)` via the legacy proxy. PR 11 -does the removal once every criterion body has been migrated to -`context.task.sandbox.run_command(...)`. - -**Files:** - -- Modify: `ergon_core/ergon_core/api/criterion/context.py` - -- [ ] **Step 1: Confirm every criterion caller uses `context.task.sandbox` not `context.run_command`** - -```bash -rg "context\.run_command\|context\.write_file\|context\.read_resource\|context\.upload_files\|context\.ensure_sandbox\|context\.execute_code\|context\.cleanup\|context\.list_resources\|context\.get_all_files" ergon_core ergon_builtins -``` - -Expected: zero production hits (only docs/tests). - -- [ ] **Step 2: Remove proxy methods and legacy private attrs from `CriterionContext`** - -Delete from `ergon_core/ergon_core/api/criterion/context.py`: -- `_runtime: CriterionRuntime | None` PrivateAttr and all blocks that set it -- `sandbox_id: str | None` field -- `with_runtime(...)` classmethod -- `has_runtime` property -- `runtime` property -- `_require_runtime()` method -- All proxy methods: `ensure_sandbox`, `upload_files`, `write_file`, `run_command`, `execute_code`, `cleanup`, `read_resource`, `read_resource_by_id`, `list_resources`, `get_all_files_for_task`, `list_output_files` - -The resulting class is a pure data carrier: - -```python -class CriterionContext(BaseModel): - task: Task - worker_result: WorkerOutput - run_id: UUID - execution_id: UUID -``` - -- [ ] **Step 3: Remove `CriterionRuntime` import from `context.py`** - -After Step 2, `CriterionRuntime` has no callers in `context.py`. Remove the import. -Check it has no remaining callers in production: - -```bash -rg "CriterionRuntime" ergon_core ergon_builtins ergon_cli -``` - -Expected: zero production hits. - -## Task 1.4: Add `Criterion.from_definition` Classmethod - -**Deferred from PR 5.** `Task.from_definition`'s object-bound evaluators -path routes rubric-shaped evaluator JSON through `Evaluator.from_definition`, -which handles `Rubric` subclasses fine (`criteria` are `exclude=True` so -they don't appear in the JSON and don't need deserialization). However, -bare `Criterion` subclasses that might be stored directly as evaluators -would have no `from_definition` entry point. Add it so the pattern is -complete and consistent before the PR 11 audit. - -**Files:** - -- Modify: `ergon_core/ergon_core/api/criterion/criterion.py` - -- [ ] **Step 1: Add `from_definition` classmethod** - -```python -@classmethod -def from_definition(cls, criterion_json: TaskDefinitionJson) -> "Criterion": - """Reconstruct a Criterion subclass from ``_type``-discriminated JSON. - - Mirrors Worker.from_definition / Sandbox.from_definition. Called by - Task.from_definition when an evaluator entry resolves to a bare - Criterion subclass (not a Rubric). - """ - criterion_type = criterion_json.get("_type") - if not isinstance(criterion_type, str): - raise ValueError( - f"Criterion snapshot is missing the required `_type` discriminator " - f"(got {type(criterion_type).__name__}). Every persisted criterion " - f"must carry `_type`." - ) - CriterionCls = import_component(criterion_type) - return cast("Criterion", CriterionCls.model_validate(criterion_json)) -``` - -Note: this requires `Criterion` to be a Pydantic `BaseModel`. If it isn't -yet at PR 11 time, defer to a dedicated "Criterion Pydantic migration" PR -and leave a `TODO(PR N)` comment here instead. - -## Task 1.5: Retire The Legacy Worker And Evaluator Fallbacks - -After PR 10c lands (production benchmarks migrated) AND each PR 10x has -migrated its matching smoke fixture in -`tests/fixtures/smoke_components/benchmarks.py`, no benchmark anywhere -still returns `TaskSpec`. Both `_legacy_worker_bridge` and -`_legacy_evaluator_bridge` are unreachable. PR 11 deletes both files -plus their matching fallback branches. - -- [ ] **Step 1: Delete the bridge modules** - -```bash -git rm ergon_core/ergon_core/core/application/jobs/_legacy_worker_bridge.py -git rm ergon_core/ergon_core/core/application/jobs/_legacy_evaluator_bridge.py -``` - -- [ ] **Step 2: Delete the `if worker is None:` fallback in `worker_execute.py`** - -Remove the `if worker is None:` block in -`ergon_core/ergon_core/core/application/jobs/worker_execute.py` that -imports `legacy_worker_from_payload`. After PR 10c (incl. smoke -fixtures), `task.worker` is always non-None for every benchmark; the -branch is unreachable and the import is dead. - -- [ ] **Step 3: Delete the `if not task.evaluators:` fallback in `evaluate_task_run.py`** - -Symmetric to Step 2. Remove the `else` branch in -`ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` that -imports `legacy_evaluator_from_binding` and -`legacy_inject_criterion_runtime`. After PR 10c (incl. smoke fixtures), -`task.evaluators` is always populated for every benchmark; the eval-side -fallback is unreachable. - -- [ ] **Step 4: Drop the bridge entries from the dead-path audit `_XFAIL_BY_SYMBOL`** - -In `ergon_core/tests/unit/architecture/test_dead_path_audit.py`, remove -any entries for `_worker_from_payload_bridge`, `_legacy_worker_bridge`, -`_evaluator_bridge`, or `_legacy_evaluator_bridge` from -`_XFAIL_BY_SYMBOL`. Folded into the "Empty `_XFAIL_BY_SYMBOL`" sweep in -Task 4 Step 2 — call it out here so the deletion isn't lost when the -dict is collapsed. - -Verify: - -```bash -rg "_worker_from_payload_bridge|_legacy_worker_bridge|legacy_worker_from_payload|_evaluator_bridge|_legacy_evaluator_bridge|legacy_evaluator_from_binding|legacy_inject_criterion_runtime" \ - ergon_core ergon_builtins ergon_cli tests -``` - -Expected: only docs hits remain. - -## Task 2: Reset Migration Chain - -This step **wipes every revision currently on disk** under -`ergon_core/migrations/versions/`. As of the time this plan was written -that's 27 revisions, including `5f01559f2bc3_initial_schema_v2.py` and every -additive migration introduced by PRs 1 and 7. The workshop decision was -"no prod data," so this is safe in production. It is **not** safe for -contributors' local databases without an explicit downgrade — see the -developer-facing instructions below. - -### Revisions to be deleted - -Run this once before the destructive step to capture an inventory in the PR -description (the list will be longer than the snapshot below by the time -this PR lands, but the snapshot establishes the lower bound): - -```bash -ls ergon_core/migrations/versions/*.py | sort -``` - -Snapshot at the time of writing (delete all of these): - -```text -0a1b2c3d4e5f_add_thread_summary.py -11f1497a53e8_add_batch_operation_id_to_run_graph_.py -307fcca3a621_drop_run_task_state_events.py -4a71a3dc2ef5_add_blocked_to_taskexecutionstatus_enum.py -5f01559f2bc3_initial_schema_v2.py -7c9661121a86_add_triggered_by_mutation_id_to_run_.py -84519b3f8431_add_cancelled_to_taskexecutionstatus_enum.py -925ff225d97e_add_sandbox_id_to_run_task_executions.py -a1b2c3d4e5f6_unique_thread_run_topic.py -a2b3c4d5e6f7_add_copied_from_resource_id.py -a66564b89aac_rename_output_text_to_final_assistant_.py -b1c2d3e4f5a6_add_experiment_records.py -b5b36e45e5e6_add_containment_and_cancelled.py -c1d2e3f4a5b6_add_sandbox_event_tables.py -c2d3e4f5a6b7_add_sandbox_dependency_fields.py -d1e2f3a4b5c6_add_component_catalog.py -d4f5a6b7c8d9_drop_run_generation_turns.py -e2f3a4b5c6d7_add_import_reducer_tables.py -e5f6a7b8c9d0_normalize_evaluation_summary_nulls.py -e89c6c427de4_drop_run_actions_add_turn_timing.py -e96c85469899_rename_task_key_to_task_slug_and_.py -f1a2b3c4d5e6_add_run_context_events.py -f6a7b8c9d0e1_key_task_evaluations_by_node.py -f9075c2ddbc9_run_resource_append_only_log.py -``` - -Plus the additive migrations introduced earlier in this program: - -- `aabbccdd0001_add_run_graph_task_json.py` (PR 1) -- `aabbccdd0002_add_worker_output_json.py` (PR 4) -- `aabbccdd0003_add_definition_task_json.py` (PR 5) -- `aabbccdd0004_add_experiment_tag.py` (PR 6.5) -- `aabbccdd0005_definition_metadata_and_launch.py` (PR 7) -- (any `aabbccdd0006+` additive migrations PR 10a/10b/10c may have added) - -### Developer-facing downgrade step - -Every contributor (including CI runners with persistent volumes) must -downgrade to `base` **before** pulling this PR, because the new initial -migration is not a descendant of any existing revision and Alembic will -refuse to apply it on top of a live history. - -The PR description must include this exact block: - -```text -## BREAKING: developer DB reset required - -Before pulling this PR, run on every machine that has an Ergon dev DB: - - uv run alembic -c ergon_core/alembic.ini downgrade base - docker compose down -v # if you use the docker stack with named volumes - -After pulling: - - uv run alembic -c ergon_core/alembic.ini upgrade head - -There is no upgrade path from v1 data; the workshop decision was "no prod -data," and this PR enforces it. -``` - -Add the same notice as a comment at the top of the new initial migration so -`alembic history` readers can find it later. - -### Steps - -- [ ] **Step 1: Inventory existing revisions** - -```bash -ls ergon_core/migrations/versions/*.py | sort > /tmp/v2_pre_reset_revisions.txt -wc -l /tmp/v2_pre_reset_revisions.txt -``` - -Expected: matches the snapshot above plus any later additions. Attach the -file to the PR description. - -- [ ] **Step 2: Delete all existing revisions** - -```bash -git rm ergon_core/migrations/versions/*.py -``` - -- [ ] **Step 3: Create the final initial migration** - -Create `ergon_core/migrations/versions/00000000_initial_v2.py`: - -```python -"""Initial v2 schema. - -# BREAKING: this migration is not a descendant of any v1 or v2-transition -# revision. Contributors must run `alembic downgrade base` against any DB -# created before PR 11 lands. There is no data preservation path. -""" - -revision = "00000000_initial_v2" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # Schema from 02-persistence-layer.md plus the RunGraphNode and - # ExperimentDefinition final shapes defined in this PR. Generate this - # body with `alembic revision --autogenerate` against a fresh DB after - # all model deletions in Task 1 are applied, then hand-audit so that: - # - run_graph_nodes uses (run_id, task_id) as the composite PK - # - parent_task_id is the only parent reference (no parent_node_id) - # - task_json is NOT NULL with NO server_default (every insert must - # set it explicitly — the additive backfill default from PR 1 is - # gone) - # - no experiment_records table exists - # - no saved_specs table exists - ... - - -def downgrade() -> None: - raise NotImplementedError( - "v2 initial migration intentionally has no downgrade. To roll back, " - "restore from the snapshot taken in Task 2 Step 1." - ) -``` - -The PR author runs `alembic revision --autogenerate -m "initial v2"` -against a clean DB **after** Task 1 deletions, then renames the file to -`00000000_initial_v2.py` and reviews the body against the criteria above. - -- [ ] **Step 4: Verify single head** - -```bash -uv run alembic -c ergon_core/alembic.ini heads -``` - -Expected: exactly one head, `00000000_initial_v2`. - -- [ ] **Step 5: Run Alembic upgrade against a fresh dev DB** - -```bash -docker compose down -v -docker compose up -d postgres -uv run alembic -c ergon_core/alembic.ini upgrade head -``` - -Expected: schema creates with no references to deleted tables/columns; the -following grep is empty: - -```bash -uv run python -c " -from sqlalchemy import create_engine, inspect -import os -engine = create_engine(os.environ['DATABASE_URL']) -inspector = inspect(engine) -forbidden = {'experiment_records', 'saved_specs', 'definition_task_id'} -present = set(inspector.get_table_names()) -columns = {col['name'] for t in present for col in inspector.get_columns(t)} -assert not (forbidden & (present | columns)), forbidden & (present | columns) -print('clean') -" -``` - -### CI Invariant: Exactly One Alembic Head - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_single_alembic_head.py` - -```python -from pathlib import Path -import subprocess - -ROOT = Path(__file__).resolve().parents[4] - - -def test_alembic_has_exactly_one_head() -> None: - """Guards against migration chain forks introduced by stacked v2 PRs.""" - - result = subprocess.run( - [ - "uv", - "run", - "alembic", - "-c", - str(ROOT / "ergon_core" / "alembic.ini"), - "heads", - ], - check=True, - capture_output=True, - text=True, - ) - head_lines = [ - line for line in result.stdout.strip().splitlines() if line.strip() - ] - assert len(head_lines) == 1, ( - f"Expected exactly one Alembic head, got: {head_lines}" - ) - assert head_lines[0].split()[0] == "00000000_initial_v2", ( - f"Unexpected head revision: {head_lines[0]!r}" - ) -``` - -Wire this guard into `pnpm run check:be` so a stacked PR that re-introduces -the additive migration chain fails fast. - -## Task 3: Deleted Symbol Guard - -**Files:** - -- Create: `ergon_core/tests/unit/architecture/test_no_deleted_v2_symbols.py` - -- [ ] **Step 1: Add guard** - -```python -DELETED_SYMBOLS = ( - "TaskSpec", - "WorkerSpec", - "ComponentRegistry", - "saved_specs", - "ExperimentRecord", # PR 6.5 renamed to BenchmarkDefinitionRecord - "ExperimentDefineRequest", # PR 6.5 deleted - "BUILTIN_EXPERIMENT_FACTORIES", # PR 6.5 deleted - "define_benchmark_experiment", # PR 6.5 deleted - "persist_definition", # PR 6.5 renamed to persist_benchmark - "class Experiment", # PR 6.5 deleted public; PR 11 deletes domain class - "class ExperimentService", # post-PR-6.5 cleanup deleted the facade - "_evaluator_bridge", # PR 5 retired; restored as _legacy_evaluator_bridge - "_legacy_worker_bridge", # PR 11 Task 1.5 - "_legacy_evaluator_bridge", # PR 11 Task 1.5 - "_definition_task_snapshot", # PR 1 bridge helper, PR 11 deletes - "_dynamic_task_snapshot", # PR 1 bridge helper, PR 11 deletes - "_ExperimentDefinitionWriter", # v1 leftover, docstring marks PR 11 - "definition_task_id", - "EvaluateTaskRunRequest", - "CriterionExecutor", - "InngestCriterionExecutor", - "from_buffer", - "terminate_sandbox_by_id", -) -# evaluate_task_run is intentionally NOT in DELETED_SYMBOLS. Per Δ.4 it -# survives as the per-evaluator fanout target reshaped in PR 4. -# persist_benchmark, BenchmarkDefinitionRecord, and the experiment string -# column are also NOT deleted — they're PR 6.5's replacements. -KEPT_RESHAPED_SYMBOLS = ( - "evaluate_task_run", - "TaskEvaluateRequest", - "persist_benchmark", # PR 6.5 introduced - "BenchmarkDefinitionRecord", # PR 6.5 introduced -) - - -def test_deleted_v2_symbols_do_not_exist_in_production_code() -> None: - offenders = [] - for root in SEARCH_ROOTS: - for path in root.rglob("*.py"): - text = path.read_text() - for symbol in DELETED_SYMBOLS: - if symbol in text: - offenders.append(f"{path.relative_to(ROOT)} contains {symbol}") - assert offenders == [] -``` - -Allow this test file itself and docs. - -## Task 4: Flip Remaining XFails And Verify Empty Ledgers - -**Files:** - -- Modify: `ergon_core/tests/unit/architecture/test_v2_final_state_ledger.py` -- Modify: `ergon_core/tests/unit/architecture/test_dead_path_audit.py` -- Modify: `ergon_core/tests/unit/architecture/test_no_type_circumventors.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_layer_conventions.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` -- Modify: `ergon_core/tests/unit/architecture/test_no_dead_repository_methods.py` -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` - -PR 11 is the deletion gate — by definition every remaining xfail in the -final-state ledger, dead-path audit, no-type-circumventors ledger, -repository ledgers, walkthrough smoketest, and identity invariants -flips green here. - -- [ ] **Step 1: Empty `_XFAIL_BY_NAME`** - -In `test_v2_final_state_ledger.py`, delete every remaining entry from -`_XFAIL_BY_NAME` so the dict is empty: - -```python -_XFAIL_BY_NAME: dict[str, str] = {} -``` - -The entries removed in this PR are: - -```python -"prepare_definition_helper_is_removed" -"criterion_executor_is_removed" -"saved_specs_package_is_removed" -"run_graph_node_has_no_definition_task_id_column" -"worker_from_buffer_is_removed" -"terminate_sandbox_by_id_is_removed" -``` - -- [ ] **Step 2: Empty `_XFAIL_BY_SYMBOL`** - -In `test_dead_path_audit.py`, delete every remaining entry from -`_XFAIL_BY_SYMBOL` so the dict is empty: - -```python -_XFAIL_BY_SYMBOL: dict[str, str] = {} -``` - -The entries removed in this PR are: - -```python -"saved_specs" -"Worker.from_buffer" -"CriterionExecutor" -"InngestCriterionExecutor" -"_prepare_legacy_definition" -"terminate_sandbox_by_id" -``` - -- [ ] **Step 3: Flip remaining smoketest case** - -In `test_walkthrough_smoketest.py`, remove the -`@pytest.mark.xfail(reason="PR 11: full v2 lifecycle ...")` from -`test_run_completion_releases_every_acquired_sandbox` and implement the -real body: drive a three-task run through the test Inngest driver, -assert one acquire and one release per task in -`run_resource_events` (or the equivalent sandbox-event table), with no -release coming from the per-run cleanup path. - -- [ ] **Step 4: Add the "no xfails remain" architecture guard** - -Append to `test_v2_final_state_ledger.py`: - -```python -def test_no_v2_invariants_are_still_xfailed() -> None: - """PR 11 completion bar: every invariant in FINAL_STATE_ASSERTIONS - has landed. Empty `_XFAIL_BY_NAME` confirms the program is done.""" - - assert _XFAIL_BY_NAME == {}, ( - f"v2 program incomplete — invariants still xfailed: " - f"{sorted(_XFAIL_BY_NAME)}" - ) -``` - -Append to `test_dead_path_audit.py`: - -```python -def test_no_dead_paths_are_still_xfailed() -> None: - """PR 11 completion bar: every v1 dead path has been deleted.""" - - assert _XFAIL_BY_SYMBOL == {}, ( - f"v2 program incomplete — dead paths still xfailed: " - f"{sorted(_XFAIL_BY_SYMBOL)}" - ) -``` - -- [ ] **Step 2.5: Empty `_KNOWN_EXEMPTIONS`** - -In `test_no_type_circumventors.py`, delete every remaining entry from -`_KNOWN_EXEMPTIONS` so the dict is empty: - -```python -_KNOWN_EXEMPTIONS: dict[tuple[str, str], str] = {} -``` - -The only `getattr`/`hasattr` sites surviving past PR 11 are the -legitimate exemptions carrying a `# typing:` comment (the qualname -walks in `_import_component`, any external-SDK boundary). Every -formerly-violating production site has been fixed by an earlier PR per -the schedule in `00-program.md` "Ledger Files". - -Append to `test_no_type_circumventors.py`: - -```python -def test_no_known_type_circumventor_exemptions_remain() -> None: - """PR 11 completion bar: every transitional getattr/hasattr in - production code has either been replaced with typed access or - annotated with a `# typing:` comment. The exemption dict is empty.""" - - assert _KNOWN_EXEMPTIONS == {}, ( - f"v2 program incomplete — type-circumventor exemptions remain: " - f"{sorted(_KNOWN_EXEMPTIONS)}" - ) -``` - -Also flip the repository-ledger completion bars (added by PR 0.5, -xfailed until now): - -In `test_repository_layer_conventions.py` (or wherever -`_KNOWN_VIOLATORS` lives — single shared dict across the two repo -guard files), delete every remaining entry and remove the -`@pytest.mark.xfail` decorator from -`test_no_repository_violators_remain`: - -```python -def test_no_repository_violators_remain() -> None: - assert _KNOWN_VIOLATORS == {}, ( - f"Repository layer cleanup incomplete — violators remain: " - f"{sorted(_KNOWN_VIOLATORS)}" - ) -``` - -In `test_no_dead_repository_methods.py`, empty -`_KNOWN_UNUSED_FOR_NOW` (every remaining method here must have its -deletion landed by this PR, per the structural simplification audit in -Task 5) and remove the xfail decorator from -`test_no_dead_repository_methods_remain`. The entries removed here are -the methods identified during PR 0.5's initial audit -(`TaskExecutionRepository.latest_for_definition_task`, etc.) — they -either gained a caller by PR 10 or are deleted in Task 1 of this PR. - -These five assertions are the **machine-readable v2 completion bar**: -if the v2 implementation program is complete, all five dicts -(`_XFAIL_BY_NAME`, `_XFAIL_BY_SYMBOL`, `_KNOWN_EXEMPTIONS`, -`_KNOWN_VIOLATORS`, `_KNOWN_UNUSED_FOR_NOW`) are empty and all five -tests pass; if any still has entries, the program has not landed. - -## Task 5: Structural Simplification Audit - -The deletions from Tasks 1-4 leave several methods, services, and DTOs -*working* but possibly *no longer earning their existence*: indirection -layers whose original justification was the parallel old path, or -methods whose only branch is gone. Pytest cannot catch this class — -nothing about it is structurally wrong, the abstractions just stopped -paying for themselves. - -This task is a **hand-checked review pass**, not a pytest assertion. -Either dispatch a `superpowers:code-reviewer` agent scoped to the -candidate sites below, or do the audit by hand. The deliverable is a -keep / rename / inline / split decision documented in the PR description -for each candidate. - -The audit is intentionally bounded to this list — open-ended "look for -smells" reviews don't converge. Each candidate is named because -something concrete about the v2 deletions made it suspicious. - -### Candidates - -For each, ask: *does this still earn its name, its signature, its -existence?* - -- [ ] **Step 1: `WorkflowGraphRepository.node`** - - The OR predicate `(RunGraphNode.id == task_id) | (RunGraphNode.definition_task_id == task_id)` - from PR 2 collapses to `task_id == task_id` once `definition_task_id` - and the separate `id` column are gone (Task 1 schema collapse). Does - `node()` still earn its existence as a method, or does it shrink to - `session.get(RunGraphNode, (run_id, task_id))` + `await Task.from_definition(...)`? - If it stays, is "node" still the right name when the returned shape - is a `RunGraphNodeView` carrying an inflated `Task`? - -- [ ] **Step 2: `TaskExecutionService.prepare` / `_prepare_run_node`** - - PR 3 collapsed the static/dynamic branch into a single - `_prepare_run_node`. After PR 11 deletes `_prepare_legacy_definition`, - the service has one method. Does the service class still earn its - existence, or does `_prepare_run_node`'s body inline into - `worker_execute`? - -- [ ] **Step 3: Sandbox lifecycle indirection** - - PR 4 made `worker_execute` the sole owner of `sandbox.acquire` and - `sandbox.terminate` (via `lifecycle_hub` or whatever wrapper exists - today). After Task 1 deletes `terminate_sandbox_by_id`, is the wrapper - layer still earning anything, or is it now a thin pass-through to - `Sandbox.provision()` and `Sandbox.terminate()`? If thin, inline it. - -- [ ] **Step 4: `Sandbox.terminate` vs `Sandbox.detach` naming** - - Both survive PR 11 by design (lifecycle owner uses `terminate`, eval - workers use `detach`). Run a final check: is the difference named - clearly enough that a new reader doesn't have to read the docstring to - know which to call? Grep for callsites and confirm the right one is - used at each. If `detach` is ever wrong, consider renaming for clarity - (e.g. `release_local_handle`). - -- [ ] **Step 5: `RunGraphNodeView`** - - PR 2 introduced the view carrying both `node_id` and `task_id` during - the transition. After Task 1 drops `node_id` and `definition_task_id`, - the view collapses to `(run_id, task_id, parent_task_id, status, task, is_dynamic)`. - Does it still earn a separate type, or does it become - `RunGraphNode` + an inflated `Task` accessor on the row itself? - -- [ ] **Step 6: `RunTaskExecution.attempt_number`** - - After PR 4 the canonical retry id is `execution_id`. Does - `attempt_number` add anything `execution_id` ordering doesn't already - carry, or is it duplicate state? If duplicate, drop in this PR (it's - a small additional column drop). - -- [ ] **Step 7: `EvaluationService.evaluate` / `persist_success` / `persist_failure` split** - - The split made sense when `CriterionExecutor` did the running and - `EvaluationService` did the persisting. PR 4 deleted the executor; - the reshaped `evaluate_task_run` calls `evaluator.evaluate(...)` and - `EvaluationService.persist_*` directly. Is the persist/success/failure - three-method shape still right, or does it collapse to one - `persist(result_or_exc)` method? - -- [ ] **Step 8: `WorkerContext` curated surface** - - PR 9 routed `spawn_task` through the graph-native path. Re-verify the - curated method set in `01-api-surface.md` against the implementation: - any methods that became no-ops, any methods now missing. - -- [ ] **Step 9: `Worker.from_definition` vs `Worker(...)` author surface** - - PR 5 makes authors construct workers via `Worker(...)`. Is - `from_definition` discoverable as framework-internal? Is there - guidance / a docstring that prevents future authors from reaching for - it the way the v1 audit found people doing with `from_buffer`? - -- [ ] **Step 10: `PreparedTaskExecution` itself** - - After Task 1 drops `node_id`, `definition_task_id`, `worker_type`, - `assigned_worker_slug`, `model_target`, the DTO is down to ~6 fields, - all of which the caller already has. Does the DTO still earn - marshaling, or does `TaskExecutionService.prepare` return a tuple or - call directly into `worker_execute`? - -- [ ] **Step 11: Single-use private helper sweep** - - The previous ten candidates are named ahead of time. This step is - *generative*: run a one-shot audit script against the post-deletion - tree and add each finding as a row in the decision table. - - The principle: a private helper (`_foo`) whose body is ≥ 6 lines and - has exactly one call site, both in the same module, *may* be a - premature abstraction left over from when the v2 cutovers were - in-flight. The candidate is not guilty by default — many extractions - are legitimate (testability, naming, branch isolation). The audit - forces a decision rather than presuming one. - - **Run the audit:** - - ```bash - uv run python scripts/single_use_helper_audit.py \ - --paths ergon_core/ergon_core ergon_builtins/ergon_builtins ergon_cli/ergon_cli \ - --min-body-lines 6 \ - --exclude-decorators inngest.function,pytest.fixture,property,classmethod,staticmethod,model_validator,field_validator,model_serializer \ - --output /tmp/single_use_helpers.csv - ``` - - The script lives at `scripts/single_use_helper_audit.py` and is - committed as part of this PR's Task 5 work (small, self-contained, - AST-based). Its output is `(function, defining_file:line, caller_file:line, body_size)`, - one row per candidate. - - **What the script must skip** (avoid the false-positive categories - the policy explicitly carves out): - - - Decorator-registered callables: `@inngest.function`, `@pytest.fixture`, - `@pytest.mark.*`, `@property`, `@classmethod`, `@staticmethod`, - `@model_validator`, `@field_validator`, `@model_serializer`, - `@cached_property`, `@functools.cache`. - - Test files (path-excluded under `tests/`). - - Functions in `__init__.py` re-export modules. - - Functions whose call site is `__all__ = [..., "_foo", ...]` (export - list). - - **Triage:** - - For each surviving row, decide KEEP / INLINE / RENAME and add it to - the audit decision table with the same shape as the named candidates: - - ```markdown - | 11.a | `_compute_default_evaluator_index` (jobs/worker_execute.py:142) | INLINE | 8-line helper called once 30 lines below; inlining brings the constant next to its only user. | - | 11.b | `_validate_definition_handle` (experiments/launch.py:88) | KEEP | Single use today but the validation rules are explicitly subject to unit tests in test_launch_validation.py. | - | 11.c | `_persist_eval_payload` (telemetry/repository.py:156) | RENAME → `persist_evaluation` | Single use but the name is private-ish for no reason; promote and document. | - ``` - - **Authors can preempt the audit** by annotating helpers with a - `# locality: <reason>` comment at the definition site (mirrors the - `# typing:` exemption pattern from § 0.6). The script skips any - function whose def-line or the line above carries `# locality:`. Use - this for genuinely-single-use helpers that exist for testability, - naming clarity, or other documented reasons. - - **Expected scale:** A first-run audit of the v1 tree surfaces - perhaps 20–40 candidates. After this sweep, the count should stay - near zero unless future PRs reintroduce premature abstractions. - -### How to record the outcome - -For each candidate above, the PR description **must** include a -Markdown decision table: - -```markdown -## Structural Simplification Audit - -| # | Candidate | Decision | Reason | -|---|---|---|---| -| 1 | `WorkflowGraphRepository.node` | KEEP | Still earns its existence after the OR-predicate collapses — typed view + inflated Task in one method. | -| 2 | `TaskExecutionService.prepare` / `_prepare_run_node` | INLINE | One method, one branch, no service-level concern; inlined into `worker_execute`. | -| 3 | Sandbox lifecycle indirection | KEEP | `lifecycle_hub` still owns the per-run cleanup backstop. | -| ... | ... | ... | ... | -``` - -Rules: - -- **Every named candidate (1-10) and every Step 11 audit row must - appear** in the table. No omissions; a missing row blocks merge. The - Step 11 rows are numbered `11.a`, `11.b`, ... — one per surviving - audit finding. -- **Decision must be one of: KEEP / RENAME / INLINE / SPLIT.** Use a - single value — no "MAYBE", no "DEFER", no compound values. -- **Reason is one sentence.** Long rationale belongs in the commit - message or a comment in the code; the table is the at-a-glance - artifact. -- **INLINE / SPLIT / RENAME decisions ship in this PR.** A KEEP - candidate stays unchanged (and Step 11 KEEP entries should gain a - `# locality: <reason>` comment at the def site so future audits skip - them). -- **A deferred decision is not allowed** — the v2 program's exit - criterion includes "no half-finished structural decisions." - -The reviewer's job during merge: read the table, spot-check two or -three candidates against the code, and confirm the decision is the one -the table claims. For Step 11 rows, the reviewer can verify the audit -script's output matches the table by re-running the script. - -A code-reviewer dispatch can do this in one pass: - -```text -Dispatch: superpowers:code-reviewer -Scope: PR 11's structural simplification audit (see § Task 5) -Candidates: [paste the 10 candidates above] -Output: for each, KEEP / RENAME / INLINE / SPLIT with one-sentence reason -``` - -## Task 6: Run Verification - -```bash -uv run pytest ergon_core/tests/unit/architecture -q -uv run pytest ergon_core/tests/unit/api -q -uv run pytest ergon_core/tests/unit/runtime -q -uv run pytest ergon_builtins/tests/unit -q -uv run pytest ergon_cli/tests -q -``` - -After PR 11 these expectations hold: - -- `test_v2_final_state_ledger.py`: every parametrized case PASS; the - "no xfails remain" guard PASS. -- `test_dead_path_audit.py`: every parametrized case PASS; the "no dead - paths xfailed" guard PASS. -- `test_no_type_circumventors.py`: every production-code hit either - carries a `# typing:` exemption or no hits remain; the - `test_no_known_type_circumventor_exemptions_remain` guard PASS. -- `test_repository_layer_conventions.py`: every case PASS; the - `test_no_repository_violators_remain` guard PASS. -- `test_repository_companion_files.py`: every case PASS. -- `test_no_dead_repository_methods.py`: every case PASS; the - `test_no_dead_repository_methods_remain` guard PASS. -- `test_walkthrough_smoketest.py`: every case PASS, no XFAIL. -- `test_identity_invariants.py`: every case PASS, no XFAIL. -- `test_no_deleted_v2_symbols.py` (Task 3): PASS. -- `test_single_alembic_head.py` (Task 2): PASS. - -If any of these still report XFAIL/XPASS, the deletion gate has not -truly closed and PR 11 is not ready to merge. - -## PR Ledger - -Invariant landed: all v2 transition bridges are gone; the run-tier read -boundary is structurally enforced (no fallback code exists); both -xfail-ledger dicts are empty; the structural simplification audit -recorded a keep/rename/inline/split decision for each of the 10 -candidates. - -Bridge code introduced: none. - -Old path still intentionally alive: none. - -Deletion gate: this PR is the deletion gate. Completion bar is the -"no xfails remain" assertions in `test_v2_final_state_ledger.py` and -`test_dead_path_audit.py`. - -Tests added or updated: deleted-symbol guards, single-Alembic-head -guard, and the two completion-bar `test_no_*_are_still_xfailed` -assertions. - -Modules owned by this PR: cleanup across all lanes. - -PR description must include the structural-audit outcome table from -Task 5 — one line per candidate with the decision (KEEP / RENAME / -INLINE / SPLIT) and one-sentence reason. Reviewers can quickly verify -the audit was actually run by checking that table. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/13-pr-12-walkthrough-ci.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/13-pr-12-walkthrough-ci.md deleted file mode 100644 index d816f7910..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/13-pr-12-walkthrough-ci.md +++ /dev/null @@ -1,701 +0,0 @@ -# PR 12 — Walkthrough Integration And CI Hardening - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. - -**Goal:** Prove the canonical walkthrough runs end to end in the final v2 -shape and wire the guard suite into CI. - -**Architecture:** The integration test mirrors `04-walkthrough.md` directly: -author code creates an experiment, definition persists, run prepares graph -nodes, worker execution fans out criteria via `ctx.step.invoke`, every eval completes, sandbox releases, and the run -settles. - -**Tech Stack:** pytest integration markers, Postgres test container, fake -Sandbox runtime, synchronous Inngest test driver. - ---- - -## Post-PR-8 audit reconciliation - -The post-PR-8 drift audit (`_post-pr8-drift-audit.md`) assigned these items to -this PR. The first two are infrastructure prerequisites that probably warrant -either a Task 0 inside this PR or a separate "PR 12-prep" micro-PR. - -1. **Re-export `launch_run` from `ergon_core.api`.** Plan imports - `from ergon_core.api import launch_run`; the function actually lives at - `ergon_core.core.application.experiments.launch.launch_run` and is NOT - re-exported from the public API package. Either add the re-export in - `ergon_core/api/__init__.py` OR update every plan reference to use the - internal import path. The re-export option is cleaner and matches the - `persist_benchmark` public-API placement. -2. **Scaffold the synchronous Inngest test driver.** Plan assumes a driver - with `inngest_driver.run_until_terminal()`, - `step_invocations_for_function()`, `events_for()` methods exists. - Audit confirmed **no such driver exists** in the codebase. This is a - substantial prerequisite — needs research into Inngest's test-mode - client / `inngest.SDK()` and a wrapper layer in - `ergon_core/tests/integration/conftest.py`. Treat as Task 0 of this PR - OR extract into a "PR 12-prep" micro-PR. Either way, this MUST land - before Task 1 (Integration Fixtures) can be written. -3. **Decide on `read_run()` vs `read_run_state()`.** Plan calls - `await read_run(run_id)`. Audit confirmed only `read_run_state()` exists. - Either alias the existing helper as `read_run` OR update the plan to use - the existing name. Pick whichever is more consistent with the wider - read-side helper naming. -4. **Create `ergon_core/tests/integration/` directory.** Currently only - `ergon_core/tests/unit/` exists. This is a trivial mkdir + `__init__.py`, - but call it out explicitly so the first integration test commit doesn't - get blocked on a missing directory. -5. **Verify CLI commands used in the walkthrough match the post-PR-8 - surface.** Valid commands today: `ergon run status <id>`, - `ergon run list [--status] [--experiment]`, `ergon run cancel <id>`, - `ergon experiment show <UUID>`, `ergon experiment list`, - `ergon experiment tags`, `ergon experiment by-tag <tag>`. Audit any - `ergon experiment define` / `ergon experiment run` / `ergon run <benchmark>` - references — those are deleted commands. - ---- - -## Files - -**Create:** - -```text -ergon_core/tests/integration/test_walkthrough.py -ergon_core/tests/integration/conftest.py -ergon_core/tests/unit/regression/test_v1_audit_findings.py -``` - -**Modify:** - -```text -pyproject.toml -.github/workflows/ci.yml -ergon_core/tests/unit/architecture/ -``` - -## Task 1: Integration Fixtures - -**Files:** - -- Create: `ergon_core/tests/integration/conftest.py` - -- [ ] **Step 1: Add fake sandbox** - -```python -class FakeSandboxRuntime: - sandbox_id = "fake-sandbox" - - def __init__(self) -> None: - self.files: dict[str, bytes] = {} - self.commands: list[str] = [] - - async def run_command(self, cmd, *, timeout=None): - self.commands.append(" ".join(cmd) if isinstance(cmd, list) else cmd) - return CommandResult(exit_code=0, stdout="ok", stderr="") - - async def write_file(self, path: str, content: bytes) -> None: - self.files[path] = content - - async def read_file(self, path: str) -> bytes: - return self.files[path] - - async def list_files(self, path: str) -> list[str]: - return sorted(p for p in self.files if p.startswith(path)) - - async def close(self) -> None: - pass -``` - -- [ ] **Step 2: Add fake sandbox subclass** - -```python -class FakeSandbox(Sandbox): - async def provision(self) -> None: - object.__setattr__(self, "_runtime", FakeSandboxRuntime()) -``` - -- [ ] **Step 3: Add deterministic worker and criterion** - -`FakeSandbox` already exposes the runtime; add a worker that writes a file -and a criterion that reads it through `task.sandbox`: - -```python -from ergon_core.api.benchmark.task import Task -from ergon_core.api.criterion.criterion import Criterion -from ergon_core.api.criterion.context import CriterionContext -from ergon_core.api.criterion.result import CriterionResult -from ergon_core.api.worker.worker import Worker -from ergon_core.api.worker.context import WorkerContext -from ergon_core.api.worker.results import WorkerOutput - - -class FileDropWorker(Worker): - type_slug = "test-file-drop" - - output_filename: str = "result.txt" - output_content: str = "ok" - - async def execute(self, task: Task, *, context: WorkerContext): - await task.sandbox.write_file( - f"{task.sandbox.output_path}{self.output_filename}", - self.output_content.encode(), - ) - yield WorkerOutput( - final_text=self.output_content, - artifacts=({"path": self.output_filename},), - ) - - -class FileExistsCriterion(Criterion): - type_slug = "test-file-exists" - - expected_filename: str = "result.txt" - - async def evaluate(self, ctx: CriterionContext) -> CriterionResult: - files = await ctx.task.sandbox.list_files(ctx.task.sandbox.output_path) - passed = any(f.endswith(self.expected_filename) for f in files) - return CriterionResult( - passed=passed, - score=1.0 if passed else 0.0, - reason=f"file {self.expected_filename} present: {passed}", - ) -``` - -- [ ] **Step 4: Wire sandbox lifecycle counters** - -The walkthrough needs to assert that acquire/release pair up. Add an -instrumented hub fixture that replaces `SandboxLifecycleHub` for the test -session: - -```python -import pytest -from ergon_core.core.infrastructure.sandbox.lifecycle import SandboxLifecycleHub - - -class CountingSandboxHub(SandboxLifecycleHub): - """Real hub semantics with acquire/release counters and ordering log.""" - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.acquire_count = 0 - self.release_count = 0 - self.events: list[tuple[str, str]] = [] # (event, sandbox_id) - - async def acquire(self, sandbox): - result = await super().acquire(sandbox) - self.acquire_count += 1 - self.events.append(("acquire", result.sandbox_id)) - return result - - async def release(self, sandbox): - await super().release(sandbox) - self.release_count += 1 - self.events.append(("release", sandbox.sandbox_id)) - - -@pytest.fixture -def sandbox_hub(monkeypatch): - hub = CountingSandboxHub() - monkeypatch.setattr( - "ergon_core.core.infrastructure.sandbox.lifecycle.SandboxLifecycleHub", - lambda *a, **kw: hub, - ) - return hub - - -@pytest.fixture -def evaluation_log(): - """Captures the per-criterion persistence calls so the walkthrough can - assert that sandbox release happens AFTER criterion persistence.""" - - return [] - - -@pytest.fixture(autouse=True) -def _capture_criterion_persist(monkeypatch, evaluation_log): - from ergon_core.core.application.evaluation import service as eval_service - - original = eval_service.EvaluationService.persist_success - - async def _wrapped(self, *args, **kwargs): - result = await original(self, *args, **kwargs) - evaluation_log.append(("criterion_persisted", kwargs.get("task_id"))) - return result - - monkeypatch.setattr( - eval_service.EvaluationService, "persist_success", _wrapped - ) -``` - -## Task 2: Happy Path Walkthrough - -**Files:** - -- Create: `ergon_core/tests/integration/test_walkthrough.py` - -- [ ] **Step 1: Build four-task benchmark** - -Mirror `04-walkthrough.md`: research -> code -> review -> summarize, each -with a fake worker, fake sandbox, and one rubric. - -- [ ] **Step 2: Persist and launch** - -```python -handle = persist_benchmark(benchmark) -result = await launch_run(handle.definition_id) -``` - -(`persist_benchmark` is the module-level function in `ergon_core.api`; -post PR 6.5 it takes a `Benchmark` instance directly — `name`, -`description`, `metadata` come from `Benchmark.__init__`. The variable -name `experiment` from earlier drafts of this plan should be `benchmark` -throughout.) - -- [ ] **Step 3: Drive jobs synchronously** - -Use the local test driver to run prepare, worker-execute, and advance until -the run is terminal. - -- [ ] **Step 4: Assert final state** - -```python -assert run.status == "succeeded" -assert all(node.status == "completed" for node in nodes) - -# Acquire/release pair up: one of each per task in the four-task walkthrough. -assert sandbox_hub.acquire_count == 4 -assert sandbox_hub.release_count == 4 - -# Release must happen AFTER every evaluate_task_run invocation completes. -# Sandbox release is now event-driven: `execute_task` emits -# `task/completed` only after `_fan_out_evaluators` returns, and the -# sibling `sandbox_cleanup_on_completed_fn` Inngest function calls -# `terminate_sandbox_by_id` in response. Drive the test fixture to -# observe both: -for node in nodes: - task_id = node.task.task_id - n_evals = len(node.task.evaluators) - sandbox_id = node.execution.sandbox_id # stamped by sandbox_setup - - eval_completions = [ - i for i, (kind, key) in enumerate(combined_log) - if kind == "eval_complete" and key == task_id - ] - completed_emit_pos = next( - i for i, (kind, key) in enumerate(combined_log) - if kind == "emit:task/completed" and key == task_id - ) - sandbox_terminate_pos = next( - i for i, (kind, key) in enumerate(combined_log) - if kind == "sandbox_terminate" and key == sandbox_id - ) - - assert len(eval_completions) == n_evals, ( - f"task {node.task.task_slug}: expected {n_evals} eval completions, " - f"got {len(eval_completions)}" - ) - # Every eval invocation must complete before task/completed is emitted. - for eval_pos in eval_completions: - assert eval_pos < completed_emit_pos, ( - f"task {node.task.task_slug}: task/completed emitted before " - f"an evaluate_task_run invocation finished" - ) - # And sandbox termination must come AFTER task/completed (it's - # gated on the event by sandbox_cleanup_on_completed_fn). - assert completed_emit_pos < sandbox_terminate_pos, ( - f"task {node.task.task_slug}: sandbox terminated before " - f"task/completed was emitted — the sibling cleanup function " - f"is firing too early" - ) - -# Also: every step.invoke must have been awaited (synchronous fanout). -step_invokes = inngest_driver.step_invocations_for_function("execute_task") -for invocation in step_invokes: - n_evals = len(invocation.task.evaluators) - assert len(invocation.step_invokes) == n_evals - assert all(s.completed for s in invocation.step_invokes) -``` - -The ordering check is the load-bearing v1-audit regression: in v1, -sandbox release happened in `check_evaluators` before -`evaluate_task_run` finished. **PR 4's first attempt** put release in -`execute_task`'s `try/finally` (which fired on every `step.invoke` -suspension — broke smoke). **The shipped v2 fix** is event-driven: -`execute_task` emits `task/completed` only after `_fan_out_evaluators` -returns (synchronous fanout via `ctx.group.parallel`); the sibling -`sandbox_cleanup_on_completed_fn` in -`core/application/jobs/sandbox_cleanup.py` terminates the sandbox in -response to that event. The PR 4 try/finally is **gone**; do not assert -on it. - -## Task 3: Required Variants - -- [ ] **Failure cascade** - -```python -@pytest.mark.asyncio -async def test_walkthrough_failure_cascade( - walkthrough_factory, - sandbox_hub, - inngest_driver, -): - benchmark = walkthrough_factory( - # Override task-2's worker to raise; other tasks unchanged. - overrides={"code": _FailingWorker(name="code", model=None)}, - ) - - handle = persist_benchmark(benchmark) - result = await launch_run(handle.definition_id) - await inngest_driver.run_until_terminal(result.run_ids[0]) - - final = await read_run(result.run_ids[0]) - assert final.status == "failed" - - nodes_by_slug = {n.task.task_slug: n for n in final.nodes} - assert nodes_by_slug["research"].status == "completed" # ran first - assert nodes_by_slug["code"].status == "failed" # raised - # Spawn descendants of `code` cascade-fail: - for spawn_child in nodes_by_slug["code"].spawn_children: - assert spawn_child.status == "failed" - # `review` is a dependency-dependent of `code`, NOT a spawn descendant. - # It must stay PENDING per the workshop "Failure semantics" lock. - assert nodes_by_slug["review"].status == "pending" - # `summarize` has no dependency on `code` → keeps running. - assert nodes_by_slug["summarize"].status in ("completed", "running") - - # Sandbox release still pairs with acquire even on failure. - assert sandbox_hub.acquire_count == sandbox_hub.release_count -``` - -- [ ] **Dynamic spawn** - -```python -@pytest.mark.asyncio -async def test_walkthrough_dynamic_spawn( - walkthrough_factory, - inngest_driver, - session_factory, -): - benchmark = walkthrough_factory( - overrides={"research": _SpawningWorker(name="research", model=None)}, - ) - - handle = persist_benchmark(benchmark) - result = await launch_run(handle.definition_id) - await inngest_driver.run_until_terminal(result.run_ids[0]) - - final = await read_run(result.run_ids[0]) - spawned = [n for n in final.nodes if n.task.task_slug == "research-followup"] - assert len(spawned) == 1 - spawned_node = spawned[0] - assert spawned_node.is_dynamic is True - - # No definition row was created for the dynamic task. - with session_factory() as session: - rows = session.exec( - "SELECT 1 FROM experiment_definition_tasks WHERE task_slug = :s", - {"s": "research-followup"}, - ).all() - assert rows == [], "dynamic spawn must not create definition rows" - - # The dynamic node was loaded via the same graph_repo.node path used - # by static tasks — assert worker_execute was invoked with task_id = - # spawned_node.task_id (the run-tier identity), NOT a definition_task_id. - invocations = inngest_driver.events_for("worker_execute") - spawn_invocation = next( - e for e in invocations if e.payload["task_id"] == str(spawned_node.task_id) - ) - assert spawn_invocation.payload.get("definition_task_id") is None -``` - -- [ ] **Restart** - -```python -@pytest.mark.asyncio -async def test_walkthrough_restart_after_success( - walkthrough_factory, - sandbox_hub, - inngest_driver, - session_factory, -): - benchmark = walkthrough_factory() - handle = persist_benchmark(benchmark) - result = await launch_run(handle.definition_id) - await inngest_driver.run_until_terminal(result.run_ids[0]) - - initial = await read_run(result.run_ids[0]) - research_node = next(n for n in initial.nodes if n.task.task_slug == "research") - first_execution_id = research_node.latest_execution_id - initial_acquires = sandbox_hub.acquire_count - - # Trigger restart through the same management entrypoint a worker would. - await TaskManagementService().restart_task( - run_id=initial.run_id, - task_id=research_node.task.task_id, - ) - await inngest_driver.run_until_terminal(initial.run_id) - - after = await read_run(initial.run_id) - refreshed = next(n for n in after.nodes if n.task.task_slug == "research") - - # Fresh execution row with a new id. - assert refreshed.latest_execution_id != first_execution_id - - # Old execution row preserved for audit. - with session_factory() as session: - rows = session.exec( - "SELECT id FROM run_task_executions WHERE node_id = :n ORDER BY started_at", - {"n": str(research_node.id)}, - ).all() - assert len(rows) >= 2 - assert str(first_execution_id) in {str(r[0]) for r in rows} - - # New sandbox acquire/release pair landed. - assert sandbox_hub.acquire_count == initial_acquires + 1 - assert sandbox_hub.release_count == sandbox_hub.acquire_count -``` - -`_FailingWorker` and `_SpawningWorker` are tiny test doubles defined in -`conftest.py` next to `FileDropWorker` — one raises in `execute`, the other -calls `context.spawn_task(...)` once with a `research-followup` task. - -## Task 4: Regression Net - -**Files:** - -- Create: `ergon_core/tests/unit/regression/test_v1_audit_findings.py` - -- [ ] Add one test per audit finding. Each test is named after the - finding it guards so failures map back to the audit report. - -```python -import os -import subprocess -from pathlib import Path - -import pytest -from sqlalchemy import inspect, create_engine - -ROOT = Path(__file__).resolve().parents[4] -PRODUCTION_ROOTS = ( - ROOT / "ergon_core" / "ergon_core", - ROOT / "ergon_builtins" / "ergon_builtins", - ROOT / "ergon_cli" / "ergon_cli", -) - - -def _production_grep(symbol: str) -> list[Path]: - hits: list[Path] = [] - for root in PRODUCTION_ROOTS: - for path in root.rglob("*.py"): - if "tests" in path.parts or "migrations" in path.parts: - continue - if symbol in path.read_text(): - hits.append(path) - return hits - - -# 1. No ExperimentRecord table in the final v2 schema. -def test_audit_no_legacy_experiment_records_table() -> None: - engine = create_engine(os.environ["DATABASE_URL"]) - tables = set(inspect(engine).get_table_names()) - assert "experiment_records" not in tables - - -# 2. Runtime does not read definition task rows. -def test_audit_runtime_does_not_read_definition_task_rows() -> None: - worker_execute = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/worker_execute.py" - ).read_text() - assert "DefinitionRepository" not in worker_execute - assert "ExperimentDefinitionTask" not in worker_execute - assert "task_with_instance" not in worker_execute - - -# 3. Dynamic spawn has no definition row (covered end-to-end by the -# walkthrough; this is the unit-level invariant). -@pytest.mark.asyncio -async def test_audit_dynamic_spawn_has_no_definition_row( - worker_context_factory, run_graph_factory, session -): - from ergon_core.api.benchmark.task import Task - from sqlmodel import select, func - from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinitionTask, - ) - from tests.unit.runtime._test_workers import EchoWorker, EchoSandbox - - graph = run_graph_factory(nodes=[("root", None)]) - ctx = worker_context_factory(run_id=graph.run_id, task_id=graph.task_id("root")) - before = session.exec(select(func.count()).select_from(ExperimentDefinitionTask)).one() - await ctx.spawn_task( - Task( - task_slug="c", - instance_key="i", - description="d", - worker=EchoWorker(name="e", model=None), - sandbox=EchoSandbox(), - evaluators=(), - ) - ) - after = session.exec(select(func.count()).select_from(ExperimentDefinitionTask)).one() - assert after == before - - -# 4. Sandbox cleanup lives in a sibling Inngest function (PR 4 fix — -# the earlier draft of this guard expected an inline `try/finally` -# inside worker_execute or execute_task; that pattern fired -# `terminate_sandbox_by_id` on every Inngest `step.invoke` suspension -# because Inngest raises `ResponseInterrupt` (a `BaseException`) to -# suspend coroutines, which fires Python's `finally`. The shipped fix -# moves cleanup to a sibling function triggered by terminal events.) -def test_audit_sandbox_cleanup_is_event_driven() -> None: - execute_task_body = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/execute_task.py" - ).read_text() - cleanup_body = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/sandbox_cleanup.py" - ).read_text() - handler_body = ( - ROOT - / "ergon_core/ergon_core/core/infrastructure/inngest/handlers/sandbox_cleanup.py" - ).read_text() - - # execute_task must NOT terminate inline (the broken PR 4 try/finally). - assert "terminate_sandbox_by_id(task_sandbox_id)" not in execute_task_body, ( - "execute_task must NOT call terminate_sandbox_by_id inline; " - "the sibling sandbox_cleanup function does it on terminal events." - ) - # The sibling cleanup job must call terminate_sandbox_by_id. - assert "terminate_sandbox_by_id" in cleanup_body - # And its handlers must trigger on both terminal task events. - assert 'event="task/completed"' in handler_body - assert 'event="task/failed"' in handler_body - # Synchronous evaluator fanout still lives in execute_task. - assert "ctx.group.parallel" in execute_task_body - assert "evaluate_task_run" in execute_task_body - - -# 5. evaluate_task_run is registered and uses the thin id-only payload. -# (Δ.4 reshapes it; it is NOT deleted.) -def test_audit_evaluate_task_run_uses_thin_payload() -> None: - # Identity-based registration check — no attribute introspection. - # If Inngest changes the slug attribute name across versions, this - # test stays green so long as the function is in the registered - # set. - from ergon_core.core.application.jobs.evaluate_task_run import ( - evaluate_task_run, - ) - from ergon_core.core.infrastructure.inngest.registry import ALL_FUNCTIONS - - assert evaluate_task_run in ALL_FUNCTIONS, ( - "evaluate_task_run is the per-evaluator fanout target; it must " - "remain registered in ALL_FUNCTIONS." - ) - - body = ( - ROOT - / "ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py" - ).read_text() - assert "TaskEvaluateRequest" in body - assert "EvaluateTaskRunRequest" not in body # v1 payload class is gone - # No definition-tier reads: - assert "DefinitionRepository" not in body - assert "ExperimentDefinitionTask" not in body - # Same run-tier load path as worker_execute: - assert "WorkflowGraphRepository" in body - - -# 6. Single Alembic head. -def test_audit_single_alembic_head() -> None: - result = subprocess.run( - [ - "uv", - "run", - "alembic", - "-c", - str(ROOT / "ergon_core" / "alembic.ini"), - "heads", - ], - check=True, - capture_output=True, - text=True, - ) - head_lines = [ln for ln in result.stdout.strip().splitlines() if ln.strip()] - assert len(head_lines) == 1 - - -# 7. No saved_specs / from_buffer / CriterionExecutor / EvaluateTaskRunRequest -# in production code. (evaluate_task_run as a name DOES survive — see #5.) -@pytest.mark.parametrize( - "symbol", - [ - "saved_specs", - "from_buffer", - "CriterionExecutor", - "InngestCriterionExecutor", - "EvaluateTaskRunRequest", - ], -) -def test_audit_deleted_symbol_absent(symbol: str) -> None: - hits = _production_grep(symbol) - assert hits == [], f"{symbol} still appears in: {hits}" - - -# 8. CLI define routes through persist_benchmark (PR 6.5 renamed). -def test_audit_cli_define_calls_persist_benchmark() -> None: - body = (ROOT / "ergon_cli/ergon_cli/commands/experiment.py").read_text() - assert "persist_benchmark" in body - assert "persist_definition" not in body # PR 6.5 renamed - assert "define_benchmark_experiment" not in body - assert "ExperimentDefineRequest" not in body -``` - -## Task 5: CI - -**Files:** - -- Modify: `.github/workflows/ci.yml` -- Modify: `pyproject.toml` - -- [ ] Register `integration` pytest marker. -- [ ] Add architecture/regression CI job: - -```bash -uv run pytest ergon_core/tests/unit/architecture ergon_core/tests/unit/regression -q -``` - -- [ ] Add walkthrough CI job when Postgres service is available: - -```bash -uv run pytest -m integration ergon_core/tests/integration/test_walkthrough.py -q -``` - -## Task 6: Final Verification - -```bash -uv run pytest ergon_core/tests/unit/architecture -q -uv run pytest ergon_core/tests/unit/regression -q -uv run pytest -m integration ergon_core/tests/integration/test_walkthrough.py -q -``` - -## PR Ledger - -Invariant landed: final v2 behavior matches the canonical walkthrough. - -Bridge code introduced: none. - -Old path still intentionally alive: none. - -Deletion gate: all gates closed before this PR. - -Tests added or updated: walkthrough integration, v1 audit regression net, -CI jobs. - -Modules owned by this PR: tests and CI. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_post-pr8-drift-audit.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_post-pr8-drift-audit.md deleted file mode 100644 index 05cb68604..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_post-pr8-drift-audit.md +++ /dev/null @@ -1,281 +0,0 @@ -# Post-PR-8 Drift Audit (2026-05-16) - -After PR 8 (lifecycle CLI cleanup) landed, this audit checks whether the -remaining v2-stack plans (PR 9, 10a/b/c, 11, 12) still reflect the current -state of the code, or whether they have drifted in the same way PR 8's plan -had (which required ~290 lines of reconciliation before implementation). - -> **Reconciliation status:** each drift item below has been assigned to a -> specific PR plan. Every plan in `10-pr-09-*.md` through `13-pr-12-*.md` now -> has a `## Post-PR-8 audit reconciliation` section at the top listing the -> items it owns. Implementers picking up any of those PRs should read that -> section first — it patches over the drift before the existing task list -> becomes safe to execute. - -**Methodology:** six parallel Explore agents, one per plan, each given the -plan path plus a checklist of high-yield drift sources (file paths, class -existence, sync/async signatures, public API shape, migration ids, deletion -target status). Reports were then triaged here to separate *genuine drift* -from *current-state-as-starting-point* (where the agent flagged what a -PR proposes to do *as if* it were already done). - -## TL;DR - -| PR | Genuine drift verdict | Blocking? | Reconciliation effort | -|----|----|----|----| -| 9 (dynamic subtasks) | **MAJOR** | Yes — frozen WorkerContext contradicts RFC | Medium — unfreeze + add PrivateAttr | -| 10a (SWEBench) | **MINOR** | No — mostly "this is the work" | Light — one prerequisite extraction | -| 10b (ResearchRubrics) | **MEDIUM** | Yes — Criterion base class wrong shape | Medium — micro-PR to convert ABC → BaseModel + ABC | -| 10c (GDPEval) | **MINOR** | No — mostly "this is the work" | Light | -| 11 (deletion gate) | **MINOR** | No — but only valid after 9/10a-c | Light — sync xfail names | -| 12 (walkthrough CI) | **MAJOR** | Yes — missing infra | Medium — needs test driver + exports | - -Three plans need real reconciliation work before implementation: -- **PR 9** — both design questions resolved (Path A: unfreeze `WorkerContext`). -- **PR 10b** — design question resolved (Path A: `Criterion` becomes `BaseModel + ABC`). -- **PR 12** — needs Inngest test driver scaffolding; no design call, just infra. - -The benchmark migration plans (10a/10c) and deletion gate (11) are mostly fine -but have a handful of name/signature drift items that should be corrected -in-place when each PR is picked up. - -### Design calls resolved against RFC (`/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/`) - -1. **PR 9 `WorkerContext` mutability** → RFC `03-runtime.md` (lines 299–313, 457–501) - and `01-api-surface.md` (lines 288–451): `WorkerContext(BaseModel)` with - `PrivateAttr` for `_task_mgmt` / `_task_inspect` / `_resource_repo`. **NOT - frozen.** Framework injects via `_for_job` classmethod using - `object.__setattr__`. The current `model_config = {"frozen": True}` is itself - drift that needs to be undone. -2. **PR 10b `Criterion` base class** → RFC `01-api-surface.md` line 120 and lines - 1030–1082: `class Criterion(BaseModel, ABC)`. Mandatory because all public - types follow the `_type`-discriminator serialization pattern. Current pure - ABC shape is drift; conversion needs to happen before PR 10b lands. - ---- - -## PR 9 — Dynamic Subtasks (MAJOR drift) - -### What the plan assumes that isn't true - -| Plan says | Reality | -|---|---| -| `WorkerContext` is mutable with injected `_task_mgmt` / `_task_inspect` services | `WorkerContext` is a **frozen Pydantic BaseModel** (`model_config = {"frozen": True}`) | -| `WorkerContext.spawn_task(task, depends_on=()) → SpawnedTaskHandle` | Method doesn't exist; `SpawnedTaskHandle` doesn't exist | -| `ContainmentViolation` error in `api/errors.py` | Doesn't exist | -| `TaskManagementService.add_subtask(*, run_id, parent_task_id, task, depends_on)` | Actual signature: `add_subtask(session, command: AddSubtaskCommand)` — uses a command object | -| `Task` model has `created_by` field (added in PR 7) | PR 7 added `created_by` to **Benchmark**, not to Task. Misread of PR 7 scope | -| `WorkflowGraphRepository.descendants_by_parent(...)` | Method doesn't exist (the SQL CTE in the plan is correct but needs to be added) | -| `TaskInspectionService.descendant_ids(...)` | Method doesn't exist | - -### Design decision — RESOLVED via RFC - -The RFC answers this. From `03-runtime.md` §"Worker runtime API: WorkerContext" -(lines 299–313) and §"Framework-side WorkerContext construction" (lines 457–501), -plus `01-api-surface.md` §"The unifying pattern: one class per concept, runtime -as PrivateAttr" (lines 288–451): - -> `WorkerContext(BaseModel)` has public fields `(run_id, task_id, execution_id, -> definition_id)` and **PrivateAttr fields** `_task_mgmt`, `_task_inspect`, -> `_resource_repo` holding service references. The framework explicitly injects -> these via `object.__setattr__` in a `_for_job` classmethod. - -So **Path A** is the RFC-mandated direction: `WorkerContext` is a `BaseModel` -with `PrivateAttr` for services, **not** frozen. The current frozen config is -itself a drift item that needs to be removed. - -### Reconciliation work before PR 9 - -1. **Drop `model_config = {"frozen": True}`** from `WorkerContext` (it contradicts - the RFC). Add `PrivateAttr` fields for `_task_mgmt`, `_task_inspect`, and - `_resource_repo`. -2. **Add `_for_job` classmethod** on `WorkerContext` that injects service - references via `object.__setattr__` (RFC's two-phase construction pattern). -3. **Drop the `Task.created_by` assumption** — if dynamic spawns need spawner - identity, record it on the graph node or audit metadata, not on Task. -4. **Add `WorkflowGraphRepository.descendants_by_parent`** and - **`TaskInspectionService.descendant_ids`** as PR 9 tasks (the SQL CTE in the - plan is fine; just needs to be implemented, not assumed-existing). -5. **Resolve `add_subtask` signature drift** — either extend the command-object - pattern to accept a `Task` instance, or add a thin wrapper on `WorkerContext` - that builds the command. Plan should pick one and update Task 2 accordingly. - ---- - -## PR 10a — SWEBench Migration (MINOR drift) - -The audit agent flagged "SWEBench still uses TaskSpec" as drift, but that's -just the starting point PR 10a converts from. Real items: - -| Plan says | Reality | Severity | -|---|---|---| -| Shared `ManagerBackedSandboxRuntime` at `ergon_builtins/sandbox/_manager_backed.py` | Doesn't exist yet — currently inlined in `minif2f/sandbox.py` | Verify: is this PR 10a's task to extract, or assumed-already-done? | -| Worker factory pattern `make_swebench_worker()` in `workers.py` | Currently `SWEBenchReactWorker` class in `worker_factory.py` | Likely PR 10a's renaming task — verify wording | -| Toolkit is Pydantic `BaseModel` with `tools(sandbox, task)` | Currently regular class with `__init__(sandbox, workdir)`, `get_tools()` | Likely PR 10a's conversion — verify | -| Sandbox subdirectory rename to avoid module shadowing | Need: `git mv swebench_verified/sandbox swebench_verified/sandbox_template` first | Prerequisite step to call out in plan | - -### Reconciliation work before PR 10a - -Light — read the plan's Task 1 carefully and confirm it explicitly proposes -the extraction of `_ManagerBackedSandboxRuntime` from MiniF2F (audit agent -read it as if the adapter already existed; the plan likely says "create it"). -If the plan does propose creating it as part of PR 10a, treat the audit's -"missing" flag as informational, not drift. - ---- - -## PR 10b — ResearchRubrics Migration (MINOR-MEDIUM drift) - -The audit flagged real drift items that affect plan correctness: - -| Plan says | Reality | Severity | -|---|---|---| -| `JudgeCriterion` becomes a Pydantic `Criterion` subclass with `judge_model` field | Base `Criterion` is **pure ABC** (`class Criterion(ABC)`), not `BaseModel + ABC` | Real drift, design RESOLVED below | -| `ResearchRubricsBenchmark.__init__` accepts `worker_factory`/`sandbox_factory`/`evaluator_factory` kwargs | Currently takes only `limit`/`name`/`description`/`metadata` | Real — needs adding in PR 10b | -| Depends on PR 10a's `_ManagerBackedSandboxRuntime` adapter | Natural sequencing — not drift | - -### Design decision — RESOLVED via RFC - -The RFC answers this. From `01-api-surface.md` line 120 (file tree) and -§"Criterion class signature — locked [v2: locked]" (lines 1030–1082): - -> ```python -> class Criterion(BaseModel, ABC): -> type_slug: ClassVar[str] -> required_packages: ClassVar[list[str]] = [] -> -> @abstractmethod -> async def evaluate(self, context: CriterionContext) -> CriterionOutcome: ... -> -> @classmethod -> def from_definition(cls, criterion_json: TaskDefinitionJson) -> "Criterion": ... -> ``` - -So **Path A** is the RFC-mandated direction: `Criterion` must be -`class Criterion(BaseModel, ABC)` to support the `_type` discriminator -serialization pattern used across all public types. The current pure-ABC shape -is itself a drift item. - -This is **architectural** — it affects every existing `Criterion` subclass -(rubric criteria, builtins, tests). It probably warrants its own micro-PR (or -becomes Task 0 of PR 10b) rather than being snuck in as part of the -`ResearchRubricsJudgeCriterion` work. - -### Reconciliation work before PR 10b - -1. **Convert `Criterion` base from pure ABC to `class Criterion(BaseModel, ABC)`**. - Audit every existing subclass; most likely they all pass identity fields via - `__init__` today and will need to be converted to `BaseModel` field - declarations. Could be its own micro-PR before 10b lands. -2. **Add factory kwargs** (`worker_factory`, `sandbox_factory`, - `evaluator_factory`) to `ResearchRubricsBenchmark.__init__` mirroring the - MiniF2F pattern. -3. **Land PR 10a first** (natural dependency for the shared - `_ManagerBackedSandboxRuntime` adapter). - ---- - -## PR 10c — GDPEval Migration (MINOR drift) - -Same pattern as 10a — most of what the agent flagged as drift is the work the -PR proposes to do (TaskSpec → Task). Real items to verify when picking up PR 10c: - -| Plan says | Reality | Severity | -|---|---|---| -| `make_gdpeval_worker()` factory in `workers.py` | Currently `GDPEvalReactWorker(ReActWorker)` class in `worker_factory.py` | Likely the work — verify wording | -| Toolkit as Pydantic `BaseModel` | Currently regular class | Likely the work — verify wording | -| `GDPEvalBenchmark.__init__` accepts factory kwargs | Currently only takes dataset/split/limit | Real — needs adding in PR 10c | -| `GDPEvalSandbox(Sandbox)` subclass | Doesn't exist | Likely the work | -| Smoke fixture missing GDPEval row | Confirmed missing | Real — PR 10c should add | - -### Reconciliation work before PR 10c - -Light — same as 10a. Read Task 1 carefully to confirm the plan proposes the -sandbox subclass creation and factory pattern adoption as PR 10c work -(not assumed-already-done). - ---- - -## PR 11 — Deletion Final Schema (MINOR drift) - -The audit agent's "MAJOR DRIFT" verdict conflates "deletion targets still -exist" (the point of PR 11) with actual drift. Real drift items: - -| Plan says | Reality | Severity | -|---|---|---| -| `Worker.from_buffer` is a PR 11 deletion target | Already deleted | Drop from plan | -| `Worker.validate` rename to `validate_runtime_deps` is a PR 11 task | Already renamed (PR 5) | Drop from plan | -| Test xfail symbol names in `test_dead_path_audit.py` match `_worker_from_payload_bridge`, etc. | Actual xfails use different symbol names (e.g., `legacy_worker_from_payload`, `_prepare_legacy_graph_native`, `execute_task`) | Sync the symbol names | -| Plan still lists `saved_specs` package as deletion target | Still exists today; verify whether 9/10 should have removed it | Cross-check after PRs 9/10 land | - -### Reconciliation work before PR 11 - -Trivial — sync the symbol names in the plan's xfail flip checklist against -what's actually in `test_dead_path_audit.py` and `test_v2_final_state_ledger.py` -at the time PR 11 is picked up. Drop already-completed items -(`Worker.from_buffer`, `Worker.validate` rename) from the deletion list. - -The PR 11 deletion gate itself is only valid AFTER PRs 9 and 10a/b/c have -landed — that's natural sequencing, not drift. - ---- - -## PR 12 — Walkthrough CI (MAJOR drift) - -| Plan says | Reality | -|---|---| -| `from ergon_core.api import launch_run` | `launch_run` lives in `ergon_core.core.application.experiments.launch`, **not** re-exported from `ergon_core.api` | -| Synchronous Inngest test driver with `inngest_driver.run_until_terminal()`, `step_invocations_for_function()`, `events_for()` | **No such driver exists** in the codebase | -| `await read_run(run_id)` | Only `read_run_state()` exists | -| `ergon_core/tests/integration/` directory holds new tests | Directory doesn't exist (only `ergon_core/tests/unit/`) | - -### Reconciliation work before PR 12 - -Three concrete prerequisites: - -1. **Export `launch_run` from `ergon_core.api`** — either re-export it from - `ergon_core/api/__init__.py` or update every plan reference to import from - the internal module. -2. **Scaffold the synchronous Inngest test driver** — research Inngest's - testing library (likely `inngest.SDK()` or a test-mode client) and build - the wrapper helper (`run_until_terminal`, `step_invocations_for_function`, - `events_for`) in `ergon_core/tests/integration/conftest.py` BEFORE Task 1 - can run. This is a substantial prerequisite, not a quick rename. -3. **Decide on `read_run()` vs `read_run_state()`** — either alias the - existing helper, or update the plan to use the existing name. - -The plan's overall architecture (fixture-driven happy-path → variant -walkthrough) is sound; only the tooling layer is missing. - ---- - -## Recommended order of operations - -Given current state of the world (and RFC-resolved design calls): - -1. **Before starting PR 9:** unfreeze `WorkerContext` and add `PrivateAttr` fields - for services (per RFC `03-runtime.md`). Then rewrite PR 9's Task 2 / 2b / 3 - sections to match the unified pattern. ~30-60 minutes of plan editing. -2. **Before starting PR 10a:** quick read of the plan's Task 1 to confirm the - `_ManagerBackedSandboxRuntime` extraction is explicitly scoped to PR 10a - (not assumed pre-existing). Likely no edit needed. -3. **Before starting PR 10b:** convert `Criterion` ABC → `BaseModel + ABC` per - RFC `01-api-surface.md` line 1054. This touches every existing subclass and - could be its own micro-PR ("PR 10b-prep" or "Task 0 of PR 10b"). Once done, - PR 10b's `JudgeCriterion` Pydantic conversion is a one-line subclass - declaration. -4. **Before starting PR 10c:** same quick re-read as 10a; likely no edits. -5. **Before starting PR 11:** sync xfail symbol names; drop completed items - (`Worker.from_buffer`, `Worker.validate` rename). ~15 minutes. -6. **Before starting PR 12:** scaffold the Inngest test driver and re-export - `launch_run` from `ergon_core.api`. This is a meaningful prerequisite — - could be its own micro-PR or rolled into PR 12. - -Net additional reconciliation work: ~3-5 hours total, plus two micro-PRs that -could land independently: -- **micro-PR-A:** `Criterion` ABC → `BaseModel + ABC` (architectural prep for PR 10b). -- **micro-PR-B:** Inngest test driver scaffolding + `launch_run` re-export (prep for PR 12). - -The `WorkerContext` unfreeze + `PrivateAttr` work fits naturally into PR 9 -itself rather than a separate micro-PR — it's part of the dynamic-spawning -infrastructure. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_public-api-lifecycle-audit.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_public-api-lifecycle-audit.md deleted file mode 100644 index 592a62d6c..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/_public-api-lifecycle-audit.md +++ /dev/null @@ -1,238 +0,0 @@ -# Public API Lifecycle Audit Log - -Date: 2026-05-18 - -Scope: read-only audit of the public authoring API object lifecycle across -`ergon_core`, `ergon_builtins`, and tests. The audit looked for duplicate write -paths, duplicate execution paths, dead transitional paths, and places where the -v2 public API object model is not actually the runtime source of truth. - -This is an evidence log, not an implementation plan. Items marked "PR 11 -covered" are already in or adjacent to the deletion-final-schema PR. Items -marked "uncovered" need either a PR 11 checklist entry or a separate cleanup/fix -PR. - -## Executive Summary - -The duplication suspicion is well formed. The most serious issues are not just -dead code; several v2 object-bound public API paths exist but are not yet the -canonical runtime/persistence path. - -Highest-risk findings: - -1. Object-bound evaluators can be authored and serialized but never fanned out, - because execution still counts `task.evaluator_binding_keys`. -2. If evaluator fanout is fixed, evaluation persistence can still fail because - object-bound inline evaluators do not create the legacy - `ExperimentDefinitionEvaluator` rows that persistence currently requires. -3. Worker-side `Task.sandbox` may be config-only during execution; tools can bind - to a non-live sandbox unless `worker_execute` inflates the task with the live - `sandbox_id`. -4. `ReActWorker(toolkit=...)` does not rehydrate nested toolkit JSON, so - persisted v2 workers can fail at runtime before tools bind. -5. Generic `Task[Payload]` snapshots can serialize an unimportable `_type`. -6. `WorkerContext` exposes public methods that appear out of sync with current - management/inspection service APIs. -7. The registry/catalog slug path still coexists with `_type` import dispatch. - Some uses are intentional transition bridges; others are validation/routing - leftovers that need explicit keep/delete decisions. - -## Findings By Object Family - -### Criterion / Criteria - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| High | Criterion serialization is incomplete: `Criterion` has no `_type` serializer/from-definition, and nested `Criterion` fields can lose subclass fields. | `ergon_core/api/criterion/criterion.py`; `ergon_core/api/rubric/rubric.py`; `ergon_builtins/benchmarks/gdpeval/rubric.py` | No | Add `Criterion` discriminator serialization/from-definition, or ban persisted criterion objects and require authored-data rebuild patterns. | -| Medium | Two `CriterionContext` models coexist with different meanings. | `ergon_core/api/criterion/context.py`; `ergon_core/core/application/evaluation/models.py`; `_legacy_evaluator_bridge.py` imports both | Mostly | Delete or rename the internal context with `CriterionExecutor`; keep public `api.criterion.CriterionContext` as the v2 runtime context. | -| Medium | Runtime injection has two paths: object-bound sandbox runtime and legacy synthesized runtime. | `core/application/jobs/evaluate_task_run.py`; `_legacy_evaluator_bridge.py` | Yes | Remove bridge branch with TaskSpec deletion; add a guard that object-bound tasks attach live sandbox runtime when `sandbox_id` is available. | -| Medium | Duplicate criterion execution paths remain. | `EvaluationService.evaluate`; `EvaluationService.evaluate_legacy`; `InngestCriterionExecutor` | Yes | Delete legacy executor path in PR 11. | -| Low | `SandboxFileCheckCriterion` bypasses public runtime DI and connects to E2B directly. | `ergon_builtins/evaluators/criteria/sandbox_file_check.py` | No | Delete if unused; otherwise rewrite through `CriterionContext`. | -| Low | TODOs around magic runtime injection and slug/name compatibility are real API ambiguity. | `api/criterion/context.py`; `api/criterion/results.py`; telemetry `evaluation_summary.py` | No | Normalize criterion identity on `slug`; make runtime injection explicit and singular. | - -Clean checks: - -- One canonical persisted criterion outcome shape was found: - `RunTaskEvaluation.summary_json` containing `EvaluationSummary` / - `CriterionOutcomeEntry`. -- No second criterion result table or duplicate outcome row writer was found. -- Active production evaluation loops criteria inline once per evaluator; the - duplicate executor path is legacy-only and PR11-marked. - -### Rubric / Evaluator - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| Critical | Object-bound evaluators can be silently skipped. | `_fan_out_evaluators` in `core/application/jobs/execute_task.py` counts `view.task.evaluator_binding_keys`; MiniF2F/SWEBench tasks set `evaluators=(...)` without binding keys. | No | Make fanout count/select from `task.evaluators` for object-bound tasks; keep binding fallback only until PR 11 removes legacy. | -| Critical | Object-bound evaluation persistence still requires legacy evaluator rows. | `persist_benchmark` does not create `ExperimentDefinitionEvaluator` rows for inline `Task.evaluators`; `EvaluationService.persist_success/persist_failure` calls `lookup_evaluator_id`; FK is non-null. | Partially hinted | Either persist evaluator definition rows from inline evaluators, or make the FK nullable/remove lookup in the object-bound final state. | -| High | Plain `Rubric(criteria=...)` does not round-trip. | `Rubric.criteria` is `exclude=True`; `Evaluator.from_definition` just `model_validate`s dumped JSON. | Maybe related | Serialize criteria with `_type`, or make base `Rubric` non-persistable unless a subclass can rebuild criteria. | -| Medium | `EvaluationService.prepare_dispatch` is a dead v1 dispatch DTO path. | `core/application/evaluation/service.py`; callers only in tests | Not explicitly | Delete with evaluator bridge cleanup or add to PR11 ledger. | -| Low | Evaluation DTO mapping is duplicated. | `build_dashboard_evaluation_dto` and `read_models/run_snapshot.py` both map evaluation summary to dashboard DTOs. | No | Share one mapper from persisted `EvaluationSummary` to DTO. | - -Clean checks: - -- `Evaluator.model_dump` injects `_type`. -- `Task` serialization re-dumps nested evaluators with concrete subclass - schemas. -- `evaluate_task_run` prefers object-bound `task.evaluators[index]` before - legacy fallback. -- Only one canonical persisted evaluation row writer was found: - `TelemetryRepository.create_task_evaluation`. - -### Worker / WorkerContext / WorkerOutput - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| High | `WorkerContext` public facade is partially broken or stale. | `api/worker/context.py` calls service methods whose current signatures/names do not line up in `tasks/inspection.py` and `tasks/management.py`. | No | Add thin adapters or align `WorkerContext` with command/session service APIs; test every public facade method. | -| High | `ReActWorker` cannot deserialize nested `toolkit` from JSON. | `ReActWorker.toolkit: Toolkit | None`; `Worker.from_definition` directly calls `WorkerCls.model_validate(worker_json)`. Diagnostic probe hit abstract `Toolkit` instantiation. | No | Add `Toolkit.from_definition`, a field validator, or pre-validation in `Worker.from_definition`; test concrete toolkit rehydration. | -| Medium | `_resource_repo` is injected into `WorkerContext` but unused and unexposed. | `api/worker/context.py`; `worker_execute.py`; `rg _resource_repo` finds only assignments. | No | Remove injection or add explicit worker resource methods with containment semantics. | -| Medium | `persist_outputs` is marked as PR11-dead, but it is still live sandbox resource publication. | `execute_task.py`; `persist_outputs.py`; architecture dead-path audit | Misclassified | Split ledger: delete legacy WorkerOutput resource path, but keep or rename resource publication unless worker execution absorbs it. | -| Low | Dead context-derived `WorkerOutput` helper remains. | `core/application/context/output_extraction.py`; no callers except architecture test. | No | Delete after confirming no external import contract. | -| Low | `PersistOutputsResult.output_resource_ids` and `FinalizeTaskExecutionCommand.output_resource_ids` are effectively unused. | `core/application/jobs/models.py`; `execute_task.py`; `tasks/execution.py` | Adjacent | Remove stale IDs or plumb actual resource IDs if attempt-level IDs are still desired. | - -Clean checks: - -- `Worker` serialization/from-definition is coherent for the top-level worker. -- `WorkerOutput` has one live authoritative persistence path: - terminal stream item -> `WorkerOutputRepository.persist()` -> - `run_task_executions.worker_output_json`. -- Stream contract enforces exactly one terminal `WorkerOutput`. -- Sandbox file/resource publication no longer writes fake worker-output - resources. - -### Benchmark / Task - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| Critical | Object-bound inline evaluators are not launched or persisted. | MiniF2F/SWEBench create `Task(..., evaluators=(...))`; `persist_benchmark` and fanout still use `task.evaluator_binding_keys`. | No | Make object-bound tasks the source for evaluator rows and fanout. | -| High | Generic `Task[Payload]` snapshots serialize an unimportable `_type`. | `Task.model_serializer` uses `type(self).__qualname__`; MiniF2F/SWEBench instantiate `Task[Payload]`. Diagnostic probe produced an import failure for `Task[SWEBenchTaskPayload]`. | No | Serialize origin class for parametrized generics, or introduce concrete importable task subclasses. | -| High | Object-bound task worker/sandbox identity is duplicated, while execution prep still depends on legacy assignment rows. | `persist_benchmark` does not write `ExperimentDefinitionWorker` / assignment rows; graph init and prepare still derive worker metadata from those rows. | Partial | Decide whether definition worker/assignment rows are obsolete. If yes, remove read/prepare dependence; if no, populate from object-bound tasks. | -| High | ResearchRubrics and GDPEval still produce `TaskSpec`. | `ergon_builtins/benchmarks/researchrubrics/benchmark.py`; `ergon_builtins/benchmarks/gdpeval/benchmark.py` | Yes, sequencing risk | Block PR 11 deletion until PR 10b/10c migrate them to object-bound `Task`. | -| Medium | Two definition write and launch paths remain. | `persist_benchmark`; `_ExperimentDefinitionWriter.persist_definition`; `_ExperimentRunLauncher` fallback | Yes | Delete old writer/domain launch path once legacy launch is gone. | -| Medium | Two task snapshot fallback paths synthesize `TaskSpec` JSON. | `_definition_task_snapshot`; `_dynamic_task_snapshot` in graph repository | Yes | Delete both helpers and require object-bound `task_json`. | -| Medium | Dynamic subtask APIs are split. | `WorkerContext.spawn_task` writes object-bound `Task`; `SubtaskLifecycleToolkit` still creates slug/description/worker legacy nodes. | Partial | Convert tool APIs to accept/create `Task`, or retire legacy add/plan APIs. | - -Clean checks: - -- `Task` serialization tries to preserve nested worker/sandbox/evaluator - subclass fields. -- `worker_execute` prefers `task.worker`; legacy worker lookup is isolated behind - `_legacy_worker_bridge`. -- `evaluate_task_run` prefers `task.evaluators`; legacy evaluator lookup is - isolated behind `_legacy_evaluator_bridge`. -- `start_workflow` is definition-first and does not go through - `BenchmarkDefinitionRecord`. - -### Sandbox / SandboxRuntime - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| High | Worker-side v2 `Sandbox` is inflated config-only, so toolkit tools can bind to a dead sandbox. | `worker_execute` loads `graph_repo.node(...)` without `sandbox_id`; `ReActWorker` passes `task.sandbox` into tools; tools call `sandbox.run_command/write_file`. | Partial/new | In `worker_execute`, inflate task with `sandbox_id=payload.sandbox_id`, or make setup return/bind a real runtime consistently. | -| High | Acquisition still uses `BaseSandboxManager` registry, not author-defined `Sandbox.provision()`. | `sandbox_setup.py` resolves `registry.sandbox_managers` and calls `sandbox_manager.create`; public API exposes `Sandbox.provision()`. | Partial | Decide the acquisition owner. For v2, setup should load task snapshot and call `task.sandbox.provision()`, or mark manager path transitional. | -| High | Object-bound evaluation does not inject `CriterionRuntime`, while built-in criteria still call `context.ensure_sandbox/run_command/write_file`. | `evaluate_task_run` creates bare `CriterionContext`; runtime injection only happens in legacy branch; SWE/MiniF2F criteria use proxy methods. | New | Either inject runtime for object-bound contexts during transition, or migrate criteria to `context.task.sandbox` and remove proxies. | -| Medium | Authored `Sandbox.output_path` is ignored by output publishing. | Public field in `api/sandbox/sandbox.py`; publisher hard-codes `/workspace/final_output/`; `persist_outputs` never reads `task.sandbox`. | No | Thread `task.sandbox.output_path` into `SandboxResourcePublisher`, or remove the public field until honored. | -| Medium | Duplicate cleanup owners on `task/failed`. | `sandbox_cleanup.py` terminates on `task/failed`; `propagate_execution.py` also terminates on same event. | Partial | Keep termination in `sandbox_cleanup`; remove `_terminate_failed_task_sandbox` from propagation. | -| Medium | Stale run-level cleanup still tries sandbox termination via run summary. | `run_cleanup.py` reads `summary_json["sandbox_id"]` and calls `terminate_sandbox_by_id`. | Partial | Remove sandbox termination from run cleanup or constrain it to explicitly legacy records. | -| Low | Manager registry leaks creation locks. | `_creation_locks` allocated in sandbox manager; termination cleanup omits it. | No | Pop `_creation_locks[task_id]` in termination paths. | - -Clean checks: - -- `Task.from_definition(..., sandbox_id=...)` correctly attaches object-bound - sandbox runtime when used. -- Eval-side object-bound detach is scoped to the local handle and runs in - `finally`. -- Completed cleanup is gated after evaluator fanout. -- `SandboxResourcePublisher` is append-only and content-hash deduped. - -### Toolkit / Tools - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| High | `ReActWorker` nested toolkit rehydration fails. | Same as Worker finding; diagnostic rehydrate failed on abstract `Toolkit`. | No | Add toolkit discriminator rehydration and regression test. | -| High | SWE-Bench v2 editor allows path traversal outside `repo_root`. | `swebench_verified/_tools.py` builds `repo_root + "/" + path.lstrip("/")`; legacy duplicate in `_legacy_workers.py`. | Legacy duplicate yes; v2 tool no | Normalize POSIX paths and reject empty, absolute, and `..` segments. | -| Medium | Toolkit `max_tool_calls` config is exposed but unenforced. | MiniF2F/SWEBench toolkit classes define it; only round-trip tests reference it. | No | Enforce through wrappers/deps or remove/rename until real. | -| Medium | Multiple runtime-only `Toolkit` classes coexist with new Pydantic `Toolkit`. | Public `api/toolkit.py`; GDPEval/ResearchRubrics/Graph/Subtask runtime toolkit classes. | No | Rename runtime factories away from `Toolkit`, or migrate persisted benchmark-owned toolkits to the public shape. | -| Medium | Subtask lifecycle tools bypass `WorkerContext` containment facade. | TODO in `subtask_lifecycle_toolkit.py`; direct sessions/services mutate graph. | No | Rebuild around `WorkerContext` or add descendant validation before mutation/read. | -| Low | ResearchRubrics path traversal helper is duplicated. | `_workspace_path` appears in `researcher_worker.py` and `workflow_cli_react_worker.py`. | No | Move to shared ResearchRubrics path helper. | -| Low | Some SWE-Bench toolkit tests still target the old constructor/API. | `tests/integration/swebench_verified/test_toolkit.py` expects `SWEBenchToolkit(sandbox=..., workdir=...)` and `.get_tools()`. | No | Update to v2 `tools(sandbox, task)` or delete if superseded. | - -Clean checks: - -- SWE-Bench and MiniF2F author construction embeds - `ReActWorker(toolkit=...)`. -- `ReActWorker.execute()` binds `toolkit.tools(task.sandbox, task)` before - creating the pydantic-ai agent. -- `Toolkit.model_serializer` injects `_type`. - -### Registry / Serialization / Component Discovery - -This portion was audited locally after the six parallel object-family audits. - -| Severity | Finding | Evidence | PR 11 covered? | Action | -| --- | --- | --- | --- | --- | -| High | There are two component resolution models: v2 `_type` import dispatch and slug/catalog lookup. | `_serialization.import_component`; `ComponentRegistry`; `ComponentCatalogService`; legacy bridges; launch/workflow validation still use registry. | Partially | Make `_type` import dispatch canonical for object-bound runtime; list every remaining slug lookup as keep/delete. | -| Medium | `ComponentCatalogService.resolve_benchmark` and `resolve_sandbox_manager` appear unused; `resolve_evaluator` is legacy-bridge-only. | `core/application/components/catalog.py`; `rg` results | Mostly not | Delete unused methods or ledger them under PR11 bridge cleanup. | -| Medium | `ComponentRegistry.publish` / persistent catalog writes are only tested and not used by runtime outside legacy lookup. | `api/registry.py`; registry tests; catalog publish tests | Partially | Decide whether persistent component catalog survives v2. If yes, document its runtime owner; if no, delete with ComponentRegistry. | -| Medium | Slug registry still validates dynamic subtasks and workflow service even when object-bound `Task` is the target shape. | `tasks/management.py`; `workflows/service.py` | No | Replace registry slug validation with object-bound worker validation, or explicitly keep slug-only tool contracts. | -| Low | `import_component_ref` and `_row_to_ref` are free helpers with TODOs to inline. | `core/application/components/catalog.py` | No | Inline into service or keep with a locality comment. | -| Low | `_serialization.import_component` returns `type[Any]` with a TODO. | `api/_serialization.py` | No | Type it as `type[BaseModel]` or add generic/cast helpers per component class. | - -Clean checks: - -- Builtins registration is explicit and eager; no decorator scanning. -- `_type` dispatch is centralized in `_serialization.import_component`. -- Runtime job bodies are mostly protected from direct `ComponentCatalogService` - references by architecture tests; legacy bridges hold most violations. - -## Cross-Cutting Duplicates And Dead Paths - -### Uncovered or Misclassified - -- `output_extraction.py` is dead context-derived worker output logic. -- `PersistOutputsResult.output_resource_ids` and - `FinalizeTaskExecutionCommand.output_resource_ids` are stale. -- `WorkerContext._resource_repo` is injected but unused. -- `EvaluationService.prepare_dispatch` appears test-only and preserves old - evaluator binding semantics. -- `ComponentCatalogService.resolve_benchmark` / `resolve_sandbox_manager` are - unused; `resolve_evaluator` is bridge-only. -- `Sandbox.output_path` is public but ignored by output publishing. -- `SubtaskLifecycleToolkit` bypasses the newer `WorkerContext` containment - facade and keeps a parallel mutation path. - -### PR11-Covered Legacy Clusters Confirmed - -- `TaskSpec` -- `WorkerSpec` -- `_legacy_worker_bridge` -- `_legacy_evaluator_bridge` -- `CriterionExecutor` -- `InngestCriterionExecutor` -- `EvaluateTaskRunRequest` -- `EvaluationService.evaluate_legacy` -- `_definition_task_snapshot` -- `_dynamic_task_snapshot` -- `_ExperimentDefinitionWriter` -- domain `Experiment` -- `terminate_sandbox_by_id`, with caveat that current cleanup callers must be - reconciled before deletion - -## Suggested Next Actions - -1. Add a pre-PR11 blocking checklist item: object-bound evaluator fanout and - persistence must be fixed before deleting the legacy evaluator bridge. -2. Add a pre-PR11 blocking checklist item: object-bound sandbox runtime must be - live in worker execution before deleting manager/proxy transition paths. -3. Add a dedicated test for `Task.from_definition` round-tripping real MiniF2F - and SWE-Bench `Task[Payload]` snapshots, including nested worker toolkit and - evaluator objects. -4. Add a small cleanup PR for clearly dead/misclassified items: - `output_extraction.py`, stale output resource IDs, unused - `ComponentCatalogService` resolvers, and `prepare_dispatch` if tests can move. -5. Decide whether public `Toolkit`, runtime tool factories, and dynamic subtask - tools should all remain named "Toolkit". The current naming hides two - distinct lifecycles. -6. Decide whether the persistent component catalog survives v2. If it does, - document its owner; if it does not, move the remaining slug/catalog users into - PR11 deletion scope. - diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/README.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/README.md deleted file mode 100644 index a240909a3..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# PR 11 Reconciliation Audit - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -Worktree audited: -`/Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema` - -GitHub PR: `https://github.com/DeepFlow-research/ergon/pull/65` - -Head audited: `a6138755f3f1b4794b0c640c377c9e87dc42c6cf` - -## Scope - -This folder is a self-contained reconciliation audit of the PR 11 worktree at -the audited head above. It describes the code that currently exists on this -branch: the deletion work already present, the live runtime paths that remain -load-bearing, and the gaps that still keep PR 11 from being pure final-schema -cleanup. - -The purpose here is narrower and sharper: - -- Identify what dead code still remains on PR 11 head. -- Identify duplicated or split implementations that still make core hard to - reason about. -- Separate completed PR 11 deletion/migration work from current PR 11 gaps. -- Propose how to keep the remaining PR 11 work mostly deletions, while naming - the fixes that are now clearly not just mechanical deletion. - -## Main Docs - -- [dead_code.MD](dead_code.MD) lists dead, stale, and deletion-candidate code - still present on PR 11 head. -- [duplication.mD](duplication.mD) explains each core construct/domain, the - current implementation path, and the duplicate or competing logic still in - tree. -- [pr11_gap_register.md](pr11_gap_register.md) prioritizes the gaps that make - PR 11 not yet a clean final-schema deletion PR. -- [pr_stack_recut.md](pr_stack_recut.md) proposes how to edit or split the - remaining PR work so PR 11 can become honest cleanup again. -- [post_pr16_flow_digest.md](post_pr16_flow_digest.md) summarizes the expected - post-PR16 call paths for each audited core flow. - -## Follow-On PR Plans - -- [pr12-runtime-identity-and-dynamic-task-correctness.md](pr12-runtime-identity-and-dynamic-task-correctness.md) - makes runtime task identity coherent and fixes dynamic child execution. -- [pr13-evaluation-cleanup.md](pr13-evaluation-cleanup.md) deletes the - remaining evaluator v1 dispatch/fallback path. -- [pr14-public-api-registry-surface.md](pr14-public-api-registry-surface.md) - deletes the core registry and persistent component catalog, and stabilizes - the builtins toolkit surface. -- [pr15-dashboard-event-contracts.md](pr15-dashboard-event-contracts.md) - aligns backend dashboard events with generated frontend contracts and live - parsers. -- [pr16-core-debt-sweep.md](pr16-core-debt-sweep.md) performs the final dead - code, stale comment, lifecycle duplication, xfail, and ledger cleanup sweep. - -## Executive Summary - -On PR 11 head, the runtime is largely object-bound in the happy path: -`Benchmark.build_instances()` returns `Task` objects, `persist_benchmark()` -stores full task snapshots, graph runtime reads inflate from -`run_graph_nodes.task_json`, `worker_execute` runs `task.worker`, and -`evaluate_task_run` selects `task.evaluators[index]`. - -The remaining risk is no longer "we forgot to migrate everything." It is that -several final-state promises are only partially implemented: - -- final schema identity collapse is incomplete; -- some code already assumes the final `RunGraphNode.task_id` model while the - model still only has `id`; -- public registry/catalog surfaces are still exported and used; -- dynamic subtask tools still have both object-bound and slug/registry paths; -- dynamic child task workers can still receive `WorkerContext.task_id=None`, - making the public context facade unreliable for recursive workers; -- worker-authored `spawn_task()` is not memoized as an Inngest step, while - `plan_subtasks()` is, so replay can duplicate graph mutations; -- PR 11 commit `a613875` fixed the smoke parent/recursive-worker polling - deadlock by moving smoke child waiting out of worker execution, so the - follow-on runtime plan no longer needs to change smoke fixture wait - semantics; -- sandbox lifecycle behavior is public-object-bound, but observability cleanup - still assumes manager-owned lifecycle; -- evaluator binding fallback remains even though the receiver no longer - supports it; -- live dashboard graph/context events now have PR 11 compatibility parsers for - wrapped graph mutation events and backend context-part payloads, but generated - event contracts and canonical frontend field names still need alignment; -- PR 11 smoke failures likely include concrete fixture/runtime issues, not only - external flake. - -Net: PR 11 can still become mostly deletion, but only after a short stack of -targeted cleanup/fix commits or micro-PRs lands on top of the current PR 11 -branch. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/dead_code.MD b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/dead_code.MD deleted file mode 100644 index 6aca9a46a..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/dead_code.MD +++ /dev/null @@ -1,136 +0,0 @@ -# Dead Code Audit On PR 11 Head - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -## Confirmed Deleted On PR 11 Head - -The following legacy packages, bridge modules, and registry files are absent -from PR 11 head and should be treated as completed deletion work, not remaining -dead code: - -- `ergon_core.core.persistence.saved_specs` -- `ergon_core.core.application.jobs._legacy_worker_bridge` -- `ergon_core.core.application.jobs._legacy_evaluator_bridge` -- `ergon_core.core.application.jobs.check_evaluators` -- `ergon_core.core.application.evaluation.executors` -- `ergon_core.core.application.evaluation.inngest_executor` -- per-benchmark `_legacy_workers.py` files -- per-benchmark `sandbox_manager.py` files -- `ergon_builtins.registry`, `registry_core`, `registry_data` - -PR 11 has already done meaningful bridge deletion. The rest of this file lists -code that is still present on the audited branch. - -## Strong Dead-Code Candidates - -### Evaluator v1 dispatch DTO path - -`EvaluationService.prepare_dispatch()` in -`ergon_core/ergon_core/core/application/evaluation/service.py` reconstructs -evaluator dispatches from definition tables, but the production path now fans -out from `execute_task._fan_out_evaluators()` into `TaskEvaluateRequest`. The -remaining references are tests and stale comments. The related internal models -in `ergon_core/ergon_core/core/application/evaluation/models.py` are also -candidate deletions: - -- `PreparedEvaluation` -- `EvaluationDispatch` -- internal `TaskEvaluationContext` -- internal `CriterionContext` - -Keep only if a non-runtime planning API explicitly needs them; otherwise delete -with the v1 dispatch tests. - -### Evaluator binding fallback - -`Task.evaluator_binding_keys` still exists in -`ergon_core/ergon_core/api/benchmark/task.py`, and -`execute_task._fan_out_evaluators()` still falls back to it when -`task.evaluators` is empty. On PR 11, `persist_benchmark()` rejects -`evaluator_binding_keys`, and `evaluate_task_run` only indexes -`task.evaluators`. If the fallback fires, it sends evaluator jobs that the -receiver rejects. - -This should be deleted rather than retained as a bridge. - -### MiniF2F legacy toolkit - -`ergon_builtins/ergon_builtins/benchmarks/minif2f/_legacy_toolkit.py` appears -unreferenced by source search and duplicates active MiniF2F tool logic in -`ergon_builtins/ergon_builtins/benchmarks/minif2f/_tools.py`. It also contains -comments pointing at deleted `_legacy_workers.py` files. - -### Context-output extraction - -`ergon_core/ergon_core/core/application/context/output_extraction.py` rebuilds -`WorkerOutput` from context events. The live output path is now terminal stream -item to `WorkerOutputRepository.persist()` to `RunTaskExecution.worker_output_json`. -This file is marked with a TODO saying much of it may be dead after PR 11. - -### Run cleanup sandbox teardown - -`run_cleanup` is still registered/emitted, but its sandbox teardown path reads -`RunRecord.summary_json["sandbox_id"]`. The active per-task sandbox id is now -stamped on `RunTaskExecution`. Unless a run-level sandbox id is intentionally -reintroduced, this cleanup branch is stale. - -### WorkflowService lifecycle duplicates - -`WorkflowService.restart_task()` and `WorkflowService.abandon_task()` duplicate -task lifecycle operations with weaker behavior than `TaskManagementService`. -The workflow-service path directly changes status and does not apply the -terminal guards, cascade invalidation, dispatch, or containment behavior owned -by the task-management path. CLI non-dry-run paths reject these actions, so the -remaining active surface appears mostly test/legacy. - -### Persistent component catalog model - -`ergon_core/ergon_core/core/persistence/components/models.py` still defines the -old persistent component catalog table model. The source audit found no -production source references except the model/tests. PR 14 deletes this model -and its tests alongside the core registry deletion. - -### Stale completion/test artifacts - -`ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` still has an -xfailed completion-bar test for sandbox acquire/release. The PR 11 plan says -this should be un-xfailed and enforced. - -`ergon_core/tests/unit/architecture/test_repository_companion_files.py` still -has one `_KNOWN_VIOLATORS` entry, despite the PR 11 plan requiring ledger -completion. - -The PR 11 plan references `scripts/single_use_helper_audit.py`, but `rg --files -scripts` did not find it. - -## Stale Docs And Comments - -These are not runtime dead code, but they will confuse the next person reading -the branch: - -- `ergon_builtins/AGENTS.md` still documents deleted `registry_core.py` as the - source of truth. -- `ergon_builtins/ergon_builtins/benchmarks/minif2f/workers.py` and - `swebench_verified/workers.py` still mention deleted legacy bridge files. -- `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/_tools.py` says Exa - tools remain on legacy worker classes, but those classes/files are gone. -- `ergon_builtins/ergon_builtins/evaluators/criteria/code_check.py` still refers - to deleted Inngest criterion executor wiring. -- `ergon_core/ergon_core/core/application/evaluation/service.py` still mentions - `evaluate_legacy` even though the method is gone. -- TS test harness helpers still expose deleted `parent_node_id` while the - backend harness returns `parent_task_id`. - -## Live Runtime Code That Should Not Be Deleted Blindly - -`execute_task.py`, `sandbox_setup.py`, `worker_execute.py`, -`persist_outputs.py`, `evaluate_task_run.py`, and `sandbox_cleanup.py` are live -runtime pieces on PR 11 head. They should be simplified around the final -architecture, but they are not deletion targets as a group. - -`ExperimentDefinition*` rows are duplicated with object-bound `task_json`, but -they are still load-bearing for definition persistence, evaluator FK lookup, -task assignment, run initialization, and read models. They are not safe to -delete as "dead code" without a schema redesign. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/duplication.mD b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/duplication.mD deleted file mode 100644 index 65a861607..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/duplication.mD +++ /dev/null @@ -1,359 +0,0 @@ -# Duplication And Core-Construct Audit On PR 11 Head - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -## Reconciliation Map - -Use this section as the review index for the rest of the document: - -- Benchmark/task definition duplication is owned mostly by - [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md), with - evaluator-row cleanup bounded by - [PR 13](pr13-evaluation-cleanup.md) and residual dead-code sweep by - [PR 16](pr16-core-debt-sweep.md). -- Worker context identity and worker-authored dynamic actions are owned by - [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md); the builtins - toolkit import surface is owned by - [PR 14](pr14-public-api-registry-surface.md). -- Sandbox ownership and cleanup consolidation are owned by - [PR 16](pr16-core-debt-sweep.md), after PR 12 finishes identity cleanup. -- Evaluator/rubric/criterion dispatch cleanup is owned by - [PR 13](pr13-evaluation-cleanup.md). -- Registry, serialization, component discovery, and public import boundaries - are owned by [PR 14](pr14-public-api-registry-surface.md), which deletes the - core registry and persistent component catalog. -- Data access, run copying, launch provenance, and schema identity are owned by - [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md). -- Runtime orchestration identity bugs are owned by - [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md); failed-task - sandbox cleanup ownership is finished in [PR 16](pr16-core-debt-sweep.md). -- Dynamic subtask containment spans - [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) for identity - and replay safety, [PR 14](pr14-public-api-registry-surface.md) for - removing slug/registry authoring paths while keeping `SubtaskLifecycleToolkit` - worker-authored, and [PR 16](pr16-core-debt-sweep.md) for duplicate lifecycle - methods. -- Dashboard/read-model/event contract drift is owned by - [PR 15](pr15-dashboard-event-contracts.md), after PR 12 settles final task-id - vocabulary. - -The intended destination across the stack is summarized in -[post_pr16_flow_digest.md](post_pr16_flow_digest.md). - -## Benchmark And Task - -Reconciliation owner: -[PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) for task/run -identity and the definition-to-run copy shape; -[PR 13](pr13-evaluation-cleanup.md) for deciding which evaluator metadata rows -remain because evaluation still needs them; -[PR 16](pr16-core-debt-sweep.md) for deleting residual dead files/comments once -the shape is settled. - -The canonical path is now object-bound. `Benchmark.build_instances()` returns -`Mapping[str, Sequence[Task[BaseModel]]]`, and `Task` carries the runtime graph -fields plus `worker`, `sandbox`, and `evaluators`. Production MiniF2F, -SWE-Bench Verified, GDPEval, ResearchRubrics, and the smoke fixture benchmarks -all construct object-bound `Task` objects. - -Duplication remains in the persistence shape. `persist_benchmark()` stores full -`task_json`, but definition tables also store workers, evaluators, task payloads, -assignments, dependencies, and task-evaluator rows separately. Some duplication -is currently load-bearing: evaluator rows are still needed for -`RunTaskEvaluation.definition_evaluator_id`, and worker assignment rows still -feed preparation/read-model metadata. The system therefore has two partially -canonical representations of the same authored task: the object snapshot in -`task_json`, and normalized definition rows. - -There is also a smaller but real logic duplication inside definition writing: -`persist_benchmark()` validates by calling `benchmark.build_instances()`, then -calls `build_instances()` again while persisting. Dataset-backed benchmarks can -load or sample twice, and the graph validated may not be the exact graph -persisted. - -## Worker, WorkerContext, Toolkit, WorkerOutput - -Reconciliation owner: -[PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) for -`WorkerContext.task_id`, job payload identity, worker-authored `spawn_task()` -replay safety, and removal of public `node_id` dependence; -[PR 14](pr14-public-api-registry-surface.md) for keeping `Toolkit` out of core -and giving it a stable builtins/baselines import path; -[PR 16](pr16-core-debt-sweep.md) for deleting dead output extraction or stale -worker comments after the runtime path is final. - -The worker execution path is mostly canonical. `worker_execute` reads the -run-tier task view through `WorkflowGraphRepository.node(..., sandbox_id=...)`, -requires `task.sandbox.is_live`, validates `task.worker`, constructs -`WorkerContext._for_job()`, consumes the worker stream, persists context chunks, -and persists exactly one terminal `WorkerOutput`. - -Duplication remains around toolkits and manager-facing dynamic tools. -`ReActWorker(toolkit=...)` is now the active baseline pattern, but the `Toolkit` -base lives under `ergon_builtins/ergon_builtins/workers/baselines/toolkit.py` -and is not exported through a stable builtins/baselines authoring facade. -`Toolkit` should not move into `ergon_core`: core owns the generic `Worker` -contract, while ReAct/toolkit composition is a builtins baseline concern. If -toolkit authoring is part of the public surface, the stable path should remain -under builtins, not `ergon_core.api`. - -`WorkerContext` is the object-bound worker facade, but it still carries bridge -identity: both `task_id` and `node_id`. Many smoke fixtures and CLI tests still -read `context.node_id`. The PR 11 plan says this should collapse to task id, -but the branch has not done that. This is not merely a rename: job payloads, -telemetry rows, dashboard registration, task events, and test helpers still use -`node_id`. - -ResearchRubrics has an important behavioral regression candidate: the v2 -worker is a plain `ReActWorker` with `ResearchRubricsToolkit`, but the toolkit -only exposes bash/write_report/read_report. The old web/search tool language -remains in `_tools.py`, while the legacy worker files that apparently owned -those tools are gone. If the benchmark expects a sourced report, this is a -functional gap, not dead code. - -## Sandbox And Sandbox Runtime - -Reconciliation owner: -[PR 16](pr16-core-debt-sweep.md) for single-owner sandbox cleanup, actual -cancelled-task sandbox release, stale lifecycle comments, and failure -propagation not terminating sandboxes directly. [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) -must land first where cleanup payloads depend on final task identity. - -The canonical behavior path is object-bound. `sandbox_setup` loads the typed -task view and calls `task.sandbox.provision()`. `worker_execute` and -`evaluate_task_run` rehydrate the task with the live `sandbox_id`, then builtins -bind through the sandbox runtime helpers. Evaluators detach local runtime -handles; terminal cleanup is intended to happen through `sandbox_cleanup` on -`task/completed` or `task/failed`. - -The duplicate ownership problem is observability and failure cleanup. Builtin -`Sandbox.provision()` now provisions E2B directly and returns a direct runtime, -but manager event/WAL bookkeeping still assumes `BaseSandboxManager` owns -created sandboxes. Cleanup later calls manager termination, which can kill the -raw E2B sandbox but bypass manager-created/closed bookkeeping for direct -provisioned sandboxes. In addition, `TaskFailedEvent` is handled by both -failure propagation and sandbox cleanup, and failure propagation still calls -the sandbox termination path directly. That creates two registered terminators -for the same failure event. - -Cancelled-task cleanup is also incomplete. The event/job docstring promises -sandbox release on `task/cancelled`, but `TaskCleanupService` explicitly leaves -E2B release unimplemented and returns `sandbox_released=False`. A manager or -cascade cancellation can therefore keep a running sandbox alive until timeout -unless a separate terminal cleanup path happens to run. - -There are also public fields with unclear force. `Sandbox.requires_network` is -declared and set by builtins, but current provisioning passes template, -timeout, envs, and API key only. `SandboxSetupRequest.envs` and `sandbox_slug` -exist but are not consumed in the object-bound setup path. - -## Evaluator, Rubric, Criterion - -Reconciliation owner: -[PR 13](pr13-evaluation-cleanup.md) owns deletion of evaluator binding fallback, -v1 dispatch DTOs, `EvaluationService.prepare_dispatch()` as a runtime path, and -stale criterion executor references. [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) -is a prerequisite only where evaluator rows still carry bridge task/node ids. - -The evaluator execution path is now comparatively narrow. `persist_benchmark()` -mirrors inline `Task.evaluators` into evaluator rows, -`execute_task._fan_out_evaluators()` counts `task.evaluators`, and -`evaluate_task_run` reloads the run-tier task and executes -`task.evaluators[evaluator_index]` through `EvaluationService.evaluate()`. The -old executor modules and legacy evaluator bridge are absent on PR 11 head. - -The remaining duplication is stale dispatch/payload infrastructure. -`EvaluationService.prepare_dispatch()` and its related DTOs reconstruct -definition-table evaluator dispatches even though production uses -`TaskEvaluateRequest` and the run-tier task snapshot. That old path appears -test-only. `Task.evaluator_binding_keys` also remains as a public field, and -fanout still falls back to it. Since persistence rejects it and the eval worker -does not support it, this fallback is harmful rather than transitional. - -The result side is relatively stable: `CriterionOutcome` and -`EvaluationSummary` flow into `RunTaskEvaluation.summary_json`, and dashboard -DTOs are emitted from the persisted summary. There is still some compatibility -magic in outcome slug/name handling, but it is lower priority than deleting the -dead dispatch path. - -## Registry, Serialization, Component Discovery - -Reconciliation owner: -[PR 14](pr14-public-api-registry-surface.md) owns deletion of the core registry, -replacement of discovery/onboarding/test consumers, public import cleanup, -deletion of persistent component catalog files/tests, and the -builtins/baselines toolkit path. - -The `_type` path is the canonical object-bound serializer. `Task`, `Worker`, -`Sandbox`, `Criterion`, and `Evaluator` inject `module:qualname`; rehydration -goes through `ergon_core.api._serialization.import_component()` and -`import_component_subclass()`. - -But the process-local registry is still public and live: -`ergon_core.api.__init__` exports `ComponentCatalog` and `registry`, and -`ergon_core.api.registry` remains in tree. Core runtime still imports it for -workflow setup, dynamic subtask worker validation, REST startup/test harnesses, -CLI discovery, onboarding, and smoke fixtures. This is a major discrepancy -with the PR 11 deletion plan, which lists `api/registry.py` as a deletion -target. - -The current state is therefore not "registry deleted." It is "builtins registry -tables deleted, core process-local registry still used." PR 14 resolves this by -deleting the core process-local registry entirely and replacing CLI/discovery, -onboarding, REST startup, test harness, and smoke fixture consumers with -explicit imports, dependency declarations, entrypoints, or fixture-local -factories. - -## Data Access, Definition Persistence, And Run Copying - -Reconciliation owner: -[PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) owns final -schema identity, `RunGraphNode`/edge/task execution/evaluation id vocabulary, -run launch provenance, definition-to-run copying, and Alembic reset alignment. -[PR 13](pr13-evaluation-cleanup.md) may narrow evaluator definition metadata -after PR 12 decides the final FK/read-model shape. - -Definition persistence is in `persist_benchmark()`. Launch is in `launch_run()`. -Definition-to-run copying is in `WorkflowGraphRepository.initialize_from_definition()`. -Runtime reads are centered on `WorkflowGraphRepository.node()`, which is the -right architectural boundary. - -The biggest duplication is schema identity. The PR 11 plan promises final graph -identity as `(run_id, task_id)` with no `RunGraphNode.id`, but the model still -has `id` as the primary key and no `task_id` column. Telemetry still carries -both `node_id` and `task_id` on execution/evaluation rows. Some code now -references `RunGraphNode.task_id` as if the collapse had happened, while the -model has not caught up. - -There is also provenance ambiguity. `launch_run()` passes `experiment_id` as -the definition id, while `RunRecord.experiment_id` is foreign-keyed to the old -`experiments` table shape. `persist_benchmark()` writes no matching -`BenchmarkDefinitionRecord`, so fresh launches can depend on an unrelated or -missing row. The PR 11 plan's final `BenchmarkDefinitionRecord` shape does not -match the current telemetry model. - -The Alembic reset is also only partially aligned with the plan. The current -single migration uses `SQLModel.metadata.create_all()` / `drop_all()` and -revision `"00000000"`, while the PR 11 plan calls for a hand-audited final -schema and a non-downgradeable reset. - -## Runtime Orchestration And Propagation - -Reconciliation owner: -[PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) owns the -`task/ready` -> `execute_task` -> `worker_execute` identity repair and dynamic -child context correctness. [PR 16](pr16-core-debt-sweep.md) owns failure -propagation vs sandbox cleanup single-owner behavior after the identity fix. - -The live event flow is still a multi-job orchestrator: -`task/ready -> execute_task -> sandbox_setup -> worker_execute -> persist_outputs -> evaluate_task_run fanout -> task/completed|task/failed -> propagation + sandbox cleanup`. - -This is final-shape behavior despite older plan text that implied some job -files would be deleted. The duplicate area is failure cleanup: failed task -events currently reach both propagation and sandbox cleanup, and propagation -still has a direct termination call. The final owner should be `sandbox_cleanup`; -propagation should handle graph/run state only. - -The prepare failure path is still awkward. If preparation fails before an -execution row exists, `execute_task` logs loudly but leaves the graph node in a -bad state because `payload.task_id` may be `None` for dynamic subtasks and the -code still relies on bridge identity. This is another symptom of incomplete -`node_id`/`task_id` collapse. - -There is a concrete dynamic-child identity bug in the event flow. Dynamic -`task/ready` events intentionally carry `task_id=None` and the real graph id in -`node_id`. `execute_task` passes that `None` into -`WorkerExecuteJobRequest.task_id`, and `worker_execute` constructs -`WorkerContext(task_id=None, node_id=...)`. Dynamic workers using the public -facade (`spawn_task`, `subtasks`, `descendants`, containment methods) therefore -fail or see empty state, while smoke fixtures can hide the problem by bypassing -the facade and using `context.node_id` plus direct task-management services. - -PR 11 commit `a613875` removed the smoke-fixture version of this deadlock: -smoke parent and recursive workers now plan children and return, and smoke -assertions wait outside the parent worker execution path. That does not solve -the broader identity duplication above: dynamic worker payloads still carry -`task_id=None` plus `node_id`, and `WorkerContext.task_id` can still be null for -dynamic children until the PR 12 identity cleanup lands. - -## Dynamic Subtasks And Containment - -Reconciliation owner: -[PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) owns object-bound -dynamic child identity, `WorkerContext.spawn_task()` replay safety, dependency -containment, duplicate sibling guards, and `refine_task()`/`task_json` -coherence. [PR 14](pr14-public-api-registry-surface.md) owns deletion of -slug/registry authoring paths and makes `SubtaskLifecycleToolkit` -worker-authoring/object-bound only. [PR 16](pr16-core-debt-sweep.md) owns -deletion or consolidation of duplicate lifecycle methods once the API boundary -is explicit. - -There are two dynamic-subtask surfaces. The canonical v2 surface is -`WorkerContext.spawn_task(Task)` to `TaskManagementService.spawn_dynamic_task()`, -which writes a graph-native node with object-bound `task_json` and no definition -row. - -The legacy slug/registry surface remains: -`TaskManagementService.add_subtask()` and `plan_subtasks()` accept slugs and -`assigned_worker_slug`, validate through `registry.workers`, and synthesize -subtask `task_json` from the parent task and registry worker class. -`SubtaskLifecycleToolkit` exposes both object-bound context tools and explicit -slug-path tools. PR 14 deletes those slug/registry paths rather than retaining -them as admin APIs. The context path supports object-bound `add_subtask`, but -`plan_subtasks` is unavailable through context and only exists in the -slug-based path before PR 14. - -Containment is split too. `WorkerContext.cancel_task`, `refine_task`, -`restart_task`, `get_task`, and `descendants` enforce descendant checks. The -slug path in `SubtaskLifecycleToolkit` calls `TaskManagementService` directly -and can bypass the worker-context facade. PR 14 should remove that bypass and -leave `SubtaskLifecycleToolkit` as a worker-authoring object-bound surface. - -Mutation semantics are also split. `refine_task` updates -`run_graph_nodes.description` but does not update the embedded `task_json`; -execution rehydrates the authored `Task` from `task_json`, so -refine-then-restart can rerun the old description. `spawn_task` accepts -arbitrary dependency UUIDs, and repository edge creation only validates -same-run/no-cycle, not descendant containment. `spawn_dynamic_task`, -`add_subtask`, and `plan_subtasks` also do not prevent duplicate slugs against -existing siblings. Finally, `_StepAwareTaskManagementService` memoizes -`plan_subtasks` under Inngest steps but not `spawn_dynamic_task`, so public -`WorkerContext.spawn_task()` can replay and duplicate nodes/events. - -## Dashboard, Read Models, And Frontend Contracts - -Reconciliation owner: -[PR 15](pr15-dashboard-event-contracts.md) owns generated event contracts, -live parser cleanup, graph/context event drift, workflow-started task-tree -alignment, and frontend field vocabulary. [PR 12](pr12-runtime-identity-and-dynamic-task-correctness.md) -must land first for final `task_id` naming in read models and event payloads. - -The backend exposes snapshot/read-model DTOs from -`ergon_core.core.application.read_models`, live dashboard event contracts from -`ergon_core.core.infrastructure.dashboard.event_contracts`, and generated -frontend event schemas in `ergon-dashboard/src/generated/events`. - -Duplication remains between read-model snapshot builders and event DTOs. The -run snapshot path builds task maps, execution maps, resource maps, evaluation -maps, context events, and sandbox summaries from persistence rows. Dashboard -event contracts separately model task status changes, evaluation updates, -resource publication, sandbox events, graph mutations, and context events. - -The most urgent read-model issue is tied to schema identity: snapshot helpers -still key data by `node_id` while some code assumes final `task_id`. Until the -schema collapse is finished, dashboard and read-model code will continue to -carry bridge naming. If the final schema collapse proceeds, the generated -frontend contracts and REST DTOs need regeneration/checking in the same slice. - -Live dashboard events have additional contract drift. PR 11 commit `a613875` -added compatibility parsing for backend graph mutation events emitted as -`{ mutation: GraphMutationRecordDto }` and backend context-part payloads. The -remaining duplication is that this compatibility lives in handwritten frontend -parsers rather than generated event contracts. Backend graph edge payloads use -`source_task_id` / `target_task_id`, while frontend graph mutation contracts and -reducers still normalize to deleted `source_node_id` / `target_node_id` names. -`workflow.started` task-tree contracts also drift: backend requires `status` -and `level` and sends resource ids, while handwritten frontend schemas omit -those fields, expect resource objects, and add non-backend fields. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/post_pr16_flow_digest.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/post_pr16_flow_digest.md deleted file mode 100644 index 690b5557f..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/post_pr16_flow_digest.md +++ /dev/null @@ -1,356 +0,0 @@ -# Post-PR16 Core Flow Digest - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -Head used for this digest: `a6138755f3f1b4794b0c640c377c9e87dc42c6cf` - -## Purpose - -This document is the reader's-digest version of the reconciliation audit. It -does not list every debt item. Instead, it describes the core flows as they -should read after PRs 12-16 land, so reviewers can ask: "if this is where the -stack is taking us, does the final system make sense?" - -Each section has: - -- the main entry point; -- the expected step-by-step call path; -- what should be gone by the end of PR 16; -- the review question that matters most. - -## 1. Authoring A Benchmark Definition - -Entry point: benchmark author code, usually a concrete `Benchmark` subclass. - -Post-PR16 flow: - -1. The author creates a benchmark whose `build_instances()` returns concrete - `Task` objects. -2. Each `Task` directly contains its runtime objects: `worker`, `sandbox`, and - zero or more `evaluators`. -3. Each nested public object serializes itself with a `_type` discriminator: - `module:qualname`. -4. The definition persistence layer calls `build_instances()` once for the - persisted graph. -5. `persist_benchmark()` writes immutable definition metadata and stores the - complete object-bound `task_json` snapshot for each task. -6. Normalized definition rows remain only where they are deliberately - load-bearing, such as evaluator metadata/FK lookup or read-model/query - ergonomics. - -What should be gone: - -- repeated `build_instances()` calls that can validate one graph and persist - another; -- registry/spec authoring as a runtime requirement; -- `saved_specs`; -- `ExperimentRecord`; -- `_prepare_definition`; -- public examples that imply workers/evaluators are bound by slug at runtime. - -Review question: is `task_json` obviously the canonical authored task, and are -all remaining normalized definition rows justified by a current read/write need? - -## 2. Persisting And Launching A Run - -Entry point: `persist_benchmark()` followed by `launch_run()`. - -Post-PR16 flow: - -1. `persist_benchmark()` stores the immutable definition graph. -2. `launch_run()` creates a `RunRecord` with unambiguous provenance back to the - definition id/model chosen as final. -3. `WorkflowGraphRepository.initialize_from_definition()` copies definition - tasks into run-tier graph rows. -4. Each run graph node receives the canonical runtime task identity, - consistently named `task_id`. -5. Run graph rows carry their object-bound `task_json` snapshot. -6. Initial ready tasks are computed from run-tier graph edges only. -7. Runtime after launch reads from run-tier graph state, not from definition - reconstruction paths. - -What should be gone: - -- mixed `RunGraphNode.id`/`task_id` public identity vocabulary; -- payloads that use `task_id=None` and smuggle the real id through `node_id`; -- inconsistent `RunRecord.experiment_id` vs definition id provenance; -- code that references `RunGraphNode.task_id` before the model actually has the - final field; -- schema reset drift between SQLModel models and Alembic. - -Review question: can a reader follow one identity from authored task, to run -graph node, to execution/evaluation/read-model/dashboard without learning a -second public name? - -## 3. Executing A Task - -Entry point: `task/ready` event handled by `execute_task`. - -Post-PR16 flow: - -1. A `TaskReadyEvent` arrives with `run_id`, `definition_id`, and canonical - `task_id`. -2. `execute_task` calls `TaskExecutionService.prepare()` and creates a - `RunTaskExecution`. -3. `execute_task` invokes `sandbox_setup`. -4. `sandbox_setup` loads the run-tier task through - `WorkflowGraphRepository.node(run_id, task_id)` and calls - `task.sandbox.provision()`. -5. `execute_task` invokes `worker_execute` with `task_id`, `execution_id`, and - `sandbox_id`. -6. `worker_execute` reloads the same run-tier task with the live `sandbox_id`. -7. `worker_execute` validates and runs `task.worker`. -8. Context chunks are persisted as they stream. -9. Exactly one terminal `WorkerOutput` is persisted. -10. `execute_task` invokes `persist_outputs`. -11. `execute_task` fans out evaluators, waits for them, then emits - `task/completed` or `task/failed`. - -What should be gone: - -- registry lookup to choose the runtime worker for a persisted task; -- `WorkerExecuteRequest.task_id=None`; -- public `node_id` dependence in job payloads, traces, and worker context; -- worker execution paths that rebuild a task from definition rows; -- output extraction from context events as the main worker-output path. - -Review question: does task execution look like "load the object-bound task and -run its worker", or are there still hidden fallback paths deciding what worker -to run? - -## 4. Worker Context And Worker-Authored Actions - -Entry point: `WorkerContext._for_job()` inside `worker_execute`. - -Post-PR16 flow: - -1. `worker_execute` constructs `WorkerContext` with a non-null `task_id`. -2. Worker code uses the public facade: `spawn_task`, `subtasks`, - `descendants`, `get_task`, `cancel_task`, `refine_task`, `restart_task`, - `resources`. -3. Every target-taking facade method enforces containment: a worker can act on - itself or descendants, not arbitrary tasks in the run. -4. `WorkerContext.spawn_task(Task(...))` creates object-bound dynamic children. -5. `spawn_task()` is wrapped in Inngest step memoization so replay does not - duplicate graph rows or `task/ready` events. -6. Slug/registry dynamic authoring workflows are gone; worker-authored actions - use concrete `Task` objects. - -What should be gone: - -- worker-authored code reading `context.node_id`; -- public facade methods returning empty state because `task_id` is null; -- slug/registry synthesis in worker-authored dynamic task creation; -- duplicate sibling slugs; -- dependencies on non-descendant task ids; -- `refine_task()` updating graph description without updating embedded - `task_json`. - -Review question: can a worker author understand the facade without knowing -about run graph internals? - -## 5. Dynamic Subtask Propagation - -Entry point: `WorkerContext.spawn_task()` and object-bound subtask tools. - -Post-PR16 flow: - -1. A worker spawns a child by passing a concrete `Task` object. -2. `TaskManagementService.spawn_dynamic_task()` writes a run-tier graph node - with object-bound `task_json`. -3. Dependencies are checked for same-run, acyclic, and containment-valid. -4. If the new child has no unsatisfied dependencies, a `task/ready` event is - emitted with canonical `task_id`. -5. If the child depends on another task, graph propagation readies it after all - dependencies complete. -6. Parent workers do not synchronously wait inside `worker_execute` for sibling - Inngest jobs to finish. -7. PR 11's smoke behavior is preserved: parent/recursive smoke workers plan and - return; later graph/e2e assertions observe child completion. - -What should be gone: - -- dynamic `task/ready` events with `task_id=None`; -- child execution that only works because smoke fixtures bypass the facade and - use `context.node_id`; -- un-memoized worker-authored spawn replay; -- slug/registry planning exposed as a worker authoring model. - -Review question: if a dynamic child is spawned during worker execution, is its -lifecycle graph-native and replay-safe from the first write? - -## 6. Evaluating A Task - -Entry point: evaluator fanout from `execute_task`. - -Post-PR16 flow: - -1. After worker output and output resources are persisted, `execute_task` - reloads the run-tier task. -2. Evaluator count is `len(task.evaluators)`. -3. For each evaluator index, `execute_task` invokes `evaluate_task_run` with an - id-only payload: `run_id`, `task_id`, `execution_id`, `evaluator_index`. -4. `evaluate_task_run` reloads the run-tier task with the live sandbox id from - the execution row. -5. `evaluate_task_run` selects `task.evaluators[evaluator_index]`. -6. `EvaluationService.evaluate()` runs the evaluator and criteria. -7. The evaluator returns criterion outcomes and task-level summary. -8. Results persist to `RunTaskEvaluation.summary_json` and any retained - evaluator metadata/FK fields. -9. Dashboard/read models emit evaluation updates from persisted results. - -What should be gone: - -- `Task.evaluator_binding_keys`; -- fallback fanout based on binding keys; -- `EvaluationService.prepare_dispatch()` as a runtime dispatch path; -- v1 dispatch DTOs used only by tests; -- `CriterionExecutor`/Inngest criterion executor references. - -Review question: is there exactly one runtime answer to "which evaluator runs": -the evaluator object inside the task snapshot at `evaluator_index`? - -## 7. Sandbox Lifecycle - -Entry point: `sandbox_setup`, `worker_execute`, `evaluate_task_run`, terminal -task events. - -Post-PR16 flow: - -1. `sandbox_setup` loads the run-tier task and calls `task.sandbox.provision()`. -2. The sandbox object attaches its live runtime and returns `sandbox_id`. -3. `worker_execute` reloads the task with `sandbox_id`, so worker code sees a - live `task.sandbox`. -4. `evaluate_task_run` reloads the same task with `sandbox_id`, so criteria can - use the same sandbox while evaluation runs. -5. Evaluators detach local runtime handles without terminating the external - sandbox. -6. `execute_task` emits a terminal task event after worker, output persistence, - and evaluators are done. -7. `sandbox_cleanup` is the single owner for terminal sandbox release. -8. Observability events/WAL rows are emitted by that same lifecycle owner. - -What should be gone: - -- failure propagation terminating sandboxes directly; -- double cleanup on failed tasks; -- comments promising cancelled-task sandbox release if the code does not do it; -- manager-owned observability assumptions that direct `Sandbox.provision()` - bypasses; -- unused `SandboxSetupRequest` fields if they are not part of final - provisioning. - -Review question: is there exactly one owner for external sandbox termination, -and does observability follow that owner? - -## 8. Registry, Serialization, And Discovery - -Entry point: `_type` serialization for runtime and explicit package/module -discovery. - -Post-PR16 flow: - -1. Runtime task snapshots carry `_type` for `Task`, `Worker`, `Sandbox`, - `Evaluator`, and `Criterion`. -2. Rehydration imports the concrete class from `_type`. -3. `_type` is stripped before Pydantic validates constructor payloads. -4. Persisted runtime execution never needs registry lookup. -5. CLI discovery, onboarding, tests, and smoke fixtures use explicit imports, - dependency declarations, package entrypoints, or fixture-local factories. -6. Architecture guards prevent registry use from creeping back into the tree. -7. `Toolkit` has a stable builtins/baselines import path if it remains part of - the v2 ReAct authoring story. - -What should be gone: - -- builtins static registry files; -- persistent component catalog; -- `ergon_core.api.registry` as a recommended authoring path; -- `ergon_core.api.registry` as an importable registry; -- runtime worker/evaluator selection by registry slug; -- stale docs pointing at deleted registry files. -- `Toolkit` exported from `ergon_core`. - -Review question: can persisted tasks run in a fresh process using only importable -`_type` snapshots and installed packages? - -## 9. Dashboard Events And Read Models - -Entry point: REST snapshot reads and live dashboard events. - -Post-PR16 flow: - -1. Backend read models build run snapshots from persistence rows. -2. Backend live event contracts model task status, graph mutation, context, - resource, sandbox, evaluation, communication, workflow started/completed, and - cohort updates. -3. Generated frontend contracts include every live backend event shape. -4. Frontend live parsers consume generated/canonical event shapes. -5. Graph mutation payloads use final `task_id` vocabulary, including - `source_task_id` and `target_task_id`. -6. Context event live parsing and REST hydration share the same canonical - conversion path. -7. Reducers update run state from live events without stale compatibility - field names. - -What should be gone: - -- handwritten-only event shapes for graph/context events; -- frontend normalization from `source_task_id` into `source_node_id`; -- live parsers that accept backend reality only through ad hoc shims; -- workflow-started task-tree drift around `status`, `level`, and resource ids; -- test harness DTOs using deleted `parent_node_id` names. - -Review question: when backend emits a live event, is there a generated frontend -contract and one obvious parser path for it? - -## 10. Task Lifecycle Operations - -Entry point: `TaskManagementService` and object-bound worker/tool wrappers. - -Post-PR16 flow: - -1. Lifecycle operations are owned by `TaskManagementService`. -2. Restart, abandon/cancel, refine, dependency invalidation, and descendant - traversal go through one semantics layer. -3. Terminal guards prevent illegal mutation of completed/failed tasks unless an - explicit restart flow reopens them. -4. Cascade invalidation updates downstream graph status and edges. -5. Worker-facing operations enforce containment through `WorkerContext`. -6. CLI/tool operations use the same canonical lifecycle service rather than - duplicate weaker semantics. - -What should be gone: - -- weaker duplicate `WorkflowService.restart_task()` and - `WorkflowService.abandon_task()` paths; -- lifecycle behavior that bypasses containment; -- graph description updates that leave embedded `task_json` stale; -- repository companion-file ledger exceptions. - -Review question: if a task changes lifecycle state, is there one service where -the invariants live? - -## Final Post-PR16 Picture - -If PRs 12-16 land as planned, the core should read like this: - -1. Authors create object-bound tasks. -2. Persistence stores immutable definitions plus complete task snapshots. -3. Launch copies tasks into a run graph with one public runtime identity: - `task_id`. -4. Runtime jobs load run-tier tasks and execute their embedded objects. -5. Workers interact through `WorkerContext`, not graph internals. -6. Dynamic children are graph-native and replay-safe. -7. Evaluators run from `task.evaluators[index]`, not binding registries. -8. Sandbox lifecycle has one cleanup owner. -9. The core registry and persistent component catalog are gone. -10. Dashboard live and snapshot contracts use generated/canonical task-id - shapes. -11. PR 16 deletes the old scaffolding, stale comments, duplicate lifecycle - paths, and all remaining ledgers. - -The desired feeling after PR 16 is not "there is no complexity." The desired -feeling is "there is one place to look for each kind of complexity." diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr11_gap_register.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr11_gap_register.md deleted file mode 100644 index 07ea47864..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr11_gap_register.md +++ /dev/null @@ -1,143 +0,0 @@ -# PR 11 Gap Register - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -## P0 / P1 Gaps - -### Final schema identity collapse is incomplete - -The plan says `RunGraphNode` final identity is `(run_id, task_id)` with no -`id`, no `definition_task_id`, and no graph-edge definition dependency bridge. -On PR 11 head, `RunGraphNode` still has `id` as primary key and no `task_id` -column. `RunTaskExecution` and `RunTaskEvaluation` still carry both `node_id` -and `task_id`. - -Worse, some readers already reference columns that do not exist on the current -model, such as `RunGraphNode.task_id` in read/evaluation code. That is a live -runtime correctness gap. - -Decision needed: either finish the schema collapse in PR 11, or revise the PR -11 plan to keep `id` as the final runtime task identity and update all docs, -guards, and readers accordingly. The current halfway state is the riskiest -option. - -### Launch provenance and definition metadata are inconsistent - -`launch_run()` passes a definition id as `experiment_id`, but `RunRecord` -foreign-keys `experiment_id` to the old experiments/benchmark-definition -telemetry table shape. `persist_benchmark()` writes `ExperimentDefinition` -rows, not `BenchmarkDefinitionRecord` rows. The plan's final -`BenchmarkDefinitionRecord` shape does not match the current model. - -This is not dead code. It is an unresolved data-model decision across -definition persistence, run provenance, read models, and migrations. - -### Registry deletion plan does not match runtime reality - -The process-local core registry remains public and runtime-used. It is exported -from `ergon_core.api`, used by workflow service, task management dynamic -subtasks, REST/test harnesses, CLI discovery/onboarding, and smoke fixtures. - -PR 14 deletes `ergon_core/api/registry.py`; the live consumers must move to -object-bound `_type` snapshots, explicit imports, benchmark dependency -declarations, package entrypoints, or fixture-local factories. - -### Sandbox lifecycle observability has split ownership - -Behavioral provisioning is now object-bound via `Sandbox.provision()`, but -sandbox event/WAL bookkeeping still assumes manager-owned lifecycle. Direct E2B -provisioning can bypass manager created/closed events. Failure cleanup is also -duplicated: both failure propagation and sandbox cleanup handle failed-task -sandbox termination. - -Final owner should be one thing. If it is `sandbox_cleanup`, propagation should -stop terminating sandboxes directly. - -### Smoke lifecycle regressions were patched on PR 11 - -PR 11 commit `a613875` fixed the smoke lifecycle regressions that were visible -when this audit was first drafted. It changed smoke parent and recursive workers -so they plan children and return instead of polling child completion inside the -same `worker_execute` job. It also readies dependency-free dynamic children when -their parent completes, relaxes dynamic worker payloads with no model target, -and adds parser coverage for wrapped graph mutation events and backend context -part payloads. - -This removes the smoke-fixture wait-semantics work from the follow-on stack. -It does not remove the remaining identity/API risks: - -- dynamic child workers still receive `WorkerContext.task_id=None` in the - dynamic `task/ready` path, with the real graph id carried through `node_id`; -- public `WorkerContext.spawn_task()` is still not Inngest-step memoized and - can duplicate graph mutations/events on replay; -- the evaluator binding fallback can still produce eval jobs that the receiver - rejects; -- the run/read schema mismatch can still break dashboard/read-model paths. - -## P2 Gaps - -### Dead evaluator fallback and dispatch DTOs remain - -`Task.evaluator_binding_keys`, fanout fallback, `EvaluationService.prepare_dispatch()`, -and internal v1 evaluation dispatch DTOs remain despite the new per-evaluator -id-only path being canonical. - -### Dynamic subtasks still have two authoring models - -`WorkerContext.spawn_task(Task)` is the canonical object-bound path. -`TaskManagementService.add_subtask()` / `plan_subtasks()` and -`SubtaskLifecycleToolkit` slug tools still use registry synthesis. PR 14 deletes -those slug/registry authoring paths and leaves `SubtaskLifecycleToolkit` as a -worker-authoring object-bound toolkit. - -This lane has correctness bugs, not only cleanup: `refine_task` updates graph -description without updating embedded `task_json`; dependency containment is not -enforced; duplicate sibling slugs are not consistently prevented; and -`WorkflowService` restart/abandon duplicate task-management lifecycle semantics -with weaker guards. - -### Worker/task DTO bridges remain - -`PreparedTaskExecution` and `WorkerExecuteJobRequest` still carry bridge fields -such as `assigned_worker_slug`, `worker_type`, `model_target`, and `node_id`. -Some of these feed traces/read models; others are likely removable once schema -identity is decided. - -### Public toolkit surface is unstable - -`Toolkit` is part of the v2 ReAct worker authoring story, but it is not exposed -through a stable builtins/baselines package path. It should not move into -`ergon_core`: core should expose the generic `Worker` contract, while -ReAct/toolkit composition remains a builtins baseline authoring surface. - -### PR ledgers are not fully drained - -At least one repository companion-file violator remains, and the walkthrough -sandbox acquire/release guard is still xfailed. PR 16 drains these completely: -no known-violator ledgers or xfails remain. - -### Dashboard live contracts drift from backend contracts - -PR 11 commit `a613875` added compatibility parsing for backend-wrapped graph -mutation events and backend context-part payloads. The remaining gap is now -narrower: graph mutation and context events are still not generated first-class -frontend event contracts, and frontend graph edge contracts still normalize -backend `source_task_id` / `target_task_id` into `source_node_id` / -`target_node_id`. PR 15 should replace those compatibility shims with canonical -generated contracts and final task-id vocabulary. - -## PR 11 Work Already Present - -These current-state facts are important context for planning the remaining PR -11 work: - -- Production builtins are object-bound. -- Smoke fixture benchmarks are mostly object-bound. -- Inline evaluator persistence rows exist. -- `Task.from_definition()` handles nested worker/sandbox/evaluator objects. -- `worker_execute` requires a live sandbox. -- Legacy worker/evaluator bridge modules are gone. -- Criterion executor modules are gone. -- `saved_specs` is gone. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr12-runtime-identity-and-dynamic-task-correctness.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr12-runtime-identity-and-dynamic-task-correctness.md deleted file mode 100644 index 84ba6aedb..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr12-runtime-identity-and-dynamic-task-correctness.md +++ /dev/null @@ -1,183 +0,0 @@ -# PR 12 Runtime Identity And Dynamic Task Correctness Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make runtime task identity coherent after PR 11 and fix dynamic child task execution so object-bound recursive workers can rely on the public `WorkerContext` facade. - -**Architecture:** This PR implements the RFC v2 target: `task_id` is the public and runtime identity. Legacy `node_id` vocabulary is removed from models, schema, events, job payloads, worker context, telemetry, dashboard registration, and tests rather than hidden behind adapters. Dynamic subtasks must create graph-native task snapshots, pass the canonical task id into workers, and make worker-authored spawning replay-safe. - -**Tech Stack:** Python, SQLModel, Alembic, Inngest, pytest. - ---- - -## PR 11 Head Update - -PR 11 commit `a613875` fixed the smoke-fixture lifecycle deadlock by making -smoke parent and recursive workers plan children and return instead of polling -child completion inside `worker_execute`. It also readies dependency-free -dynamic children on parent completion. This PR therefore no longer owns smoke -fixture wait-semantics changes. It still owns the underlying identity cleanup: -dynamic worker payloads can still carry `task_id=None` with the real id in -`node_id`, and `WorkerContext.task_id` can still be null for dynamic children. - -## Locked Decisions - -- Final runtime identity is `task_id`. -- `node_id` bridge fields are removed outright, not retained internally, with - final end-to-end tests as the merge gate. -- `RunRecord.experiment_id` is renamed/replaced with explicit `definition_id` - provenance rather than overloaded. - -## Scope - -This PR should land immediately after PR 11. It should not change evaluator -semantics, public registry decisions, or frontend dashboard contract generation -except where they directly depend on `task_id` naming. - -## Primary Files - -- Modify: `ergon_core/ergon_core/core/persistence/graph/models.py` -- Modify: `ergon_core/ergon_core/core/application/graph/repository.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/execute_task.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/worker_execute.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/sandbox_setup.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/persist_outputs.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` -- Modify: `ergon_core/ergon_core/api/worker/context.py` -- Modify: `ergon_core/ergon_core/core/application/tasks/management.py` -- Modify: `ergon_core/ergon_core/api/worker/results.py` -- Modify: `ergon_core/ergon_core/core/application/read_models/*` -- Modify: `ergon_core/ergon_core/core/infrastructure/dashboard/event_contracts.py` -- Modify: `ergon_core/tests/unit/runtime/test_identity_invariants.py` -- Modify: `ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py` -- Modify: `ergon_core/tests/unit/runtime/test_spawned_task_handle.py` -- Modify: `ergon_core/tests/unit/runtime/test_child_function_payloads.py` -- Modify: `ergon_core/tests/unit/runtime/test_graph_worker_identity.py` -- Modify: `ergon_core/tests/unit/runtime/test_worker_context_containment.py` -- Modify: `ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py` - -## Code TODOs / Comments To Remove - -When PR 12 lands, remove or rewrite the code comments that describe -`node_id` as a temporary bridge rather than leaving them for PR 16. Expected -cleanup targets include: - -- `ergon_core/ergon_core/api/worker/context.py`: remove the transitional - `task_id` field description that says PR 11 will make it required, remove - the `node_id` bridge field and its `RunGraphNode.id` description, and delete - worker-context tests/fixtures that use `context.node_id` as a public escape - hatch. -- `ergon_core/ergon_core/core/application/graph/repository.py`: remove comments - that explain `RunGraphNode.id` or parent/source/target node columns as the - pre-PR-12 identity shape once the repository operates on `task_id`. -- `ergon_core/ergon_core/core/application/tasks/execution.py`: remove - `TODO(PR 11)` comments about bridge metadata and worker assignment rows if - the PR has moved runtime worker selection entirely onto the task snapshot. -- `ergon_core/ergon_core/core/application/graph/propagation.py`: remove - `execution_id # TODO: dead param` if the identity rewrite proves the - parameter unused. -- Dashboard and e2e helper comments/tests that mention `parent_node_id`, - `source_node_id`, `target_node_id`, or "node_id join" should be removed when - the backend vocabulary changes. If frontend parser compatibility still needs - work, leave only the PR 15-owned parser note, not a backend identity TODO. - -## Tasks - -### Task 1: Lock The Identity Decision In Tests - -- [ ] Add failing tests proving `RunGraphNode` exposes `task_id` as the canonical runtime identifier and does not require public callers to use `node_id`. -- [ ] Extend `ergon_core/tests/unit/runtime/test_identity_invariants.py` to assert graph, execution, evaluation, task events, and worker context all agree on the same task id for a static task. -- [ ] Add a dynamic child variant proving a spawned child receives `WorkerContext.task_id == spawned_task_id`. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema -uv run pytest ergon_core/tests/unit/runtime/test_identity_invariants.py -q -``` - -Expected before implementation: fail on missing or inconsistent `task_id` fields. - -### Task 2: Finish The Graph Model Collapse - -- [ ] Update `RunGraphNode` so `(run_id, task_id)` is the runtime identity. -- [ ] Remove `RunGraphNode.id` as the public/runtime identity; do not leave a public/internal `node_id` bridge behind. -- [ ] Update `RunGraphEdge` to store `source_task_id` and `target_task_id`. -- [ ] Update `RunTaskExecution`, `RunTaskEvaluation`, and related repositories so `task_id` is canonical and old `node_id` bridge fields are removed. -- [ ] Rename or replace `RunRecord.experiment_id` with `definition_id` so run provenance points at the immutable definition without overloading the old experiment table relationship. -- [ ] Update the final Alembic reset to match the selected schema. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py ergon_core/tests/unit/runtime/test_graph_traversal.py -q -``` - -Expected after implementation: graph read/write tests pass using `task_id`. - -### Task 3: Repair Job Payloads And Event Flow - -- [ ] Update `TaskReadyEvent`, `WorkerExecuteJobRequest`, sandbox setup requests, output persistence requests, and evaluator requests so the public task identifier is always `task_id`. -- [ ] Remove payload branches where dynamic tasks send `task_id=None` and smuggle the real id through `node_id`. -- [ ] Update `execute_task` so static and dynamic tasks produce the same downstream payload shape. -- [ ] Update `worker_execute` so `WorkerContext._for_job()` receives the canonical task id for every task. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_child_function_payloads.py ergon_core/tests/unit/runtime/test_graph_worker_identity.py -q -``` - -Expected after implementation: static and dynamic children use the same task-id path. - -### Task 4: Make Worker-Authored Spawning Replay-Safe - -- [ ] Add a failing test showing repeated Inngest replay of `WorkerContext.spawn_task()` does not create duplicate graph nodes or duplicate child-ready events. -- [ ] Extend `_StepAwareTaskManagementService` or the equivalent step wrapper so `spawn_dynamic_task()` is memoized under `ctx.step.run` / `ctx.step.invoke`, matching `plan_subtasks()`. -- [ ] Keep idempotency at the worker facade boundary; do not require worker authors to pass custom replay keys. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py ergon_core/tests/unit/runtime/test_spawned_task_handle.py -q -``` - -Expected after implementation: replay emits one child task and one handle. - -### Task 5: Preserve The PR 11 Smoke Lifecycle Fix While Cleaning Identity - -- [ ] Keep the PR 11 `a613875` smoke behavior: smoke parent and recursive workers plan children and return instead of polling child completion inside `worker_execute`. -- [ ] Add or update a full-stack unit/integration smoke proving dependency-free dynamic children become ready after parent completion. -- [ ] Add coverage proving this behavior does not require worker code to read `context.node_id` once PR 12 identity cleanup is complete. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py ergon_core/tests/unit/runtime/test_smoke_topology_drift.py -q -``` - -Expected after implementation: recursive smoke topology still passes, and public worker code no longer needs direct `context.node_id` escape hatches. - -### Task 6: Update Read Models And Architecture Guards - -- [ ] Update run snapshots, dashboard registration, and telemetry DTOs to expose `task_id` consistently. -- [ ] Remove or tighten tests that permit `node_id` as public runtime vocabulary. -- [ ] Add architecture guard coverage preventing new application-layer public DTOs from adding `node_id`. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_run_read_service.py ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py -q -``` - -Expected after implementation: read models use task id vocabulary and architecture guards pass. - -## Acceptance Criteria - -- Dynamic children receive a non-null `WorkerContext.task_id`. -- Worker-authored `spawn_task()` is Inngest replay-safe. -- Public runtime DTOs, events, tests, and telemetry no longer expose `node_id`. -- Run provenance is explicit through `definition_id`. -- Schema reset and SQLModel models agree on the chosen identity. -- Unit tests covering static tasks, dynamic tasks, read models, and recursive smoke behavior pass. -- Final end-to-end smoke coverage passes on the branch before merge. - -## Do Not Include - -- Evaluator dispatch deletion. -- Registry deletion work. -- Dashboard frontend parser rewrites beyond backend contract naming required by this identity change. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr13-evaluation-cleanup.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr13-evaluation-cleanup.md deleted file mode 100644 index 9ad61855f..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr13-evaluation-cleanup.md +++ /dev/null @@ -1,147 +0,0 @@ -# PR 13 Evaluation Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete the remaining evaluator v1 dispatch path and make object-bound `Task.evaluators` the only runtime evaluator source. - -**Architecture:** Evaluation fanout starts from the run-tier task snapshot, counts inline evaluator objects, and sends id-only `TaskEvaluateRequest` payloads. The receiver reloads the task snapshot and executes `task.evaluators[evaluator_index]`. Definition evaluator rows may remain only as persistence/read-model metadata and FK targets, not as runtime dispatch sources. - -**Tech Stack:** Python, Pydantic, SQLModel, Inngest, pytest. - ---- - -## Scope - -This PR should follow PR 12 so task identity vocabulary is stable. It should -keep normalized evaluator persistence rows for `RunTaskEvaluation` -FK/read-model/query ergonomics. It deletes runtime dispatch duplication without -trying to collapse evaluator metadata into `task_json`. - -## Primary Files - -- Modify: `ergon_core/ergon_core/api/benchmark/task.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/execute_task.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` -- Modify: `ergon_core/ergon_core/core/application/evaluation/service.py` -- Modify: `ergon_core/ergon_core/core/application/evaluation/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/experiment_definition_writer.py` -- Modify: `ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py` -- Modify: `ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py` -- Modify: `ergon_core/tests/unit/runtime/test_rubric_evaluation_service.py` -- Modify: `ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py` -- Modify: `ergon_core/tests/unit/runtime/test_evaluation_context_schemas.py` - -## Code TODOs / Comments To Remove - -When PR 13 lands, delete the comments and TODOs that only existed to mark the -temporary evaluator bridge. Expected cleanup targets include: - -- `ergon_core/ergon_core/core/application/jobs/models.py`: remove the - `TaskEvaluateRequest` docstring sentence that says the binding-key fallback - remains only until PR 11. -- `ergon_core/ergon_core/core/application/jobs/execute_task.py`: remove the - `_fan_out_evaluators()` legacy binding-key fallback docstring and the - `TODO(PR 11)` comment above the fallback branch when the branch is deleted. -- `ergon_core/ergon_core/core/application/evaluation/service.py`: remove - comments that refer to `evaluate_legacy`, v1 executor-based signatures, or - PR 11 deleting those methods after the v1 dispatch path is gone. -- `ergon_builtins/ergon_builtins/evaluators/criteria/code_check.py` and any - remaining evaluator files: remove comments that describe deleted - `CriterionExecutor`, Inngest criterion executor, or bridge wiring. -- `ergon_core/ergon_core/api/rubric/evaluator.py` and benchmark rubric files: - resolve TODOs about unused runtime dependency validation if this PR proves - those hooks are obsolete; otherwise leave a precise non-transitional comment - explaining the live public contract. - -## Tasks - -### Task 1: Prove Binding Fallback Is Unsupported - -- [ ] Add a failing test that constructs a task with `evaluator_binding_keys` and no inline `evaluators`. -- [ ] Assert `persist_benchmark()` rejects that task with a clear object-bound evaluator error, or assert model construction no longer accepts the field after deletion. -- [ ] Add a runtime fanout test proving `execute_task` does not emit evaluator jobs when a task has no inline evaluators. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema -uv run pytest ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py ergon_core/tests/unit/core/application/jobs/test_execute_task_object_bound_fanout.py -q -``` - -Expected before implementation: failure on fallback behavior or accepted legacy field. - -### Task 2: Delete `Task.evaluator_binding_keys` - -- [ ] Remove `evaluator_binding_keys` from `Task`. -- [ ] Remove serialization/deserialization compatibility for that field. -- [ ] Replace any tests or fixtures that used binding keys with inline evaluator objects. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_definition_task_payload_typing.py ergon_core/tests/unit/runtime/test_run_graph_task_snapshot.py -q -``` - -Expected after implementation: task snapshots round-trip through inline evaluator objects only. - -### Task 3: Remove Runtime Fallback Fanout - -- [ ] Delete the fallback branch in `execute_task._fan_out_evaluators()` that derives evaluator count from binding keys or definition rows. -- [ ] Keep fanout behavior simple: `for evaluator_index in range(len(task.evaluators))`. -- [ ] Ensure zero inline evaluators means zero evaluator jobs and valid task completion. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_execute_task_evaluator_fanout.py ergon_core/tests/unit/runtime/test_evaluation_score_aggregation.py -q -``` - -Expected after implementation: fanout count is tied only to `task.evaluators`. - -### Task 4: Delete V1 Dispatch DTOs And Service Method - -- [ ] Delete `EvaluationService.prepare_dispatch()` after source search confirms no production path calls it. -- [ ] Delete stale internal DTOs from `ergon_core/ergon_core/core/application/evaluation/models.py`: `PreparedEvaluation`, `EvaluationDispatch`, internal `TaskEvaluationContext`, and internal `CriterionContext`. -- [ ] Replace test coverage with direct coverage of `TaskEvaluateRequest` and `EvaluationService.evaluate()`. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_rubric_evaluation_service.py ergon_core/tests/unit/runtime/test_evaluation_context_schemas.py -q -``` - -Expected after implementation: evaluation service tests cover execution, not dispatch preparation. - -### Task 5: Tighten Dynamic Evaluation Mapping - -- [ ] Add a test where a dynamic task has two inline evaluators and the second evaluator persists to the correct `definition_evaluator_id` or documented dynamic fallback id. -- [ ] Ensure `evaluate_task_run` maps `evaluator_index` through the run-tier task snapshot and definition evaluator metadata without consulting deleted dispatch DTOs. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_dynamic_task_evaluation_mapping.py -q -``` - -Expected after implementation: dynamic task evaluator mapping is deterministic. - -### Task 6: Clean Stale Comments And Guards - -- [ ] Remove comments mentioning deleted `CriterionExecutor`, `inngest_executor`, or legacy evaluator bridge wiring. -- [ ] Add an architecture test or import-boundary assertion preventing runtime jobs from importing deleted evaluation dispatch modules. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_import_boundaries.py ergon_core/tests/unit/architecture/test_runtime_read_boundaries.py -q -``` - -Expected after implementation: no runtime path can revive v1 evaluator dispatch. - -## Acceptance Criteria - -- `Task.evaluator_binding_keys` is gone. -- Runtime evaluator fanout depends only on inline `Task.evaluators`. -- `evaluate_task_run` executes the evaluator from the run-tier task snapshot. -- V1 dispatch DTOs and tests are deleted or renamed as non-runtime read-model helpers. -- Existing rubric/evaluation summary tests pass. - -## Do Not Include - -- Reworking rubric scoring semantics. -- Removing normalized evaluator definition rows. -- Dashboard UI changes except generated/read-model updates caused by deleted fields. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr14-public-api-registry-surface.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr14-public-api-registry-surface.md deleted file mode 100644 index ecaad5f3d..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr14-public-api-registry-surface.md +++ /dev/null @@ -1,155 +0,0 @@ -# PR 14 Public API And Registry Deletion Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete the core process-local registry and make the v2 public authoring surface fully object-bound. - -**Architecture:** Object snapshots using `_type` discriminators are the canonical runtime serialization path. No core process-local registry remains for runtime, discovery, onboarding, tests, CLI, or admin composition. Replacement discovery uses explicit imports, benchmark dependency declarations, package entrypoints, or local fixture factories. Public core imports guide authors toward `Benchmark`, `Task`, `Worker`, `Sandbox`, `Evaluator`, `Criterion`, and object-bound dynamic task APIs. ReAct `Toolkit` authoring, if public, belongs under a stable builtins/baselines package path rather than `ergon_core`. - -**Tech Stack:** Python, pytest, import-boundary tests, CLI tests. - ---- - -## Scope - -This PR is a registry deletion and public-surface PR. It should not change -runtime task identity or evaluator dispatch; those belong to PR 12 and PR 13. - -## Primary Files - -- Modify: `ergon_core/ergon_core/api/__init__.py` -- Delete: `ergon_core/ergon_core/api/registry.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/toolkit.py` -- Modify: `ergon_core/ergon_core/core/application/tasks/management.py` -- Modify: `ergon_core/ergon_core/core/application/workflow/service.py` -- Modify: `ergon_cli/tests/unit/state/test_subtask_lifecycle_toolkit.py` -- Modify: `ergon_cli/tests/unit/cli/test_workflow_cli.py` -- Delete or replace: `ergon_core/tests/unit/registry/test_core_registry_boundary.py` -- Delete or replace: `ergon_core/tests/unit/registry/test_component_registry.py` -- Delete: `ergon_core/tests/unit/registry/test_component_catalog_model.py` -- Modify: `ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py` - -## Code TODOs / Comments To Remove - -When PR 14 lands, registry-related comments should disappear with the registry -rather than moving into PR 16. Expected cleanup targets include: - -- `ergon_builtins/AGENTS.md`: remove or rewrite all references to - `ergon_builtins/registry_core.py`, `WORKERS`, `BENCHMARKS`, `EVALUATORS`, - and `SANDBOX_MANAGERS` as registry tables. -- `ergon_core/ergon_core/api/worker/context.py`: remove the typing comment - that says the "real objects" create a cycle through `ergon_core.api.registry` - once the registry import path is deleted. -- CLI, onboarding, smoke fixture, and REST startup files: remove comments that - describe registry discovery as a temporary bridge or fallback. -- Builtins benchmark docs and comments: remove references to - `_legacy_workers.py`, registry-string bridges, and PR 11 deleting registry - compatibility where those comments are only explaining deleted migration code. -- Registry tests and architecture ledgers: delete comments that allow - `ComponentCatalog`, `registry.workers`, or `ergon_core.api.registry` as known - transitional exceptions. - -## Locked Decisions - -- Delete the process-local core registry entirely. -- Delete the persistent component catalog model/tests. -- Keep `Toolkit` out of core; expose it only through a stable - builtins/baselines import path if it remains public. -- Treat `SubtaskLifecycleToolkit` as a worker-authoring toolkit, not an - operator/admin slug-registry surface. - -## Tasks - -### Task 1: Replace Registry Discovery Consumers - -- [ ] Add failing tests showing no production, CLI, onboarding, REST startup, smoke fixture, or test harness path imports `ergon_core.api.registry`. -- [ ] Replace CLI/onboarding discovery with explicit benchmark imports, benchmark dependency declarations, package entrypoints, or fixture-local factories. -- [ ] Add a test proving persisted task execution uses `_type` rehydration rather than registry worker lookup. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema -uv run pytest ergon_core/tests/unit/runtime/test_import_boundaries.py ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -q -``` - -Expected before implementation: failure while any registry import remains. - -### Task 2: Stabilize Builtins Toolkit Import - -- [ ] Expose `Toolkit` through a stable builtins/baselines package path if `ReActWorker(toolkit=...)` remains the intended authoring pattern. -- [ ] Do not export `Toolkit` from `ergon_core.api`; core should not depend on or present builtins ReAct composition as a core primitive. -- [ ] Create or document the builtins import path and use it consistently. -- [ ] Update builtins and examples to import from the stable path. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_import_boundaries.py ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -q -``` - -Expected after implementation: toolkit imports are stable under builtins/baselines, core remains free of toolkit exports, and object-bound benchmarks do not need registry lookup. - -### Task 3: Remove Registry Lookup From Dynamic Subtask Creation - -- [ ] Update `TaskManagementService.spawn_dynamic_task()` and worker-facing dynamic APIs so object-bound `Task` objects are the only worker-authored creation path. -- [ ] Delete slug/registry dynamic task creation paths instead of moving them to an admin method. -- [ ] Add tests proving `WorkerContext.spawn_task(Task(...))` succeeds with no registry registration. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py ergon_core/tests/unit/runtime/test_worker_context_containment.py -q -``` - -Expected after implementation: worker-authored dynamic subtasks do not consult the registry. - -### Task 4: Make `SubtaskLifecycleToolkit` Worker-Authoring Only - -- [ ] Delete or replace `SubtaskLifecycleToolkit` methods that accept worker slugs or registry worker names. -- [ ] Ensure `SubtaskLifecycleToolkit` exposes object-bound worker-facing methods only. -- [ ] Update CLI/toolkit tests so they construct concrete `Task` objects rather than selecting workers by slug. -- [ ] Run: - -```bash -uv run pytest ergon_cli/tests/unit/state/test_subtask_lifecycle_toolkit.py ergon_cli/tests/unit/cli/test_workflow_cli.py -q -``` - -Expected after implementation: `SubtaskLifecycleToolkit` is a worker-authoring object-bound toolkit, not an admin/registry escape hatch. - -### Task 5: Delete Persistent Component Catalog - -- [ ] Confirm source references to `ergon_core/ergon_core/core/persistence/components/models.py`. -- [ ] Delete the persistent component catalog model and its tests. -- [ ] Remove public exports such as `ComponentCatalog` that present the catalog as part of the authoring/runtime surface. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_import_boundaries.py -q -``` - -Expected after implementation: persistent component catalog code is gone. - -### Task 6: Update Public API Docs And Guards - -- [ ] Update RFC implementation docs so they say builtins static registries and the core process-local registry are deleted. -- [ ] Remove stale `AGENTS.md` references to deleted `registry_core.py`. -- [ ] Update public `__all__` exports to remove registry and component catalog surfaces. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_import_boundaries.py ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -q -``` - -Expected after implementation: public import surface and guardrails agree. - -## Acceptance Criteria - -- `ergon_core.api.registry` is deleted. -- Runtime, CLI, onboarding, REST startup, smoke fixtures, and test harnesses do not use registry lookup. -- Worker-authored dynamic tasks use object-bound `Task` inputs. -- `Toolkit` has a stable builtins/baselines import path and is not exported from core. -- Persistent component catalog is deleted. - -## Do Not Include - -- Schema identity work. -- Evaluator fallback deletion. -- Dashboard event parser rewrites. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr15-dashboard-event-contracts.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr15-dashboard-event-contracts.md deleted file mode 100644 index bc4d8b8b7..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr15-dashboard-event-contracts.md +++ /dev/null @@ -1,158 +0,0 @@ -# PR 15 Dashboard Event Contract Alignment Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Align live dashboard event payloads, generated frontend contracts, handwritten parsers, and run-state reducers after the PR 11/12 runtime vocabulary changes. - -**Architecture:** Backend dashboard event contracts are the source of truth. Every backend live event must have a generated frontend contract. Handwritten parser logic may normalize transport details, but it must be tested against generated schemas and final `task_id` vocabulary. REST hydration adapters may remain separate, but live events must not bypass canonical conversion for graph mutations or context events. - -**Tech Stack:** Python, Pydantic, TypeScript, Zod, Next.js dashboard tests, pytest, pnpm. - ---- - -## PR 11 Head Update - -PR 11 commit `a613875` added compatibility parsing for two live-event drift -cases: backend-wrapped graph mutation events and backend context-part payloads. -This PR should not redo that short-term patch. It should turn those shims into -canonical generated contracts and final field names. - -## Scope - -This PR should follow PR 12 because it depends on final `task_id` naming. It can -run in parallel with PR 13/14 if the backend event contract files are stable. -It is limited to contracts, reducers, fixtures, and tests; it should not include -visual dashboard component redesign. - -## Primary Files - -- Modify: `ergon_core/ergon_core/core/infrastructure/dashboard/event_contracts.py` -- Modify: `ergon_core/tests/unit/runtime/test_graph_mutation_contracts.py` -- Modify: `ergon_core/tests/unit/runtime/test_context_event_contracts.py` -- Modify: `ergon_core/tests/unit/dashboard/test_event_contract_types.py` -- Modify: `ergon-dashboard/scripts/generate-event-contracts.mjs` -- Modify: `ergon-dashboard/src/generated/events/*` -- Modify: `ergon-dashboard/src/lib/contracts/events.ts` -- Modify: `ergon-dashboard/src/lib/contracts/contextEvents.ts` -- Modify: `ergon-dashboard/src/lib/runEvents.ts` -- Modify: `ergon-dashboard/src/lib/run-state/reducers.ts` -- Modify: `ergon-dashboard/tests/contracts/contracts.test.ts` -- Modify: `ergon-dashboard/tests/contracts/context-events.contract.test.ts` -- Modify: `ergon-dashboard/tests/contracts/run-state-roundtrip.contract.test.ts` - -## Code TODOs / Comments To Remove - -When PR 15 lands, remove the dashboard parser TODOs and compatibility comments -that the generated contracts replace. Expected cleanup targets include: - -- `ergon-dashboard/src/lib/contracts/events.ts`: remove `TODO(E2b)` comments - for generated evaluation, thread/message, context, and workflow-started event - schemas once every backend live event has a generated frontend contract. -- `ergon-dashboard/src/features/graph/contracts/graphMutations.*` and reducer - tests: remove comments and fixtures that normalize backend - `source_task_id`/`target_task_id` into deleted `source_node_id`/ - `target_node_id` names. -- `ergon-dashboard/tests/helpers/testHarnessClient.ts` and - `ergon-dashboard/tests/helpers/backendHarnessClient.ts`: replace - `parent_node_id` fixture/helper vocabulary with `parent_task_id` after PR 12. -- Backend/dashboard contract tests: delete temporary comments that describe - backend-wrapped graph mutations or backend context-part payloads as - compatibility shims once those shapes are canonical generated contracts. -- Any e2e assertion comments that explain a "node_id join" should be removed or - rewritten to the final task-id contract. - -## Tasks - -### Task 1: Snapshot Backend Event Shapes - -- [ ] Add backend tests that serialize representative graph mutation, context event, workflow started, task status, evaluation, sandbox, and resource events. -- [ ] Assert graph edge payloads use `source_task_id` and `target_task_id`. -- [ ] Assert context events use the canonical context payload shape emitted by backend runtime. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema -uv run pytest ergon_core/tests/unit/runtime/test_graph_mutation_contracts.py ergon_core/tests/unit/runtime/test_context_event_contracts.py ergon_core/tests/unit/dashboard/test_event_contract_types.py -q -``` - -Expected before implementation: fail where tests expose current backend/frontend drift. - -### Task 2: Generate Missing Event Contracts - -- [ ] Extend backend event contract export so graph mutation and context events are first-class generated frontend contracts. -- [ ] Update `ergon-dashboard/scripts/generate-event-contracts.mjs` only if the generator cannot consume the backend schema as-is. -- [ ] Regenerate frontend event files. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema/ergon-dashboard -pnpm run generate:contracts:events -``` - -Expected after implementation: generated files include graph mutation and context event contracts, or the equivalent canonical event union. - -### Task 3: Replace Compatibility Shims With Canonical Contracts - -- [ ] Keep the PR 11 behavior that accepts nested backend `{ mutation: ... }` graph events, but move the accepted shape into generated/frontend canonical contracts. -- [ ] Remove the long-term need for `source_node_id` / `target_node_id` compatibility by making frontend graph contracts and reducers consume `source_task_id` / `target_task_id`. -- [ ] Keep the PR 11 behavior that accepts backend context-part payloads, but move normalization into a shared canonical converter or generated contract path used by both REST hydration and live events. -- [ ] Run: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema/ergon-dashboard -pnpm test -- tests/contracts/contracts.test.ts tests/contracts/context-events.contract.test.ts -``` - -Expected after implementation: live parser contract tests pass with backend-shaped fixtures without relying on stale node-id field names. - -### Task 4: Align Workflow Started Task Tree - -- [ ] Update backend or frontend contract expectations so `workflow.started` task-tree nodes agree on required `status`, `level`, and resource-id fields. -- [ ] Do not invent frontend-only fields in the contract layer; derive UI fields inside run-state selectors or reducers. -- [ ] Run: - -```bash -pnpm test -- tests/contracts/run-state-roundtrip.contract.test.ts -``` - -Expected after implementation: a backend workflow-started fixture round-trips through frontend run-state without manual patching. - -### Task 5: Update Reducers And Fixtures - -- [ ] Update run-state reducers to consume canonical graph/context events. -- [ ] Update dashboard fixtures and e2e harness client DTOs from `parent_node_id` to `parent_task_id` if PR 12 removed public node vocabulary. -- [ ] Run: - -```bash -pnpm test -- src/lib/run-state/reducers.test.ts tests/contracts/run-state-roundtrip.contract.test.ts -pnpm test -- tests/e2e/live.smoke.spec.ts -``` - -Expected after implementation: live smoke and reducer tests consume canonical event payloads. - -### Task 6: Add Drift Guards - -- [ ] Add a test that fails when any backend dashboard event has no generated frontend contract. -- [ ] Add a test that fails when handwritten frontend parser fields diverge from generated event fields. -- [ ] Run: - -```bash -pnpm test -- tests/contracts/contracts.test.ts -uv run pytest ergon_core/tests/unit/dashboard/test_event_contract_types.py -q -``` - -Expected after implementation: future backend event additions require corresponding frontend contract coverage. - -## Acceptance Criteria - -- Live dashboard graph events parse backend-emitted nested graph mutation payloads. -- Live context events share canonical parsing with REST hydration. -- Frontend contracts use `task_id` vocabulary consistently. -- Contract tests pass in Python and TypeScript. -- Dashboard e2e smoke can hydrate and update a run with live events. - -## Do Not Include - -- Visual redesign of dashboard components. -- Runtime schema migration work. -- Registry or evaluator cleanup. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr16-core-debt-sweep.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr16-core-debt-sweep.md deleted file mode 100644 index 4db1cc1a6..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr16-core-debt-sweep.md +++ /dev/null @@ -1,180 +0,0 @@ -# PR 16 Core Debt Sweep Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove residual dead code, stale comments, duplicate lifecycle paths, and unfinished PR ledgers after the main v2 runtime/public-surface decisions have landed. - -**Architecture:** This is intentionally a sweep PR. It should not make new architecture decisions. It deletes code already made unreachable by PRs 11-15, tightens guards, implements cancelled-task sandbox release, and removes all lingering xfails/ledgers so the v2 core is easier to reason about. - -**Tech Stack:** Python, pytest, architecture tests, docs. - ---- - -## Scope - -This PR should land after PRs 12-15. It should leave no known-violator ledgers, -xfails, or unresolved cleanup exceptions behind. - -## Primary Files - -- Delete or modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/_legacy_toolkit.py` -- Delete or modify: `ergon_core/ergon_core/core/application/context/output_extraction.py` -- Modify: `ergon_core/ergon_core/core/application/workflow/service.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/sandbox_cleanup.py` -- Modify: `ergon_core/ergon_core/core/application/tasks/management.py` -- Modify: `ergon_core/tests/unit/runtime/test_workflow_service.py` -- Modify: `ergon_core/tests/unit/runtime/test_failed_task_sandbox_cleanup.py` -- Modify: `ergon_core/tests/unit/runtime/test_sandbox_cleanup.py` -- Modify: `ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py` -- Modify: `ergon_core/tests/unit/architecture/test_repository_companion_files.py` -- Modify: `ergon_builtins/AGENTS.md` -- Modify: stale comments in builtins and core files identified by `dead_code.MD` - -## Code TODOs / Comments To Remove - -PR 16 is the final cleanup pass. It should not carry forward broad "maybe -later" TODOs created by the v2 migration. Expected cleanup targets include: - -- `ergon_core/ergon_core/core/application/context/output_extraction.py`: delete - the file if it is dead, or replace the file-level TODO with a precise live - responsibility if it remains as a read-model adapter. -- `ergon_core/ergon_core/core/application/graph/propagation.py`: resolve the - remaining service/repository-ownership TODO if it still exists after PR 12 - and PR 16's lifecycle consolidation. -- `ergon_core/ergon_core/core/application/jobs/__init__.py`, - `ergon_core/ergon_core/core/application/evaluation/__init__.py`, - `ergon_core/ergon_core/core/application/graph/models.py`, and - `ergon_core/ergon_core/core/application/evaluation/scoring.py`: delete or - narrow broad module-structure TODOs for domains this PR actually touches; - leave only specific, non-migration follow-ups if a larger refactor is out of - scope. -- Sandbox cleanup files and tests: remove comments that promise cancelled-task - release without implementing it, and remove failed-task duplicate-terminator - notes once `sandbox_cleanup` is the only owner. -- Architecture tests and ledgers: delete remaining known-violator comments, - xfail reasons, and `landing_pr="PR 11"` expectations that no longer describe - the current stack. -- Builtins/core stale comments discovered by the final search for - `legacy bridge`, `_legacy_workers`, `registry_core`, `CriterionExecutor`, - `inngest criterion`, `sandbox_manager`, and `evaluate_legacy` should either - be gone or live only in historical RFC prose, not production code. - -## Tasks - -### Task 1: Re-Audit Dead-Code Candidates Against Current Head - -- [ ] Run source searches for each candidate in `dead_code.MD`. -- [ ] Record only current-head facts in the PR description; do not rely on old audit wording. -- [ ] Use this command set: - -```bash -cd /Users/charliemasters/.config/superpowers/worktrees/ergon/codex-v2-pr-11-deletion-final-schema -rg -n "output_extraction|_legacy_toolkit|prepare_dispatch|evaluator_binding_keys|ComponentCatalog|restart_task\\(|abandon_task\\(" ergon_core ergon_builtins ergon_cli -``` - -Expected before deletion: every deletion target has no runtime references or a clear replacement task in this plan. - -### Task 2: Delete MiniF2F Legacy Toolkit If Unused - -- [ ] Delete `ergon_builtins/ergon_builtins/benchmarks/minif2f/_legacy_toolkit.py` if it remains unreferenced. -- [ ] Remove stale comments in MiniF2F files pointing at deleted legacy workers. -- [ ] Run: - -```bash -uv run pytest ergon_builtins/tests/unit/architecture/test_object_bound_benchmarks_no_registry.py -q -``` - -Expected after implementation: MiniF2F object-bound benchmark tests pass with no legacy toolkit. - -### Task 3: Delete Or Rename Context Output Extraction - -- [ ] If `output_extraction.py` is unreferenced, delete it and any tests that only cover the dead path. -- [ ] If still referenced by an external contract, rename it to a read-model adapter and document why it exists. -- [ ] Ensure terminal output persistence still flows through `WorkerOutputRepository.persist()` and `RunTaskExecution.worker_output_json`. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py ergon_core/tests/unit/runtime/test_persist_outputs_resources.py -q -``` - -Expected after implementation: worker output tests pass without dead extraction helpers. - -### Task 4: Consolidate Workflow Lifecycle Methods - -- [ ] Delete `WorkflowService.restart_task()` and `WorkflowService.abandon_task()` after moving remaining callers to `TaskManagementService`. -- [ ] Ensure terminal guards, cascade invalidation, dispatch, and containment checks live in one service. -- [ ] Update CLI dry-run/non-dry-run tests to call the canonical service path. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_workflow_service.py ergon_cli/tests/unit/state/test_workflow_cli_tool.py -q -``` - -Expected after implementation: restart/abandon semantics are covered once and are not weaker through workflow service. - -### Task 5: Make Sandbox Cleanup Single-Owner - -- [ ] Remove failed-task sandbox termination from propagation because `sandbox_cleanup` owns terminal sandbox release. -- [ ] Add tests proving a failed task emits exactly one cleanup action and one terminal sandbox event. -- [ ] Implement and test sandbox release for cancelled tasks. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime/test_failed_task_sandbox_cleanup.py ergon_core/tests/unit/runtime/test_sandbox_cleanup.py -q -``` - -Expected after implementation: failed-task cleanup has one owner and cancelled tasks release sandboxes. - -### Task 6: Drain Remaining Ledgers And Xfails - -- [ ] Remove the remaining `_KNOWN_VIOLATORS` entry from `ergon_core/tests/unit/architecture/test_repository_companion_files.py` by adding the missing companion file or deleting the stale exception. -- [ ] Un-xfail the walkthrough sandbox acquire/release completion guard and make it pass. -- [ ] Delete any remaining known-violator ledgers instead of moving them to follow-ups. -- [ ] Run: - -```bash -uv run pytest ergon_core/tests/unit/architecture/test_repository_companion_files.py ergon_core/tests/unit/runtime/test_walkthrough_smoketest.py -q -``` - -Expected after implementation: no undocumented ledger exceptions remain. - -### Task 7: Clean Stale Docs And Comments - -- [ ] Update `ergon_builtins/AGENTS.md` so it no longer points at deleted registry files. -- [ ] Remove comments in builtins/core files that reference deleted bridge modules, deleted criterion executors, or deleted sandbox managers. -- [ ] Run: - -```bash -rg -n "legacy bridge|_legacy_workers|registry_core|CriterionExecutor|inngest criterion|sandbox_manager|evaluate_legacy" ergon_core ergon_builtins docs -``` - -Expected after implementation: matches are either gone or intentionally describe historical RFC context. - -### Task 8: Final Verification Sweep - -- [ ] Run focused backend suites touched by this PR. -- [ ] Run import-boundary and architecture tests. -- [ ] Run diff checks. -- [ ] Commands: - -```bash -uv run pytest ergon_core/tests/unit/runtime ergon_core/tests/unit/architecture ergon_builtins/tests/unit/architecture -q -git diff --check -``` - -Expected after implementation: all touched suites pass and no whitespace/check errors remain. - -## Acceptance Criteria - -- Dead files listed in this plan are deleted or explicitly documented as live. -- Duplicate lifecycle paths are collapsed to a single owner. -- Failed-task sandbox cleanup is single-owner. -- Cancelled tasks release sandboxes. -- PR ledgers and xfails are fully drained. -- Stale comments no longer mislead readers about deleted v1 paths. - -## Do Not Include - -- New schema identity work. -- New evaluator architecture. -- New dashboard contracts. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr17-remove-legacy-compatibility-notes.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr17-remove-legacy-compatibility-notes.md deleted file mode 100644 index 4d2143e63..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr17-remove-legacy-compatibility-notes.md +++ /dev/null @@ -1,51 +0,0 @@ -# PR 17 Remove Legacy Compatibility Notes - -Date: 2026-05-18 - -## Why - -PR 16 removed the last broad bridge/debt structures, but several active source -paths still preserved pre-v2 compatibility semantics. The most confusing parts -were public variables named `experiment_id` even though the value was now the -canonical v2 definition id, plus read-model and test-harness paths that used -`BenchmarkDefinitionRecord` as a compatibility row beside -`ExperimentDefinition`. - -This PR makes the v2 contract explicit: active runtime, API, CLI, test, and -dashboard code speaks in terms of definition identity. There are no source/test -references to old compatibility paths, and no active read model falls back from -`ExperimentDefinition` to the older benchmark-definition row. - -## How - -- Added architecture guards for active source/test references to compatibility - language and for `experiment_id` / `experimentId` naming. -- Removed small public API aliases and fallback fields: - `Task.evaluator_binding_keys`, `Criterion.validate_deps()`, and - `ResearchRubricsJudgeCriterion(model=...)` / `.model`. -- Renamed active REST/event/dashboard DTO identity fields from experiment - vocabulary to `definition_id` / `definitionId` where they represent - definition identity. -- Removed `BenchmarkDefinitionRecord` read-model and test-harness - compatibility writes. Cohort/display metadata for harness-created - definitions now lives in `ExperimentDefinition.metadata_json`. -- Updated cohort/run read models, workflow completion/failure tracing, CLI - filters, RL rollout creation, and integration fixtures to use canonical - definition rows directly. -- Regenerated dashboard event schemas, OpenAPI, and generated REST/event Zod - contracts from the backend after the rename. - -## Gotchas - -- The `experiments` REST route and UI route names still exist as product - language, but the route parameter and payload identity fields are - `definition_id` / `definitionId`. -- Cohort membership no longer comes from `BenchmarkDefinitionRecord.cohort_id`. - New test-harness cohorts stamp `ExperimentDefinition.metadata_json["cohort_id"]`. - This avoids a schema migration in PR 17 while removing the compatibility row. -- `BenchmarkDefinitionRecord` still exists as a persistence model because - deleting the table/model is schema work. This PR removes active compatibility - reads/writes from source paths covered by the guards. -- Dashboard generated files were regenerated after writing - `ergon-dashboard/src/generated/rest/openapi.json` from `app.openapi()`. - Review generated diffs together with the backend DTO changes. diff --git a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr_stack_recut.md b/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr_stack_recut.md deleted file mode 100644 index 17dbff704..000000000 --- a/docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/reconciliation_plan/pr_stack_recut.md +++ /dev/null @@ -1,147 +0,0 @@ -# PR Stack Recut Proposal - -Date: 2026-05-18 - -Branch audited: `codex/v2-pr-11-deletion-final-schema` - -## Goal - -Keep PR 11 as close as possible to deletion/final-schema cleanup, while not -pretending that unresolved data-model and runtime gaps are mechanical deletion. - -## Current State - -PR 11 already contains a lot of valid deletion work: legacy bridge modules, -saved specs, executor modules, builtins registries, benchmark sandbox managers, -and old legacy workers are gone. The object-bound happy path is real. - -The remaining work falls into three categories: - -1. Mechanical deletion still appropriate for PR 11. -2. Small correctness fixes that can land on the PR 11 branch before final - deletion. -3. Architectural decisions that need to be finished in the follow-on stack - rather than hidden inside PR 11. - -## Suggested Micro-PRs Or Commit Groups - -### PR 11a: Smoke/runtime correctness patch - -Purpose: get the current branch to a trustworthy baseline before schema churn. - -Status on PR 11 head `a613875`: the smoke lifecycle portion of this patch has -landed. Smoke parent and recursive workers now plan children and return instead -of polling siblings inside `worker_execute`, dependency-free dynamic children -are readied on parent completion, dynamic worker payloads tolerate no model -target, object-bound `_type` rehydration strips discriminator fields before -Pydantic validation, and dashboard compatibility parsers cover wrapped graph -mutation/context-part payloads. - -Include: - -- Fix the GDPEval smoke worker registration mismatch. -- Restore or explicitly replace ResearchRubrics search/web tools if the - benchmark contract requires them. -- Fix dynamic `WorkerContext.task_id` for dynamic children by passing the - canonical graph id into the public facade, not only `node_id`. -- Make public `WorkerContext.spawn_task()` replay-safe under Inngest, matching - the step-aware treatment already added for `plan_subtasks`. -- Preserve the landed smoke contract where parents plan children and return; - move any remaining child-completion assertions outside parent worker polling. -- Delete or hard-error the evaluator binding fallback so it cannot invoke - unsupported eval jobs. -- Remove duplicate failed-task sandbox termination from propagation if - `sandbox_cleanup` is the owner. - -This is a correctness patch, not final-schema deletion. - -### PR 14: Registry deletion - -Purpose: settle the mismatch between the PR 11 deletion list and live registry -usage by deleting the core process-local registry entirely. - -Include: - -- Replace workflow-service payload model lookup with task-json-derived payload - handling. -- Replace dynamic subtask slug worker synthesis with object-bound Task creation - only. -- Move smoke fixtures, CLI discovery, onboarding, REST startup, and test - harnesses away from registry lookup. -- Remove public `registry` and `ComponentCatalog` exports. -- Delete persistent component catalog model/tests. -- Add architecture guards proving no source path imports `ergon_core.api.registry`. - -### PR 11c: Schema identity decision and implementation - -Purpose: resolve `node_id` vs `task_id` for real. - -- Add final `RunGraphNode.task_id` composite identity, remove `id`, remove - edge/telemetry bridge columns, update all repositories, read models, events, - tests, and generated contracts. -- Remove `node_id` from `WorkerContext`, job payloads, task events, telemetry - rows, dashboard registration, and test helpers. -- Regenerate final migration explicitly and verify generated frontend/backend - contracts. -- Rename/replace `RunRecord.experiment_id` with `definition_id`. - -### PR 11d: Definition metadata cleanup - -Purpose: decide how much normalized definition metadata remains beside -`task_json`. - -Include: - -- Decide whether `ExperimentDefinitionWorker`, `ExperimentDefinitionEvaluator`, - task assignment, task evaluator, and `task_payload_json` remain final schema - or are bridge-only. -- Retain normalized evaluator metadata rows for evaluator FK/read-model/query - ergonomics. -- Do not remove evaluator rows in the cleanup stack unless a later dedicated - schema PR replaces the evaluation persistence FK strategy. -- Fix launch provenance between `ExperimentDefinition`, `BenchmarkDefinitionRecord`, - and the new `RunRecord.definition_id`. - -### PR 11e: Mechanical deletion and ledgers - -Purpose: the actual PR 11 deletion pass after the above is settled. - -Include: - -- Delete dead evaluator dispatch DTOs/service/tests. -- Delete `Task.evaluator_binding_keys`. -- Delete `minif2f/_legacy_toolkit.py`. -- Delete `output_extraction.py` if no external import contract remains. -- Delete persistent component catalog if the registry decision says it is dead. -- Remove stale docs/comments. -- Drain remaining architecture ledgers and un-xfail walkthrough completion - guard. No known-violator ledgers or xfails remain. - -### PR 12a or PR 11f: Dashboard contract alignment - -Purpose: prevent schema/identity cleanup from leaving the live dashboard on -stale event shapes. - -Include: - -- Generate graph/context dashboard events as first-class frontend contracts, or - update handwritten live parsers to match backend payloads exactly. -- Replace frontend `source_node_id` / `target_node_id` graph vocabulary with - `source_task_id` / `target_task_id` if PR 11 keeps that final naming. -- Align `workflow.started` task-tree fields with backend generated schemas. -- Update TS test-harness DTOs from `parent_node_id` to `parent_task_id`. - -## Recommendation - -Do not merge PR 11 as-is. It is not just missing a few final deletes; it has a -mixed schema identity state and live registry mismatch. The fastest honest path -is probably: - -1. land a small smoke/runtime correctness commit group on the PR 11 branch; -2. delete the core registry and persistent component catalog; -3. choose and implement one runtime identity model; -4. then perform the remaining deletion/ledger cleanup. - -That still preserves the spirit of PR 11: it becomes the branch where old paths -die. It just stops asking the deletion PR to also quietly invent unresolved -architecture. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/README.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/README.md deleted file mode 100644 index be0c3402c..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/README.md +++ /dev/null @@ -1,192 +0,0 @@ ---- -status: landed -opened: 2026-05-18 -author: charlie + agent -architecture_refs: - - ../../../architecture/01_public_api.md - - ../../../architecture/02_runtime_lifecycle.md - - ../../../architecture/04_persistence.md -supersedes: [] -superseded_by: null ---- - -# RFC: Core Domain Structure Standardization - -## Purpose - -This RFC records the target architecture for cleaning up -`ergon_core.core` after the v2 authoring/runtime stack landed. - -Earlier versions of this folder included speculative option documents. Those -have been replaced with concrete PRDs plus audit evidence. The implementation -direction is now settled enough that the next step should be writing executable -PR plans from these PRDs. - -## Problem - -`ergon_core.core` has the right high-level layers, but the boundaries are -currently uneven: - -- `persistence/` contains table models, but also some application command DTOs, - repositories, lifecycle vocabulary, and compatibility schemas. -- `domain/` is nominal and does not contain real domain behavior. -- `infrastructure/` sometimes owns application behavior such as dashboard - view assembly, cohort recomputation, and resource append policy. -- REST should be a thin inbound adapter under `infrastructure/http/`, but - needs boundary enforcement. -- `application/` contains most real behavior, but runtime logic is split across - `tasks`, `workflows`, and `graph`; view-building logic is mixed with - command-side services and deprecated cohort compatibility. - -## Target Direction - -Use a layered, port-and-adapter shape: - -```text -inbound adapters -> application use cases / views -application use cases -> application ports -> infrastructure implementations -``` - -Ownership rules: - -- Application services own use cases and invariants. -- Application should not depend on concrete infrastructure. Where a use case - needs an external system, application declares a narrow port/protocol and the - composition root injects an implementation. -- Persistence owns SQLModel tables, database/session setup, storage-safe types, - and low-level row validation. -- Persistence should not define application command DTOs, domain repositories, - runtime lifecycle policy, or compatibility services. -- Infrastructure owns external adapters and framework glue. It may implement - application ports, but it should not own business rules, views, or - persistence semantics that already exist in application. -- HTTP adapters own REST translation only. -- Inngest handlers are thin framework adapters into application jobs/use cases. -- Views own read-only contract builders and must not mutate runtime state. -- If a duplicate operation appears outside application, first check whether - application already owns that operation shape. Reuse or extract that logic - instead of reimplementing it under a tidier path. - -## PRDs - -The PRDs are ordered by recommended implementation sequence. - -0. [`prds/00-move-vs-delete-decisions.md`](prds/00-move-vs-delete-decisions.md) - records the scrutiny pass for moved schemas/repositories: which ones are - real contracts, which ones should be merged, and which ones should die. -1. [`prds/01-persistence-schema-boundary.md`](prds/01-persistence-schema-boundary.md) - makes persistence a true storage-definition layer and fixes schema concept - debt. -2. [`prds/02-shared-context-contract.md`](prds/02-shared-context-contract.md) - deletes the anemic `domain/` package by moving context stream schemas to - `core/shared/context_parts.py`. -3. [`prds/03-infrastructure-adapter-boundary.md`](prds/03-infrastructure-adapter-boundary.md) - makes dashboard, sandbox, Inngest, and tracing true adapters. -4. [`prds/04-rest-inbound-adapter-boundary.md`](prds/04-rest-inbound-adapter-boundary.md) - moves REST under `infrastructure/http/` as a thin inbound adapter instead - of application logic. -5. [`prds/05-application-runtime-restructure.md`](prds/05-application-runtime-restructure.md) - consolidates duplicated runtime lifecycle behavior across tasks, workflows, - and graph. -6. [`prds/06-views-dashboard-contracts.md`](prds/06-views-dashboard-contracts.md) - replaces loose read models with a stricter view/dashboard-contract - boundary. -7. [`prds/07-deprecated-cohorts-legacy-experiments.md`](prds/07-deprecated-cohorts-legacy-experiments.md) - isolates and removes cohort and legacy experiment-record compatibility. -8. [`prds/08-job-composition-modules.md`](prds/08-job-composition-modules.md) - promotes jobs into semantic composition modules so Inngest handlers stay - thin and application use cases can receive infrastructure implementations - through ports. -9. [`prds/09-final-core-folder-state.md`](prds/09-final-core-folder-state.md) - defines the final `ergon_core.core` package layout and architecture-test - acceptance criteria for the full refactor. -10. [`prds/10-application-domain-layout-convention.md`](prds/10-application-domain-layout-convention.md) - defines the post-stack convention for application domain folders, public - facades, temporary characterization tests, and permanent import-boundary - tests. - -## Implementation Plan - -The engineering PR stack lives in -[`implementation-plan/00-program.md`](implementation-plan/00-program.md). -Each PR plan is intentionally separate from the PRDs and includes the "what", -"why", "how", planned implementation steps, acceptance criteria, and test gates -for that slice. - -## Implementation Status - -The implementation-plan stack has landed as of 2026-05-19. PR12 added the -final architecture gates for the accepted `ergon_core.core` folder shape, -deleted roots, deleted compatibility names, import boundaries, and the docs -status recorded here. Future changes should update `docs/architecture/*` and -the architecture gates when they intentionally alter the landed layout. - -## Evidence - -The audit documents are evidence for the PRDs. They are not implementation -plans and should not be treated as competing options. - -- [`audits/current-structure.md`](audits/current-structure.md): current package - map and boundary overview. -- [`audits/runtime-domain-merge-audit.md`](audits/runtime-domain-merge-audit.md): - evidence for merging or standardizing runtime domains. -- [`audits/infrastructure-application-boundary-audit.md`](audits/infrastructure-application-boundary-audit.md): - dashboard/sandbox/Inngest/tracing duplication audit. -- [`audits/persistence-boundary-audit.md`](audits/persistence-boundary-audit.md): - persistence-layer ownership audit. -- [`audits/schema-concept-debt-audit.md`](audits/schema-concept-debt-audit.md): - deprecated tables, duplicated identity fields, stale columns, and schema - concept debt. - -## Recommended Order Of Operations - -### 1. Persistence And Schema Concepts - -Clean persistence first because it is the foundation for every other boundary: - -- move application concepts out of persistence; -- fix stale schema references; -- route compatibility-only tables through named `application/compat/*` modules; -- enforce the boundary with tests. - -### 2. Shared Context Contract - -Delete `domain/` as a quick win by moving the context stream schemas to -`core/shared/context_parts.py`. - -### 3. Infrastructure Adapters - -Clean up dashboard, sandbox, Inngest, and tracing after persistence and shared -contracts are clearer. - -### 4. REST Inbound Adapter - -Move REST route modules under `infrastructure/http/` as thin translators into -application services and views. Do not move route logic into application -packages. - -### 5. Application Restructure - -Only after the outer boundaries stop leaking, bite off the larger application -deduplication work: runtime domain consolidation, views, dashboard -contracts, job composition modules, and deprecated cohort/legacy experiment -removal. - -## Implementation Notes - -- Each PRD should become one or more small implementation plans before code - changes begin. -- Every implementation plan should include characterization tests before moves, - import-boundary tests after moves, and explicit grep checks for deleted - compatibility paths. -- Behavior changes should be isolated from mechanical package moves whenever - possible. -- Compatibility-only concepts must be named as such in code while they remain. - -## On Acceptance - -When this RFC is accepted: - -- write executable PR plans for the PRDs in order; -- update `docs/architecture/*` with the accepted ownership rules; -- add or update architecture tests that enforce the new package boundaries. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/current-structure.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/current-structure.md deleted file mode 100644 index e0ca4cb8a..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/current-structure.md +++ /dev/null @@ -1,375 +0,0 @@ -# Current Core Structure - -Date audited: 2026-05-18 - -Branch audited: `origin/codex/v2-pr-16-core-debt-sweep` - -## Summary - -`ergon_core.core` currently uses a layered split: - -```text -core/ - application/ - domain/ - infrastructure/ - persistence/ - rest_api/ - rl/ - shared/ -``` - -This is a good high-level frame. The unevenness is inside the layers: - -- `application/` contains most real domain behavior. -- `domain/` is almost empty. -- `jobs/` is a cross-domain orchestration folder. -- `tasks`, `workflows`, and `graph` have overlapping runtime lifecycle - responsibilities. -- `read_models` is a query-side contract layer, but it lives beside command - services without a naming distinction. - -## Top-Level Layers - -### `application/` - -This is the active core of the system. It owns use cases, orchestration, -runtime services, job payload handlers, read models, and most business rules. - -Current subdomains: - -```text -application/ - communication/ - context/ - evaluation/ - events/ - experiments/ - graph/ - jobs/ - read_models/ - resources/ - tasks/ - workflows/ -``` - -Observed pattern: each subdomain uses a loose mix of `models.py`, `errors.py`, -`service.py`, and more specific files. The pattern is useful but not consistent -enough to make ownership obvious without reading code. - -### `domain/` - -This is currently very thin. The visible source is mostly: - -```text -domain/ - generation/ - context_parts.py -``` - -There are also stale cache-only directories from deleted domain experiments -code. As of this audit, `domain/` is not where most runtime invariants live. -Those invariants live in application services. - -`context_parts.py` is also not a domain model in the usual sense. It defines -worker/context stream payload contracts used by worker APIs, persistence, -dashboard views, replay, tests, and RL extraction. The better home is: - -```text -shared/ - context_parts.py -``` - -That move keeps the import boundary that motivated the original placement: -persistence can import the schemas without importing `ergon_core.api`. It also -lets the codebase delete `domain/` instead of preserving an anemic one-file -layer. - -### `persistence/` - -This owns SQLModel tables, repository primitives, database setup, and DB-adjacent -types. - -Current subdomains: - -```text -persistence/ - context/ - definitions/ - graph/ - imports/ - shared/ - telemetry/ -``` - -This layer is comparatively coherent. It mirrors the v2 persistence model: -definition tables, run graph tables, context event tables, import metadata, and -telemetry rows. - -There are stale or cache-only remnants under: - -```text -persistence/components/ -persistence/saved_specs/ -``` - -These should be deleted from the working tree if they are only generated cache -artifacts or empty packages left behind by PR 11/16 cleanup. - -### `infrastructure/` - -This owns adapters and framework glue. - -Current subdomains: - -```text -infrastructure/ - dashboard/ - inngest/ - sandbox/ - tracing/ - dependencies.py -``` - -This is a good shape. The main boundary concern is that Inngest jobs live in -`application/jobs`, while Inngest registration lives in -`infrastructure/inngest`. That split is defensible, but the job functions should -remain thin entrypoints into application services. - -### `rest_api/` - -This owns FastAPI routers and HTTP translation: - -```text -rest_api/ - app.py - cohorts.py - experiments.py - rollouts.py - runs.py - test_harness.py -``` - -The desirable invariant is that these modules call application services and -read models, not persistence or infrastructure directly except for boundary -composition. - -### `rl/` - -This is a separate RL-specific subsystem: - -```text -rl/ - checkpoint.py - eval_runner.py - extraction.py - rewards.py - rollout_service.py - rollout_types.py - vllm_manager.py -``` - -It currently sits top-level inside core. That may be right if RL is a first-class -core capability. If it is optional or adapter-like, this is a future boundary to -review. - -### `shared/` - -Small shared support: - -```text -shared/ - context_parts.py - json_types.py - settings.py - utils.py -``` - -This is acceptable, but `shared` should stay small. New business concepts should -not land here just because they are used by two modules. - -`context_parts.py` is an acceptable shared module because it is a genuine -cross-cutting schema contract. It should remain narrow: context stream payload -schemas only, not event builders, persistence models, dashboard-only DTOs, or -worker execution logic. - -## Application Subdomains - -### `application/experiments` - -Current role: - -- persist benchmark/experiment definitions; -- launch runs from definitions; -- model experiment run requests/results; -- expose experiment handles. - -Tension: - -- The v2 system often uses "definition" as the precise concept, while this - folder is named "experiments". -- It sits close to `persistence/definitions` and `application/workflows`, so - new code can be unsure whether definition launch belongs here or in workflows. - -### `application/workflows` - -Current role: - -- workflow orchestration service; -- run lifecycle commands; -- run creation helpers; -- workflow-level errors/models. - -Tension: - -- Some task lifecycle behavior historically lived here and later moved toward - `tasks`. -- It overlaps with `graph` whenever the operation is "advance this run graph". - -### `application/tasks` - -Current role: - -- task execution service; -- task management operations such as cancel/refine/restart/spawn; -- task inspection and cleanup; -- task command/result models. - -Tension: - -- It owns user/task-facing lifecycle operations, but these operations often - need graph traversal and workflow dispatch. -- This folder is the natural owner of public worker-context task operations, - but not necessarily the owner of graph propagation. - -### `application/graph` - -Current role: - -- run graph lookup; -- traversal; -- propagation; -- graph repository interface; -- graph command/data models. - -Tension: - -- Some files look like pure graph utilities. -- Some files are workflow lifecycle policies in graph clothing. -- Persistence graph tables live in `persistence/graph`, so the application - graph package must avoid becoming a second persistence layer. - -### `application/jobs` - -Current role: - -- Inngest job entrypoints and payload handlers; -- workflow start/complete/fail; -- task execute/worker/evaluate; -- sandbox setup/cleanup; -- output persistence; -- propagation. - -Tension: - -- This folder is organized by execution mechanism, not by domain. -- It cuts across workflows, tasks, evaluation, sandbox, resources, and graph. -- That is fine if job functions stay thin. It becomes hard to reason about if - job functions own business rules. - -### `application/evaluation` - -Current role: - -- evaluator/criterion execution service; -- scoring and result models; -- evaluation errors. - -Tension: - -- It is one of the cleaner domains after PR 13. -- It still touches persistence telemetry and task/evaluator definition metadata, - so tests should keep guarding the runtime path. - -### `application/read_models` - -Current role: - -- query-side DTOs for runs, cohorts, experiments, resources, and run snapshots; -- dashboard/API-facing aggregation from persistence rows. - -Tension: - -- This is not command-side application logic. It is a contract-building layer. -- It is close to frontend generated contracts and should be treated as such. -- Deleting remaining compatibility around `BenchmarkDefinitionRecord` will - leave `views` as the read-only contract-building boundary. - -### `application/resources` - -Current role: - -- resource DTOs and repository-facing application logic. - -Tension: - -- Some resource publication behavior also lives in jobs and sandbox - infrastructure. The ownership split should stay explicit: - jobs trigger publication, infrastructure adapts external files, resources - owns persisted resource semantics. - -### `application/context` - -Current role: - -- context event models. - -Tension: - -- Very small package. It may belong with context persistence/read models unless - more application behavior lands here. - -### `application/communication` - -Current role: - -- message/communication models, service, and errors. - -Tension: - -- Appears self-contained, but should be reviewed for whether it is core runtime - behavior or an adapter-facing support domain. - -### `application/events` - -Current role: - -- internal application event payloads for task/runtime/infrastructure events. - -Tension: - -- Similar to `jobs`, this is cross-domain by nature. It may be best kept as a - shared application boundary rather than merged into a single domain. - -## Non-Source Artifacts Found - -The audited tree includes generated/local artifacts: - -```text -__pycache__/ -.DS_Store -``` - -There are also cache-only/stale-looking directories: - -```text -application/components/ -domain/experiments/ -persistence/components/ -persistence/saved_specs/ -``` - -Some architecture tests already assert that deleted v2 paths such as -`saved_specs` and component catalog files should be absent. These artifacts -should be cleaned up as a hygiene PR or as part of PR17/PR18 if they are still -present in the working tree. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/infrastructure-application-boundary-audit.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/infrastructure-application-boundary-audit.md deleted file mode 100644 index 41f15138a..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/infrastructure-application-boundary-audit.md +++ /dev/null @@ -1,360 +0,0 @@ -# Infrastructure/Application Boundary Audit - -Date: 2026-05-18 - -Audited against PR 16 head in the main checkout. - -## Boundary Rule - -The target direction is port-and-adapter style: - -```text -inbound adapters -> application use cases / views -application use cases -> application ports -> infrastructure implementations -``` - -Application owns business operations. Infrastructure adapts external systems -and frameworks. If infrastructure needs an operation that application already -knows how to perform, infrastructure must delegate to that application boundary -or implement a narrow application-declared port. It should not reimplement a -parallel copy locally. - -Infrastructure should own framework and external-system adapters: - -- Inngest function registration and transport concerns; -- dashboard event transport; -- E2B SDK calls and sandbox command/file proxies; -- tracing sinks and span export; -- process startup wiring. - -Application should own use-case decisions: - -- runtime lifecycle and propagation policy; -- run graph view assembly; -- resource persistence semantics; -- event payload contracts that are product/API contracts rather than transport - implementation details; -- deprecated compatibility domains such as cohorts until they are removed. - -The useful question during cleanup is therefore not only "which folder should -this live in?" It is: - -> Does application already have a service, view, repository, or policy for -> this operation shape? - -If yes, remove the duplicate infrastructure-side implementation and route -through that application operation. If no, extract the operation into -application first, then keep infrastructure as the adapter. - -## Dashboard - -The dashboard boundary is currently the largest infrastructure/application -mix. `infrastructure/dashboard/emitter.py` is partly a transport adapter: it -builds typed event contracts and sends them through the Inngest client. That -part belongs in infrastructure. But the module also owns application behavior: -`emit_cohort_updated_for_run()` reaches into the cohort read-model service, -recomputes cohort state, fetches a summary, and emits it. That is command/query -logic behind an infrastructure import path. - -`infrastructure/dashboard/event_contracts.py` is not just infrastructure glue. -It defines generated frontend contracts and imports view DTOs such as -`RunTaskEvaluationDto`, communication DTOs, `CohortSummaryDto`, and -`GraphMutationRecordDto`. These models are product-facing event DTOs shared by -backend and dashboard generation. Keeping them under `infrastructure` makes the -contracts look like an implementation detail of the emitter, when they are -actually part of the dashboard/API contract surface. - -The tree-building path is duplicated across application and infrastructure. -`application/jobs/start_workflow.py` imports dashboard contract types -(`TaskTreeNode`, `WorkerRef`) and contains `_build_task_tree_for_run()`, which -queries `RunGraphNode`, `RunGraphEdge`, and definition worker rows, then adapts -them to the dashboard tree. That is view assembly, not job orchestration. -It overlaps with the proposed `views` boundary and with the -existing run snapshot builders under `application/read_models`. PRD 06 deletes -`WorkerRef`, `TaskTreeNode`, and `_build_task_tree_for_run()` by changing -`workflow.started` to carry the existing `RunSnapshotDto` view. - -Dashboard emission is also wired into application services directly. Task -execution, task management, worker execution, evaluation, communication, and -workflow completion all import `get_dashboard_emitter()` or the emitter type. -The calls are pragmatic today, but they mean command services know the concrete -dashboard transport. A cleaner target is an application-level event publisher -or listener interface, with the dashboard emitter registered as one -infrastructure subscriber. - -This should not be implemented as a mechanical folder move. The dashboard event -path should reuse the run snapshot view rather than carrying its own -second graph-to-tree implementation. Likewise, cohort refresh should not remain -a dashboard helper with a new import path; it should become a deprecated -application cohort compatibility operation until the dashboard deletes cohorts. - -Recommended target: - -```text -views/dashboard_events/ - contracts.py # generated dashboard event DTOs - cohort_events.py # temporary deprecated cohort refresh/summary emission helper - graph_mutations.py # RunGraphMutation -> GraphMutationRecordDto mapping - -infrastructure/dashboard/ - emitter.py # sends already-built event DTOs through Inngest - provider.py # process-level emitter wiring -``` - -This move should be paired with the view/cohort refactor. Cohort events -should remain clearly marked deprecated until the dashboard refactor removes -cohort concepts. - -Directionality rule for dashboard: - -- application may define dashboard event contracts or a `DashboardEventPublisher` - port; -- infrastructure may implement that port by sending through Inngest; -- application services should not import the concrete `DashboardEmitter`; -- dashboard infrastructure should not import cohort services or view - repositories directly. - -## Sandbox - -The sandbox code is closer to a real infrastructure adapter, but it still mixes -three responsibilities. - -`infrastructure/sandbox/manager.py` owns E2B lifecycle for the manager-style -sandbox path: create/connect/kill, directory bootstrap, dependency install -hooks, input upload/download helpers, in-process registries, and event sink -emission. Those E2B SDK interactions belong in infrastructure. The risk is that -the v2 public `Sandbox.provision()` path now also provisions sandboxes directly, -so the codebase has two sandbox ownership paths: object-bound public sandbox -instances and `BaseSandboxManager`. The application jobs call the public -sandbox path in `sandbox_setup.py`, while cleanup still terminates through -`terminate_external_sandbox()` -> `BaseSandboxManager.terminate_by_sandbox_id()`. -That bridge is useful during the current transition, but the final owner should -be explicit: application lifecycle decides when to terminate; infrastructure -performs termination by sandbox id. - -`infrastructure/sandbox/event_sink.py` is a mostly healthy adapter boundary. -It defines a `SandboxEventSink` protocol plus no-op, dashboard, Postgres, and -compound sinks. The duplication risk is not the protocol. The risk is that the -Postgres sink writes telemetry rows directly while other telemetry writes are -owned by application repositories/services. If sandbox telemetry is intended to -be append-only infrastructure observability, this is acceptable. If those rows -feed dashboard/read-model product state, the write should move behind an -application telemetry/resource service. - -`infrastructure/sandbox/instrumentation.py` is also a reasonable adapter. It -wraps E2B command/file/run-code calls and emits command events. It should stay -infra as long as it does not decide task status, run state, or resource -semantics. - -`infrastructure/sandbox/resource_publisher.py` is the real duplication hotspot. -It scans sandbox directories, reads files through the sandbox adapter, hashes -content, writes a content-addressed blob, deduplicates rows, guesses MIME type, -and appends `RunResource` rows through `RunResourceRepository`. The external -filesystem reads and blob writes are infrastructure concerns, but the append -policy, dedup semantics, resource kind mapping, and DTO creation are -application/resource concerns. `application/jobs/persist_outputs.py` already -states that `SandboxResourcePublisher` is the single authoritative path from -sandbox to resources, but that authority currently lives under -`infrastructure`. - -This should be fixed by splitting operation shape from adapter shape. The -application operation is "publish these sandbox outputs as run resources with -canonical dedup, kind, MIME, metadata, and append-only row semantics." The -infrastructure operations are "list/read files in a sandbox" and "write bytes -to a blob store." If resource logic already exists in -`application/resources/repository.py` or read-model resource helpers, reuse or -extend it there. Do not leave a second resource append policy inside sandbox -infrastructure under a tidier name. - -Recommended target: - -```text -application/resources/ - service.py # publish sandbox outputs; owns dedup/resource semantics - repository.py # append/list/get RunResource rows - models.py # RunResourceView and command/result DTOs - -infrastructure/sandbox/ - files.py # E2B file listing/reading adapter - blob_store.py # content-addressed local blob store - instrumentation.py # command/file proxy event emission - lifecycle.py # terminate external sandbox by id - manager.py # E2B manager compatibility path, if still needed -``` - -The next implementation should not stuff this into the runtime-domain merge. -It is a separate resource boundary cleanup with tests around dedup, MIME -selection, blob path stability, and `persist_outputs` behavior. - -Directionality rule for sandbox: - -- application owns when to provision, publish, and terminate; -- application may depend on `SandboxFileReader`, `BlobStore`, and - `SandboxTerminator` ports; -- infrastructure implements those ports with E2B and local filesystem code; -- infrastructure sandbox modules should not append `RunResource` rows directly - once the application resource service exists. - -## Inngest - -The Inngest boundary is mostly clean after PR 16. Handler modules under -`infrastructure/inngest/handlers` primarily parse `ctx.event.data`, configure -function metadata, and delegate to `application/jobs/*`. That is the right -shape for infrastructure. - -There are two cleanup notes. First, `infrastructure/inngest/contracts.py` is a -re-export layer for application job models. That is not harmful, but it is -more indirection than ownership. Job payload/result schemas already live in -`application/jobs/models.py`; handlers can import them directly unless the team -wants a deliberate transport contract facade. - -Second, `cancel_orphan_subtasks.py` still carries a TODO asking whether each -handler should become a module that owns its own logic and contracts. The audit -answer is no for infrastructure handlers: they should stay adapters. If logic -needs a clearer home, it should move from `application/jobs` into the runtime -application domain, not into Inngest handler modules. - -This is a good example of the intended directionality. Inngest handlers are -inbound adapters. They may know about Inngest triggers, retries, cancellation -configuration, output types, and `ctx.event.data`. They should not become owners -of task lifecycle, propagation, cleanup, or event fanout policy. If a handler -needs reusable behavior, that behavior belongs in application runtime/jobs and -the handler calls it. - -Recommended target: - -```text -infrastructure/inngest/ - client.py - registry.py - handlers/ - -application/jobs/ - models.py # keep job payload/result contracts here while jobs exist - -application/runtime/ - ... # eventual owner of reusable runtime lifecycle policy -``` - -## Tracing - -Tracing is mostly healthy infrastructure. `infrastructure/tracing` owns -deterministic trace/span id generation, context factories, attribute -normalization, no-op and OpenTelemetry sinks, and exported facade functions. -Application jobs emit `CompletedSpan` records at use-case boundaries, which is -acceptable observability code rather than duplicated domain behavior. - -The one boundary smell is that `contexts.py` encodes the runtime span -hierarchy: workflow root, task execute, sandbox setup, worker execute, persist -outputs, propagation, workflow completion/failure, evaluation task, and -criterion spans. That mirrors runtime lifecycle concepts. This is acceptable as -long as tracing context factories remain pure naming/id helpers and do not make -lifecycle decisions. If the runtime domain merge lands, the tracing hierarchy -should be reviewed against the new runtime module names, but it does not need -to move into application. - -Recommended target: keep tracing under infrastructure. Add a short ownership -comment to `contexts.py` if this RFC becomes an implementation plan: - -> Trace context factories may mirror runtime concepts, but must stay pure: -> deterministic ids, parent/child links, and relational ids only. They must not -> inspect persistence, start jobs, emit dashboard events, or choose lifecycle -> transitions. - -Directionality rule for tracing: - -- application use cases may emit span facts through a tracing port; -- infrastructure owns id generation, normalization, and sink implementation; -- tracing modules may mirror lifecycle names for observability, but must not - become an alternate lifecycle model. - -## REST App Startup - -`rest_api/app.py` initializes infrastructure adapters during FastAPI lifespan: -database setup, rollout service, dashboard emitter provider, sandbox event -sink composition, and Inngest serving. This is appropriate startup wiring. - -The only caveat is that it currently wires the sandbox event sink only onto -`DefaultSandboxManager`. If concrete sandbox manager subclasses remain after -the public object-bound sandbox path settles, startup should install the sink -for every live manager class or delete the manager-class registry entirely. -This is not application duplication, but it is a lifecycle wiring risk. - -## Cross-Cutting Duplications To Resolve - -1. Dashboard event DTOs are generated product contracts but live under - infrastructure. Move or rename them into a view/contract boundary. - -2. Dashboard workflow tree assembly lives inside `application/jobs/start_workflow.py` - and imports dashboard DTOs directly. Before moving it, compare it with - existing run snapshot/read-model view logic and extract any shared - graph traversal or task summary behavior once. - -3. Cohort recompute-and-emit logic lives under `infrastructure/dashboard`. - Move it to a deprecated cohort/view compatibility operation until the - dashboard removes cohorts. Do not let the dashboard emitter own recompute - policy. - -4. Application services import the concrete dashboard emitter. Introduce a - small application event/listener interface if this keeps spreading. - -5. Sandbox resource publication owns application resource semantics under an - infrastructure path. Split sandbox file/blob adapters from the application - resource service policy, and reuse existing `application/resources` logic - rather than creating a second append/dedup implementation. - -6. Sandbox lifecycle still bridges object-bound public sandboxes through - manager termination. Keep the application decision point in cleanup jobs, - but make the infrastructure termination path explicitly id-based and - manager-compatibility-only. - -7. Inngest handler TODOs should resolve toward thinner handlers and stronger - application runtime services, not toward handler-owned domain modules. - -## Suggested PR Slices - -### PR A: Dashboard View Boundary - -Move dashboard event contracts and workflow tree assembly into an application -view/contract package. Leave `DashboardEmitter` as infrastructure -transport. Characterize `workflow.started`, graph mutation, task status, -evaluation update, context event, and communication event payloads before -moving imports. As part of the characterization, identify whether existing -run snapshot/read-model code already implements graph traversal, task summary, -or resource summary logic that the dashboard tree can reuse. - -### PR B: Deprecated Cohort Boundary - -Move cohort event refresh logic out of infrastructure and into an explicitly -deprecated cohort compatibility module. Add the file-level TODO already agreed -for the dashboard refactor: cohorts are confirmed deprecated in v2 and should -be removed from the UI/backend compatibility layer later. - -### PR C: Sandbox Resource Service Split - -Create an application resource publication service that owns dedup, resource -kind, MIME, metadata, and row append semantics. Keep E2B file reads and blob -filesystem writes in infrastructure. Update `persist_outputs` and toolkit -publish paths to call the application service. The service should reuse -`RunResourceRepository` and existing resource DTOs rather than reimplementing -row append/dedup policy inside sandbox infrastructure. - -### PR D: Inngest Cleanup And Boundary Tests - -Delete the handler TODO, keep `infrastructure/inngest/contracts.py` only for -transport/framework contracts, and add architecture tests for: - -- `infrastructure/inngest/handlers` may import `application/jobs`, but not the - other way around; -- infrastructure dashboard transport may not import cohort services; -- infrastructure sandbox resource adapters may not append `RunResource` rows - directly after PR C. -- application services may not import concrete dashboard/Inngest/E2B/tracing - implementations except in composition roots or explicitly approved bootstrap - modules. - -### PR E: Tracing Ownership Comments - -Add lightweight docstrings/comments and tests, if useful, that tracing context -factories are pure observability helpers. This can be folded into whichever PR -touches runtime module names. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/persistence-boundary-audit.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/persistence-boundary-audit.md deleted file mode 100644 index e0deac8af..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/persistence-boundary-audit.md +++ /dev/null @@ -1,367 +0,0 @@ -# Persistence Boundary Audit - -Date: 2026-05-18 - -Audited against PR 16 head in the main checkout. - -## Summary - -`core/persistence` earns its keep as a separate layer, but not as a product -domain. It should remain the storage schema and concrete data-access boundary: -SQLModel table models, database session setup, storage enums/types, migrations, -and low-level row validation belong here. - -It should not become the owner of runtime/business behavior. Application should -own use cases, views, command models, and repository interfaces/semantics. -Persistence can implement concrete row access, but it should not quietly define -alternate business APIs beside application repositories. - -The current split is close enough to preserve, but uneven: - -- most persistence files are table definitions and JSON column accessors; -- `application/graph/repository.py`, `application/tasks/repository.py`, and - `application/resources/repository.py` contain the real application-facing - repository semantics; -- `persistence/telemetry/repository.py` is a second repository style; PRD 01 - deletes it and folds its two single-consumer methods into - `EvaluationService`; -- `persistence/graph/status_conventions.py` contains runtime lifecycle - vocabulary and helper functions, which is application/runtime policy rather - than storage schema; -- `persistence/context/event_payloads.py` duplicates context-part aliases and - already has a TODO saying to remove it; -- `persistence/telemetry/models.py` is a large mixed schema module covering - runs, old benchmark definition records, cohorts, communication, training, - rollout batches, resources, evaluations, and sandbox WAL rows. - -## What Lives In Persistence Today - -### `persistence/definitions` - -This package stores immutable authored definition tables: - -- `ExperimentDefinition`; -- `ExperimentDefinitionWorker`; -- `ExperimentDefinitionEvaluator`; -- `ExperimentDefinitionInstance`; -- `ExperimentDefinitionTask`; -- definition task dependencies, assignments, and evaluator bindings. - -This is the right kind of persistence package. It is table/schema heavy, with -light JSON accessors for storage fields. The main architectural drift is -historical naming: v2 says `definition`, while table classes still carry -`ExperimentDefinition*`. That is not an immediate duplication bug, but it -should be reconciled with the planned `application/definitions` naming. - -There is one known schema duplication: `ExperimentDefinitionTask` stores both -`task_payload_json` and full object-bound `task_json`. That mirrors the v2 stack -history. The audit does not propose deleting it in this RFC, but the ownership -should be explicit: if `task_json` is the canonical authored task snapshot, -payload-specific readers should not become a second authoring path. - -### `persistence/graph` - -This package stores run-tier graph tables: - -- `RunGraphNode`; -- `RunGraphEdge`; -- `RunGraphAnnotation`; -- `RunGraphMutation`. - -The SQLModel table definitions belong in persistence. The append-only mutation -and annotation rows are storage primitives that application graph services can -use for audit/replay. - -The questionable file is `status_conventions.py`. It defines `PENDING`, -`READY`, `RUNNING`, `COMPLETED`, `FAILED`, `CANCELLED`, `BLOCKED`, -`TERMINAL_STATUSES`, and helper functions such as -`is_terminal_node_status()`. Those names describe runtime lifecycle policy, not -database storage. Application services already import these conventions as -runtime vocabulary. PRD 01 moves them to -`application/runtime/status.py`, where runtime services and view schemas -can share one vocabulary without persistence owning lifecycle policy. - -### `persistence/telemetry` - -This is the broadest package. It currently contains: - -- launch/provenance rows: `BenchmarkDefinitionRecord`, `RunRecord`; -- task execution rows: `RunTaskExecution`; -- resource rows: `RunResource`; -- evaluation rows: `RunTaskEvaluation`; -- deprecated cohort rows: `ExperimentCohort`, `ExperimentCohortStats`; -- communication rows: `Thread`, `ThreadMessage`; -- training and rollout rows: `TrainingSession`, `TrainingMetric`, - `RolloutBatch`, `RolloutBatchRun`; -- sandbox observability rows: `SandboxCommandWalEntry`, `SandboxEvent`; -- `TelemetryRepository` for evaluation row reads/writes; -- `EvaluationSummary` JSON schema for `RunTaskEvaluation.summary_json`. - -The table models belong in persistence, but the package is too semantically -wide to read as one coherent domain. It is really "run telemetry and auxiliary -run-adjacent tables." That is acceptable as a storage grouping, but application -should not treat `persistence.telemetry` as one product domain. - -The duplicated repository shape is `TelemetryRepository`. Application already -has domain-specific repositories: - -- `application/graph/repository.py`; -- `application/tasks/repository.py`; -- `application/resources/repository.py`. - -Evaluation persistence should not create another repository for two -single-consumer methods. PRD 01 deletes `TelemetryRepository`, folds -`get_task_evaluations()` and `create_task_evaluation()` into private -`EvaluationService` helpers, and keeps only table models under persistence. - -`CreateTaskEvaluation` is also an application command object living inside a -persistence model module. PRD 01 deletes it because it only exists to shuttle -arguments into the single-consumer repository. Persistence table modules may -expose row constructors or JSON validators, but command DTOs do not belong -there. - -`EvaluationSummary` is a harder call. It is the canonical JSON schema for a -persisted evaluation summary, but it imports public API criterion evidence. -That makes persistence depend upward into the public API. A better target is: - -```text -application/evaluation/summary.py -``` - -or, if it must be shared by API, persistence, views, and dashboard: - -```text -core/shared/evaluation_summary.py -``` - -Persistence can then validate `summary_json` against that shared/application -schema without owning evaluation semantics. - -### `persistence/context` - -This stores context stream rows: - -- `RunContextEvent`; -- context event payload aliases. - -`RunContextEvent` belongs in persistence as a SQLModel table. But -`event_payloads.py` duplicates aliases over `ContextPartChunkLog`, and the file -already says it should be killed. Once `core/domain/generation/context_parts.py` -moves to `core/shared/context_parts.py`, this package should import that shared -contract directly. `ContextEventPayload` should not survive the move; consumers -should annotate payloads as `ContextPartChunkLog`. - -### `persistence/imports` - -This package contains reducer/drop-manifest storage: - -- `RunReducer`; -- `RunReducerFootprint`; -- `RunDropsManifest`. - -These look like storage rows for imported/public rollout cards and reducer -provenance, but no production writer/reader remains in v2 and -`RunReducer.node_id` points at a non-existent graph column. PRD 01 deletes the -package rather than renaming it. - -### `persistence/shared` - -This package owns database/session helpers and storage-level shared types: - -- `db.py`; -- `enums.py`; -- `ids.py`; -- `types.py`. - -This is useful, but `enums.py` mixes storage enums with product/runtime enums: -`RunStatus`, `TaskExecutionStatus`, `TrainingStatus`, and `RunResourceKind`. -Some of these are persisted values and therefore need stable storage names. -However, when application services use them as lifecycle policy, the source of -truth should move to application/shared contracts and persistence should import -the storage-safe enum rather than own the product concept by default. - -## Where Duplication Exists - -### Repository Ownership - -There are two repository styles: - -1. persistence-level `TelemetryRepository`; -2. application-level repositories for graph, task execution, worker output, and - resources. - -The application-level style is the better target. It gives the repository a -business-facing name and lets the service own invariants. Persistence should -provide row models and perhaps low-level storage adapters, not a generic -telemetry repository with command DTOs. - -Recommendation resolved in PRD 01: delete -`persistence/telemetry/repository.py`, inline its two methods as private -`EvaluationService` helpers, delete `CreateTaskEvaluation`, and keep -`RunTaskEvaluation` as the storage table. - -### Direct SQL In Application Services - -Many application services and views directly import SQLModel rows and run -queries. That is not automatically wrong in this codebase: views need -query composition, and repository extraction has not been standardized yet. -But it means persistence is already not the sole data-access domain. - -The target should be pragmatic: - -- command-side mutations should go through application repositories/services; -- views may run direct optimized reads if they remain read-only and - contract-focused; -- infrastructure should not run product queries when an application - service/view already exists. - -This matches the earlier infrastructure rule: do not duplicate application -operations in adapters. - -### Status And Lifecycle Vocabulary - -`persistence/graph/status_conventions.py` is used as runtime status vocabulary. -That keeps constants near the table but makes persistence look like it owns -lifecycle semantics. The runtime merge/refactor should move these constants to -the application runtime domain or a narrow shared runtime contract. - -Until that happens, do not create another status vocabulary elsewhere. The bug -would be having both `persistence.graph.status_conventions` and -`application.runtime.status` diverge. Move once, then update imports. - -### Evaluation Summary Schema - -`persistence/telemetry/evaluation_summary.py` is intentionally the canonical -schema for persisted evaluation summaries, but it is not purely persistence. -It describes evaluator/criterion output semantics and imports -`ergon_core.api.criterion.CriterionEvidence`. - -Recommendation: make evaluation summary a shared/application evaluation -contract, then have persistence table models validate against it. Do not leave -the canonical evaluator result schema hidden in a storage package if dashboard, -API, and evaluator services all depend on it. - -### Context Payload Aliases - -`persistence/context/event_payloads.py` duplicates context stream type aliases -over `ContextPartChunkLog`. PRD 02 deletes the alias file, moves -`ContextEventType` to `core/shared/context_parts.py`, and updates consumers to -use `ContextPartChunkLog` directly. - -### Deprecated Cohorts And Legacy Definition Records - -`BenchmarkDefinitionRecord`, `ExperimentCohort`, and `ExperimentCohortStats` -are still table models. They should remain in persistence while rows exist and -dashboard compatibility needs them. But application should mark their services -as deprecated compatibility, not let telemetry persistence imply they are -first-class v2 domains. - -## Does Persistence Earn Its Keep? - -Yes, as a layer. No, as an independent product domain. - -It earns its keep because: - -- the storage schema is large and materially different from application DTOs; -- SQLModel table definitions, foreign keys, JSON columns, and timestamp/default - mechanics need one obvious home; -- migrations and final schema audits need a stable package to inspect; -- storage-level row validation should be close to the row definitions. - -It should remain separate from application because application code should not -be cluttered with table declarations and SQLAlchemy column mechanics. - -But it should be kept intentionally narrow: - -- persistence owns tables, storage enums/types, low-level row validation, and - database setup; -- application owns command models, use-case services, lifecycle policy, - view assembly, and application-facing repositories; -- infrastructure owns external adapters and implements application-declared - ports; -- rest/api/dashboard should consume application services and views, not - persistence rows directly except in explicit view modules. - -## Recommended Target Shape - -```text -persistence/ - definitions/ - models.py - graph/ - models.py - telemetry/ - models.py # or split into run/models.py, evaluation/models.py, etc. later - context/ - models.py - shared/ - db.py - ids.py - types.py - -application/ - definitions/ - repository.py - runtime/ - status.py # moved from persistence.graph.status_conventions - graph_repository.py - evaluation/ - repository.py # replaces persistence.telemetry.repository - summary.py # or shared/evaluation_summary.py - resources/ - repository.py - communication/ - repository.py # if communication service keeps growing direct SQL - views/ - ... -``` - -This does not require an immediate large move. The first useful cleanup is to -set the rule and stop adding new repositories or command DTOs under -`persistence`. - -## Suggested PR Slices - -### PR A: Persistence Ownership Tests - -Add import-boundary tests that encode the intended split: - -- `persistence/*/models.py` may import `shared` and storage-safe schemas; -- persistence table modules should not import concrete infrastructure; -- persistence should not define new application command DTOs; -- infrastructure should not query persistence rows directly when application - services and views exist. - -This PR can also delete the empty/stale `persistence/components` package if it -is tracked and unused. - -### PR B: Evaluation Repository And Summary Boundary - -Delete `TelemetryRepository` and `CreateTaskEvaluation`. Move `EvaluationSummary`, -`CriterionOutcomeEntry`, and `EvalCriterionStatus` to -`application/evaluation/summary.py`, and delete -`RunTaskEvaluation.parsed_summary()` so persistence no longer imports the -semantic evaluation schema. - -### PR C: Runtime Status Boundary - -Move `persistence/graph/status_conventions.py` to -`application/runtime/status.py`. Update imports in graph services, task -services, jobs, views, dashboard contracts, fixtures, and tests in one -slice. - -### PR D: Context Payload Alias Deletion - -After context parts move to `core/shared/context_parts.py`, remove -`persistence/context/event_payloads.py` unless a DB-specific event payload type -is still needed. Update dashboard contracts and read models to import the -shared context stream contracts directly. - -### PR E: Telemetry Model Split Or Documentation - -Do not split `persistence/telemetry/models.py` just for tidiness. First decide -whether cohorts, training, rollout batches, sandbox WAL, communication, and -run/task/evaluation/resource tables are all still active v2 storage. If they -are, either document `telemetry` as "run-adjacent storage" or split it into -subpackages by storage family. If some are deprecated, mark those rows and their -application services as compatibility-only until deleted. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/runtime-domain-merge-audit.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/runtime-domain-merge-audit.md deleted file mode 100644 index ee98709c0..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/runtime-domain-merge-audit.md +++ /dev/null @@ -1,409 +0,0 @@ -# Runtime Domain Merge Audit - -## Why This Needs Attention - -The current split between: - -```text -application/tasks/ -application/workflows/ -application/graph/ -``` - -is one of the larger remaining sources of duplication in core. The split made -sense while the team was still churning on the mental model of "definition", -"run graph", "task", and "workflow". After PRs 11-17, that model should be -stable enough to pay down the structure. - -The issue is not only file location. The issue is that lifecycle concepts are -spread across three packages, so readers have to infer the "why" from the "how" -of the code: - -- graph code sometimes owns lifecycle policy; -- workflow code sometimes owns task/query/resource behavior; -- task management code sometimes owns graph traversal and propagation-adjacent - invalidation; -- multiple modules know how to resolve run definition identity and dispatch - `task/ready` events. - -That makes the code harder to review and increases the chance that future -changes add another partial lifecycle path instead of extending the canonical -one. - -## What Lives In The Three Domains Today - -### `application/graph` - -Current responsibilities: - -- `repository.py`: structural run graph writes, node/edge queries, - definition-to-run copying, task snapshot inflation, and graph mutation WAL. -- `propagation.py`: readiness calculation, task status updates, edge - satisfaction/invalidation, blocking downstream successors, reactivating - dynamic children, and workflow terminal-state detection. -- `traversal.py`: containment-descendant traversal. -- `lookup.py`: task/node lookup helper. -- `models.py`: graph DTOs and mutation payloads. - -Tension: - -`repository.py` is a good structural graph boundary. `propagation.py` is not -just graph structure; it is runtime lifecycle policy. It decides what happens -after completion/failure and when downstream work becomes eligible. - -### `application/tasks` - -Current responsibilities: - -- `management.py`: worker/manager task mutations, dynamic task spawning, - cancellation, orphan cancellation, descendant blocking, refinement, restart, - downstream invalidation, edge reset, ready-event dispatch, and cancelled-event - dispatch. -- `execution.py`: task execution application service and status event emission. -- `inspection.py`: task inspection helpers. -- `cleanup.py`: task cleanup and cancellation cleanup result semantics. -- `repository.py`: task execution/output repository helpers. -- `models.py`: command/result DTOs for task management and cleanup. - -Tension: - -`management.py` is the real command-side lifecycle service for task operations, -but it also owns pieces that look like graph lifecycle policy: downstream -invalidation, edge reset, descendant traversal, terminal-status rules, and -ready-event dispatch. - -### `application/workflows` - -Current responsibilities: - -- `service.py`: run initialization/finalization, propagation wrappers, task and - dependency inspection, resource visibility/materialization, CLI-style graph - mutation previews, blocker/next-action DTOs, node/resource lookup helpers, - ready-event dispatch, and sandbox upload path construction. -- `runs.py`: run creation/cancellation/latest-run helpers. -- `orchestration.py`: workflow command/result DTOs. -- `models.py`: workflow/task/resource/dependency read and mutation DTOs. - -Tension: - -`WorkflowService` is doing too much. It is partly run lifecycle, partly runtime -inspection, partly resource service, partly CLI mutation adapter, and partly -propagation facade. It overlaps with both `tasks` and `graph`. - -## Duplication And Friction To Investigate - -### Status And Lifecycle Policy - -The same conceptual rules appear in multiple places: - -- terminal vs non-terminal task states; -- readiness; -- edge pending/satisfied/invalidated semantics; -- completion/failure propagation; -- cancellation and restart invalidation; -- when a task may move from terminal back to pending; -- when cancelled dynamic children may reactivate. - -The most obvious overlap is: - -- `WorkflowService.propagate()` sets a task completed, then calls - `graph.propagation.on_task_completed_or_failed()`; -- `WorkflowService.propagate_failure()` sets a task failed, then calls the same - propagation helper; -- `TaskManagementService.restart_task()` implements a separate but related - downstream invalidation and edge-reset model. - -This looks like one lifecycle domain split across three packages. - -### Run Definition Identity Resolution - -Both task management and workflow service resolve a run's definition id from -`RunRecord.workflow_definition_id`. Jobs also carry definition ids through -payloads. - -This should probably be a single runtime helper or service boundary so all -runtime code answers "what definition does this run belong to?" the same way. - -### Descendant Traversal - -Descendant logic exists in several forms: - -- `application/graph/traversal.py`; -- `WorkflowGraphRepository.descendants_by_parent()`; -- `TaskManagementService._count_non_terminal_descendants()`; -- `TaskManagementService._invalidate_downstream()`; -- `WorkflowService._descendant_ids()` for resource scopes. - -Some of these are containment traversal; some are dependency-edge traversal. -The code would be easier to reason about if those two graph concepts were named -and separated explicitly: - -- containment descendants: parent/child hierarchy; -- dependency descendants: downstream through dependency edges. - -### Node/Task DTO Vocabulary - -Multiple DTO families describe overlapping concepts: - -- `GraphNodeDto`; -- `RunGraphNodeView`; -- `WorkflowTaskRef`; -- `TaskDescriptor`; -- `WorkflowMutationRef`; -- graph mutation payload DTOs. - -These may all be justified, but their roles should be documented: - -- structural graph DTO; -- hydrated runtime task view; -- CLI/view task reference; -- job orchestration descriptor; -- mutation/audit payload. - -Without that explanation, the duplication looks accidental. - -### Ready/Cancellation Event Dispatch - -`TaskReadyEvent` and `TaskCancelledEvent` dispatch decisions appear close to -task management, workflow service, and jobs. Dispatch should have one clear -owner: - -- lifecycle service decides the domain event should happen; -- job/adapter layer sends it to Inngest after commit; -- or the service is explicitly allowed to dispatch after commit. - -The current shape mixes those ideas. - -### Resource Visibility And Materialization - -`WorkflowService` owns resource listing, resource reads, workspace inspection, -resource-copy naming, sandbox destination validation, and sandbox upload. That -is a lot of resource behavior for a workflow service. - -This may want its own runtime resource service so workflow/run lifecycle is not -responsible for file visibility and sandbox copy policy. - -## Merge Target: `application/runtime` - -One possible target is: - -```text -application/runtime/ - __init__.py - errors.py - models.py - graph_repository.py - graph_traversal.py - lifecycle.py - task_management.py - run_lifecycle.py - inspection.py - resources.py -``` - -The important part is not the exact filenames. The important part is the -ownership split. - -### `graph_repository.py` - -Owns structural graph persistence and WAL: - -- copy definition graph into run graph; -- add nodes and edges; -- update node/edge fields; -- enforce structural graph invariants such as referential integrity and cycles; -- append graph mutation records; -- hydrate a run-tier task snapshot. - -Does not own: - -- task status transition policy; -- run lifecycle; -- user authorization/containment permissions; -- event dispatch. - -### `lifecycle.py` - -Owns runtime state-transition policy: - -- initial readiness; -- task completion/failure propagation; -- edge satisfaction/invalidation; -- downstream blocking; -- terminal run detection; -- cancellation/restart transition rules that are graph-wide rather than a - single command's input validation. - -This is where the "why" should live. A reader should be able to understand the -state machine by reading this module's class and method docstrings. - -### `task_management.py` - -Owns command-side task mutations: - -- spawn dynamic object-bound tasks; -- cancel a task; -- refine a task; -- restart a task; -- cancel or block descendants in response to lifecycle decisions. - -This service should call lifecycle/graph services for graph-wide policy instead -of reimplementing propagation concepts locally. - -### `run_lifecycle.py` - -Owns run-level lifecycle: - -- initialize a run graph from a definition; -- mark run executing; -- finalize run score/status; -- fail/cancel run; -- locate latest run for a definition if that remains core behavior. - -This should stay distinct from task-level lifecycle. - -### `inspection.py` - -Owns view runtime inspection for tasks/dependencies/blockers/next actions: - -- list/get tasks; -- list dependencies; -- compute blockers; -- compute suggested next actions; -- return task workspace summaries if resources remain separate. - -This should not mutate graph state. - -### `resources.py` - -Owns runtime resource visibility and materialization policy: - -- list resources by scope; -- resolve resource producer; -- read resource bytes; -- compute safe sandbox destination paths; -- materialize/copy resource into sandbox; -- persist import resource records. - -This removes file/sandbox resource policy from `WorkflowService`. - -## Documentation Standard If We Merge - -The new modules should not repeat the current pattern where readers infer -purpose by reading implementation details. Each major class/function should -carry a short "what and why" docstring. - -### Module Docstrings - -Each runtime module should start with: - -```python -"""Runtime lifecycle policy for run graph execution. - -This module owns state-transition decisions for run graph nodes and edges: -which tasks are initially ready, how completion satisfies dependencies, how -failure blocks downstream work, and when a run is terminal. - -It deliberately does not own structural graph persistence or Inngest event -delivery. Structural writes go through RuntimeGraphRepository; job modules turn -returned domain events into Inngest sends after commit. -""" -``` - -The docstring should state: - -- what the module owns; -- what it deliberately does not own; -- why the boundary exists. - -### Class Docstrings - -Each service class should explain the domain role, not merely list methods. - -Example: - -```python -class RuntimeLifecycleService: - """State machine for run graph execution. - - The service translates task terminal events into graph consequences: - satisfied or invalidated edges, newly ready tasks, blocked successors, and - run terminal-state decisions. Keeping this policy here prevents task - management, workflow jobs, and graph persistence from each inventing their - own propagation rules. - """ -``` - -### Function Docstrings - -Public methods should answer: - -1. What state transition or query does this perform? -2. Why does this rule exist? -3. What side effects occur? -4. What does the caller still own? - -Example: - -```python -async def propagate_completion(...): - """Apply the graph consequences of one task completing successfully. - - Completion satisfies all outgoing dependency edges from the completed task. - A downstream task becomes ready only when every upstream source task is - currently completed. Dependency-free dynamic children are also activated - here because they are intentionally created after their parent begins - running and therefore cannot be discovered during initial run setup. - - Returns the task ids that should receive `task/ready` events. The caller - owns transaction commit and event delivery. - """ -``` - -This is the standard we want because the hard part of this subsystem is not the -SQL. The hard part is the lifecycle reasoning. - -## Suggested Audit Before Implementation - -Before writing the implementation plan, audit these exact questions: - -1. Which `WorkflowService` methods are command/mutation methods versus read-only - inspection methods? -2. Which `TaskManagementService` helpers are actually lifecycle policy? -3. Which `graph.propagation` helpers still read definition-tier tables and can - now be run-tier only? -4. Which traversal helpers are containment traversal versus dependency-edge - traversal? -5. Which methods dispatch Inngest events directly, and should they instead - return domain events? -6. Which DTOs are structural, view, job orchestration, or audit payloads? -7. Which tests cover restart invalidation, failure blocking, cancelled-child - reactivation, and resource-scope traversal? - -## Possible Implementation Order - -1. Add characterization tests for lifecycle behavior before moving files. -2. Extract a `RuntimeLifecycleService` from `graph/propagation.py`. -3. Move terminal run detection into that lifecycle service. -4. Move restart downstream invalidation from `TaskManagementService` into the - lifecycle service, while keeping the `restart_task` command in task - management. -5. Split `WorkflowService` read-only inspection methods into a runtime - inspection service. -6. Split resource visibility/materialization into a runtime resource service. -7. Rename or re-export old imports only if needed for one PR; remove shims in - the follow-up. - -Each step should be behavior-preserving. - -## Acceptance Criteria For The Merge - -- There is one owner for lifecycle state-transition policy. -- There is one owner for structural graph persistence and graph mutation WAL. -- There is one owner for task-management commands. -- There is one owner for run-level lifecycle. -- Resource materialization no longer lives in a general workflow service. -- Module/class/function docstrings explain the ownership and "why", not only - the mechanics. -- Architecture tests prevent job handlers, REST handlers, or persistence code - from growing new lifecycle policy. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/schema-concept-debt-audit.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/schema-concept-debt-audit.md deleted file mode 100644 index 0e931c006..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/audits/schema-concept-debt-audit.md +++ /dev/null @@ -1,626 +0,0 @@ -# Schema Concept Debt Audit - -Date: 2026-05-18 - -Audited against PR 16 head in the main checkout. - -## Purpose - -This audit looks above the level of "which package owns persistence?" and asks -whether the schema still carries concepts that no longer match v2: - -- old tables that are no longer populated by the canonical path; -- tables that are active only for dashboard/test-harness compatibility; -- duplicated identity columns; -- fields that encode pre-object-bound worker/evaluator/sandbox selection; -- invalid or suspicious foreign keys; -- product concepts that have been deprecated in Python but still shape the - database and frontend contracts. - -## Classification - -### Keep - -Concept is active and aligned with v2. - -### Keep, But Rename Or Document - -Concept is active but the name reflects older vocabulary. - -### Compatibility Only - -Concept is still read or written, but only for dashboard, CLI, RL, test harness, -or transitional compatibility. It should not be used by new core runtime code. - -### Suspect - -Concept appears stale, invalid, or under-used. It needs a focused verification -before deletion. - -### Delete Candidate - -No active production writer/reader was found, or the concept directly -duplicates a canonical v2 path. - -## Table Families - -### Definition Tables: Keep - -```text -experiment_definitions -experiment_definition_workers -experiment_definition_evaluators -experiment_definition_instances -experiment_definition_tasks -experiment_definition_task_dependencies -experiment_definition_task_assignments -experiment_definition_task_evaluators -``` - -These remain the immutable authored definition tier. The canonical v2 authoring -path writes them through `application/experiments/definition_writer.py`, and -runtime preparation copies them into run-tier graph rows. - -Debt: - -- The Python/table vocabulary still says `experiment_definition`, while the - architecture now mostly says `definition`. This is naming debt, not dead - schema. -- `experiment_definition_tasks.task_payload` and - `experiment_definition_tasks.task_json` duplicate authored task state. The - full object-bound `task_json` is canonical for runtime reconstruction, while - `task_payload` remains as payload-specific compatibility/read convenience. - New code should not treat `task_payload` as an alternate authoring source. -- Worker assignment and task-evaluator join rows duplicate information that is - also inside object-bound `task_json`. Some duplication is still load-bearing: - worker definition rows feed execution metadata and evaluator rows feed - `RunTaskEvaluation.definition_evaluator_id`. The final target should be - documented: either keep normalized rows as read/index/provenance tables, or - simplify once no runtime path needs them. - -Recommended action: - -- Keep all definition tables for now. -- Mark `task_payload` as compatibility/derived from `task_json`. -- Decide whether normalized worker/evaluator assignment tables are permanent - provenance/index tables or cleanup targets. - -### Run Graph Tables: Keep - -```text -run_graph_nodes -run_graph_edges -run_graph_annotations -run_graph_mutations -``` - -These are the v2 run-tier runtime graph. `run_graph_nodes.task_id` is now the -canonical runtime task identity, and `run_graph_nodes.task_json` is the runtime -task snapshot used by `WorkflowGraphRepository.node()`. - -Debt: - -- Some application/CLI/test code still says `node_id` when it means `task_id`. - The schema mostly completed the identity collapse, but naming debt remains in - method parameters, DTOs, tests, and comments. -- The schema comments in `RunGraphNode.status` say graph status is free-form - and experiment-owned, while `persistence/graph/status_conventions.py` now - provides core runtime status literals. That is conceptual drift: status - semantics have become core runtime vocabulary. -- `RunGraphAnnotation` and `RunGraphMutation` are valid storage concepts, but - they should be kept read-only/append-only from application graph services. - Dashboard graph events should project these rows rather than invent a - parallel event shape. - -Recommended action: - -- Keep run graph tables. -- Move status vocabulary out of persistence into runtime/shared. -- Continue the `node_id` -> `task_id` naming cleanup in code/contracts. - -### Legacy `experiments` Table: Compatibility Only - -```text -experiments # BenchmarkDefinitionRecord -``` - -This table is the largest concept-debt item. It is not the canonical v2 -definition table, but it is still active: - -- `application/read_models/experiments.py` reads it as the legacy fallback path - when no `ExperimentDefinition` row exists. -- cohort read models join through it. -- `complete_workflow.py` and `fail_workflow.py` update it when a run's - `definition_id` points at a `BenchmarkDefinitionRecord`. -- `rl/rollout_service.py` still writes `BenchmarkDefinitionRecord` rows for RL - rollout batches. -- `rest_api/test_harness.py` writes same-id legacy rows for cohort/dashboard - compatibility. -- CLI experiment tag commands read its `experiment` tag. - -Debt: - -- The table name `experiments` collides with the v2 Python concept where an - "experiment" is more like a composition/tag/grouping of runs, not the - immutable definition itself. -- `BenchmarkDefinitionRecord` duplicates fields now represented by - `ExperimentDefinition`, definition instances, object-bound task snapshots, - and `RunRecord`. -- Several fields are pre-v2 selection defaults rather than canonical runtime - state: `default_worker_team_json`, `default_evaluator_slug`, - `default_model_target`, `sandbox_slug`, `dependency_extras_json`, - `sample_selection_json`, and `design_json`. -- The `experiment` string tag is useful for grouping runs, but it is attached - to a deprecated definition-shaped row instead of a v2 run grouping model. - -Recommended action: - -- Treat `BenchmarkDefinitionRecord` as compatibility-only, not a first-class v2 - schema. -- Stop new canonical authoring/launch paths from writing it. -- Move RL rollout and test-harness/dashboard compatibility off this table or - isolate the writes behind an explicitly deprecated compatibility module. -- Replace CLI tag filtering with `RunRecord.experiment` before deleting. - -### Cohort Tables: Compatibility Only, Frontend-Visible - -```text -experiment_cohorts -experiment_cohort_stats -``` - -These are still live because the REST API and dashboard use them: - -- `/cohorts` routes expose list/detail/update endpoints. -- `ExperimentCohortService` creates, updates, lists, and recomputes cohort - stats. -- dashboard generated REST contracts still include cohort status. -- test harness resolves and seeds cohorts. - -Debt: - -- We have already agreed cohorts are deprecated in v2. They remain because the - dashboard has not yet moved to the newer "experiment as collection/tag of - runs" view. -- Cohort membership is not attached to canonical `ExperimentDefinition` or - `RunRecord` directly. It flows through deprecated `BenchmarkDefinitionRecord`. -- `ExperimentCohortStats` denormalizes aggregate values that can be recomputed - from runs/evaluations. It may be fine as a cache, but the ownership should be - explicit. -- `ExperimentCohortStatus` is exported into frontend contracts even though the - concept is scheduled for removal. - -Recommended action: - -- Keep until the dashboard refactor removes cohort UI and endpoints. -- Mark table/service/API/frontend contracts as deprecated compatibility. -- Do not build new runtime features on cohort membership. -- When deleting, delete the frontend contracts/routes and - `BenchmarkDefinitionRecord.cohort_id` in the same cleanup slice. - -### Run Records: Keep, But Identity Fields Need Cleanup - -```text -runs -``` - -`RunRecord` is active and central. It anchors run status, provenance, -instance key, summaries, and lifecycle timestamps. - -Debt: - -- It carries both `definition_id` and `workflow_definition_id`. In canonical v2 - launches both point at `ExperimentDefinition.id`. In compatibility paths, - `definition_id` may point at a `BenchmarkDefinitionRecord`, while - `workflow_definition_id` points at `ExperimentDefinition`. -- Several callers now treat `workflow_definition_id` as the canonical - definition id. Others still use `definition_id` to find legacy experiments or - cohorts. -- `worker_team_json`, `evaluator_slug`, `sandbox_slug`, - `dependency_extras_json`, and `assignment_json` are mostly launch/selection - metadata from the pre-object-bound runtime. The active runtime reads - worker/sandbox/evaluators from run-tier task snapshots, not these fields. - Some fields still feed CLI/read-model displays or sandbox setup payloads. -- `summary_json` is a mixed bag: checkpoint metadata, final score/cost/error - summaries, sandbox summaries, and other run-level facts can all land there. - That makes it flexible but conceptually under-specified. -- `ergon_cli/commands/run.py` still filters through `RunRecord.experiment_id`, - but the model no longer defines that field. That is an actual stale reference - rather than just naming debt. - -Recommended action: - -- Keep `runs`. -- Make `RunRecord.definition_id` the canonical FK to `ExperimentDefinition` and - delete `workflow_definition_id` after backfill. -- Mark `worker_team_json`, `evaluator_slug`, `sandbox_slug`, and - `dependency_extras_json` compatibility/display-only in field descriptions. -- Keep `model_target` and `assignment_json` active. -- Replace CLI filtering that references `RunRecord.experiment_id`. -- Split or type run `summary_json` if it continues accumulating unrelated - concepts. - -### Task Execution Rows: Keep - -```text -run_task_executions -``` - -This table is active and aligned with v2 execution telemetry. It records -attempts per `(run_id, task_id)`, status, sandbox id, final assistant message, -errors, and worker output. - -Debt: - -- `definition_worker_id` points at normalized definition worker rows even - though object-bound workers are also serialized in task snapshots. This is - still useful as provenance/read-model metadata, but it should be documented - as derived/index/provenance rather than runtime source of truth. -- `worker_output_json` has an in-code TODO saying it may move behind a lazy - `CriterionContext.worker_output()` accessor or dedicated output store. That - is real concept debt: worker output is currently stored on execution rows - because evaluation needs it. -- Status values come from `TaskExecutionStatus`, while graph node status has a - separate vocabulary. That is okay if documented: execution status and graph - status are different concepts. It becomes debt if code treats them - interchangeably. - -Recommended action: - -- Keep. -- Document `definition_worker_id` as provenance. -- Resolve the worker output storage TODO in the criterion-context redesign. - -### Evaluation Rows: Keep, But Summary Contract Should Move - -```text -run_task_evaluations -``` - -This table is active. Evaluation jobs write rows, read models consume them, and -dashboard task evaluation events carry their DTOs. - -Debt: - -- `definition_evaluator_id` points to normalized definition evaluator rows - while task snapshots also carry inline evaluators. This is still load-bearing - for provenance, but it is duplicated conceptually. -- Evaluation summary schema lives in `persistence/telemetry/evaluation_summary.py` - even though it describes evaluator/criterion semantics and is consumed by - application/read-model code. -- `CreateTaskEvaluation` is an application command DTO living in persistence; - PRD 01 deletes it rather than moving it because it only feeds a - single-consumer repository that is also being deleted. - -Recommended action: - -- Keep `run_task_evaluations`. -- Move evaluation summary/command DTOs into application or shared. -- Document `definition_evaluator_id` as provenance/index derived from the - object-bound evaluator. - -### Resource Rows: Keep - -```text -run_resources -``` - -This table is active and aligned with v2 append-only resource publication. - -Debt: - -- Resource publication policy currently lives partly under - `infrastructure/sandbox/resource_publisher.py`. -- `kind` is a string validated against `RunResourceKind`; if resource kinds are - product concepts, the enum should live with application/shared contracts and - persistence should consume it. -- `file_path` points at local blob-store paths. That is practical but couples - resource rows to one storage implementation. If blob backends expand, split - URI/storage backend metadata. - -Recommended action: - -- Keep. -- Move publication semantics to application resources service as described in - the infrastructure boundary audit. - -### Context Event Rows: Keep, Alias File Delete Candidate - -```text -run_context_events -``` - -The table is active: worker execution persists context chunks, run snapshots -read them, dashboard context events stream them, and RL extraction consumes -them. - -Debt: - -- `persistence/context/event_payloads.py` duplicates aliases over - `ContextPartChunkLog`; the alias should be deleted and consumers should use - `ContextPartChunkLog` directly. -- The model still imports context part contracts from - `core.domain.generation.context_parts`, which we already plan to move to - `core/shared/context_parts.py`. - -Recommended action: - -- Keep `run_context_events`. -- Delete or collapse `event_payloads.py` after the shared context contract - move. - -### Communication Tables: Keep, But Consider Domain Split - -```text -threads -thread_messages -``` - -These are active through `CommunicationService`, run snapshots, and dashboard -thread message events. - -Debt: - -- They live inside the broad telemetry model file even though communication is - an application domain with its own service and DTOs. -- Communication service runs direct SQL and emits dashboard events directly. - That is application/infrastructure boundary debt rather than table death. - -Recommended action: - -- Keep. -- If telemetry models are split, move these table classes to a communication - storage family. -- If communication grows, introduce an application communication repository. - -### Training Tables: Keep If RL Training UI Remains - -```text -training_sessions -training_metrics -``` - -These are read through REST `/runs/training/*` endpoints and dashboard training -UI components. I did not find active writers in this audit outside the model -comments, but the tables support a visible product surface. - -Debt: - -- The audit found REST reads and dashboard pages, but no in-repo production - writer. -- `TrainingSession.experiment_definition_id` uses old experiment-definition - naming but points at canonical `ExperimentDefinition`. -- Training status enum lives in persistence shared enums. - -Recommended action: - -- Treat `TrainingSession` and `TrainingMetric` as gated deletion candidates. -- If training observability remains a product surface, add/identify the writer - and move the training DTOs to `views/training.py`. -- If training observability is stale, delete the tables, REST endpoints, - generated contracts, and dashboard training UI together. - -### Rollout Batch Tables: Keep - -```text -rollout_batches -rollout_batch_runs -``` - -These are active through `RolloutService`. They provide durable batch state for -RL trainers and join rollout batches to generated run ids. - -Debt: - -- `RolloutService.submit()` still creates a `BenchmarkDefinitionRecord` per RL - batch, then creates `RunRecord.definition_id=experiment.id` and - `RunRecord.workflow_definition_id=request.definition_id`. That keeps the - legacy `experiments` table alive for RL. -- Batch status literals are local to the persistence table validator, while RL - request/response types also have `BatchStatus`. - -Recommended action: - -- Keep rollout batch tables. -- Stop using `BenchmarkDefinitionRecord` as RL batch provenance. Either make - `RolloutBatch` the provenance container or add explicit run grouping metadata. -- Use one status contract for rollout batches. - -### Sandbox WAL/Event Tables: Keep, But Clarify Ownership - -```text -sandbox_command_wal_entries -sandbox_events -``` - -These are actively written by `PostgresSandboxEventSink` and read by e2e test -helpers. They are observability tables, not core runtime state. - -Debt: - -- `run_id` carries no FK and comments acknowledge teardown quirks where - synthetic entries may use `run_id=task_id`. -- They live in telemetry models but are written directly from infrastructure. - That is acceptable only if they remain observability/WAL rows, not product - state. -- Sandbox event kinds are raw strings rather than typed event literals. - -Recommended action: - -- Keep as observability, but document that they are not runtime source of - truth. -- Keep sandbox WAL/event rows in `persistence/telemetry/models.py` during this - RFC; a later telemetry split may move them mechanically, without changing - ownership. -- Fix the run-id teardown quirk before exposing sandbox WAL/event rows through - dashboard/read-model views. - -### Import Reducer Tables: Suspect - -```text -run_reducers -run_reducer_footprints -run_drops_manifests -``` - -I found table definitions but no active production writer/reader in the scanned -code paths. The package docstring says they are for imported/public rollout -cards, but they appear disconnected from the current runtime. - -There is also a concrete schema bug: - -```python -node_id: UUID | None = Field(default=None, foreign_key="run_graph_nodes.id") -``` - -`RunGraphNode` no longer has an `id` column; its primary key is -`(run_id, task_id)`. This FK points at a non-existent schema concept. - -Recommended action: - -- Treat as suspect/high-priority verification. -- If unused, delete the tables/models before they become accidental API. -- If active outside this scan, rename the package around the real product - concept and fix the FK to `(run_id, task_id)` or remove the node FK. - -## Field-Level Concept Debt - -### `definition_id` vs `workflow_definition_id` - -Current meaning: - -- `RunRecord.workflow_definition_id` is the canonical v2 definition id used to - initialize/read a run. -- `RunRecord.definition_id` sometimes points at the same definition id, but in - compatibility/RL flows can point at `BenchmarkDefinitionRecord.id`. - -This is confusing enough that callers have to know which vintage of run they -are reading. It also makes `experiment_id`/`definition_id` naming in API DTOs -hard to reason about. - -Target: - -- choose one canonical definition FK for runtime; -- if legacy provenance is still needed, rename or isolate it as - `legacy_experiment_record_id` or a compatibility table relation. - -### `task_payload` vs `task_json` - -Current meaning: - -- `task_json` is the full object-bound task snapshot and is runtime-canonical. -- `task_payload` is payload-only compatibility/read convenience. - -Target: - -- keep only if readers need payload-specific access without loading the full - object-bound task; -- otherwise plan deletion after all readers use `task_json`. - -### Worker/Evaluator/Sandbox Selection Fields - -Duplicated fields: - -- `BenchmarkDefinitionRecord.default_worker_team_json`; -- `BenchmarkDefinitionRecord.default_evaluator_slug`; -- `BenchmarkDefinitionRecord.sandbox_slug`; -- `RunRecord.worker_team_json`; -- `RunRecord.evaluator_slug`; -- `RunRecord.sandbox_slug`; -- `RunTaskExecution.definition_worker_id`; -- `RunTaskEvaluation.definition_evaluator_id`; -- object-bound `Task.worker`, `Task.sandbox`, `Task.evaluators` inside - `task_json`. - -Target: - -- object-bound task snapshots should drive runtime execution; -- normalized worker/evaluator ids may remain as provenance/index/read-model - metadata; -- run/default fields should be display/compatibility only or deleted. - -### `node_id` vs `task_id` - -The schema largely moved to `task_id`, but code still uses `node_id` names in -application services, tests, CLI helpers, dashboard compatibility, and the -invalid reducer FK. This is naming debt and sometimes schema debt. - -Target: - -- use `task_id` for runtime graph identity everywhere public; -- reserve `node` only for internal graph row language if the team still wants - it, but do not expose both names for the same id. - -### Cohort Status And Experiment Tags - -`ExperimentCohortStatus` and `BenchmarkDefinitionRecord.experiment` are both -frontend/CLI-visible grouping concepts, but neither matches the v2 statement -that experiments are a collection/tag of runs carried by Python composition. - -Target: - -- define one run grouping concept; -- move dashboard/CLI to it; -- delete cohort tables and legacy experiment tags after migration. - -## Highest-Risk Findings - -1. `RunReducer.node_id` references `run_graph_nodes.id`, but that column does - not exist on the PR 16 schema. - -2. `ergon_cli/commands/run.py` references `RunRecord.experiment_id`, but the - model defines `definition_id` and `workflow_definition_id`, not - `experiment_id`. - -3. `BenchmarkDefinitionRecord` is deprecated by architecture but still active - in RL rollout, cohort/dashboard compatibility, test harness, and CLI tags. - It needs an explicit migration plan rather than ad hoc deletion. - -4. Cohorts are deprecated by design but still frontend-visible through REST and - generated dashboard contracts. - -5. `RunRecord.definition_id` and `RunRecord.workflow_definition_id` encode two - different eras of provenance in one active table. - -## Suggested PR Slices - -### PR A: Broken/Stale Reference Fixes - -- Fix or delete `RunReducer.node_id` FK. -- Fix CLI run filtering that references `RunRecord.experiment_id`. -- Add schema/source tests that catch nonexistent FK target columns and stale ORM - attribute references. - -### PR B: Legacy Experiment Record Isolation - -- Create a compatibility module around `BenchmarkDefinitionRecord` usages. -- Move RL rollout away from writing `BenchmarkDefinitionRecord`. -- Mark all remaining reads/writes as compatibility-only. -- Replace `BenchmarkDefinitionRecord.experiment` tags with - `RunRecord.experiment`. - -### PR C: Cohort Deprecation Migration - -- Mark `/cohorts`, cohort DTOs, frontend contracts, and cohort tables as - deprecated. -- Plan dashboard replacement around v2 run grouping. -- Delete cohort tables only after the dashboard no longer depends on them. - -### PR D: Run Identity And Selection Field Cleanup - -- Decide canonical `RunRecord` definition identity. -- Classify `worker_team_json`, `evaluator_slug`, `sandbox_slug`, - `dependency_extras_json`, and `assignment_json`. -- Remove, rename, or document each as compatibility/display/provenance. - -### PR E: Import Reducer Deletion - -- Delete reducer/drop-manifest tables in PRD 01. The scanned code paths have no - production writer/reader, and `RunReducer.node_id` points at a non-existent - graph column. - -### PR F: Training Tables Gate - -- Keep training only if the implementation identifies or adds the production - writer for `TrainingSession` and `TrainingMetric`. -- Otherwise remove the training dashboard/API surface with the tables. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/00-program.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/00-program.md deleted file mode 100644 index 34dfcc2ba..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/00-program.md +++ /dev/null @@ -1,118 +0,0 @@ -# Core Refactor PR Stack - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement each PR plan task-by-task. - -## Goal - -Turn the core-domain-structure-standardization PRDs into an ordered engineering -stack with reviewable PR boundaries. - -## Source PRDs - -- `../prds/00-move-vs-delete-decisions.md` -- `../prds/01-persistence-schema-boundary.md` -- `../prds/02-shared-context-contract.md` -- `../prds/03-infrastructure-adapter-boundary.md` -- `../prds/04-rest-inbound-adapter-boundary.md` -- `../prds/05-application-runtime-restructure.md` -- `../prds/06-views-dashboard-contracts.md` -- `../prds/07-deprecated-cohorts-legacy-experiments.md` -- `../prds/08-job-composition-modules.md` -- `../prds/09-final-core-folder-state.md` - -## Stack Order - -1. [`01-persistence-evaluation-and-status.md`](01-persistence-evaluation-and-status.md) -2. [`02-persistence-schema-concept-cleanup.md`](02-persistence-schema-concept-cleanup.md) -3. [`03-shared-context-and-domain-deletion.md`](03-shared-context-and-domain-deletion.md) -4. [`04-views-package-foundation.md`](04-views-package-foundation.md) -5. [`05-dashboard-contracts-and-publisher-port.md`](05-dashboard-contracts-and-publisher-port.md) -6. [`06-rest-http-adapter-boundary.md`](06-rest-http-adapter-boundary.md) -7. [`07-sandbox-resource-publishing-boundary.md`](07-sandbox-resource-publishing-boundary.md) -8. [`08-legacy-experiment-and-cohort-isolation.md`](08-legacy-experiment-and-cohort-isolation.md) -9. [`09-legacy-cohort-and-experiment-deletion.md`](09-legacy-cohort-and-experiment-deletion.md) -10. [`10-job-composition-modules.md`](10-job-composition-modules.md) -11. [`11-application-runtime-restructure.md`](11-application-runtime-restructure.md) -12. [`12-final-folder-state-and-architecture-gates.md`](12-final-folder-state-and-architecture-gates.md) - -## Coverage Matrix - -| PR | Primary PRDs Covered | Notes | -| --- | --- | --- | -| PR 01 | PRD 00, PRD 01, PRD 09 | Evaluation repository deletion, evaluation summary move, runtime status move. | -| PR 02 | PRD 00, PRD 01, PRD 07 | Dead import tables, run identity, training gate, rollout status, compatibility table isolation foundations. | -| PR 03 | PRD 00, PRD 02, PRD 09 | Shared context contract, alias deletion, `domain/` deletion. | -| PR 04 | PRD 06, PRD 09 | `views/` foundation and `application/read_models` split. | -| PR 05 | PRD 03, PRD 06 | Dashboard contracts, dashboard publisher port, workflow-started snapshot contract. | -| PR 06 | PRD 04, PRD 09 | REST moves under `infrastructure/http`. | -| PR 07 | PRD 03 | Sandbox resource publishing boundary. | -| PR 08 | PRD 07, PRD 01 | Compatibility isolation for cohorts and legacy experiment records. | -| PR 09 | PRD 07, PRD 09 | Cohort and legacy experiment deletion after frontend/CLI/RL replacements. | -| PR 10 | PRD 08, PRD 03, PRD 09 | Job-local composition modules and Inngest handler deletion. | -| PR 11 | PRD 05, PRD 09 | Runtime consolidation under `application/runtime`. | -| PR 12 | PRD 09 | Final architecture gates, docs, and shim deletion. | - -## Why This Order - -The stack pays down foundations before high-motion package moves. Persistence -cleanup comes first because stale schemas and command DTOs leak into almost -every other boundary. The shared context move comes next because it deletes the -nominal `domain/` package and removes alias debt used by views, dashboard, RL, -and persistence. The `views/` package is then created before dashboard and REST -move toward it. Job composition waits until the ports, views, and HTTP adapter -boundaries exist. The large runtime consolidation comes late, after outer-layer -duplication has stopped pulling runtime code in several directions. - -## Stack-Wide Rules - -- Each PR should preserve behavior unless its plan explicitly says otherwise. -- Each PR must add or update architecture tests before deleting old homes. -- Compatibility concepts must be named `compat` or `deprecated` while they - remain. -- Do not leave import shims unless the PR plan explicitly permits a temporary - shim and names the later PR that deletes it. -- Do not move a schema only because its folder is wrong. First check whether an - existing view or DTO already expresses the same contract. - -## Stack-Wide Test Gates - -Run the narrow tests named in each PR plan. Before merging the stack, run: - -```bash -pytest ergon_core/tests/unit/architecture -q -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/unit/dashboard -q -pytest ergon_core/tests/unit/persistence -q -pytest ergon_core/tests/smoke -q -``` - -If the dashboard contract generator is available in the branch, also run the -schema export/drift check after PRs 05, 06, 09, and 12. - -## Final Shape - -The stack should converge on: - -```text -ergon_core/core/ - application/ - infrastructure/ - jobs/ - persistence/ - views/ - rl/ - shared/ -``` - -No final PR should leave these packages: - -- `core/domain` -- `core/rest_api` -- `core/application/jobs` -- `core/application/read_models` -- `core/application/graph` -- `core/application/tasks` -- `core/application/workflows` -- `core/infrastructure/inngest/handlers` diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/01-persistence-evaluation-and-status.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/01-persistence-evaluation-and-status.md deleted file mode 100644 index 29563d516..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/01-persistence-evaluation-and-status.md +++ /dev/null @@ -1,79 +0,0 @@ -# PR 01: Persistence Evaluation And Status Cleanup - -## What - -Remove application semantics from persistence where the replacement is already -obvious: evaluation summary schemas, the single-consumer telemetry repository, -and graph status vocabulary. - -## Why - -Persistence currently owns more than storage shape. `TelemetryRepository` and -`CreateTaskEvaluation` form a tiny repository around one evaluation service, -`evaluation_summary.py` models evaluation-domain semantics over a JSON column, -and `graph/status_conventions.py` owns runtime lifecycle vocabulary. Keeping -these in persistence makes later runtime and views work inherit the wrong -ownership boundary. - -## How - -- Delete `core/persistence/telemetry/repository.py`. -- Delete `CreateTaskEvaluation` from - `core/persistence/telemetry/models.py`. -- Add private helpers to `core/application/evaluation/service.py`: - `_list_task_evaluations(...)` and `_create_task_evaluation(...)`. -- Move `core/persistence/telemetry/evaluation_summary.py` to - `core/application/evaluation/summary.py`. -- Move dashboard/read DTO conversion that is duplicated between evaluation - service and run snapshots into - `core/application/evaluation/dto_mapping.py`. -- Delete `RunTaskEvaluation.parsed_summary()` from - `core/persistence/telemetry/models.py`. -- Move `core/persistence/graph/status_conventions.py` to - `core/application/runtime/status.py`. -- Update imports in application graph/tasks/workflows, dashboard contracts, - dashboard emitter, smoke fixtures, and tests. - -## Plan - -1. Add characterization tests for `EvaluationService.evaluate()` persistence: - one test should assert that a `RunTaskEvaluation` row is written with the - same `summary_json`, score, pass/fail, task execution id, task id, and - definition evaluator id as today. -2. Add a unit test for the new `evaluation_row_to_dto(...)` mapper using a - stored `RunTaskEvaluation` row with multiple criterion outcomes. -3. Add an architecture test that `core.persistence.telemetry.models` does not - import `core.application.evaluation.summary`. -4. Inline `TelemetryRepository.get_task_evaluations()` and - `TelemetryRepository.create_task_evaluation()` into private helpers on - `EvaluationService`. -5. Replace `CreateTaskEvaluation(...)` construction with explicit keyword - arguments. -6. Move `EvaluationSummary`, `CriterionOutcomeEntry`, and - `EvalCriterionStatus` into `application/evaluation/summary.py`. -7. Replace all imports of - `core.persistence.telemetry.evaluation_summary`. -8. Replace `RunTaskEvaluation.parsed_summary()` call sites with - `EvaluationSummary.model_validate(evaluation.summary_json)`. -9. Move status constants/helpers into `application/runtime/status.py`. -10. Delete the old persistence files after imports are clean. - -## Acceptance Criteria - -- No production code imports `core.persistence.telemetry.repository`. -- No production code references `CreateTaskEvaluation`. -- No production code imports `core.persistence.telemetry.evaluation_summary`. -- No production code imports `core.persistence.graph.status_conventions`. -- Evaluation summary validation still happens through `EvaluationSummary`. -- Runtime status vocabulary lives under `core.application.runtime.status`. - -## Tests - -```bash -pytest ergon_core/tests/unit/evaluation -q -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/unit/dashboard/test_event_contract_types.py -q -pytest ergon_core/tests/unit/architecture -q -rg -n "TelemetryRepository|CreateTaskEvaluation|persistence\\.telemetry\\.evaluation_summary|persistence\\.graph\\.status_conventions" ergon_core/ergon_core -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/02-persistence-schema-concept-cleanup.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/02-persistence-schema-concept-cleanup.md deleted file mode 100644 index 33600b2a6..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/02-persistence-schema-concept-cleanup.md +++ /dev/null @@ -1,74 +0,0 @@ -# PR 02: Persistence Schema Concept Cleanup - -## What - -Remove or isolate stale persistence concepts that are either invalid in the v2 -schema or compatibility-only: import reducer tables, run definition identity -debt, training observability ambiguity, rollout status strings, and legacy -experiment/cohort table helpers. - -## Why - -PR 01 fixes ownership where the replacement is clear. This PR handles the -remaining schema concept debt so later adapter and runtime moves do not need to -carry broken identities or dead tables forward. - -## How - -- Delete `core/persistence/imports/models.py` and - `core/persistence/imports/`. -- Make `RunRecord.definition_id` the canonical runtime definition id. -- Backfill/update runtime reads and writes away from - `RunRecord.workflow_definition_id`, then delete that field. -- Remove stale `RunRecord.experiment_id` references. -- Add field descriptions marking `RunRecord.worker_team_json`, - `evaluator_slug`, `sandbox_slug`, and `dependency_extras_json` as - compatibility/display-only until later PRs remove reads. -- Decide the training surface: - - either identify/add the production writer and move training DTOs to - `views/training.py`; - - or delete `TrainingSession`, `TrainingMetric`, `/runs/training/*`, - generated contracts, and dashboard training UI together. -- Create `core/shared/rollout_status.py` and use it from - `core/rl/rollout_types.py` and `RolloutBatch.status`. -- Move non-table legacy experiment/cohort helpers into - `application/compat/*` only where needed for temporary callers. - -## Plan - -1. Add schema tests that fail on foreign keys pointing at missing columns. -2. Add source tests that fail on `RunRecord.experiment_id` references. -3. Delete reducer/drop-manifest persistence models and imports. -4. Update migration/schema creation imports so reducer tables are absent. -5. Change launch/run creation paths to write `RunRecord.definition_id`. -6. Change read paths to use `RunRecord.definition_id`. -7. Delete `workflow_definition_id` once no code reads or writes it. -8. Remove broken CLI run filtering that joins through `experiment_id`; leave - experiment-tag filtering for PR 08/09. -9. Mark display-only run selection fields in model descriptions. -10. Resolve the training gate in one direction; do not leave a half-moved - training surface. -11. Introduce `RolloutStatus` in `core/shared/rollout_status.py`. -12. Update rollout models and persistence validation to use the shared status. - -## Acceptance Criteria - -- `core/persistence/imports/` is gone. -- No schema field references a non-existent target column. -- No production code references `RunRecord.experiment_id`. -- `RunRecord.definition_id` is the canonical runtime definition id. -- `workflow_definition_id` is gone. -- Training is either explicitly retained with a writer and `views/training.py`, - or deleted across backend/frontend/contracts. -- Rollout status vocabulary is shared, not persistence-local. - -## Tests - -```bash -pytest ergon_core/tests/unit/persistence -q -pytest ergon_core/tests/unit/architecture -q -pytest ergon_core/tests/unit/rl -q -pytest ergon_cli/tests -q -rg -n "workflow_definition_id|experiment_id|persistence/imports|RunReducer|RunDropsManifest" ergon_core ergon_cli -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/03-shared-context-and-domain-deletion.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/03-shared-context-and-domain-deletion.md deleted file mode 100644 index 14c13cc6c..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/03-shared-context-and-domain-deletion.md +++ /dev/null @@ -1,63 +0,0 @@ -# PR 03: Shared Context Contract And Domain Deletion - -## What - -Move context stream schemas from the nominal `core/domain` package into -`core/shared/context_parts.py`, delete redundant aliases, and remove the -`core/domain` package. - -## Why - -`core/domain` currently contains one shared contract and no real domain logic. -The context stream schema is used by public worker contracts, persistence, -dashboard events, RL extraction, and tests. `shared/` is the correct home, but -only if the aliases `WorkerYield` and `ContextEventPayload` die at the same -time. - -## How - -- Move `core/domain/generation/context_parts.py` to - `core/shared/context_parts.py`. -- Move `ContextEventType` from - `core/persistence/context/event_payloads.py` into - `core/shared/context_parts.py`. -- Replace `WorkerYield` with `ContextPartChunk`. -- Replace `ContextEventPayload` with `ContextPartChunkLog`. -- Delete `core/persistence/context/event_payloads.py`. -- Delete `core/domain/generation/__init__.py`, `core/domain/__init__.py`, and - the `core/domain/` package. - -## Plan - -1. Add an architecture test asserting `core/domain` does not exist. -2. Add a source test asserting `WorkerYield` and `ContextEventPayload` have no - production or test references. -3. Move the context part models into `core/shared/context_parts.py`. -4. Update production imports from: - - `core.domain.generation.context_parts` - - `core.persistence.context.event_payloads` -5. Update test and fixture imports listed in PRD 02. -6. Replace type annotations that use aliases with the real model names. -7. Delete the old domain and persistence alias modules. -8. Regenerate dashboard/event schemas if the contract export path changes. - -## Acceptance Criteria - -- No import references `core.domain.generation.context_parts`. -- No import references `core.persistence.context.event_payloads`. -- No code references `WorkerYield` or `ContextEventPayload`. -- `RunContextEvent.parsed_payload()` still validates stored payloads. -- RL extraction still reads context events. -- `core/domain` is absent. - -## Tests - -```bash -pytest ergon_core/tests/unit/persistence/test_context_event_repository.py -q -pytest ergon_core/tests/unit/runtime/test_context_event_contracts.py -q -pytest ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py -q -pytest ergon_core/tests/unit/state/test_context_part_stream.py -q -pytest ergon_core/tests/unit/architecture -q -rg -n "domain\\.generation\\.context_parts|persistence\\.context\\.event_payloads|WorkerYield|ContextEventPayload" ergon_core tests -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/04-views-package-foundation.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/04-views-package-foundation.md deleted file mode 100644 index 892d20852..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/04-views-package-foundation.md +++ /dev/null @@ -1,68 +0,0 @@ -# PR 04: Views Package Foundation - -## What - -Replace `application/read_models` with a top-level `core/views` package for -read-only DTO builders and API/dashboard view services. - -## Why - -`read_models` does not earn a home under `application` because it is neither -command-side use case logic nor persistence. It builds read-only contracts for -REST, dashboard, CLI, and tests. Pulling it into `views/` gives dashboard, -REST, and job composition a stable target before those packages move. - -## How - -- Create `core/views/`. -- Split `application/read_models/models.py` instead of moving it wholesale. -- Move run snapshot DTOs to `views/runs/models.py`. -- Move `application/read_models/runs.py` to `views/runs/service.py`. -- Move `application/read_models/run_snapshot.py` to - `views/runs/snapshot.py`. -- Move experiment DTOs/service to `views/experiments/models.py` and - `views/experiments/service.py`. -- Move `application/read_models/resources.py` to `views/resources.py`. -- Move `application/read_models/errors.py` to `views/errors.py`. -- Leave cohort compatibility split for PR 08 unless needed to unblock imports. -- Move training DTOs only if PR 02 kept training observability. - -## Plan - -1. Add architecture tests for `core/views`: - - views may read persistence rows; - - views must not call `session.add`, `session.commit`, or start jobs; - - views must not import concrete infrastructure. -2. Create `core/views/__init__.py`, `views/runs/`, and - `views/experiments/`. -3. Move run DTOs and update imports in REST routes, dashboard code, and tests. -4. Move run read service and snapshot builder. -5. Move experiment DTOs and read service. -6. Move resource DTO/read helpers. -7. Move read-model errors. -8. Update import-boundary tests and model field description tests. -9. Delete `application/read_models` files that are fully migrated. -10. Leave an explicit failure if `application/read_models` still contains only - cohort compatibility that PR 08 owns. - -## Acceptance Criteria - -- Normal run, experiment, resource, and error views import from `core.views`. -- `application/read_models/models.py`, `runs.py`, `run_snapshot.py`, - `experiments.py`, `resources.py`, and `errors.py` are gone. -- View modules are read-only by architecture test. -- REST/dashboard consumers use the new imports. -- Cohort compatibility is either still clearly isolated for PR 08 or already - moved into `views/compat` and `application/compat`. - -## Tests - -```bash -pytest ergon_core/tests/unit/read_models -q -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/unit/dashboard -q -pytest ergon_core/tests/unit/architecture -q -rg -n "application\\.read_models\\.(models|runs|run_snapshot|experiments|resources|errors)" ergon_core tests -rg -n "session\\.add\\(|session\\.commit\\(" ergon_core/ergon_core/core/views -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/05-dashboard-contracts-and-publisher-port.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/05-dashboard-contracts-and-publisher-port.md deleted file mode 100644 index 2629d9dcc..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/05-dashboard-contracts-and-publisher-port.md +++ /dev/null @@ -1,72 +0,0 @@ -# PR 05: Dashboard Contracts And Publisher Port - -## What - -Move dashboard event contracts and dashboard DTO construction out of -infrastructure, replace direct application imports of `DashboardEmitter` with a -publisher port, and delete duplicate workflow tree schemas. - -## Why - -Dashboard infrastructure should transport events, not build application DTOs. -The current workflow-started tree duplicates run snapshot logic through -`WorkerRef`, `TaskTreeNode`, and frontend-only recursive schemas. This PR makes -dashboard contracts part of `views/` and makes infrastructure a publisher -implementation. - -## How - -- Move `core/infrastructure/dashboard/event_contracts.py` to - `core/views/dashboard_events/contracts.py`. -- Create `core/application/ports/dashboard.py` with - `DashboardEventPublisher`. -- Simplify `DashboardEmitter` to `publish(event: InngestEventContract)`. -- Extract graph mutation mapping to - `views/dashboard_events/graph_mutations.py`. -- Extract context event mapping to - `views/dashboard_events/context_events.py`. -- Extract cohort dashboard event shape to - `views/dashboard_events/cohorts.py`, but keep emission orchestration in - compatibility code until PR 08/09. -- Delete `WorkerRef`, `TaskTreeNode`, `_WORKER_SLUG_NS`, - `_worker_ref_for_slug()`, and `_build_task_tree_for_run()`. -- Change `DashboardWorkflowStartedEvent` to carry `snapshot: RunSnapshotDto`. -- Update frontend event parsing to consume the generated snapshot contract. - -## Plan - -1. Add tests for dashboard event contract import location. -2. Add a run snapshot based workflow-started contract test. -3. Add tests for graph mutation row-to-DTO mapping shared by live events and - `RunReadService.list_mutations()`. -4. Move dashboard contracts into `views/dashboard_events/contracts.py`. -5. Update `scripts/export_contract_schemas.py::CONTRACTS_MODULE`. -6. Create `DashboardEventPublisher` protocol. -7. Narrow `DashboardEmitter` to transport-only publishing. -8. Move graph mutation and context event mappers into views. -9. Replace workflow tree assembly with snapshot event construction. -10. Update frontend contract parser/types to delete manual - `WorkerRefSchema` and `TaskTreeNodeSchema`. -11. Regenerate dashboard contracts. -12. Delete old dashboard contract module and old workflow tree helpers. - -## Acceptance Criteria - -- Dashboard event contracts live in `core/views/dashboard_events/contracts.py`. -- `core/infrastructure/dashboard/event_contracts.py` is gone. -- Application services do not import concrete `DashboardEmitter`. -- `DashboardWorkflowStartedEvent` uses `snapshot: RunSnapshotDto`. -- `WorkerRef` and `TaskTreeNode` are gone in backend and frontend. -- Graph mutation DTO mapping is shared between snapshot/API reads and live - dashboard events. - -## Tests - -```bash -pytest ergon_core/tests/unit/dashboard -q -pytest ergon_core/tests/unit/runtime/test_graph_mutation_contracts.py -q -pytest ergon_core/tests/unit/runtime/test_context_event_contracts.py -q -pytest ergon_core/tests/unit/architecture -q -rg -n "WorkerRef|TaskTreeNode|_build_task_tree_for_run|infrastructure\\.dashboard\\.event_contracts|DashboardEmitter" ergon_core ergon-dashboard/src -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/06-rest-http-adapter-boundary.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/06-rest-http-adapter-boundary.md deleted file mode 100644 index 6edd9ec47..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/06-rest-http-adapter-boundary.md +++ /dev/null @@ -1,69 +0,0 @@ -# PR 06: REST HTTP Adapter Boundary - -## What - -Move top-level `core/rest_api` into `core/infrastructure/http` and keep route -modules as thin HTTP adapters over application services and views. - -## Why - -REST is framework glue, not a core domain. Keeping it top-level makes it look -like application logic and lets route-local test harness and cohort behavior -leak into product concepts. This PR makes HTTP routing an infrastructure -adapter while preserving public REST paths. - -## How - -- Move: - - `core/rest_api/app.py` -> `core/infrastructure/http/app.py` - - `core/rest_api/runs.py` -> `core/infrastructure/http/routes/runs.py` - - `core/rest_api/experiments.py` -> - `core/infrastructure/http/routes/experiments.py` - - `core/rest_api/rollouts.py` -> - `core/infrastructure/http/routes/rollouts.py` - - `core/rest_api/test_harness.py` -> - `core/infrastructure/http/routes/test_harness.py` - - `core/rest_api/cohorts.py` -> - `core/infrastructure/http/routes/cohorts.py` temporarily for PR 08/09. -- Keep route-local `Test*Dto` classes in the test harness route module. -- Move reusable test writes into - `core/application/testing/test_harness_service.py`. -- Move legacy cohort marker writes into - `core/application/compat/cohorts.py`. -- Update imports in app startup, tests, scripts, and dashboard proxy tests. - -## Plan - -1. Add an architecture test that final route modules may import FastAPI, - shared settings, application services, and views, but not SQLModel table - classes directly. -2. Add an architecture test that `core/rest_api` does not exist. -3. Add a bootstrap exception test/docstring for - `core/infrastructure/http/app.py`. -4. Move app and route modules. -5. Update import paths in tests and startup wiring. -6. Split reusable test harness write behavior into application testing service. -7. Route cohort marker writes through compatibility helpers. -8. Fix the `slug_by_node_id` typo in test harness read-state mapping while the - file is touched. -9. Delete top-level `core/rest_api`. - -## Acceptance Criteria - -- No top-level `core/rest_api` package remains. -- Public REST paths do not change. -- Route modules are thin translation code. -- Route modules depend on application services and views, not low-level table - models. -- Test harness DTOs remain route-local. -- Reusable test writes live in application testing/compat helpers. - -## Tests - -```bash -pytest ergon_core/tests/unit/rest_api -q -pytest ergon_core/tests/unit/architecture -q -pytest ergon-dashboard/tests -q -rg -n "core\\.rest_api|ergon_core\\.core\\.rest_api" ergon_core ergon-dashboard tests -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/07-sandbox-resource-publishing-boundary.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/07-sandbox-resource-publishing-boundary.md deleted file mode 100644 index 17c265427..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/07-sandbox-resource-publishing-boundary.md +++ /dev/null @@ -1,62 +0,0 @@ -# PR 07: Sandbox Resource Publishing Boundary - -## What - -Split sandbox filesystem/blob adapters from application resource append/dedup -semantics. - -## Why - -`core/infrastructure/sandbox/resource_publisher.py` currently knows how to -read sandbox files, write blobs, and decide how `RunResource` rows are appended -or deduplicated. The first two are infrastructure adapter work. The resource -row policy is application behavior and should be shared by jobs and future -resource flows. - -## How - -- Keep filesystem/blob details in infrastructure: - `_list_sandbox_dir()`, `_read_sandbox_file()`, `_write_blob()`, and - `_blob_path()`. -- Create `core/application/ports/resources.py` for sandbox file/blob adapter - protocols if the concrete infrastructure dependency would otherwise leak. -- Create `core/application/resources/publishing.py` with - `RunResourcePublishService`. -- Move `RunResource` append/dedup writes from infrastructure into that service. -- Update `persist_outputs` job to call the application service with injected - sandbox reader/blob writer implementations. -- Add architecture tests preventing infrastructure sandbox modules from adding - `RunResource` rows directly. - -## Plan - -1. Add characterization tests for current `SandboxResourcePublisher.sync()` and - `publish_value()` row behavior. -2. Add application service tests for resource append/dedup semantics. -3. Define narrow ports for listing/reading sandbox files and writing blobs. -4. Implement `RunResourcePublishService`. -5. Move DB writes out of infrastructure. -6. Update `application/jobs/persist_outputs.py` or the later job module to use - the service. -7. Leave infrastructure sandbox code responsible only for filesystem/blob - adapter behavior. -8. Add architecture test coverage for the boundary. - -## Acceptance Criteria - -- Sandbox infrastructure no longer appends `RunResource` rows directly. -- `RunResourcePublishService` owns resource append/dedup semantics. -- Persist-output behavior is unchanged. -- Infrastructure remains responsible for E2B/filesystem/blob operations. -- Architecture tests prevent resource persistence policy from moving back into - infrastructure. - -## Tests - -```bash -pytest ergon_core/tests/unit/resources -q -pytest ergon_core/tests/unit/runtime/test_persist_outputs.py -q -pytest ergon_core/tests/unit/architecture -q -rg -n "RunResource\\(|session\\.add\\(.*RunResource|resource_publisher" ergon_core/ergon_core/core/infrastructure -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/08-legacy-experiment-and-cohort-isolation.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/08-legacy-experiment-and-cohort-isolation.md deleted file mode 100644 index 5b1eabd97..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/08-legacy-experiment-and-cohort-isolation.md +++ /dev/null @@ -1,65 +0,0 @@ -# PR 08: Legacy Experiment And Cohort Isolation - -## What - -Route all remaining cohort and legacy experiment-record behavior through -explicit compatibility modules without deleting the frontend/backend surfaces -yet. - -## Why - -Cohorts and `BenchmarkDefinitionRecord` are deprecated, but still active in -dashboard, CLI, test harness, RL rollout, and completion/failure paths. Deleting -them before replacement would break users. This PR makes every remaining usage -obviously temporary and prepares the deletion PR. - -## How - -- Add nullable `RunRecord.experiment: str | None` for v2 run grouping. -- Create `core/application/compat/legacy_experiments.py`. -- Create `core/application/compat/cohorts.py`. -- Move legacy fallback experiment detail builders into - `legacy_experiments.py`. -- Move `ExperimentCohortService` into - `DeprecatedCohortCompatibilityService`. -- Route HTTP cohort endpoints and test harness marker writes through compat. -- Move deprecated cohort event emission orchestration into compat. -- Stop RL rollout from creating `BenchmarkDefinitionRecord`; use - `RolloutBatch` and canonical `RunRecord.definition_id`. -- Update CLI experiment tag reads to prefer `RunRecord.experiment`. - -## Plan - -1. Add tests showing canonical experiment reads use `ExperimentDefinition` - before legacy fallback. -2. Add tests around `RunRecord.experiment` grouping. -3. Add tests that RL rollout does not write `BenchmarkDefinitionRecord`. -4. Create `application/compat` modules with docstrings marking deletion owner. -5. Move legacy experiment helper logic. -6. Move cohort service logic into the deprecated compatibility service. -7. Route HTTP/test harness/dashboard cohort emission through compat. -8. Update CLI experiment listing and lookup to use `RunRecord.experiment`. -9. Add architecture/source tests limiting `BenchmarkDefinitionRecord` and - cohort table imports to compat, persistence table definitions, and tests. - -## Acceptance Criteria - -- All non-table `BenchmarkDefinitionRecord` behavior is behind - `application/compat/legacy_experiments.py`. -- All cohort behavior is behind `application/compat/cohorts.py` or - `views/compat/cohorts.py`. -- RL rollout provenance no longer creates `BenchmarkDefinitionRecord`. -- CLI grouping can use `RunRecord.experiment`. -- New runtime/application code does not depend on cohorts. - -## Tests - -```bash -pytest ergon_core/tests/unit/read_models -q -pytest ergon_core/tests/unit/rl -q -pytest ergon_core/tests/unit/rest_api -q -pytest ergon_cli/tests -q -pytest ergon_core/tests/unit/architecture -q -rg -n "BenchmarkDefinitionRecord|ExperimentCohort|ExperimentCohortStats" ergon_core/ergon_core/core ergon_cli -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/09-legacy-cohort-and-experiment-deletion.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/09-legacy-cohort-and-experiment-deletion.md deleted file mode 100644 index 6cb34631c..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/09-legacy-cohort-and-experiment-deletion.md +++ /dev/null @@ -1,62 +0,0 @@ -# PR 09: Legacy Cohort And Experiment Deletion - -## What - -Delete deprecated cohort surfaces and legacy experiment-record compatibility -after PR 08 has isolated them. - -## Why - -PR 08 makes the compatibility debt explicit. This PR removes it, including the -frontend/dashboard surfaces, generated contracts, backend routes, compatibility -services, and persistence tables. Without this deletion PR, the stack would -leave v1 grouping concepts preserved under cleaner names. - -## How - -- Delete dashboard cohort pages, hooks, server-data helpers, and API proxy - routes. -- Replace dashboard grouping with `RunRecord.experiment`. -- Delete generated cohort dashboard event contracts and REST entries. -- Delete backend cohort HTTP route. -- Delete `ExperimentCohort`, `ExperimentCohortStats`, and related status - types. -- Delete `BenchmarkDefinitionRecord` after all CLI/RL/test/dashboard callers - use v2 definitions and run grouping. -- Delete compatibility modules once no production code imports them. - -## Plan - -1. Add frontend tests or route checks for the replacement run grouping surface. -2. Add backend tests proving run grouping uses `RunRecord.experiment`. -3. Delete dashboard cohort pages and hooks. -4. Delete frontend API proxy route for cohorts. -5. Regenerate frontend/backend contracts without cohort events/routes. -6. Delete backend cohort HTTP route. -7. Delete cohort compatibility service and view modules. -8. Delete cohort tables and schema references. -9. Delete `BenchmarkDefinitionRecord` and its compatibility module once no - production references remain. -10. Remove smoke/e2e navigation through cohort pages. - -## Acceptance Criteria - -- No production code references `ExperimentCohort`, - `ExperimentCohortStats`, or `ExperimentCohortStatus`. -- No production code references `BenchmarkDefinitionRecord`. -- Dashboard has no cohort pages, hooks, generated event contracts, or REST - proxy routes. -- CLI and dashboard grouping use `RunRecord.experiment`. -- Compatibility modules from PR 08 are deleted or test-only. - -## Tests - -```bash -pytest ergon_core/tests/unit/persistence -q -pytest ergon_core/tests/unit/rest_api -q -pytest ergon_cli/tests -q -pytest ergon_core/tests/unit/architecture -q -pnpm --dir ergon-dashboard test -rg -n "BenchmarkDefinitionRecord|ExperimentCohort|CohortUpdatedEvent|cohorts" ergon_core ergon_cli ergon-dashboard/src -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/10-job-composition-modules.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/10-job-composition-modules.md deleted file mode 100644 index cd4c5c4ad..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/10-job-composition-modules.md +++ /dev/null @@ -1,69 +0,0 @@ -# PR 10: Job Composition Modules - -## What - -Merge split `application/jobs/*` and `infrastructure/inngest/handlers/*` into -job-local `core/jobs/<domain>/<job>/` modules with `contract.py`, `job.py`, and -`inngest.py`. - -## Why - -Each event job is currently spread across application functions, shared model -catch-alls, event contract files, and Inngest handler files. That hides the -unit of behavior and weakens dependency direction. A job-local module gives a -reader the wire contract, composition logic, and framework wrapper in one -folder while still keeping business services below the job layer. - -## How - -- Create `core/jobs/`. -- Move each application job and matching Inngest handler into the semantic job - package listed in PRD 08. -- Split `application/jobs/models.py` into local job `contract.py` files. -- Move job-specific event contracts out of - `application/events/task_events.py` and - `application/events/infrastructure_events.py`. -- Keep event names, function ids, retry config, and cancellation rules stable. -- Make `job.py` accept concrete infrastructure through ports or explicit - adapter parameters. -- Make `inngest.py` the only place in the job package that imports Inngest and - concrete infrastructure implementations. - -## Plan - -1. Add architecture tests for `core/jobs/**/contract.py`, - `core/jobs/**/job.py`, and `core/jobs/**/inngest.py`. -2. Add a registry test that all previous Inngest function ids and event names - are still registered. -3. Move one low-risk job first, such as `run_cleanup`, to prove the pattern. -4. Move workflow jobs: start, complete, fail. -5. Move task jobs: execute, propagate, evaluate, cleanup cancelled, cancel - orphans. -6. Move sandbox jobs: setup and cleanup. -7. Move resource persist-output job. -8. Split job models into local contracts as each job moves. -9. Update registry imports. -10. Delete `application/jobs/`, `application/jobs/models.py`, and - `infrastructure/inngest/handlers/`. - -## Acceptance Criteria - -- Each job folder contains `contract.py`, `job.py`, and `inngest.py`. -- `application/jobs/` is gone. -- `infrastructure/inngest/handlers/` is gone. -- `application/jobs/models.py` is gone. -- Event names, function ids, retries, and cancellation semantics are stable. -- `contract.py` files have no infrastructure/persistence/service imports. -- `job.py` files have no concrete infrastructure imports. -- `inngest.py` files contain no SQLModel queries or business decisions. - -## Tests - -```bash -pytest ergon_core/tests/unit/inngest -q -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/smoke -q -pytest ergon_core/tests/unit/architecture -q -rg -n "core\\.application\\.jobs|application/jobs|infrastructure/inngest/handlers|application\\.events\\.task_events|application\\.events\\.infrastructure_events" ergon_core tests -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/11-application-runtime-restructure.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/11-application-runtime-restructure.md deleted file mode 100644 index 39f4d2bc6..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/11-application-runtime-restructure.md +++ /dev/null @@ -1,78 +0,0 @@ -# PR 11: Application Runtime Restructure - -## What - -Merge `application/tasks`, `application/workflows`, and `application/graph` -into `application/runtime`, removing duplicated lifecycle, traversal, -ready-dispatch, and run identity helpers. - -## Why - -Runtime behavior is the largest remaining source of duplication after the -outer boundaries are cleaned. A reviewer should not need to decide whether task -lifecycle belongs in `tasks`, `workflows`, or `graph`. This PR creates a single -runtime owner with docstrings that explain the invariant behind each service. - -## How - -- Move graph repository/traversal into - `application/runtime/graph_repository.py` and - `application/runtime/graph_traversal.py`. -- Move propagation/lifecycle behavior into - `application/runtime/lifecycle.py`. -- Move workflow initialization/finalization and run creation/cancellation into - `application/runtime/run_lifecycle.py`. -- Move task management, execution, inspection, cleanup, and execution - repository into matching runtime modules. -- Move runtime DTOs/errors into `application/runtime/models.py` and - `application/runtime/errors.py`. -- Move resource materialization methods into - `application/runtime/resources.py`. -- Merge duplicate ready dispatch into - `application/runtime/events.py::RuntimeEventDispatcher.dispatch_task_ready`. -- Merge duplicate definition lookup into - `application/runtime/run_identity.py::definition_id_for_run`. -- Rename runtime public parameters from `node_id` to `task_id`. -- Delete slug-based dynamic task APIs after callers use - `spawn_dynamic_task(Task)`. - -## Plan - -1. Add architecture tests preventing reintroduction of - `application/tasks`, `application/workflows`, and `application/graph`. -2. Add characterization tests for dynamic task spawning, task completion - propagation, failure propagation, restart/refine, and task inspection. -3. Add tests for `task_id` naming in public/runtime service APIs. -4. Create `application/runtime/` package and move graph repository/traversal. -5. Move task execution repository and execution service. -6. Move task management/inspection/cleanup. -7. Move workflow/run lifecycle. -8. Merge propagation and ready-dispatch helpers. -9. Merge definition-id lookup helpers. -10. Delete slug-based dynamic task APIs and update callers/tests. -11. Rename runtime API parameters from `node_id` to `task_id`. -12. Add module/class/function docstrings that explain ownership and invariants. -13. Delete empty old packages. - -## Acceptance Criteria - -- `application/runtime/*` is the only owner for run graph lifecycle, task - lifecycle, propagation, execution row writes, dynamic task mutation, and - runtime resource views. -- `application/tasks`, `application/workflows`, and `application/graph` are - gone. -- No duplicate descendant traversal helper remains. -- No duplicate ready-dispatch helper remains. -- `task_id` is the public/runtime identity term. -- Slug-based dynamic task APIs are gone. -- Runtime smoke tests pass. - -## Tests - -```bash -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/smoke -q -pytest ergon_core/tests/unit/architecture -q -rg -n "application\\.(tasks|workflows|graph)|node_id|add_subtask\\(|plan_subtasks\\(|WorkflowService\\.add_task|_dispatch_task_ready|_resolve_definition_id" ergon_core/ergon_core/core tests -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/12-final-folder-state-and-architecture-gates.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/12-final-folder-state-and-architecture-gates.md deleted file mode 100644 index 0ea1dc7f7..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/implementation-plan/12-final-folder-state-and-architecture-gates.md +++ /dev/null @@ -1,79 +0,0 @@ -# PR 12: Final Folder State And Architecture Gates - -## What - -Enforce the final `ergon_core.core` package layout with architecture tests, -delete remaining shims, and update architecture/RFC documentation to match the -landed code. - -## Why - -The prior PRs do the actual cleanup. The final PR prevents quiet regression: -old package roots should not come back, shared contracts should not become a -junk drawer, persistence should not regain application semantics, and jobs -should not absorb reusable business logic. - -## How - -- Add final folder-state tests from PRD 09. -- Delete any temporary import shims left by earlier PRs. -- Update `docs/architecture/*` to describe the accepted ownership rules. -- Update this RFC to mark the implementation-plan stack as landed. -- Add source checks for deleted aliases, repositories, and compatibility types. - -## Plan - -1. Add or update tests asserting these packages do not exist: - - `core/domain` - - `core/rest_api` - - `core/application/jobs` - - `core/application/read_models` - - `core/application/graph` - - `core/application/tasks` - - `core/application/workflows` - - `core/infrastructure/inngest/handlers` -2. Add tests asserting these final homes exist: - - `core/application/runtime` - - `core/views` - - `core/jobs` - - `core/infrastructure/http` - - `core/shared/context_parts.py` -3. Add import-boundary tests: - - persistence does not import application, infrastructure, jobs, or views; - - runtime/evaluation/views do not import jobs; - - job contracts do not import services, persistence sessions, or - infrastructure; - - job wrappers do not run direct SQLModel queries; - - infrastructure does not own view builders or resource append policy. -4. Add source checks for: - - `ContextEventPayload` - - `WorkerYield` - - `TelemetryRepository` - - `CreateTaskEvaluation` - - `BenchmarkDefinitionRecord` - - `ExperimentCohort` -5. Delete remaining import shims. -6. Update architecture docs and RFC status text. - -## Acceptance Criteria - -- The final top-level shape is exactly: - `application`, `infrastructure`, `jobs`, `persistence`, `views`, `rl`, - `shared`. -- Deleted package roots cannot be reintroduced without failing tests. -- Deleted compatibility names have no production references. -- Architecture docs describe the implemented layout. -- Full unit/smoke architecture gates pass. - -## Tests - -```bash -pytest ergon_core/tests/unit/architecture -q -pytest ergon_core/tests/unit/runtime -q -pytest ergon_core/tests/unit/dashboard -q -pytest ergon_core/tests/unit/persistence -q -pytest ergon_core/tests/smoke -q -rg -n "ContextEventPayload|WorkerYield|TelemetryRepository|CreateTaskEvaluation|BenchmarkDefinitionRecord|ExperimentCohort" ergon_core/ergon_core -find ergon_core/ergon_core/core -maxdepth 2 -type d | sort -``` - diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/00-move-vs-delete-decisions.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/00-move-vs-delete-decisions.md deleted file mode 100644 index 5295031e2..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/00-move-vs-delete-decisions.md +++ /dev/null @@ -1,42 +0,0 @@ -# PRD 00: Move Versus Delete Decisions - -## Goal - -Prevent the standardization PRDs from preserving dead or duplicate schemas just -because a folder move looks tidy. - -## Decisions - -| Item | Current Location | Verdict | Reason | -| --- | --- | --- | --- | -| `TelemetryRepository` | `core/persistence/telemetry/repository.py` | Delete, inline two private helpers into `EvaluationService`. | It has one production consumer and only wraps `RunTaskEvaluation` reads/writes for that service. Moving it would create a tiny fake repository. | -| `CreateTaskEvaluation` | `core/persistence/telemetry/models.py` | Delete. | It only shuttles arguments into `TelemetryRepository.create_task_evaluation()`. Once that repository dies, explicit keyword arguments are clearer. | -| `EvaluationSummary`, `CriterionOutcomeEntry`, `EvalCriterionStatus` | `core/persistence/telemetry/evaluation_summary.py` | Keep, move to `application/evaluation/summary.py`. | These are active semantic schemas for evaluation summaries. They are not storage tables, and read views need the same validation. | -| `RunTaskEvaluation.parsed_summary()` | `core/persistence/telemetry/models.py` | Delete. | It makes persistence import evaluation semantics. Application read paths should call `EvaluationSummary.model_validate(row.summary_json)` explicitly. | -| `WorkerYield` | `core/domain/generation/context_parts.py` | Delete. | It is only an alias for `ContextPartChunk`. | -| `ContextEventPayload` | `core/persistence/context/event_payloads.py` | Delete. | It is only an alias for `ContextPartChunkLog`. | -| `ContextEventType` | `core/persistence/context/event_payloads.py` | Keep, move to `core/shared/context_parts.py`. | It is an active event type literal used by read-model and dashboard contracts. | -| `WorkerRef` | `core/infrastructure/dashboard/event_contracts.py` | Delete during PRD 06. | It exists only to support `workflow.started.task_tree`; `RunSnapshotDto` already carries worker id/slug fields without a separate worker ref schema. | -| `TaskTreeNode` | `core/infrastructure/dashboard/event_contracts.py` | Delete during PRD 06. | It duplicates run snapshot task view and forces handwritten frontend recursive Zod schemas. | -| `DashboardWorkflowStartedEvent.task_tree` | `core/infrastructure/dashboard/event_contracts.py` | Replace with `snapshot: RunSnapshotDto`. | Reuses the existing run view instead of keeping a second graph-to-tree implementation. | -| `Dashboard*Event` contracts | `core/infrastructure/dashboard/event_contracts.py` | Keep, move to `views/dashboard_events/contracts.py`. | They are active generated frontend contracts, not emitter implementation details. | -| `CohortUpdatedEvent` | `core/infrastructure/dashboard/event_contracts.py` | Keep temporarily, delete in PRD 07. | It is active only while cohort UI/contracts remain. | -| `Test*Dto` classes | `core/rest_api/test_harness.py` -> `core/infrastructure/http/routes/test_harness.py` | Keep route-local. | They are Playwright/test-harness wire shapes, not product views. | -| `TrainingSession`, `TrainingMetric`, training DTOs/routes | `persistence/telemetry/models.py`, `application/read_models/models.py`, `rest_api/runs.py` -> `infrastructure/http/routes/runs.py` | Gated deletion. | Core has read endpoints and dashboard pages, but no in-repo production writer was found. If training is retained, make the writer explicit and move DTOs to `views/training.py`; if not, delete backend routes, generated contracts, and dashboard training UI together. | -| `RunReducer`, `RunReducerFootprint`, `RunDropsManifest` | `core/persistence/imports/models.py` | Delete. | No production writer/reader remains, and `RunReducer.node_id` references a non-existent graph column. | -| `Graph status` literals/helpers | `core/persistence/graph/status_conventions.py` | Keep, move to `application/runtime/status.py`. | They are active runtime lifecycle vocabulary, not persistence schema. | -| `BenchmarkDefinitionRecord` | `persistence/telemetry/models.py` | Keep temporarily behind `application/compat/legacy_experiments.py`, then delete in PRD 07. | Active compatibility table for dashboard/test/CLI/RL paths. | -| Cohort tables/services/DTOs | `persistence/telemetry/models.py`, `application/read_models/cohorts.py` | Keep temporarily behind `application/compat/cohorts.py`, then delete in PRD 07. | Active only for deprecated cohort UI and test harness compatibility. | - -## Rule For Future PR Plans - -When a PRD says "move schema", the implementation plan must first ask: - -- Is this schema used by more than one production path? -- Is it a product/API contract, or just a local helper shape? -- Does an existing view already express the same data? -- Will moving it preserve duplicated logic that should be deleted instead? - -If the answer points to a local helper, delete or inline it. If the answer -points to an active external contract, move it only when the old owner is wrong -and no existing contract can replace it. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/01-persistence-schema-boundary.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/01-persistence-schema-boundary.md deleted file mode 100644 index 9b197d422..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/01-persistence-schema-boundary.md +++ /dev/null @@ -1,168 +0,0 @@ -# PRD 01: Persistence And Schema Boundary - -## Goal - -Make `ergon_core.core.persistence` a true storage-definition layer and remove -business/application concepts that currently live there by accident. - -## Target State - -`persistence/` owns: - -- SQLModel table definitions; -- database/session setup; -- storage-safe enums/types and ids; -- low-level row validation for JSON columns and persisted enum values; -- schema artifacts needed by migrations and final schema audits. - -`persistence/` does not own: - -- application command DTOs; -- domain repositories; -- runtime lifecycle status policy; -- evaluation summary semantics; -- compatibility-service logic; -- infrastructure writes that duplicate application services. - -Application-facing repositories live under their application domain. Persistence -table modules may expose row helpers, but command semantics live above them. - -## Required Moves - -### Evaluation Persistence - -- Delete `core/persistence/telemetry/repository.py` rather than moving it. - `TelemetryRepository` has only one production consumer, - `core/application/evaluation/service.py`, and only two methods. -- Delete `CreateTaskEvaluation` from `core/persistence/telemetry/models.py` - rather than moving it. It exists only to pass arguments from - `EvaluationService` back into the single-consumer repository. -- Add private helpers to `EvaluationService`: - `_list_task_evaluations(session, run_id)` and - `_create_task_evaluation(session, *, run_id, task_execution_id, task_id, - definition_evaluator_id, score, passed, feedback, summary_json)`. -- Keep `RunTaskEvaluation` in `core/persistence/telemetry/models.py`; it is the - storage table the evaluation service writes. -- Do not create `core/application/evaluation/repository.py` in this PRD. - -### Evaluation Summary Contract - -- Move `core/persistence/telemetry/evaluation_summary.py` to - `core/application/evaluation/summary.py`. -- Move `CriterionOutcomeEntry`, `EvaluationSummary`, and - `EvalCriterionStatus` together. -- Update these imports to the application path: - `core/application/evaluation/service.py`, - `core/application/read_models/models.py`, - `core/application/read_models/cohorts.py`, and tests. -- Delete `RunTaskEvaluation.parsed_summary()` from - `core/persistence/telemetry/models.py`; application read paths should call - `EvaluationSummary.model_validate(evaluation.summary_json)` explicitly. -- Merge the duplicated DTO mapping in - `EvaluationService.build_dashboard_evaluation_dto()` and - `read_models/run_snapshot._task_keyed_evaluations()` into - `core/application/evaluation/dto_mapping.py::evaluation_row_to_dto(...)`. -- Keep the schema in `application/evaluation`, not persistence, because it is - evaluation-domain semantics over a JSON column rather than storage shape. - -### Runtime Status Contract - -- Create `core/application/runtime/status.py`. -- Move the contents of `core/persistence/graph/status_conventions.py` into - `core/application/runtime/status.py`. -- Update imports in `application/graph`, `application/tasks`, - `application/workflows`, dashboard event contracts, dashboard emitter, smoke - fixtures, and propagation/restart tests to import from - `core.application.runtime.status`. -- Delete `core/persistence/graph/status_conventions.py`; graph table models - keep string columns and do not own runtime lifecycle vocabulary. - -### Context Event Payload Alias - -- Delete `core/persistence/context/event_payloads.py`. -- PRD 02 moves `ContextEventType` into `core/shared/context_parts.py` and - deletes `ContextEventPayload`; persistence, read-model, and dashboard code - should annotate payloads as `ContextPartChunkLog` directly. - -### Import/Reducer Tables - -- Delete `core/persistence/imports/models.py` and the - `core/persistence/imports/` package. -- Remove `RunReducer` and `RunReducerFootprint` from schema creation. -- Reason: no production writer/reader remains, and `RunReducer.node_id` - references `run_graph_nodes.id`, which is not the canonical v2 graph - identity. - -### Run Identity And Legacy Experiment Fields - -- Make `RunRecord.definition_id` the canonical FK to - `experiment_definitions.id`. -- Backfill `RunRecord.definition_id` from `RunRecord.workflow_definition_id`, - update all reads/writes to use `definition_id`, then delete - `workflow_definition_id`. -- Delete stale references to `RunRecord.experiment_id`; for CLI run filtering, - remove the broken `--experiment` join until PRD 07 adds run grouping through - `RunRecord.experiment`. -- Keep `RunRecord.model_target` active; `TaskExecutionService` still uses it as - a fallback model target. -- Keep `RunRecord.assignment_json` active as the run-level launch/provenance - metadata bag. -- Keep `RunRecord.worker_team_json`, `RunRecord.evaluator_slug`, - `RunRecord.sandbox_slug`, and `RunRecord.dependency_extras_json` in PRD 01, - but mark them compatibility/display-only in `RunRecord` field descriptions. - PRD 05 removes runtime reads of `sandbox_slug`; PRD 07 removes legacy - dashboard/CLI display dependencies. - -### Training And Rollout Tables - -- Treat `TrainingSession` and `TrainingMetric` as gated deletion candidates, - not folder-move candidates. Core has REST reads and dashboard pages for them, - but the audit found no in-repo production writer. -- If training observability is still a product surface, make the writer - explicit and move `TrainingCurvePointDto`, `TrainingSessionDto`, and - `TrainingMetricDto` to `views/training.py`. -- If training observability is stale, delete `TrainingSession`, - `TrainingMetric`, `/runs/training/*`, generated REST contract entries, and - dashboard training UI together. -- Move rollout batch status vocabulary out of persistence-local strings into a - shared rollout status enum, `core/shared/rollout_status.py`, and have both - `core/rl/rollout_types.py` and `RolloutBatch.status` use that enum. - -### Compatibility Tables - -- Keep `BenchmarkDefinitionRecord`, `ExperimentCohort`, and - `ExperimentCohortStats` only as compatibility tables until PRD 07 removes - their writers, dashboard contracts, REST routes, and frontend pages. -- Move all non-table helpers and service logic for those tables into - `core/application/compat/legacy_experiments.py` and - `core/application/compat/cohorts.py`; persistence keeps only the SQLModel - classes while the compatibility window is open. - -## Non-Goals - -- Do not restructure `application/tasks`, `application/workflows`, and - `application/graph` in this slice. -- Do not delete cohort/dashboard compatibility until the dashboard PRD has a - replacement. -- Do not split all telemetry tables purely for aesthetics. - -## Acceptance Criteria - -- New tests enforce that persistence model modules do not import concrete - infrastructure or define application command DTOs. -- No production code references `RunRecord.experiment_id`. -- No schema field references a non-existent target column. -- Evaluation persistence is reached through `EvaluationService`, with no - separate single-consumer telemetry repository. -- Status constants used by runtime services no longer live under - `persistence/graph`. -- `core/persistence/imports/` is gone. -- Training tables are either backed by an explicit writer and moved to a - training view, or deleted together with REST/frontend consumers. -- Any remaining legacy/compatibility tables are reachable only through - `core/application/compat/*`. - -## Evidence - -- [`../audits/persistence-boundary-audit.md`](../audits/persistence-boundary-audit.md) -- [`../audits/schema-concept-debt-audit.md`](../audits/schema-concept-debt-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/02-shared-context-contract.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/02-shared-context-contract.md deleted file mode 100644 index 9035f2209..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/02-shared-context-contract.md +++ /dev/null @@ -1,98 +0,0 @@ -# PRD 02: Shared Context Contract And Domain Package Deletion - -## Goal - -Delete the anemic `core/domain` package and move the context stream contract to -a narrow shared home. - -## Target State - -Context stream schemas live at: - -```text -ergon_core/core/shared/context_parts.py -``` - -They are treated as a cross-cutting contract used by: - -- public worker/API contracts; -- persistence context events; -- dashboard/schema generation; -- RL extraction; -- tests. - -`core/domain/` does not exist unless a future RFC introduces real pure-domain -logic. - -## Required Moves - -- Move `core/domain/generation/context_parts.py` to - `core/shared/context_parts.py`. -- Add `ContextEventType` to `core/shared/context_parts.py`. -- Delete `WorkerYield = ContextPartChunk` while moving the file. Worker stream - annotations should use `ContextPartChunk` directly. -- Update imports from `ergon_core.core.domain.generation.context_parts` to - `ergon_core.core.shared.context_parts`. -- Update imports from `ergon_core.core.persistence.context.event_payloads` to - `ergon_core.core.shared.context_parts`. -- Replace every `ContextEventPayload` annotation with `ContextPartChunkLog`. -- Delete `core/domain/generation/__init__.py`, `core/domain/__init__.py`, and - the `core/domain/` package after the move. The package has no source besides - `generation/context_parts.py`. -- Delete `core/persistence/context/event_payloads.py`. It only re-exports the - context payload alias and event type literal. The event type moves to shared; - the payload alias dies. -- Update architecture tests to assert `core/domain` is absent or empty. - -## Import Updates - -Production imports that move to `core.shared.context_parts`: - -- `api/worker/worker.py` -- `core/rl/extraction.py` -- `core/application/jobs/worker_execute.py` -- `core/application/context/events.py` -- `core/persistence/context/models.py` -- `core/application/read_models/models.py` -- `core/infrastructure/dashboard/event_contracts.py` -- `core/infrastructure/dashboard/emitter.py` - -Test and fixture imports that move to `core.shared.context_parts`: - -- `ergon_core/tests/unit/persistence/test_context_event_repository.py` -- `ergon_core/tests/unit/runtime/test_context_event_contracts.py` -- `ergon_core/tests/unit/architecture/test_model_field_descriptions.py` -- `ergon_core/tests/unit/runtime/test_import_boundaries.py` -- `ergon_core/tests/unit/runtime/test_worker_execute_stream_contract.py` -- `ergon_core/tests/unit/state/test_context_part_stream.py` -- `tests/fixtures/smoke_components/smoke_base/leaf_base.py` -- `tests/fixtures/smoke_components/smoke_base/recursive.py` -- `tests/fixtures/smoke_components/smoke_base/worker_base.py` - -Architecture tests to update: - -- `ergon_core/tests/unit/architecture/test_public_api_boundaries.py` should - expect `core/shared/context_parts.py`. -- `ergon_core/tests/unit/architecture/test_core_schema_sources.py` should stop - requiring `domain` as a layout root and should assert `core/domain` is absent. - -## Non-Goals - -- Do not change the serialized context event payload shape. -- Do not redesign worker context streaming. -- Do not move unrelated shared schemas into `core/shared`. - -## Acceptance Criteria - -- No production import references `core.domain.generation.context_parts`. -- No production or test code references `WorkerYield` or `ContextEventPayload`. -- `RunContextEvent.parsed_payload()` still validates persisted context payloads. -- Dashboard context event contracts still generate successfully. -- RL extraction tests still consume context events. -- Architecture tests prevent reintroducing a nominal `domain/` package for - shared schemas. - -## Evidence - -- [`../audits/current-structure.md`](../audits/current-structure.md) -- [`../audits/persistence-boundary-audit.md`](../audits/persistence-boundary-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/03-infrastructure-adapter-boundary.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/03-infrastructure-adapter-boundary.md deleted file mode 100644 index eeb534fbb..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/03-infrastructure-adapter-boundary.md +++ /dev/null @@ -1,138 +0,0 @@ -# PRD 03: Infrastructure Adapter Boundary - -## Goal - -Make infrastructure a set of external/framework adapters. Infrastructure should -implement application-declared ports and should not reimplement application -business operations. - -## Target State - -Dependency direction is: - -```text -inbound adapters -> application use cases / views -application use cases -> application ports -> infrastructure implementations -``` - -Infrastructure owns: - -- Inngest function registration and transport details; -- dashboard event transport; -- E2B sandbox/file/command adapters; -- blob-store implementations; -- tracing sinks and id helpers; -- process startup wiring. - -Infrastructure does not own: - -- graph view assembly; -- cohort recomputation; -- resource append/dedup policy; -- runtime lifecycle policy; -- application DTO construction when a view exists. - -## Required Moves - -### Dashboard Contracts And Publishing - -- Move `core/infrastructure/dashboard/event_contracts.py` to - `core/views/dashboard_events/contracts.py`. -- Move every `Dashboard*Event` contract with that file. -- Do not move `WorkerRef` and `TaskTreeNode` as durable schemas. PRD 06 - replaces `workflow.started.task_tree` with the existing `RunSnapshotDto` - view, which deletes both helper schemas and the frontend's manual - recursive Zod mirror. -- Create `core/application/ports/dashboard.py` with a - `DashboardEventPublisher` protocol that publishes already-built dashboard - event contracts. -- Keep `core/infrastructure/dashboard/emitter.py`, but narrow it to a concrete - implementation of that protocol. It should expose one transport method such - as `publish(event: InngestEventContract)` and send - `inngest.Event(name=event.name, data=event.model_dump(mode="json"))`. -- Remove event-construction methods from `DashboardEmitter`. Application jobs - and services should call view builders, then pass the completed event - to the dashboard publisher port. -- Replace direct application imports of concrete `DashboardEmitter` in - `application/tasks/management.py`, `application/communication/service.py`, - `application/tasks/execution.py`, `application/jobs/worker_execute.py`, - `application/jobs/evaluate_task_run.py`, - `application/jobs/cleanup_cancelled_task.py`, - `application/jobs/start_workflow.py`, and - `application/jobs/complete_workflow.py` with the dashboard publisher port or - composition-provided helper. - -### Dashboard View Helpers - -- Delete `_WORKER_SLUG_NS`, `_worker_ref_for_slug()`, and - `_build_task_tree_for_run()` from `application/jobs/start_workflow.py` after - PRD 06 changes workflow-started events to carry `RunSnapshotDto`. -- Replace the inline workflow-started tree assembly in `start_workflow.py` - with a call to the run snapshot view. -- Move context-event payload assembly from `DashboardEmitter.on_context_event()` - into `views/dashboard_events/context_events.py`. -- Move graph mutation row-to-event assembly from `DashboardEmitter` into - `views/dashboard_events/graph_mutations.py`. -- Merge the graph mutation row-to-DTO logic used by - `DashboardEmitter.graph_mutation()` and - `RunReadService.list_mutations()` so both call the same view mapper. - -### Deprecated Cohort Emission - -- Move module-level `emit_cohort_updated_for_run()` from - `core/infrastructure/dashboard/emitter.py` to - `core/application/compat/cohorts.py` as - `emit_deprecated_cohort_updated_for_run(...)`. -- That compatibility function should recompute/build `CohortUpdatedEvent | - None` and publish through `DashboardEventPublisher`. -- Delete the infrastructure import of `experiment_cohort_service`. - -### Sandbox Resource Publishing - -- Split `core/infrastructure/sandbox/resource_publisher.py`. -- Keep filesystem/blob adapter methods in infrastructure: - `_list_sandbox_dir()`, `_read_sandbox_file()`, `_write_blob()`, and - `_blob_path()`. -- Move DB/resource semantics from `SandboxResourcePublisher.sync()` and - `SandboxResourcePublisher.publish_value()` into - `core/application/resources/publishing.py::RunResourcePublishService`. -- Move the `RunResource` append/dedup writes currently inside infrastructure - into that application service. -- Update `application/jobs/persist_outputs.py` to call - `RunResourcePublishService`, injecting sandbox reader/blob writer ports from - infrastructure composition. - -### Inngest And Tracing - -- Delete the stale TODO in - `core/infrastructure/inngest/handlers/cancel_orphan_subtasks.py`; the handler - already delegates to application jobs. -- Clarify the docstring in - `core/infrastructure/inngest/handlers/sandbox_cleanup.py`: the handler - registers terminal-event adapters, while - `application/jobs/sandbox_cleanup.py` owns cleanup sequencing. -- Keep `core/infrastructure/tracing/contexts.py` in infrastructure. Add an - architecture test that it imports only tracing id/type helpers and standard - UUID utilities; it must not import application, persistence, sessions, - settings, or clocks. - -## Non-Goals - -- Do not remove dashboard or Inngest. -- Do not change event names or frontend contracts without the dashboard - view PRD. -- Do not redesign sandbox public APIs in this slice. - -## Acceptance Criteria - -- Application services no longer import concrete `DashboardEmitter` except via - explicit composition/bootstrap paths. -- Dashboard infrastructure does not import cohort services. -- Sandbox infrastructure does not append `RunResource` rows directly after the - resource service split. -- Inngest handlers remain adapters around application jobs/use cases. -- Architecture tests encode the allowed import direction. - -## Evidence - -- [`../audits/infrastructure-application-boundary-audit.md`](../audits/infrastructure-application-boundary-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/04-rest-inbound-adapter-boundary.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/04-rest-inbound-adapter-boundary.md deleted file mode 100644 index a6f205a02..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/04-rest-inbound-adapter-boundary.md +++ /dev/null @@ -1,96 +0,0 @@ -# PRD 04: REST As Thin Inbound Adapter - -## Goal - -Move REST routes under `infrastructure/http/` and keep them as thin inbound -adapters that call application services and views. REST should not become -application logic. - -## Target State - -`infrastructure/http/` owns: - -- HTTP route registration; -- request parsing and validation; -- mapping application errors to HTTP errors; -- response model selection; -- FastAPI app lifespan and HTTP composition wiring. - -`infrastructure/http/` does not own: - -- runtime lifecycle policy; -- persistence queries when an application view/service exists; -- cohort compatibility behavior; -- dashboard-specific contract assembly; -- business DTO transformations that should be shared by CLI/dashboard/API. - -The current top-level `core/rest_api/` package is temporary. The final home is -`core/infrastructure/http/`, not `application`, because route modules are -framework adapters rather than business use cases. - -## Required Moves - -### Route Decisions - -| Route file | Decision | -| --- | --- | -| `core/rest_api/runs.py` -> `core/infrastructure/http/routes/runs.py` | Move. It already calls `RunReadService`; update imports when PRD 06 moves read models to views. | -| `core/rest_api/experiments.py` -> `core/infrastructure/http/routes/experiments.py` | Move. It already calls `ExperimentReadService` and `run_experiment`; update imports when view/service modules move. | -| `core/rest_api/cohorts.py` -> `core/infrastructure/http/routes/cohorts.py` | Move only as deprecated compatibility until PRD 07. Route it through `core/application/compat/cohorts.py`, then delete it with the frontend cohort cleanup. | -| `core/rest_api/rollouts.py` -> `core/infrastructure/http/routes/rollouts.py` | Move. It is a transport wrapper around `RolloutService`/`VLLMManager`, not route-owned query logic. | -| `core/rest_api/app.py` -> `core/infrastructure/http/app.py` | Move as the HTTP composition/bootstrap module. Add a docstring naming its persistence/infrastructure imports as bootstrap exceptions. | -| `core/rest_api/test_harness.py` -> `core/infrastructure/http/routes/test_harness.py` | Move route-local DTOs. Split only the reusable write/query behavior; do not promote Playwright-only DTOs into application views. | - -### Test Harness Split - -- Keep `TestGraphNodeDto`, `TestEvaluationDto`, `TestGraphMutationDto`, - `TestExecutionDto`, `TestRunStateDto`, `TestCohortRunDto`, and - `TestCohortIdDto` in `core/infrastructure/http/routes/test_harness.py`. - They are used only as Playwright/test-harness wire shapes; moving them to - application would create a fake product view. -- Move the read/query construction behind `read_run_state`, - `read_cohort_id`, and `read_cohort_runs` only if another test harness or - dashboard utility reuses it. Otherwise keep it route-local and document the - whole module as a test-only adapter exception. -- Move generic test write behavior behind `seed_run` and `reset_test_rows` into - `core/application/testing/test_harness_service.py`. -- Keep `submit_cohort` as a REST test-harness entry point, but delegate its - legacy cohort and `BenchmarkDefinitionRecord` marker writes to - `core/application/compat/cohorts.py`. -- Move legacy cohort marker writes from the test harness into - `core/application/compat/cohorts.py::write_legacy_cohort_marker`. -- Fix the existing `slug_by_node_id` typo while moving `read_run_state`: the - local map is named `slug_by_task_id`, and graph mutation/evaluation/execution - mapping should use task ids consistently. -- Leave `core/infrastructure/http/routes/test_harness.py` as HTTP translation - over those application services and views. - -### Route Boundary Tests - -- Add an architecture test that route modules may import application services, - application views, FastAPI, and shared settings, but not SQLModel table - classes directly. -- Exempt `core/infrastructure/http/app.py` from that import rule and document - it as the REST bootstrap/composition root. -- Add an architecture test that top-level `core/rest_api` does not exist after - the move. - -## Non-Goals - -- Do not change public REST paths in this slice. -- Do not remove cohort endpoints until the cohort deprecation PRD lands. -- Do not merge REST routes into application packages. - -## Acceptance Criteria - -- REST route functions are thin enough to read as translation code. -- Route modules depend on application services and views, not low-level - table models. -- No top-level `core/rest_api` package remains. -- HTTP tests continue to pass without behavior changes. -- Architecture tests distinguish REST inbound adapters from application logic. - -## Evidence - -- [`../audits/infrastructure-application-boundary-audit.md`](../audits/infrastructure-application-boundary-audit.md) -- [`../audits/schema-concept-debt-audit.md`](../audits/schema-concept-debt-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/05-application-runtime-restructure.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/05-application-runtime-restructure.md deleted file mode 100644 index 6b82f8583..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/05-application-runtime-restructure.md +++ /dev/null @@ -1,124 +0,0 @@ -# PRD 05: Application Runtime Restructure - -## Goal - -After outer boundaries are cleaned up, consolidate duplicated runtime lifecycle -logic across `application/tasks`, `application/workflows`, and -`application/graph`. - -## Target State - -Runtime application code has one obvious home for: - -- run graph repository/traversal; -- task lifecycle mutations; -- workflow initialization/finalization; -- propagation and invalidation; -- task inspection; -- ready-event dispatch; -- run definition identity lookup; -- resource materialization from a task/run perspective. - -Merge `application/tasks`, `application/workflows`, and `application/graph` -into: - -```text -application/runtime/ - graph_repository.py - graph_traversal.py - lifecycle.py - task_management.py - task_execution.py - task_inspection.py - propagation.py - run_lifecycle.py - resources.py - models.py - errors.py -``` - -Lifecycle ownership must not remain split across three competing domains. - -## Required Moves - -### File Moves - -| Source | Target | Notes | -| --- | --- | --- | -| `application/graph/repository.py::WorkflowGraphRepository` | `application/runtime/graph_repository.py::RuntimeGraphRepository` | Keep persistence-facing graph row operations here. | -| `application/graph/traversal.py` | `application/runtime/graph_traversal.py` | Merge `descendants()` and `descendant_ids()` here. | -| `WorkflowGraphRepository.descendants_by_parent()` | `application/runtime/graph_traversal.py` | Delete the repository helper after `TaskInspectionService` uses the traversal module. | -| `application/graph/propagation.py` | `application/runtime/lifecycle.py::RuntimeLifecycleService` | Move `get_initial_ready_tasks()`, `on_task_completed_or_failed()`, `_block_successors_bfs()`, and terminal checks here. | -| `application/workflows/service.py::initialize()` and `finalize()` | `application/runtime/run_lifecycle.py` | Own workflow initialization/finalization and run terminal state. | -| `application/workflows/runs.py` | `application/runtime/run_lifecycle.py` | Move `create_run()`, `cancel_run()`, and `latest_run_for_definition()`. | -| `application/tasks/management.py::TaskManagementService` | `application/runtime/task_management.py` | Keep object-bound dynamic task mutation here. | -| `application/tasks/execution.py::TaskExecutionService` | `application/runtime/task_execution.py` | Move `_emit_task_status()` with the service. | -| `application/tasks/inspection.py::TaskInspectionService` | `application/runtime/task_inspection.py` | Use `runtime/graph_traversal.py` for descendants. | -| `application/tasks/repository.py::TaskExecutionRepository` | `application/runtime/task_execution_repository.py` | Keep execution row writes here. | -| `application/tasks/cleanup.py` | `application/runtime/task_cleanup.py` | Keep cleanup policy with runtime task lifecycle. | -| `application/workflows/orchestration.py` | `application/runtime/models.py` | Move runtime command/result DTOs here. | -| `application/tasks/models.py`, runtime-only DTOs from `application/graph/models.py`, runtime-only DTOs from `application/workflows/models.py` | `application/runtime/models.py` | Keep public view/event DTOs out of runtime. | -| `application/graph/errors.py`, `application/tasks/errors.py`, `application/workflows/errors.py` | `application/runtime/errors.py` | Merge runtime errors into one module. | -| `application/workflows/service.py` resource methods | `application/runtime/resources.py::RuntimeResourceService` | Move `materialize_resource()`, `list_resources()`, `read_resource_bytes()`, and workspace helpers; reuse `application/resources/RunResourceRepository`. | - -`GraphMutationRecordDto` is not runtime-only. It is used by REST mutation reads -and dashboard event contracts, so keep it in a graph/view contract module -instead of burying it in `application/runtime/models.py`. - -### Deletions And Merges - -- Delete slug-based dynamic task APIs after callers/tests use - `spawn_dynamic_task(Task)`: `TaskManagementService.add_subtask()`, - `TaskManagementService.plan_subtasks()`, and `WorkflowService.add_task()`. -- Merge duplicate ready dispatch helpers into - `application/runtime/events.py::RuntimeEventDispatcher.dispatch_task_ready`. - Delete `_dispatch_task_ready()` from `TaskManagementService` and - `WorkflowService`. -- Merge duplicate definition lookup helpers into - `application/runtime/run_identity.py::definition_id_for_run`. Delete - `_resolve_definition_id()` from `TaskManagementService` and - `WorkflowService`. -- Move runtime status vocabulary to `application/runtime/status.py` as created - by PRD 01. - -### Identity Cleanup - -- Rename runtime parameters from `node_id` to `task_id` in - `RuntimeGraphRepository.update_node_status()`, `update_node_field()`, - `get_node()`, `get_incoming_edges()`, `get_outgoing_edges()`, - `_get_node_row()`, and `_require_node_exists()`. -- Delete `GraphNodeLookup.node_id()` after propagation callers pass - `task_id` directly. -- Remove the runtime read of `RunRecord.sandbox_slug` in - `application/jobs/execute_task.py` only after sandbox identity is present in - the object-bound `Task.sandbox` snapshot for every launch path. - -### Documentation In Code - -- Add module docstrings to each `application/runtime/*` module explaining the - operation it owns and why the behavior lives there. -- Add class/function docstrings to `RuntimeLifecycleService`, - `RuntimeGraphRepository`, `RuntimeResourceService`, and - `TaskManagementService.spawn_dynamic_task()` that describe the invariant they - enforce, not just the mechanics. - -## Non-Goals - -- Do not start this before persistence, shared-contract, infrastructure, and - REST boundaries are under control. -- Do not change external worker/public API behavior as part of file moves. -- Do not delete dashboard/cohort compatibility here unless already migrated. - -## Acceptance Criteria - -- A reviewer can identify the owner for runtime lifecycle changes without - choosing between `tasks`, `workflows`, and `graph`. -- No duplicate descendant traversal or ready-dispatch helper remains. -- `task_id` is the public/runtime identity term. -- Runtime tests and walkthrough smoke tests pass. -- Architecture tests prevent reintroducing parallel runtime lifecycle paths. - -## Evidence - -- [`../audits/runtime-domain-merge-audit.md`](../audits/runtime-domain-merge-audit.md) -- [`../audits/current-structure.md`](../audits/current-structure.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/06-views-dashboard-contracts.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/06-views-dashboard-contracts.md deleted file mode 100644 index 840eb6aab..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/06-views-dashboard-contracts.md +++ /dev/null @@ -1,166 +0,0 @@ -# PRD 06: Views And Dashboard Contracts - -## Goal - -Replace the loose `application/read_models` concept with a stricter view -boundary and move dashboard-facing contract assembly out of infrastructure and -jobs. - -## Target State - -View modules: - -- read persistence rows; -- assemble API/dashboard DTOs; -- do not mutate runtime state; -- do not start jobs; -- do not own lifecycle decisions. - -Dashboard event contracts and workflow tree assembly live in `views/`, not in -dashboard infrastructure or job modules. - -Target structure: - -```text -views/ - runs/ - models.py - service.py - snapshot.py - graph_tasks.py - experiments/ - models.py - service.py - resources.py - training.py - compat/ - cohorts.py - dashboard_events/ - contracts.py - graph_mutations.py - context_events.py - cohorts.py -``` - -## Required Moves - -### View Package Split - -- Split `application/read_models/models.py`; do not move the mixed module - wholesale. -- Move run snapshot DTOs (`RunTaskDto`, `RunResourceDto`, - `RunExecutionAttemptDto`, `RunTaskEvaluationDto`, - `RunEvaluationCriterionDto`, `RunSandboxDto`, `RunContextEventDto`, - `RunSnapshotDto`) to `views/runs/models.py`. -- Move `application/read_models/runs.py` to - `views/runs/service.py`. -- Move `application/read_models/run_snapshot.py` to - `views/runs/snapshot.py`. -- Move experiment DTOs and read service from `application/read_models/experiments.py` - to `views/experiments/models.py` and - `views/experiments/service.py`. -- Move `application/read_models/resources.py` to - `views/resources.py`. -- Move `application/read_models/errors.py` to `views/errors.py`. -- Do not move `application/read_models/cohorts.py` into normal views; - split it into `application/compat/cohorts.py` for writes/recompute and - `views/compat/cohorts.py` for temporary read-only - `CohortSummaryDto` / `CohortDetailDto` assembly until frontend deletion. -- If PRD 01 retains training observability, move `TrainingCurvePointDto`, - `TrainingSessionDto`, and `TrainingMetricDto` to - `views/training.py`. - -### Dashboard Contract Source - -- Move `core/infrastructure/dashboard/event_contracts.py` to - `views/dashboard_events/contracts.py` as specified in PRD - 03. -- Update `scripts/export_contract_schemas.py::CONTRACTS_MODULE` to - `ergon_core.core.views.dashboard_events.contracts`. -- Update tests importing the old contract module: - `ergon_core/tests/unit/dashboard/test_event_contract_types.py`, - `ergon_core/tests/unit/architecture/test_model_field_descriptions.py`, - `ergon_core/tests/unit/runtime/test_context_event_contracts.py`, and - `ergon_core/tests/unit/runtime/test_graph_mutation_contracts.py`. -- Update architecture tests that currently require the old infrastructure event - contract location: - `ergon_core/tests/unit/architecture/test_public_api_boundaries.py` and - `ergon_core/tests/unit/architecture/test_core_schema_sources.py`. - -### Workflow Tree And Snapshot Overlap - -- Delete `WorkerRef`, `TaskTreeNode`, `_WORKER_SLUG_NS`, - `_worker_ref_for_slug()`, and `_build_task_tree_for_run()`. -- Change `DashboardWorkflowStartedEvent` to carry `snapshot: RunSnapshotDto` - instead of `task_tree: TaskTreeNode`. -- Update the frontend workflow-started handler to initialize run state from - `RunSnapshotDto.tasks` and related maps instead of parsing a recursive - `TaskTreeNode`. -- Delete the manual frontend `WorkerRefSchema`, `TaskTreeNodeSchema`, and - `TaskTreeNode` type from `ergon-dashboard/src/lib/contracts/events.ts`. -- Extract the graph/task helper overlap between the deleted - `_build_task_tree_for_run()` and `run_snapshot._build_task_map()` into - `views/runs/graph_tasks.py` before deleting the old tree - builder. -- `graph_tasks.py` should own helpers such as `build_children_by_parent()`, - `build_depends_on_by_target()`, and `worker_by_binding()`. -- `start_workflow.py` should call `build_workflow_started_event(...)` from the - view layer and pass the completed event to the dashboard publisher. - -### Graph Mutations - -- Move graph mutation row-to-event mapping from `DashboardEmitter` to - `views/dashboard_events/graph_mutations.py`. -- Add `graph_mutation_record_from_row(row: RunGraphMutation) -> - GraphMutationRecordDto`. -- Add `dashboard_graph_mutation_event_from_row(row: RunGraphMutation) -> - DashboardGraphMutationEvent`. -- Update `RunReadService.list_mutations()` to use - `graph_mutation_record_from_row()` so snapshot/API reads and live dashboard - events share one mapper. - -### Context Events - -- Move context event mapping from `DashboardEmitter.on_context_event()` to - `views/dashboard_events/context_events.py`. -- Add `context_event_to_dashboard_event(...)` and reuse the same task/execution - mapping assumptions as `run_snapshot._context_events_by_task()`. - -### Cohort Events - -- Move cohort dashboard event assembly to - `views/dashboard_events/cohorts.py::cohort_updated_event_from_summary`. -- Recompute and emission orchestration remain in `application/compat/cohorts.py`; - the view module only turns read-only summary DTOs into - `CohortUpdatedEvent`. - -### Dashboard Transport - -- Simplify `DashboardEmitter` to transport only, with a method shaped like - `emit(event: InngestEventContract) -> None`. -- Delete event-specific DTO construction from infrastructure after the - view builders above exist. - -## Non-Goals - -- Do not redesign the frontend event schema unless drift is discovered. -- Do not delete cohorts in this PRD; isolate them through the deprecated - compatibility PRD. -- Do not turn views into command services. - -## Acceptance Criteria - -- Dashboard event schemas still generate and drift checks pass. -- `start_workflow` no longer builds dashboard tree DTOs inline. -- Dashboard infrastructure no longer imports view DTO dependencies beyond the - event payloads it sends. -- View modules are read-only by architecture test. -- `rg -n "get_session\\(|session\\.add\\(|session\\.commit\\(" - ergon_core/ergon_core/core/views` returns no writes outside - documented compatibility helpers. -- Existing run snapshot/dashboard tests pass. - -## Evidence - -- [`../audits/infrastructure-application-boundary-audit.md`](../audits/infrastructure-application-boundary-audit.md) -- [`../audits/schema-concept-debt-audit.md`](../audits/schema-concept-debt-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/07-deprecated-cohorts-legacy-experiments.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/07-deprecated-cohorts-legacy-experiments.md deleted file mode 100644 index 7cca009c0..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/07-deprecated-cohorts-legacy-experiments.md +++ /dev/null @@ -1,124 +0,0 @@ -# PRD 07: Deprecated Cohorts And Legacy Experiment Records - -## Goal - -Isolate and then remove v1/v2-bridge grouping concepts: cohorts and -`BenchmarkDefinitionRecord`. - -## Target State - -The v2 product concept is: - -- an immutable definition describes authored work; -- a run executes one definition instance; -- an experiment is a collection/tag/composition of runs, not a legacy - definition-shaped row; -- dashboard and CLI grouping use `RunRecord.experiment: str | None` as the v2 - grouping tag. - -Cohorts and `BenchmarkDefinitionRecord` are not long-lived core domains. - -## Required Moves - -### V2 Run Grouping - -- Add nullable `RunRecord.experiment: str | None`. -- Use this string as the v2 run grouping tag for Pythonic composition, CLI - grouping, and dashboard run grouping. -- Do not create `application/cohorts` as a permanent domain and do not create a - replacement cohort table. - -### Legacy Experiment Record Isolation - -- Keep `BenchmarkDefinitionRecord` temporarily as a compatibility table only. -- Move all non-table legacy experiment helpers to - `application/compat/legacy_experiments.py`. -- Move the legacy fallback detail builder from - `application/read_models/experiments.py::_legacy_benchmark_definition_record_detail` - into `application/compat/legacy_experiments.py`. -- Update canonical experiment read paths to read `ExperimentDefinition` first - and call the compatibility module only for legacy rows. -- Delete `BenchmarkDefinitionRecord` after RL rollout, test harness, CLI, - dashboard, `application/jobs/complete_workflow.py`, and - `application/jobs/fail_workflow.py` no longer read or write it. - -### Cohort Compatibility Isolation - -- Keep `ExperimentCohort`, `ExperimentCohortStats`, and - `ExperimentCohortStatus` temporarily as compatibility tables only. -- Move `ExperimentCohortService` from `application/read_models/cohorts.py` to - `application/compat/cohorts.py::DeprecatedCohortCompatibilityService`. -- Leave read-only DTO assembly in `views/dashboard_events/cohorts.py` - and `views` after PRD 06. -- Route `core/infrastructure/http/routes/cohorts.py` through - `DeprecatedCohortCompatibilityService` and mark the route module deprecated. -- Move test harness cohort marker writes to - `application/compat/cohorts.py::write_legacy_cohort_marker`. -- Move module-level `emit_cohort_updated_for_run()` from - `core/infrastructure/dashboard/emitter.py` to - `application/compat/cohorts.py::emit_deprecated_cohort_updated_for_run`. - -### RL Rollout Provenance - -- Stop `core/rl/rollout_service.py` from creating `BenchmarkDefinitionRecord`. -- Use `RolloutBatch` as rollout provenance. -- Set `RunRecord.definition_id = request.definition_id` for rollout runs. -- After PRD 01 removes `workflow_definition_id`, do not write it. -- Store rollout policy/model metadata on `RolloutBatch` or - `RunRecord.assignment_json`, not on `BenchmarkDefinitionRecord`. - -### CLI Replacement - -- Delete `BenchmarkDefinitionRecord.experiment` reads from - `ergon_cli/commands/experiment.py`. -- Reimplement experiment tag listing and lookup against - `RunRecord.experiment`. -- Delete the broken `RunRecord.experiment_id` join in - `ergon_cli/commands/run.py`. -- Reimplement `ergon run --experiment <tag>` as - `RunRecord.experiment == <tag>` after the new field exists. - -### Dashboard And Frontend Deletion Gate - -- Delete cohort UI/proxy/generated-contract dependencies before deleting - backend cohort routes/tables: - `ergon-dashboard/src/app/cohorts/page.tsx`, - `ergon-dashboard/src/app/cohorts/[cohortId]/page.tsx`, - `ergon-dashboard/src/app/cohorts/[cohortId]/runs/[runId]/page.tsx`, - `ergon-dashboard/src/app/api/cohorts/route.ts`, - `ergon-dashboard/src/hooks/useCohorts.ts`, - `ergon-dashboard/src/lib/server-data/cohorts.ts`, - generated `CohortUpdatedEvent`, generated REST contract entries for cohort - endpoints, and smoke/e2e routes that navigate through cohorts. -- Replace dashboard cohort grouping with run grouping over - `RunRecord.experiment`. -- Delete backend cohort REST routes, compatibility service, tables, and - dashboard cohort event contracts after the frontend no longer imports them. - -### Training Is Not Cohort Compatibility - -- Do not fold `TrainingSession` and `TrainingMetric` into this PRD's - cohort/legacy experiment deletion. PRD 01 owns the separate decision: either - keep training observability with an explicit writer and a training view, - or delete backend and frontend training surfaces together. - -## Non-Goals - -- Do not delete tables before dashboard and CLI replacements land. -- Do not invent a broad experiment-management product surface in this cleanup. -- Do not make `application/cohorts` a permanent domain. - -## Acceptance Criteria - -- All cohort and legacy experiment record usages are either deleted or routed - through an explicitly named deprecated compatibility module. -- No new runtime service depends on cohort membership. -- RL rollout provenance no longer requires `BenchmarkDefinitionRecord`. -- Dashboard no longer imports/generated-types for cohort contracts before - tables/routes are deleted. -- CLI experiment grouping uses `RunRecord.experiment`. - -## Evidence - -- [`../audits/schema-concept-debt-audit.md`](../audits/schema-concept-debt-audit.md) -- [`../audits/persistence-boundary-audit.md`](../audits/persistence-boundary-audit.md) diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/08-job-composition-modules.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/08-job-composition-modules.md deleted file mode 100644 index fd7d0b583..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/08-job-composition-modules.md +++ /dev/null @@ -1,291 +0,0 @@ -# PRD 08: Job Composition Modules - -## Goal - -Merge the split `application/jobs/*` and `infrastructure/inngest/handlers/*` -shape into job-local composition modules with better locality of behavior. - -One job/event should be understandable from one folder. That folder may contain -both the framework adapter and the application composition function, but the -internal file boundaries still enforce dependency direction. - -## Problem - -The current event path is spread across several unrelated-looking places: - -```text -application/jobs/start_workflow.py -application/jobs/models.py -application/events/task_events.py -infrastructure/inngest/handlers/start_workflow.py -``` - -That split hides the real unit of behavior. `start_workflow` is not just an -application function and not just an Inngest handler; it is a runtime job that: - -- receives a stable event payload; -- composes application services and views; -- uses infrastructure implementations through ports; -- exposes an Inngest registration wrapper. - -Today a reader has to chase the event contract, result DTO, job logic, and -handler registration across multiple packages. The result is low locality and -weak ownership for event-specific models. - -## Target State - -Create a composition boundary for runtime jobs: - -```text -core/jobs/ -``` - -`core/jobs` is neither pure application nor pure infrastructure. It is the edge -where an event-specific use case composes application logic with injected -infrastructure implementations. - -Each job lives in a semantic package: - -```text -core/jobs/ - workflow/ - start/ - contract.py - job.py - inngest.py - complete/ - contract.py - job.py - inngest.py - fail/ - contract.py - job.py - inngest.py - task/ - execute/ - contract.py - job.py - inngest.py - propagate/ - contract.py - job.py - inngest.py - evaluate/ - contract.py - job.py - inngest.py - cleanup_cancelled/ - contract.py - job.py - inngest.py - cancel_orphans/ - contract.py - job.py - inngest.py - sandbox/ - setup/ - contract.py - job.py - inngest.py - cleanup/ - contract.py - job.py - inngest.py - resources/ - persist_outputs/ - contract.py - job.py - inngest.py - run/ - cleanup/ - contract.py - job.py - inngest.py -``` - -## Module Roles - -### `contract.py` - -Owns the event wire contract and local job result DTOs. - -Allowed imports: - -- `application/events/base.py` for `InngestEventContract`; -- `shared/*`; -- Pydantic/typing/stdlib. - -Forbidden imports: - -- concrete infrastructure; -- SQLModel sessions/table queries; -- runtime services/repositories; -- dashboard emitters; -- sandbox managers. - -This file replaces both scattered event definitions and the shared -`application/jobs/models.py` catch-all. - -### `job.py` - -Owns the event-specific composition logic. - -Allowed imports: - -- application services, runtime services, views, and ports; -- persistence row models when the current application pattern requires direct - session reads/writes; -- shared types; -- this job's `contract.py`. - -Forbidden imports: - -- concrete infrastructure implementations such as `DashboardEmitter`, E2B - managers, Inngest clients, blob writers, or concrete tracing exporters. - -`job.py` may accept infrastructure behavior through application ports: - -```python -async def run_start_workflow_job( - event: WorkflowStartedEvent, - *, - dashboard: DashboardEventPublisher, -) -> WorkflowStartResult: - ... -``` - -### `inngest.py` - -Owns framework registration and adapter wiring. - -Allowed imports: - -- Inngest framework/client helpers; -- this job's `contract.py`; -- this job's `job.py`; -- concrete infrastructure implementations needed to satisfy ports. - -Forbidden behavior: - -- direct SQL queries; -- graph/runtime lifecycle decisions; -- dashboard DTO construction; -- sandbox/blob/resource persistence policy. - -`inngest.py` should read as a thin adapter: - -```python -@inngest_client.create_function(...) -async def start_workflow_fn(ctx: inngest.Context) -> WorkflowStartResult: - return await run_start_workflow_job( - WorkflowStartedEvent.model_validate(ctx.event.data), - dashboard=get_dashboard_publisher(), - ) -``` - -## Required Moves - -### Replace The Split Job/Handler Shape - -- Move `application/jobs/start_workflow.py` and - `infrastructure/inngest/handlers/start_workflow.py` into - `core/jobs/workflow/start/job.py` and `core/jobs/workflow/start/inngest.py`. -- Move `application/jobs/complete_workflow.py` and - `infrastructure/inngest/handlers/complete_workflow.py` into - `core/jobs/workflow/complete/`. -- Move `application/jobs/fail_workflow.py` and - `infrastructure/inngest/handlers/fail_workflow.py` into - `core/jobs/workflow/fail/`. -- Move `application/jobs/execute_task.py` and - `infrastructure/inngest/handlers/execute_task.py` into - `core/jobs/task/execute/`. -- Move `application/jobs/propagate_execution.py` and - `infrastructure/inngest/handlers/propagate_execution.py` into - `core/jobs/task/propagate/`. -- Move `application/jobs/evaluate_task_run.py` and - `infrastructure/inngest/handlers/evaluate_task_run.py` into - `core/jobs/task/evaluate/`. -- Move `application/jobs/cleanup_cancelled_task.py` and - `infrastructure/inngest/handlers/cleanup_cancelled_task.py` into - `core/jobs/task/cleanup_cancelled/`. -- Move `application/jobs/cancel_orphan_subtasks.py` and - `infrastructure/inngest/handlers/cancel_orphan_subtasks.py` into - `core/jobs/task/cancel_orphans/`. -- Move `application/jobs/sandbox_setup.py` and - `infrastructure/inngest/handlers/sandbox_setup.py` into - `core/jobs/sandbox/setup/`. -- Move `application/jobs/sandbox_cleanup.py` and - `infrastructure/inngest/handlers/sandbox_cleanup.py` into - `core/jobs/sandbox/cleanup/`. -- Move `application/jobs/persist_outputs.py` and - `infrastructure/inngest/handlers/persist_outputs.py` into - `core/jobs/resources/persist_outputs/`. -- Move `application/jobs/run_cleanup.py` and - `infrastructure/inngest/handlers/run_cleanup.py` into - `core/jobs/run/cleanup/`. - -### Split Job Contracts - -- Delete `application/jobs/models.py`. -- Move each request/result model from `application/jobs/models.py` into the - matching job package `contract.py`. -- Move job-specific event contracts from `application/events/task_events.py` - into the matching job package `contract.py`. -- Move run cleanup/cancellation event contracts from - `application/events/infrastructure_events.py` into the matching job package - `contract.py`. -- Keep `application/events/base.py` only if it remains the shared - `InngestEventContract` base. If the base is used only by jobs after this - refactor, move it to `core/jobs/base.py`. -- Do not create a new shared `core/jobs/models.py` or `core/jobs/events.py`. - -### Preserve External Contracts - -- Keep Inngest event names stable. -- Keep Inngest function ids stable. -- Keep retry and cancellation settings stable. -- Update handler registry imports to point at the new `core/jobs/**/inngest.py` - modules. - -### Composition Rules - -- Replace direct application imports of concrete infrastructure with job-level - injection where the job is the natural composition point. -- Keep pure application services below the job layer. Runtime services should - not import `core/jobs`. -- Keep concrete infrastructure adapters below `infrastructure/*`. They may - implement ports, but they should not import job packages except from - `core/jobs/**/inngest.py` wrappers. - -## Non-Goals - -- Do not create a generic job framework, decorator DSL, or registry abstraction. -- Do not change event names, function ids, retries, or cancellation semantics. -- Do not move dashboard view logic into `inngest.py`. -- Do not move runtime lifecycle logic into `inngest.py`. -- Do not make `core/jobs` a dumping ground for reusable services; reusable - behavior still belongs in application services, views, or ports. - -## Acceptance Criteria - -- A reader can inspect one `core/jobs/<domain>/<job>/` folder and find the - event contract, result DTO, job composition logic, and Inngest wrapper. -- `application/jobs/` is gone. -- `infrastructure/inngest/handlers/` is gone or reduced to one-release import - shims with deletion tasks in the same stack. -- `application/jobs/models.py` is gone. -- Job contracts are local to their owning job package. -- Architecture tests enforce: - - `contract.py` has no infrastructure/persistence/service imports; - - `job.py` has no concrete infrastructure imports; - - `inngest.py` has no SQLModel table queries or business logic; - - runtime/application services do not import `core/jobs`. -- Existing Inngest event names, function ids, retries, cancellation rules, and - end-to-end runtime behavior remain stable. - -## Evidence - -- Current flat application job files under `core/application/jobs/`. -- Current mirrored handler files under `core/infrastructure/inngest/handlers/`. -- [`03-infrastructure-adapter-boundary.md`](03-infrastructure-adapter-boundary.md) - for the application-port/infrastructure-implementation rule. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/09-final-core-folder-state.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/09-final-core-folder-state.md deleted file mode 100644 index ec39eec54..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/09-final-core-folder-state.md +++ /dev/null @@ -1,376 +0,0 @@ -# PRD 09: Final Core Folder State - -## Goal - -Define the intended `ergon_core.core` package layout after the core refactor so -the implementation stack has a concrete architectural acceptance target. - -This document is not another refactor slice. It is the final-state map that -PRDs 01-08 should converge on. - -## Final Top-Level Shape - -```text -ergon_core/core/ - application/ - infrastructure/ - jobs/ - persistence/ - views/ - rl/ - shared/ -``` - -Deleted top-level packages: - -- `domain/` - -Top-level packages that must not exist after the refactor: - -- `application/jobs/` -- `application/read_models/` -- `application/graph/` -- `application/tasks/` -- `application/workflows/` -- `infrastructure/inngest/handlers/` -- `rest_api/` - -## Final Package Responsibilities - -### `application/` - -Owns business use cases, runtime services, ports, and compatibility modules. - -```text -application/ - communication/ - compat/ - cohorts.py - legacy_experiments.py - context/ - evaluation/ - dto_mapping.py - models.py - scoring.py - service.py - summary.py - experiments/ - ports/ - dashboard.py - resources.py - sandbox.py - resources/ - errors.py - models.py - publishing.py - repository.py - runtime/ - errors.py - events.py - graph_repository.py - graph_traversal.py - lifecycle.py - models.py - resources.py - run_identity.py - run_lifecycle.py - status.py - task_cleanup.py - task_execution.py - task_execution_repository.py - task_inspection.py - task_management.py -``` - -Deleted from `application/`: - -- `jobs/` -- `read_models/` -- `graph/` -- `tasks/` -- `workflows/` - -Rules: - -- `application/runtime/*` owns run graph lifecycle, task lifecycle, propagation, - dynamic task mutation, execution row writes, and runtime resource views. -- `application/ports/*` declares external effects needed by application or - jobs; it does not import concrete infrastructure. -- `application/compat/*` is explicitly temporary and may reference deprecated - tables while PRD 07 is open. -- `application/evaluation/*` owns evaluation semantics and summary DTO mapping. - -### `views/` - -Owns read-only contract builders for REST/dashboard/API consumers. - -```text -views/ - errors.py - resources.py - compat/ - cohorts.py - dashboard_events/ - contracts.py - context_events.py - graph_mutations.py - cohorts.py - experiments/ - models.py - service.py - runs/ - graph_tasks.py - models.py - service.py - snapshot.py - training.py -``` - -Rules: - -- View modules may read persistence rows. -- View modules must not mutate runtime state. -- View modules must not start jobs. -- `views/compat/cohorts.py` exists only while frontend cohort surfaces - remain. -- `views/training.py` exists only if PRD 01 keeps training observability - and identifies/adds a real writer. - -### `jobs/` - -Owns event-local composition modules. - -```text -jobs/ - base.py - workflow/ - start/ - contract.py - job.py - inngest.py - complete/ - contract.py - job.py - inngest.py - fail/ - contract.py - job.py - inngest.py - task/ - execute/ - contract.py - job.py - inngest.py - propagate/ - contract.py - job.py - inngest.py - evaluate/ - contract.py - job.py - inngest.py - cleanup_cancelled/ - contract.py - job.py - inngest.py - cancel_orphans/ - contract.py - job.py - inngest.py - sandbox/ - setup/ - contract.py - job.py - inngest.py - cleanup/ - contract.py - job.py - inngest.py - resources/ - persist_outputs/ - contract.py - job.py - inngest.py - run/ - cleanup/ - contract.py - job.py - inngest.py -``` - -Rules: - -- `contract.py` contains event wire contracts and local result DTOs only. -- `job.py` composes application services, views, and ports and must not import - concrete infrastructure. -- `inngest.py` registers the Inngest function and wires concrete - infrastructure implementations into `job.py`. -- Runtime/application services must not import `core.jobs`. - -### `infrastructure/` - -Owns external/framework adapters and process/bootstrap wiring. - -```text -infrastructure/ - dashboard/ - emitter.py - provider.py - http/ - app.py - routes/ - experiments.py - rollouts.py - runs.py - test_harness.py - inngest/ - client.py - registry.py - sandbox/ - blob_store.py - event_sink.py - file_reader.py - manager.py - tracing/ - contexts.py - ids.py - otel.py - sink.py - types.py -``` - -Deleted from `infrastructure/`: - -- `inngest/handlers/` -- `dashboard/event_contracts.py` - -Rules: - -- Infrastructure may implement application ports. -- Infrastructure must not own view builders, business rules, runtime lifecycle, - cohort recomputation, or resource append/dedup policy. -- Inngest registration lives in `core/jobs/**/inngest.py`, not in - `infrastructure/inngest/handlers`. -- `infrastructure/http/` owns REST translation and FastAPI composition; it - must call application services and views instead of defining product logic. - -### `persistence/` - -Owns SQLModel tables, database setup, migrations-facing models, storage-safe -enums/types, and low-level JSON validation. - -```text -persistence/ - context/ - models.py - definitions/ - models.py - graph/ - models.py - shared/ - db.py - enums.py - ids.py - types.py - telemetry/ - models.py -``` - -Deleted from `persistence/`: - -- `components/` -- `imports/` -- `saved_specs/` -- `context/event_payloads.py` -- `graph/status_conventions.py` -- `telemetry/repository.py` -- `telemetry/evaluation_summary.py` - -Rules: - -- Persistence must not define application command DTOs. -- Persistence must not define runtime lifecycle policy. -- Persistence must not import concrete infrastructure. -- Persistence must not own compatibility services. - -### `shared/` - -Owns narrow cross-cutting contracts and utilities. - -```text -shared/ - context_parts.py - json_types.py - rollout_status.py - settings.py - utils.py -``` - -Rules: - -- Shared is not a junk drawer. -- Shared schemas must be used by multiple real layers. -- `ContextEventPayload` and `WorkerYield` do not exist; consumers use - `ContextPartChunkLog` and `ContextPartChunk` directly. - -### `rl/` - -Owns RL/rollout control-plane logic that is not general runtime application -behavior. - -```text -rl/ - eval_runner.py - extraction.py - rollout_service.py - rollout_types.py - vllm_manager.py -``` - -Rules: - -- RL rollout provenance uses `RolloutBatch` and canonical `RunRecord.definition_id`. -- RL must not write `BenchmarkDefinitionRecord`. -- RL may read shared context/evaluation contracts through their final homes. - -## Architecture Test Acceptance Criteria - -Add or update tests that assert: - -- `core/domain` does not exist. -- `core/application/jobs`, `core/application/read_models`, - `core/application/graph`, `core/application/tasks`, and - `core/application/workflows` do not exist. -- `core/infrastructure/inngest/handlers` does not exist. -- `core/infrastructure/dashboard/event_contracts.py` does not exist. -- `core/rest_api` does not exist. -- `core/infrastructure/http/app.py` exists as the FastAPI composition root. -- `core/infrastructure/http/routes/*` imports application services and views, - but not SQLModel table models directly. -- `core/persistence/imports`, `core/persistence/components`, - `core/persistence/saved_specs`, `core/persistence/context/event_payloads.py`, - `core/persistence/graph/status_conventions.py`, - `core/persistence/telemetry/repository.py`, and - `core/persistence/telemetry/evaluation_summary.py` do not exist. -- `core/jobs/**/contract.py` does not import infrastructure, persistence - sessions, SQLModel table models, or application services. -- `core/jobs/**/job.py` does not import concrete infrastructure. -- `core/jobs/**/inngest.py` does not query SQLModel tables or import runtime - repositories directly. -- `core/application/runtime`, `core/application/evaluation`, and - `core/views` do not import `core.jobs`. -- `core/infrastructure` does not import `core.persistence.telemetry.models` - except in explicitly approved bootstrap/test helpers. -- `core/persistence` does not import `core.application`, `core.infrastructure`, - `core.jobs`, or `core.views`. -- `rg -n "ContextEventPayload|WorkerYield|TelemetryRepository|CreateTaskEvaluation"` - returns no production references. -- `rg -n "BenchmarkDefinitionRecord" ergon_core/ergon_core/core` returns only - `application/compat/*`, persistence table definitions while the compatibility - window is open, and tests. After PRD 07, it returns no production references. - -## Evidence - -- PRDs 01-08 define the source moves/deletions that produce this final state. -- [`../audits/current-structure.md`](../audits/current-structure.md) records - the current pre-refactor folder shape. diff --git a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/10-application-domain-layout-convention.md b/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/10-application-domain-layout-convention.md deleted file mode 100644 index 0134be2a6..000000000 --- a/docs/rfcs/active/2026-05-18-core-domain-structure-standardization/prds/10-application-domain-layout-convention.md +++ /dev/null @@ -1,667 +0,0 @@ -# PRD 10: Application Domain Layout Convention - -## Goal - -Standardize surviving `core/application/*` domains after the core refactor so -each domain has a clear public surface, private internals, and enforceable -cross-domain import rules. - -## Timing - -This PRD should be implemented after the current structural stack lands, -especially: - -- `implementation-plan/04-views-package-foundation.md`; -- `implementation-plan/08-legacy-experiment-and-cohort-isolation.md`; -- `implementation-plan/10-job-composition-modules.md`; -- `implementation-plan/11-application-runtime-restructure.md`; -- `implementation-plan/12-final-folder-state-and-architecture-gates.md`. - -Before those PRs land, the application tree still contains temporary package -roots such as `application/jobs`, `application/read_models`, -`application/graph`, `application/tasks`, and `application/workflows`. This PRD -is not intended to standardize those temporary roots; it standardizes the -post-stack application domains. - -## Pattern - -Use a lightweight hexagonal/clean-architecture convention without forcing empty -files. - -For each substantial domain: - -```text -application/<domain>/ - __init__.py - service.py # public behavior facade / use cases - models.py # command/result DTOs used by behavior - errors.py # public domain/application exceptions - repository.py # domain-shaped data access, when needed - policies.py # pure business decisions, when needed - mappers.py # row/DTO transformations, when needed -``` - -For larger domains, split internals by noun: - -```text -application/<domain>/ - service.py - models.py - errors.py - repositories/ - ... - policies/ - ... - mappers/ - ... -``` - -Allowed cross-domain imports: - -- `application.<domain>.service`; -- `application.<domain>.models`; -- `application.<domain>.errors`; -- `application.<domain>` only if `__init__.py` intentionally re-exports those - public symbols; -- domain-specific public modules recorded in the architecture test allowlist. - For example, `application.evaluation.summary` may be public because views - need the evaluation summary contract. - -Forbidden cross-domain imports: - -- `application.<domain>.repository`; -- `application.<domain>.repositories.*`; -- `application.<domain>.policy`; -- `application.<domain>.policies.*`; -- `application.<domain>.mapper`; -- `application.<domain>.mappers.*`; -- `application.<domain>.dto_mapping`; -- `application.<domain>._*`; -- any other module that the domain marks as internal. - -The aim is not to make every domain look identical. The aim is to make every -domain's public behavior obvious and to stop other domains from reaching into -private helpers. - -## State Tests - -Add permanent architecture tests that enforce the convention. - -### Public Surface Import Test - -Create or extend an architecture test such as: - -```text -ergon_core/tests/unit/architecture/test_application_domain_boundaries.py -``` - -It should parse imports under `ergon_core.core.application.*` and fail when a -module in one domain imports a private module from another domain. - -Default allowed imports: - -```python -from ergon_core.core.application.evaluation.service import EvaluationService -from ergon_core.core.application.evaluation.models import EvaluationServiceResult -from ergon_core.core.application.evaluation.errors import EvaluationError -``` - -Domain-specific allowed imports: - -```python -from ergon_core.core.application.evaluation.summary import EvaluationSummary -from ergon_core.core.application.evaluation.scoring import EvaluationScoreSummary -``` - -Example forbidden imports: - -```python -from ergon_core.core.application.evaluation.dto_mapping import evaluation_row_to_dto -from ergon_core.core.application.resources.repository import RunResourceRepository -from ergon_core.core.application.runtime.graph_repository import RuntimeGraphRepository -``` - -Same-domain imports remain allowed. For example, -`application.evaluation.service` may import -`application.evaluation.mappers`. - -### Domain Layout Test - -Add a test that checks every first-level package under `core/application/` -belongs to one of these categories: - -- active domain: `communication`, `context`, `evaluation`, `experiments`, - `resources`, `runtime`; -- boundary package: `ports`; -- temporary compatibility package: `compat`; -- deleted roots that must not exist: - `jobs`, `read_models`, `graph`, `tasks`, `workflows`, `events`. - -### Internal Module Naming Test - -Add a test that treats these names as internal unless imported from the same -domain: - -- `repository.py`; -- `repositories/*`; -- `policy.py`; -- `policies/*`; -- `mapper.py`; -- `mappers/*`; -- `dto_mapping.py`; -- `_*.py`. - -### Public Facade Test - -Add a test that each active domain has a short module-level docstring in -`service.py` describing: - -- what use cases this domain owns; -- what invariants it protects; -- which modules are its public surface. - -This is deliberately docstring-based because the repeated failure mode in this -codebase is not only wrong imports, but also unclear ownership. - -## Temporary Refactor Tests - -For each domain refactor, add characterization tests before moving files or -changing imports. These tests can live under: - -```text -ergon_core/tests/unit/application_refactor/ -``` - -They are allowed to be deleted after the domain is standardized and covered by -normal domain tests plus the permanent architecture tests. Their job is to keep -the refactor mechanical and state-preserving. - -## Domain Sections - -### Runtime - -#### Current Folder Path And Logic - -Current pre-stack logic is spread across: - -```text -application/graph/ -application/tasks/ -application/workflows/ -``` - -These packages collectively own run graph persistence operations, graph -traversal, propagation, ready-task detection, task lifecycle mutation, task -execution rows, task inspection, workflow initialization/finalization, runtime -resource materialization, and CLI/operator mutation helpers. They also contain -duplicate helpers for descendant traversal, ready-event dispatch, and -definition-id lookup. - -After implementation-plan PR 11, the current path should become: - -```text -application/runtime/ -``` - -with modules such as `graph_repository.py`, `graph_traversal.py`, -`lifecycle.py`, `run_lifecycle.py`, `task_management.py`, -`task_execution.py`, `task_inspection.py`, `task_cleanup.py`, -`task_execution_repository.py`, `resources.py`, `events.py`, -`run_identity.py`, `models.py`, `errors.py`, and `status.py`. - -#### Intended Folder Path - -```text -application/runtime/ - __init__.py - service.py - models.py - errors.py - status.py - repositories/ - graph.py - task_execution.py - policies/ - propagation.py - traversal.py - invalidation.py - resources.py - events.py - run_identity.py -``` - -`service.py` should be the public facade for runtime use cases that other -domains need. It can delegate internally to smaller services such as task -management and run lifecycle, but cross-domain callers should not reach into -graph repositories or traversal policies directly. - -#### Plan For Getting There - -- Keep the PR 11 `application/runtime` consolidation as the first step. -- Introduce `RuntimeService` or a small set of public facade services in - `runtime/service.py` after duplicated logic is already colocated. -- Move graph and task-execution repositories under `runtime/repositories/`. -- Move pure traversal/propagation/invalidation helpers under - `runtime/policies/`. -- Keep public command/result DTOs in `runtime/models.py`. -- Keep lifecycle statuses in `runtime/status.py`. -- Replace cross-domain imports of `runtime.graph_repository`, - `runtime.graph_traversal`, `runtime.task_management`, and - `runtime.task_execution_repository` with facade calls on - `runtime.service` unless the caller is still inside `runtime`. - -#### Temporary Refactor Test Suite - -- Dynamic task spawn preserves task JSON, parent/child edges, dependency edges, - and ready events. -- Task completion propagation preserves successor status transitions. -- Task failure propagation preserves descendant blocking behavior. -- Restart/refine preserves invalidation and downstream reset behavior. -- Runtime resource materialization returns the same bytes/location metadata as - before the move. -- Operator unblock and restart still emit the same graph mutations. - -Delete these characterization tests after runtime has stable unit coverage -against the public facade and the permanent architecture tests are in place. - -### Evaluation - -#### Current Folder Path And Logic - -```text -application/evaluation/ - errors.py - models.py - scoring.py - service.py -``` - -After PR 01 this domain should also own: - -```text -application/evaluation/summary.py -application/evaluation/dto_mapping.py -``` - -The domain currently owns evaluator execution orchestration, evaluation result -persistence, score aggregation, summary construction, and dashboard DTO -mapping. Some helper functions live at module top level inside `service.py`, -which makes it tempting for other code to import behavior that should be -private. - -#### Intended Folder Path - -```text -application/evaluation/ - __init__.py - service.py - models.py - errors.py - summary.py - scoring.py - mappers.py -``` - -`service.py`, `models.py`, `errors.py`, `summary.py`, and `scoring.py` are -public. `mappers.py` is internal unless a future views PR explicitly makes a -mapping function a public contract. - -#### Plan For Getting There - -- Move `dto_mapping.py` to `mappers.py` if it is only used within evaluation - and views can use a service/facade instead. -- Keep `summary.py` public because views and tests need the semantic schema. -- Keep `scoring.py` public if multiple domains use score summaries; otherwise - make it internal by moving it under `policies/scoring.py`. -- Move top-level helper functions in `service.py` behind either - `EvaluationService` methods or internal `mappers.py`/`policies.py` helpers. -- Replace cross-domain imports of evaluation internals with imports from - `evaluation.service`, `evaluation.models`, `evaluation.errors`, - `evaluation.summary`, or `evaluation.scoring`. - -#### Temporary Refactor Test Suite - -- Persisted success creates the same `RunTaskEvaluation` row. -- Persisted failure creates the same failure summary. -- Score aggregation produces the same pass/fail/max-score behavior. -- Dashboard/run snapshot evaluation DTOs are byte-for-byte equivalent before - and after mapper moves. - -Delete these once evaluation service, summary, and scoring tests cover the -public behavior. - -### Experiments - -#### Current Folder Path And Logic - -```text -application/experiments/ - definition_writer.py - errors.py - handles.py - launch.py - models.py - service.py -``` - -This domain owns definition persistence, experiment handles, launch/run -creation entry points, and the public `run_experiment(...)` convenience -service. Read-side experiment DTOs move to `views/experiments` in PR 04. -Legacy experiment-record fallback moves to `application/compat` in PR 08. - -#### Intended Folder Path - -```text -application/experiments/ - __init__.py - service.py - models.py - errors.py - repositories/ - definitions.py - launch.py - handles.py -``` - -`service.py`, `models.py`, `errors.py`, and `handles.py` are public. -`repositories/*` is internal to the experiments domain. `launch.py` should -either remain public as an explicit launch facade or be merged into -`service.py`; do not leave both as parallel public entry points for the same -operation. - -#### Plan For Getting There - -- Decide whether `launch_run(...)` is a public use case. If yes, re-export it - through `experiments/service.py` or `experiments/__init__.py`; if no, make it - internal to `service.py`. -- Move definition writer persistence helpers into - `experiments/repositories/definitions.py` if they are data-access shaped. -- Keep command/result DTOs in `experiments/models.py`. -- Keep `DefinitionHandle` in `handles.py` if it remains a public product - concept; otherwise merge it into `models.py`. -- Remove any direct imports of definition writer internals from other domains. - -#### Temporary Refactor Test Suite - -- Persisting a benchmark definition writes the same definition rows. -- Launching a run initializes the same runtime graph and emits the same - workflow-started event. -- Definition handles round-trip through public APIs. -- Legacy experiment fallback remains covered by compat tests until PR 09 - deletes it. - -Delete these after experiments has stable definition persistence and launch -service tests. - -### Resources - -#### Current Folder Path And Logic - -```text -application/resources/ - errors.py - models.py - repository.py -``` - -After PR 07 this domain should also own: - -```text -application/resources/publishing.py -``` - -The domain currently owns run resource data access and resource DTOs/errors. -Resource append/dedup semantics are still partly in sandbox infrastructure -until PR 07 moves them into application. - -#### Intended Folder Path - -```text -application/resources/ - __init__.py - service.py - models.py - errors.py - repository.py - publishing.py -``` - -`service.py`, `models.py`, and `errors.py` are public. `repository.py` and -`publishing.py` are internal unless the public facade deliberately exposes -their operations. - -#### Plan For Getting There - -- Add `ResourceService` in `resources/service.py` as the public facade for - resource use cases that other domains need. -- Keep `RunResourceRepository` in `repository.py`, but make cross-domain - callers go through `ResourceService`. -- Keep `RunResourcePublishService` in `publishing.py`, but treat it as internal - to the resources domain after PR 07 unless jobs need it as a composition - boundary. -- Move reusable command/result DTOs to `models.py`; keep view DTOs in - `views/resources.py`. -- Replace direct imports of `RunResourceRepository` from outside `resources` - with facade calls. - -#### Temporary Refactor Test Suite - -- Listing resources by run/execution returns the same rows. -- Latest-by-path and find-by-hash behavior is unchanged. -- Append/dedup semantics match PR 07 characterization tests. -- Resource size visibility checks remain in views or service as decided by the - implementation. - -Delete these once `ResourceService` has normal unit coverage. - -### Communication - -#### Current Folder Path And Logic - -```text -application/communication/ - errors.py - models.py - service.py -``` - -The domain owns communication thread/message creation and read DTOs for -worker/context communication. It is already close to the intended shape, but -its DTOs should be classified: command/result models stay here; dashboard/API -read contracts should move to `views` if they become broader read models. - -#### Intended Folder Path - -```text -application/communication/ - __init__.py - service.py - models.py - errors.py -``` - -No repository file is needed unless communication data access becomes reused -outside the service. - -#### Plan For Getting There - -- Audit `models.py` and classify each DTO as command/result vs view contract. -- Keep create/send command models in `communication/models.py`. -- Move read-only dashboard/API view DTOs to `views` only if they are - consumed outside communication service boundaries. -- Add a module docstring to `service.py` naming this as the public facade. -- Ensure other domains call `communication.service`, not private helpers. - -#### Temporary Refactor Test Suite - -- Creating a communication message persists the same thread/message rows. -- Listing thread summaries and messages returns the same shape. -- Error translation remains stable. - -Delete these if existing communication tests already cover the same behavior. - -### Context - -#### Current Folder Path And Logic - -```text -application/context/ - events.py -``` - -This domain owns context event persistence sequencing, turn-id extraction, event -listeners, and read helpers for context events. Shared context payload schemas -move to `core/shared/context_parts.py` in PR 03. - -#### Intended Folder Path - -```text -application/context/ - __init__.py - service.py - models.py - errors.py -``` - -If there are no domain-specific command/result DTOs or errors, omit -`models.py` and `errors.py`. `service.py` should replace `events.py` as the -public behavior facade. - -#### Plan For Getting There - -- Rename `events.py` to `service.py` if `ContextEventService` remains the - public facade. -- Move any local command/result DTOs to `models.py` only if they exist. -- Keep payload schemas in `core/shared/context_parts.py`. -- Ensure views/dashboard code consumes context read behavior through the public - service or through `views`, not by importing private helper methods. - -#### Temporary Refactor Test Suite - -- Persisting chunks creates the same `RunContextEvent` rows and sequence - numbers. -- Turn-id extraction produces the same values. -- Listener callbacks still receive emitted events. -- `get_for_execution` and `get_for_run` return the same ordering. - -Delete these after context service tests cover the public facade. - -### Compatibility - -#### Current Folder Path And Logic - -After PR 08, compatibility logic should live under: - -```text -application/compat/ - cohorts.py - legacy_experiments.py -``` - -These modules isolate deprecated cohort and legacy experiment-record behavior -until PR 09 deletes it. - -#### Intended Folder Path - -```text -application/compat/ - __init__.py - cohorts.py - legacy_experiments.py -``` - -This is intentionally not a permanent domain. It has no standard repository or -service layout because every module should carry its own deletion path. - -#### Plan For Getting There - -- Add module docstrings that name the deleting PR/condition. -- Keep imports of compatibility modules out of normal domains except where the - PR stack explicitly allows a transitional fallback. -- Delete the package in PR 09 if no compatibility behavior remains. - -#### Temporary Refactor Test Suite - -- Legacy experiment fallback still works while compatibility exists. -- Cohort compatibility routes and dashboard events still work until deletion. -- Source tests ensure compatibility imports do not spread into new runtime - services. - -Delete the tests with the compatibility package. - -### Ports - -#### Current Folder Path And Logic - -After infrastructure cleanup, application ports should live under: - -```text -application/ports/ - dashboard.py - resources.py - sandbox.py -``` - -Ports describe external effects that application services/jobs need without -depending on concrete infrastructure. - -#### Intended Folder Path - -```text -application/ports/ - __init__.py - dashboard.py - resources.py - sandbox.py -``` - -Ports are boundary contracts, not a behavior domain. They may be imported by -application services and jobs. Infrastructure implements them. - -#### Plan For Getting There - -- Keep each port module narrow and named after the external effect. -- Do not put command services, repositories, or DTO mappers in `ports`. -- If a port grows multiple unrelated methods, split it by operation rather than - creating a broad manager protocol. -- Add tests that concrete infrastructure implementations satisfy the protocol - structurally where practical. - -#### Temporary Refactor Test Suite - -- Dashboard publisher implementation emits the same event payload. -- Sandbox/blob/resource adapter protocols are satisfied by concrete - infrastructure classes. -- Jobs can receive fake port implementations in unit tests. - -Keep fake-port tests if they remain useful for job unit tests. - -## Acceptance Criteria - -- Every active `application/*` domain has a documented public surface. -- Cross-domain imports use only `service.py`, `models.py`, `errors.py`, or - explicit package re-exports. -- Internal modules such as repositories, policies, and mappers are only - imported from inside their own domain. -- Temporary characterization tests exist for each domain before its refactor - starts. -- Characterization tests are deleted or replaced by normal unit tests after - each domain reaches the target layout. -- Permanent architecture tests enforce the convention after the refactor. - -## Non-Goals - -- Do not standardize temporary roots that the existing PR stack deletes. -- Do not force empty `repository.py`, `models.py`, or `errors.py` files into - tiny domains. -- Do not move persistence rows, view DTOs, infrastructure adapters, or job - contracts into application domains to make the tree look symmetrical. -- Do not treat this PRD as a mandate for one giant PR. Each domain should be - refactored in its own small PR after the structural stack lands. - -## Evidence - -- [`../implementation-plan/00-program.md`](../implementation-plan/00-program.md) -- [`09-final-core-folder-state.md`](09-final-core-folder-state.md) -- [`../audits/runtime-domain-merge-audit.md`](../audits/runtime-domain-merge-audit.md) -- [`../audits/current-structure.md`](../audits/current-structure.md) diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/01-smell-remediation-map.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/01-smell-remediation-map.md deleted file mode 100644 index 57cbfa5f1..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/01-smell-remediation-map.md +++ /dev/null @@ -1,847 +0,0 @@ -# CLI Smell Remediation Map - -This document expands the CLI domain RFC into concrete cleanup items. Each item -is written as: - -- problem; -- solution; -- proposed code; -- proposed location / domain. - -It is intentionally implementation-facing, but not yet a step-by-step plan. -Once the RFC is accepted, this file should become the input to the -implementation plan under `docs/superpowers/plans/`. - -## 1. Direct Persistence Access In CLI Commands - -### Problem - -`ergon_cli` currently reaches into SQLModel sessions and persistence tables for -some observation commands. This makes the CLI a second read-model layer and -couples terminal output to persistence internals. - -Current examples: - -- `ergon_cli/commands/run.py` imports `ensure_db`, `get_session`, - `BenchmarkDefinitionRecord`, `RunRecord`, and `select`. -- `ergon_cli/commands/experiment.py` imports `get_session`, - `BenchmarkDefinitionRecord`, and `select`. - -This violates the desired boundary: CLI should translate terminal input into -typed calls, while `ergon_core` owns views/runtime services and persistence -shape. - -### Solution - -Move read/query behavior into `ergon_core` views services and runtime services. -The CLI domain services should call those services and render typed DTOs. - -### Proposed code - -Core-side service targets after -[DeepFlow-research/ergon#91](https://github.com/DeepFlow-research/ergon/pull/91): - -```text -ergon_core/ergon_core/core/views/runs/service.py -ergon_core/ergon_core/core/views/experiments/service.py -ergon_core/ergon_core/core/application/runtime/run_records.py -``` - -Possible typed APIs: - -```python -class RunListFilter(BaseModel): - limit: int = 20 - status: str | None = None - experiment: str | None = None - - -class RunReadService: - def list_runs(self, filters: RunListFilter) -> tuple[RunSummaryDto, ...]: ... - def get_run(self, run_id: UUID) -> RunDetailDto | None: ... -``` - -```python -class ExperimentReadService: - def list_tags(self) -> tuple[str, ...]: ... - def list_definitions_by_tag(self, tag: str) -> tuple[ExperimentDefinitionRowDto, ...]: ... -``` - -CLI-side domain service: - -```python -class RunsCliService: - def list(self, command: RunListCommand) -> RunListView: ... - def status(self, command: RunStatusCommand) -> RunStatusView: ... - def cancel(self, command: RunCancelCommand) -> RunCancelView: ... -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/domains/runs/models.py -ergon_cli/ergon_cli/domains/runs/service.py -ergon_cli/ergon_cli/domains/experiments/models.py -ergon_cli/ergon_cli/domains/experiments/service.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_no_persistence_access.py -``` - -The guard should fail on direct `get_session`, `session.exec`, `session.get`, -`select(`, or persistence model imports inside `ergon_cli/ergon_cli`. - -## 2. `argparse.Namespace` As Domain Data - -### Problem - -Command handlers accept raw `argparse.Namespace` values and pass them into -logic. That keeps command inputs weakly typed and allows UUID parsing, enum -validation, default handling, and optional fields to happen ad hoc. - -This is how bugs like parser/executor naming drift happen. Before the PR #91 -base, `workflow.py` had parser/handler drift around parent graph identity. On -the PR #91 base, the CLI uses task identity, so the regression guard should be -for `--parent-task-id` and `args.parent_task_id`. - -### Solution - -Convert `Namespace` into typed command models immediately inside -`domains/*/commands.py`. Domain services and lower helpers should accept only -typed command objects. - -### Proposed code - -Example: - -```python -class RunStatusCommand(BaseModel): - run_id: UUID - - -def handle_status(args: argparse.Namespace, service: RunsCliService) -> int: - command = RunStatusCommand(run_id=args.run_id) - view = service.status(command) - render_key_values(view) - return ExitCode.OK -``` - -For workflow: - -```python -class WorkflowTaskTreeCommand(BaseModel): - parent_task_id: UUID | None = None - wait_seconds: float = 0 - output_format: OutputFormat = OutputFormat.TEXT -``` - -### Proposed location / domain - -Every domain: - -```text -ergon_cli/ergon_cli/domains/<domain>/models.py -ergon_cli/ergon_cli/domains/<domain>/commands.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_namespace_boundary.py -``` - -The guard should allow `argparse.Namespace` only in `parser.py`, -`commands.py`, and tests. - -## 3. Mixed Rendering And Behavior - -### Problem - -Handlers often fetch data, perform mutations, format rows, and print output in -one function. This makes tests rely on captured stdout and makes JSON output or -alternate renderers harder to add later. - -### Solution - -Domain services return typed view/result models. Shared renderers print those -views. Command modules coordinate service + renderer + exit code. - -### Proposed code - -Shared output primitives: - -```python -class TableView(BaseModel): - headers: tuple[str, ...] - rows: tuple[tuple[str, ...], ...] - - -def render_table(view: TableView) -> None: ... -def render_key_values(view: KeyValueView) -> None: ... -def render_json(value: BaseModel | Mapping[str, object]) -> None: ... -``` - -Domain view example: - -```python -class RunStatusView(BaseModel): - run_id: UUID - status: str - benchmark_type: str - workflow_definition_id: UUID - instance_key: str - evaluator_slug: str | None = None - model_target: str | None = None - created_at: datetime | None = None - started_at: datetime | None = None - completed_at: datetime | None = None - error_message: str | None = None -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/shared/output.py -ergon_cli/ergon_cli/domains/*/models.py -ergon_cli/ergon_cli/domains/*/service.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_rendering_boundary.py -``` - -The guard should allow `print(` only in `shared/output.py`, prompt modules, -and explicitly local streaming code such as E2B build log callbacks. - -## 4. Static Duplicated Catalogues - -### Problem - -The CLI has static catalogue data in `ergon_cli/discovery/__init__.py` for -benchmarks, workers, and evaluators. Onboarding separately carries benchmark -requirements. Builtins also declare benchmark metadata on benchmark classes. - -These sources can drift. - -### Solution - -Prefer one typed source of truth. For benchmarks, consume metadata from -`Benchmark` subclasses: `type_slug`, name/description metadata, and -`onboarding_deps` / `BenchmarkRequirements`. For workers and evaluators, either -define an explicit builtins metadata API or move the static listing into -`ergon_builtins` so the CLI is only a renderer. - -### Proposed code - -Builtins catalogue API: - -```python -class BenchmarkCatalogueEntry(BaseModel): - slug: str - name: str - description: str - requirements: BenchmarkRequirements - - -def list_builtin_benchmarks() -> tuple[BenchmarkCatalogueEntry, ...]: ... -``` - -CLI service: - -```python -class BenchmarkCliService: - def list(self) -> BenchmarkListView: - entries = list_builtin_benchmarks() - return BenchmarkListView.from_entries(entries) -``` - -### Proposed location / domain - -Core/builtins source: - -```text -ergon_builtins/ergon_builtins/benchmarks/catalogue.py -``` - -CLI consumer: - -```text -ergon_cli/ergon_cli/domains/benchmarks/service.py -ergon_cli/ergon_cli/domains/benchmarks/models.py -``` - -Onboarding consumer: - -```text -ergon_cli/ergon_cli/domains/onboarding/profile.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_no_static_builtin_catalogue.py -``` - -## 5. `main.py` As A Central Knowledge Dump - -### Problem - -`main.py` currently defines parser flags for every command and owns dispatch -maps. This turns it into a command registry, parser catalogue, and entrypoint -all at once. - -### Solution - -Make `main.py` and optional `app.py` composition roots. Each domain registers -its own parser and handler. - -### Proposed code - -```python -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(...) - subparsers = parser.add_subparsers(dest="command") - for registrar in DOMAIN_REGISTRARS: - registrar(subparsers) - return parser -``` - -```python -DOMAIN_REGISTRARS = ( - register_benchmark_parser, - register_doctor_parser, - register_eval_parser, - register_experiment_parser, - register_ingestion_parser, - register_onboarding_parser, - register_run_parser, - register_stack_parser, - register_training_parser, - register_workflow_parser, -) -``` - -Each parser should set a callable on the parsed namespace: - -```python -parser.set_defaults(handler=handle_run_status) -``` - -Then dispatch becomes: - -```python -handler = getattr(args, "handler", None) -if handler is None: - parser.print_help() - return ExitCode.OK -return await maybe_await(handler(args)) -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/main.py -ergon_cli/ergon_cli/app.py -ergon_cli/ergon_cli/domains/*/parser.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_main_is_composition_root.py -``` - -The guard should fail on domain-specific `add_argument` calls in `main.py`. - -## 6. Thin Bootstrap No-Ops And Legacy Names - -### Problem - -`ergon_cli/bootstrap.py` contains a no-op -`register_and_publish_builtins()`, and `main.py` conditionally calls it. The -name implies registry publishing still exists, even though object-bound -definitions no longer require it. - -This is compatibility-shaped code without behavior. - -### Solution - -Delete the no-op if no command needs it. If some command does need preparation, -replace it with an explicit command preparation hook whose name describes the -current behavior. - -### Proposed code - -Preferred deletion: - -```text -delete ergon_cli/ergon_cli/bootstrap.py -remove calls from main.py -``` - -If preparation is needed: - -```python -class CliRuntimePreparation(BaseModel): - needs_builtin_catalogue: bool = False - needs_database: bool = False - - -def prepare_runtime(preparation: CliRuntimePreparation) -> None: ... -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/app.py -ergon_cli/ergon_cli/shared/runtime.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_no_dead_bootstrap.py -``` - -## 7. Inconsistent Sync / Async Boundaries - -### Problem - -The CLI entrypoint runs through `asyncio.run`, but some lower-level helpers -also call `asyncio.run` internally. The workflow command currently bridges -async manage actions from sync dispatch code. This can break if reused inside -an existing event loop and obscures which command paths are async. - -### Solution - -Make async boundaries explicit at the command dispatch layer. Domain handlers -may be sync or async, but lower-level domain executors should not call -`asyncio.run`. - -### Proposed code - -Shared dispatch helper: - -```python -async def invoke_handler(handler: CliHandler, args: argparse.Namespace) -> int: - result = handler(args) - if inspect.isawaitable(result): - return await result - return result -``` - -Workflow executor: - -```python -async def execute_workflow_command( - command: str, - *, - context: WorkflowCliContext, - session_factory: Callable[[], Session], - service: WorkflowService, -) -> WorkflowCommandOutput: - ... - return await dispatch_workflow_command(...) -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/app.py -ergon_cli/ergon_cli/domains/workflow/executor.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_no_nested_asyncio_run.py -``` - -Allow `asyncio.run` only in `main.py`. - -## 8. Workflow Parser / Executor Naming Drift - -### Problem - -The workflow parser and handler have already shown naming drift around parent -graph identity: - -- older parser code defined `--parent-node-id`; -- handler code read `args.parent_task_id`; -- PR #91 standardizes the human CLI surface on `--parent-task-id`. - -This is exactly the kind of bug typed command models prevent. - -### Solution - -Split workflow into: - -- top-level injected context model; -- nested parser; -- nested command models; -- async executor. - -Every parser action should map to a typed command object before execution. - -### Proposed code - -```python -class WorkflowCliContext(BaseModel): - run_id: UUID - node_id: UUID - execution_id: UUID - sandbox_task_key: UUID - benchmark_type: str -``` - -```python -class WorkflowInspectTaskTreeCommand(BaseModel): - parent_task_id: UUID | None = None - wait_seconds: float = 0 - output_format: OutputFormat = OutputFormat.TEXT -``` - -```python -async def inspect_task_tree( - command: WorkflowInspectTaskTreeCommand, - *, - context: WorkflowCliContext, - session: Session, - service: WorkflowService, -) -> WorkflowCommandOutput: ... -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/domains/workflow/context.py -ergon_cli/ergon_cli/domains/workflow/models.py -ergon_cli/ergon_cli/domains/workflow/parser.py -ergon_cli/ergon_cli/domains/workflow/executor.py -``` - -Tests: - -```text -ergon_cli/tests/unit/domains/workflow/test_parser_models.py -ergon_cli/tests/unit/domains/workflow/test_executor.py -``` - -## 9. Optional Dependency Handling Inline In Commands - -### Problem - -`train.py` probes for `ergon_infra` and then imports training modules inside -the handler. The delayed import is justified because `ergon_infra` is optional, -but this pattern will become scattered if more optional domains appear. - -### Solution - -Centralize optional dependency checks in a shared helper that produces typed -CLI errors. Keep the actual optional imports at the domain service boundary. - -### Proposed code - -```python -class MissingOptionalDependency(CliError): - package: str - install_hint: str -``` - -```python -def require_module(module_name: str, *, install_hint: str) -> None: - if importlib.util.find_spec(module_name) is None: - raise MissingOptionalDependency( - package=module_name, - install_hint=install_hint, - ) -``` - -Training service: - -```python -class TrainingCliService: - def train_local(self, command: TrainLocalCommand) -> TrainLocalResult: - require_module("ergon_infra", install_hint="pip install ergon-cli[training]") - from ergon_infra.training.config import TrainingConfig - from ergon_infra.training.trl_runner import run_trl_training - ... -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/shared/dependencies.py -ergon_cli/ergon_cli/shared/errors.py -ergon_cli/ergon_cli/domains/training/service.py -``` - -## 10. Local Machine Effects Are Not Modeled - -### Problem - -`stack`, `doctor`, `onboard`, and `benchmark setup` perform real local effects: - -- subprocess execution; -- `.env` writes; -- E2B template builds; -- Docker daemon checks; -- TCP health checks; -- optional package installation. - -Today much of that code prints as it goes and returns bare exit codes. - -### Solution - -Represent local effects and their outcomes as typed result models. Streaming -output is still allowed for long-running operations such as E2B builds, but -the command should also return a structured final result. - -### Proposed code - -```python -class StackStartResult(BaseModel): - compose_file: Path - services: tuple[ServiceEndpoint, ...] - exit_code: int -``` - -```python -class DoctorReport(BaseModel): - checks: tuple[DoctorCheckResult, ...] - - @property - def ok(self) -> bool: - return all(check.status == CheckStatus.PASS for check in self.checks) -``` - -```python -class BenchmarkTemplateSetupResult(BaseModel): - slug: str - template_name: str - template_id: str - build_id: str - built_at: datetime - registry_path: Path -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/domains/stack/models.py -ergon_cli/ergon_cli/domains/doctor/models.py -ergon_cli/ergon_cli/domains/benchmarks/models.py -ergon_cli/ergon_cli/domains/onboarding/models.py -``` - -## 11. Ad Hoc Error Handling And Exit Codes - -### Problem - -Commands mix several error strategies: - -- print usage and return `1`; -- raise `SystemExit`; -- print to stderr and return a code; -- log an error; -- catch broad exceptions in domain code. - -This makes behavior inconsistent and makes command tests focus on incidental -output rather than expected failure modes. - -### Solution - -Use shared CLI error and exit-code models. Command modules catch known -`CliError` subclasses and render them uniformly. - -### Proposed code - -```python -class ExitCode(IntEnum): - OK = 0 - ERROR = 1 - USAGE = 2 - DEPENDENCY = 3 -``` - -```python -class CliError(Exception): - exit_code: ExitCode = ExitCode.ERROR - - -class UsageError(CliError): - exit_code = ExitCode.USAGE - - -class DependencyMissingError(CliError): - exit_code = ExitCode.DEPENDENCY -``` - -```python -def render_error(error: CliError) -> None: ... -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/shared/errors.py -ergon_cli/ergon_cli/shared/exit_codes.py -ergon_cli/ergon_cli/shared/output.py -ergon_cli/ergon_cli/app.py -``` - -## 12. Logging Versus Printing Is Inconsistent - -### Problem - -Some commands use `print`, others use logging for user-facing CLI output. -For a CLI, direct terminal output is fine, but it should be centralized and -deliberate. - -### Solution - -Use logging for diagnostics and debug logs. Use shared renderers for -user-facing command output. - -### Proposed code - -```python -def render_view(view: CliView, *, format: OutputFormat = OutputFormat.TEXT) -> None: - ... -``` - -Command modules: - -```python -view = service.status(command) -render_view(view, format=command.output_format) -return ExitCode.OK -``` - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/shared/output.py -ergon_cli/ergon_cli/shared/logging.py -``` - -Architecture guard may be combined with the rendering-boundary guard. - -## 13. Command Vocabulary Lags Current Architecture - -### Problem - -The CLI still has some vocabulary that feels inherited from older shapes: - -- `experiment` exists as a command group, while authoring is Python-only; -- `worker list` and `evaluator list` are static views; -- `benchmark setup` only covers a subset of template-bearing benchmarks; -- no clear naming distinction exists between observe/setup/admin surfaces and - authoring surfaces. - -This does not mean the command names are wrong, but it should be explicit that -they are observation/setup/admin commands, not authoring commands. - -### Solution - -Document CLI command intent by domain: - -- observe: `run`, `experiment`, workflow inspect commands; -- setup: `onboard`, `doctor`, `benchmark setup`, `start`, `stop`; -- runtime/admin: `run cancel`; -- agent toolkit only: workflow manage commands, owned by `ergon_builtins`; -- delegated tools: `ingest`, `eval`, `train`. - -Do not introduce CLI authoring aliases. - -### Proposed code - -Domain metadata: - -```python -class CliDomainMetadata(BaseModel): - name: str - kind: Literal["observe", "setup", "admin", "delegated"] - authoring_surface: bool = False -``` - -Parser registration can expose this metadata for architecture tests and -potential help grouping later. - -### Proposed location / domain - -```text -ergon_cli/ergon_cli/domains/<domain>/metadata.py -ergon_cli/ergon_cli/app.py -``` - -Architecture guard: - -```text -ergon_cli/tests/unit/architecture/test_no_cli_authoring_surface.py -``` - -## 14. CLI Tests Lack Architecture Guardrails - -### Problem - -The CLI has useful unit tests, but it lacks the density of architectural -guardrails present in `ergon_core`. Without tests, cleanup rules can regress -silently. - -### Solution - -Add CLI architecture tests as part of the refactor, not after. - -### Proposed code - -Test files: - -```text -ergon_cli/tests/unit/architecture/test_domain_layout.py -ergon_cli/tests/unit/architecture/test_main_is_composition_root.py -ergon_cli/tests/unit/architecture/test_namespace_boundary.py -ergon_cli/tests/unit/architecture/test_no_persistence_access.py -ergon_cli/tests/unit/architecture/test_no_nested_asyncio_run.py -ergon_cli/tests/unit/architecture/test_no_cli_authoring_surface.py -ergon_cli/tests/unit/architecture/test_rendering_boundary.py -``` - -Suggested assertions: - -- `main.py` has no domain-specific `add_argument` calls. -- `argparse.Namespace` appears only in parser/command boundary modules. -- `ergon_cli` does not import persistence models or session helpers. -- `asyncio.run` appears only in `main.py`. -- user-facing `print` appears only in shared renderers, prompts, and explicit - streaming exceptions. -- no deleted authoring commands return. -- every domain has `parser.py` and `commands.py`. - -### Proposed location / domain - -```text -ergon_cli/tests/unit/architecture/ -``` - -## Priority Order - -Recommended implementation order: - -1. Split parser registration out of `main.py`. -2. Add typed command/result models. -3. Centralize output, errors, and exit codes. -4. Move direct DB access into core views/runtime services. -5. Clean workflow parser/executor typing. -6. Delete no-op bootstrap compatibility code. -7. Consolidate benchmark/onboarding/discovery catalogues. -8. Add architecture tests as each rule becomes true. - -This order gives quick improvements to code shape while delaying the most -behavior-sensitive work, namely replacing direct CLI queries with core -views/runtime services. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/02-target-folder-shape.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/02-target-folder-shape.md deleted file mode 100644 index a18f9278a..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/02-target-folder-shape.md +++ /dev/null @@ -1,411 +0,0 @@ -# Target CLI Folder Shape - -This document records the intended `ergon_cli` source and test layout after -the CLI domain refactor. Paths are repo-relative from `ergon/`. - -## Source Tree - -```text -ergon_cli/ - pyproject.toml - - ergon_cli/ - __init__.py - main.py - app.py - - shared/ - __init__.py - dependencies.py - env.py - errors.py - exit_codes.py - logging.py - output.py - parsing.py - - domains/ - __init__.py - - benchmarks/ - __init__.py - commands.py - models.py - parser.py - service.py - templates.py - - doctor/ - __init__.py - checks.py - commands.py - models.py - parser.py - service.py - - eval/ - __init__.py - commands.py - models.py - parser.py - service.py - - experiments/ - __init__.py - commands.py - models.py - parser.py - service.py - - ingestion/ - __init__.py - commands.py - parser.py - - onboarding/ - __init__.py - commands.py - env_writer.py - installer.py - models.py - parser.py - profile.py - prompts.py - service.py - - runs/ - __init__.py - commands.py - models.py - parser.py - service.py - - stack/ - __init__.py - commands.py - models.py - parser.py - service.py - - training/ - __init__.py - commands.py - models.py - parser.py - service.py - - workflow/ - __init__.py - commands.py - context.py - executor.py - models.py - parser.py -``` - -## File Responsibilities - -### Entrypoint - -```text -ergon_cli/ergon_cli/main.py -``` - -Console script target. It should call `asyncio.run(app.run(argv))` and contain -no domain-specific parser registration or command behavior. - -```text -ergon_cli/ergon_cli/app.py -``` - -Composition root. Owns: - -- top-level parser construction; -- domain parser registration list; -- handler dispatch; -- async/sync handler invocation; -- top-level `CliError` rendering. - -It should not contain subcommand-specific flags or business logic. - -### Shared - -```text -ergon_cli/ergon_cli/shared/dependencies.py -``` - -Optional dependency checks, such as `require_module("ergon_infra", ...)`. - -```text -ergon_cli/ergon_cli/shared/env.py -``` - -Small environment/file helpers shared by CLI domains. Domain-specific `.env` -writing remains in onboarding. - -```text -ergon_cli/ergon_cli/shared/errors.py -``` - -Shared `CliError` hierarchy. - -```text -ergon_cli/ergon_cli/shared/exit_codes.py -``` - -`ExitCode` enum and constants. - -```text -ergon_cli/ergon_cli/shared/logging.py -``` - -CLI logging setup for diagnostics. User-facing output should go through -`shared/output.py`. - -```text -ergon_cli/ergon_cli/shared/output.py -``` - -Table, key-value, JSON, and error renderers. - -```text -ergon_cli/ergon_cli/shared/parsing.py -``` - -Reusable parser helpers: UUID coercion, positive integer validation, output -format enums, and common argparse helpers. - -### Domain File Contract - -For each domain: - -```text -ergon_cli/ergon_cli/domains/<domain>/parser.py -``` - -Registers argparse subcommands for that domain and sets handler callables. - -```text -ergon_cli/ergon_cli/domains/<domain>/commands.py -``` - -Accepts `argparse.Namespace`, converts it to typed command models, calls the -domain service, invokes shared renderers, and returns an exit code. - -```text -ergon_cli/ergon_cli/domains/<domain>/models.py -``` - -Typed command models and typed result/view models. - -```text -ergon_cli/ergon_cli/domains/<domain>/service.py -``` - -CLI-domain behavior and delegation into `ergon_core`, `ergon_builtins`, -`ergon_ingestion`, or `ergon_infra`. - -## Domain-Specific Notes - -### Benchmarks - -```text -ergon_cli/ergon_cli/domains/benchmarks/templates.py -``` - -E2B template specs and setup helpers for `ergon benchmark setup <slug>`. -This remains CLI-local because it builds local/remote template artifacts and -writes user config. - -Longer-term, benchmark listing should consume a typed builtins catalogue rather -than a CLI-local static table. - -### Doctor - -```text -ergon_cli/ergon_cli/domains/doctor/checks.py -``` - -Pure-ish health check functions returning typed check results. Rendering lives -outside this file. - -### Ingestion - -```text -ergon_cli/ergon_cli/domains/ingestion/ -``` - -This domain intentionally has no `service.py` or `models.py` in the first -target shape. It is a mounting/delegation adapter for -`ergon_ingestion.cli.handle_ingest`. Add models/service only if -`ergon_ingestion` later exposes typed command APIs. - -### Onboarding - -```text -ergon_cli/ergon_cli/domains/onboarding/profile.py -ergon_cli/ergon_cli/domains/onboarding/env_writer.py -ergon_cli/ergon_cli/domains/onboarding/installer.py -ergon_cli/ergon_cli/domains/onboarding/prompts.py -``` - -These are moved from the current `ergon_cli/onboarding/` package. They remain -domain-local because interactive prompts, `.env` writing, and `uv pip install` -are CLI setup concerns. - -### Workflow - -```text -ergon_cli/ergon_cli/domains/workflow/context.py -``` - -Typed injected workflow context: run id, node id, execution id, -sandbox-task key, and benchmark type. - -```text -ergon_cli/ergon_cli/domains/workflow/executor.py -``` - -Nested workflow command executor. Should be async and should not call -`asyncio.run`. - -```text -ergon_cli/ergon_cli/domains/workflow/models.py -``` - -Nested workflow command models and output models. - -## Compatibility Modules To Delete - -These current paths should disappear by the end of the refactor: - -```text -ergon_cli/ergon_cli/bootstrap.py -ergon_cli/ergon_cli/commands/ -ergon_cli/ergon_cli/discovery/ -ergon_cli/ergon_cli/onboarding/ -ergon_cli/ergon_cli/rendering/ -``` - -If compatibility imports are needed during migration, keep them for one PR -only and delete them in the next slice. - -## Test Tree - -```text -ergon_cli/ - tests/ - unit/ - architecture/ - __init__.py - test_domain_layout.py - test_main_is_composition_root.py - test_namespace_boundary.py - test_no_cli_authoring_surface.py - test_no_nested_asyncio_run.py - test_no_persistence_access.py - test_rendering_boundary.py - - domains/ - __init__.py - - benchmarks/ - __init__.py - test_benchmark_commands.py - test_benchmark_setup.py - test_benchmark_parser.py - - doctor/ - __init__.py - test_doctor_checks.py - test_doctor_commands.py - test_doctor_parser.py - - eval/ - __init__.py - test_eval_commands.py - test_eval_parser.py - - experiments/ - __init__.py - test_experiment_commands.py - test_experiment_parser.py - - ingestion/ - __init__.py - test_ingestion_parser.py - - onboarding/ - __init__.py - test_env_writer.py - test_onboard_profile.py - test_onboarding_commands.py - test_onboarding_parser.py - - runs/ - __init__.py - test_run_commands.py - test_run_parser.py - - stack/ - __init__.py - test_stack_commands.py - test_stack_parser.py - - training/ - __init__.py - test_training_commands.py - test_training_parser.py - - workflow/ - __init__.py - test_workflow_commands.py - test_workflow_executor.py - test_workflow_parser.py - - shared/ - __init__.py - test_dependencies.py - test_errors.py - test_output.py - test_parsing.py -``` - -## Final Import Shape - -The intended high-level dependency direction is: - -```text -main.py - -> app.py - -> domains/*/parser.py - -> domains/*/commands.py - -> domains/*/models.py - -> domains/*/service.py - -> shared/output.py - -> shared/errors.py - -> ergon_core / ergon_builtins / ergon_ingestion / ergon_infra -``` - -Forbidden directions: - -```text -shared/ -> domains/* -domains/*/models.py -> ergon_core persistence models -domains/*/parser.py -> ergon_core persistence models -domains/*/service.py -> argparse.Namespace -main.py -> domains/*/models.py -main.py -> domains/*/service.py -``` - -Allowed direct external dependencies: - -- `domains/runs/service.py` may call `ergon_core` run read/cancel APIs. -- `domains/experiments/service.py` may call `ergon_core` experiment read APIs. -- `domains/benchmarks/service.py` may call `ergon_builtins` catalogue APIs and - the E2B SDK through `templates.py`. -- `domains/eval/service.py` may call `ergon_core.core.rl.eval_runner`. -- `domains/training/service.py` may import `ergon_infra` after an optional - dependency check. -- `domains/ingestion/commands.py` may delegate to `ergon_ingestion.cli` until - a typed ingestion API exists. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/README.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/README.md deleted file mode 100644 index 7b7584605..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/README.md +++ /dev/null @@ -1,447 +0,0 @@ ---- -status: active -opened: 2026-05-19 -author: codex -architecture_refs: - - ../../../architecture/01_public_api.md - - ../../../architecture/06_builtins.md - - ../../../architecture/07_testing.md -supersedes: [] -superseded_by: null ---- - -# RFC: CLI Domain Structure Standardization - -## Problem - -`ergon_cli` currently works, but its internal shape does not match the -standards that `ergon_core` and `ergon_builtins` are converging on. -The CLI package is organized as a flat `commands/` bag plus a large central -`main.py` parser. As a result, command modules mix argument parsing, domain -validation, direct database access, rendering, optional dependency handling, -and calls into `ergon_core`, `ergon_ingestion`, or `ergon_infra`. - -That shape has several costs: - -- `argparse.Namespace` travels too far into the implementation, so command - inputs stay weakly typed. -- `main.py` knows every flag for every command, turning the entrypoint into a - central registry and making unrelated changes conflict-prone. -- Some commands reach directly into SQLModel sessions and persistence models, - rather than going through core views/runtime services. -- Output formatting is interleaved with query and mutation logic. -- CLI-local concerns such as Docker Compose startup, `.env` writing, and E2B - template setup sit beside observation commands with no shared boundary - language. - -This is not just aesthetic debt. The rest of the repo is moving toward explicit -domain boundaries, typed public contracts, architecture tests, and thin -adapters. The CLI should follow the same direction: it is an inbound terminal -adapter, not an alternate domain layer. - -## Proposal - -Restructure `ergon_cli` into domain-sliced packages. Each CLI domain owns its -parser registration, typed command/result models, command translation, and -thin service wrapper. Shared cross-domain utilities live under `shared/`. - -Target structure: - -```text -ergon_cli/ - ergon_cli/ - main.py - app.py - - shared/ - env.py - errors.py - exit_codes.py - output.py - parsing.py - - domains/ - benchmarks/ - parser.py - commands.py - service.py - models.py - templates.py - - doctor/ - parser.py - commands.py - checks.py - models.py - - eval/ - parser.py - commands.py - service.py - models.py - - experiments/ - parser.py - commands.py - service.py - models.py - - ingestion/ - parser.py - commands.py - - onboarding/ - parser.py - commands.py - service.py - profile.py - env_writer.py - installer.py - prompts.py - - runs/ - parser.py - commands.py - service.py - models.py - - stack/ - parser.py - commands.py - service.py - models.py - - training/ - parser.py - commands.py - service.py - models.py - - workflow/ - parser.py - commands.py - context.py - executor.py - models.py -``` - -`main.py` should become a composition root. It should build the top-level -parser, call per-domain parser registration functions, dispatch to per-domain -command handlers, and contain no domain-specific flags. - -Sketch: - -```python -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="ergon", - description="Ergon experiment orchestration", - ) - subparsers = parser.add_subparsers(dest="command") - - register_benchmark_parser(subparsers) - register_doctor_parser(subparsers) - register_eval_parser(subparsers) - register_experiment_parser(subparsers) - register_ingestion_parser(subparsers) - register_onboarding_parser(subparsers) - register_run_parser(subparsers) - register_stack_parser(subparsers) - register_training_parser(subparsers) - register_workflow_parser(subparsers) - - return parser -``` - -Each domain follows the same four-file pattern unless it is genuinely tiny: - -- `parser.py` registers argparse subcommands for that domain only. -- `commands.py` translates `argparse.Namespace` into typed command models and - calls the domain service. -- `service.py` performs CLI-local work or delegates to the relevant - `ergon_core`, `ergon_ingestion`, or `ergon_infra` API. -- `models.py` defines Pydantic models or dataclasses for command input and - rendered result views. - -The conversion from `Namespace` to typed command models happens at the domain -boundary. No service or lower-level helper accepts `argparse.Namespace`. - -Example shape for `ergon run status <run-id>`: - -```text -argparse.Namespace - -> RunStatusCommand(run_id=UUID(...)) - -> RunsCliService.status(command) - -> RunStatusView - -> shared.output renders text/json/table -``` - -## Domain Boundaries - -### `domains/runs` - -Owns: - -- `ergon run list` -- `ergon run status` -- `ergon run cancel` - -The domain should not issue SQLModel queries directly. Read paths should call -`ergon_core.core.views.runs` services. Cancellation should continue to call the -core runtime cancellation function so event emission remains in one place. - -### `domains/experiments` - -Owns: - -- `ergon experiment list` -- `ergon experiment show` -- `ergon experiment tags` -- `ergon experiment by-tag` - -Experiment observation should be backed by -`ergon_core.core.views.experiments.service.ExperimentReadService` or a small -views-side method for tag queries. The CLI should not own the SQL shape of -experiment definition or run tables. - -### `domains/benchmarks` - -Owns: - -- `ergon benchmark list` -- `ergon benchmark setup <slug>` - -Template setup is legitimately CLI-local because it builds E2B templates and -writes local config under the user's environment. The benchmark catalogue -should not drift from builtins/onboarding metadata. If benchmark requirements -are declared on the `Benchmark` subclass, the CLI should consume that typed -source instead of maintaining a second static list. - -### `domains/workflow` - -Owns contextual workflow inspection commands, if the human CLI keeps them for -operator debugging: - -- resource listing and reading -- task tree and dependency inspection -- next-action inspection - -Workflow management commands such as task creation, dependency mutation, -restart/abandon actions, and sandbox resource materialization are not a human -CLI domain. The CLI-shaped management grammar belongs in `ergon_builtins` as an -agent toolkit over `WorkerContext`. - -If workflow inspection remains in `ergon_cli`, it deserves strong typing because -it has nested command parsing and injected context. The injected context should -be represented as a typed `WorkflowCliContext` model. Nested workflow commands -must not be allowed to override injected runtime scope. - -### `domains/onboarding` - -Owns interactive setup, `.env` writing, and extras installation. -This domain already has useful internal concepts (`OnboardProfile`, -`write_env`, prompt helpers); move them under the domain folder and tighten the -typed contract around benchmark/provider requirements. - -### `domains/doctor` - -Owns environment health checks. Checks should return typed `DoctorCheckResult` -models, and rendering should happen separately. This makes doctor output stable -while keeping individual checks small and testable. - -### `domains/stack` - -Owns `ergon start` and `ergon stop`. This is a CLI-local adapter around Docker -Compose. It should remain simple, but service methods should return typed -results rather than directly mixing subprocess orchestration and printing. - -### `domains/eval` - -Owns checkpoint evaluation commands. It should remain a thin adapter around -`ergon_core.core.rl.eval_runner`, with typed command inputs for checkpoint -paths, benchmark slugs, evaluator slugs, model bases, limits, polling -intervals, and optional checkpoint hooks. - -### `domains/training` - -Owns training commands. Optional dependency checks for `ergon_infra` belong at -this boundary. The service should translate CLI flags into -`ergon_infra.training.config.TrainingConfig` and then delegate. - -### `domains/ingestion` - -Owns the CLI mounting point for ingestion. Short term, this can remain a thin -delegation adapter to `ergon_ingestion.cli.handle_ingest`. Long term, if -`ergon_ingestion` exposes typed command APIs, this domain can translate CLI -input into those APIs. Do not duplicate ingestion internals inside -`ergon_cli`. - -## Invariants affected - -This RFC introduces these CLI invariants: - -- The CLI is an inbound adapter. It translates terminal input into typed - command models and delegates domain behavior to CLI-local services or core - views/runtime services. -- `argparse.Namespace` does not cross beyond `domains/*/commands.py`. -- `main.py` is a composition root. It does not contain domain-specific flags, - SQL queries, output formatting, Docker orchestration, template setup, or - optional dependency logic. -- `ergon_cli` does not directly query or mutate persistence state. It uses - `ergon_core` views/runtime services or a narrow core API function. -- CLI services return typed result/view models. Rendering lives in - `ergon_cli.shared.output`. -- Authoring remains Python-only. CLI observation commands must not reintroduce - `ergon experiment define`, `ergon experiment run`, or equivalent flag-based - authoring surfaces. -- CLI-local infrastructure commands (`start`, `stop`, `doctor`, `onboard`, - `benchmark setup`) may perform local machine effects, but those effects are - isolated to their domain services and are represented by typed command - models. - -These invariants align with: - -- `docs/architecture/01_public_api.md`: contributor-facing authoring goes - through typed API objects, not through a growing CLI flag surface. -- `docs/architecture/06_builtins.md`: built-in benchmark authoring is - benchmark-owned and object-bound; CLI commands are observation/setup - surfaces. -- `docs/architecture/07_testing.md`: architecture tests should enforce - package boundaries and keep test fixtures out of production packages. - -## Migration - -Migrate in small PRs. Mechanical moves should be separated from behavior -changes. - -### PR 1: Shared CLI primitives - -Add: - -- `ergon_cli/shared/output.py` -- `ergon_cli/shared/errors.py` -- `ergon_cli/shared/exit_codes.py` -- `ergon_cli/shared/parsing.py` -- `ergon_cli/domains/__init__.py` - -Move table rendering from `ergon_cli/rendering` into `shared.output`, keeping a -compatibility import if needed for one PR. - -### PR 2: Parser registration slices - -Move parser registration out of `main.py` one domain at a time. Preserve the -same public CLI syntax. `main.py` should call each domain's -`register_<domain>_parser(...)`. - -Acceptance: - -- `ergon --help` and all existing subcommand helps remain equivalent. -- `main.py` no longer contains subcommand-specific arguments. - -### PR 3: Typed command models - -Introduce command models for each domain. Convert `Namespace` to a command -model in `commands.py` before calling services. - -Acceptance: - -- No service method accepts `argparse.Namespace`. -- Domain tests validate UUID/path/enum coercion at the command boundary. - -### PR 4: Remove direct DB access from CLI - -Move direct SQLModel queries out of `ergon_cli` for runs and experiments. -Add or reuse core views/runtime services for: - -- experiment tags -- definitions by experiment tag -- run listing filters -- run listing by definition id -- run status detail - -Acceptance: - -- `rg "get_session|session.exec|session.get|select\\(" ergon_cli/ergon_cli` - returns no offenders, except tests if needed. - -### PR 5: Workflow domain cleanup - -Move workflow command parsing/execution into `domains/workflow`. Fix typed -context handling and keep the nested parser isolated from top-level scope -flags. - -Acceptance: - -- Nested workflow commands reject user-supplied context flags. -- `WorkflowCliContext` is the only source of run/task/execution/sandbox scope. -- Existing workflow CLI tests move with the domain. - -### PR 6: Architecture tests and docs - -Add architecture tests that enforce the new boundary: - -- `argparse.Namespace` appears only in `domains/*/commands.py` and parser - modules. -- `main.py` imports only app/parser composition and dispatch helpers. -- no direct SQLModel session/query access in `ergon_cli`. -- no CLI authoring commands are registered. -- every domain has `parser.py` and `commands.py`; larger domains also have - `service.py` and `models.py`. - -Update `docs/architecture/` with a CLI inbound-adapter section or add a new -CLI-focused architecture page if the architecture tree wants that layer -explicitly documented. - -## Alternatives considered - -### Keep the flat `commands/` package and only split `main.py` - -Rejected as insufficient. Moving parser registration out of `main.py` reduces -one source of chaos, but it leaves weak typing, direct DB access, and mixed -rendering/behavior inside command modules. - -### Use horizontal packages: `parsers/`, `services/`, `models/`, `commands/` - -Rejected for this repo. Horizontal layers scatter one domain across multiple -top-level directories. The rest of Ergon is moving toward domain ownership: -benchmarks own their task schemas, sandboxes, toolkits, worker factories, -criteria, and rubrics. The CLI should mirror that shape. - -### Move all CLI behavior into `ergon_core` - -Rejected. `ergon_core` should not learn about terminal concerns, Docker -Compose commands, prompt UX, `.env` writing, or E2B template setup. The right -direction is for CLI services to call core views/runtime services through -typed APIs, not for core to absorb CLI-specific responsibilities. - -### Switch from `argparse` to Typer or Click - -Deferred. A framework switch may improve ergonomics later, but it would not -fix the underlying boundary problem. The first refactor should keep public CLI -syntax stable and make the current argparse implementation clean and typed. - -## Open questions - -1. Should `ergon_cli` get its own `docs/architecture/09_cli.md`, or should CLI - invariants live inside existing public API / builtins docs? -2. Should CLI result models be Pydantic models everywhere, or should simple - dataclasses be allowed for purely internal views? -3. Should `benchmark list` consume benchmark metadata from - `Benchmark.onboarding_deps` / builtins discovery instead of a CLI-local - static table in the first migration stack, or should that wait for a - follow-up? -4. Should ingestion continue delegating to `ergon_ingestion.cli` indefinitely, - or should `ergon_ingestion` expose a typed command API for the CLI domain to - consume? -5. Should architecture tests impose a line-count or complexity budget per CLI - module, or is the domain layout plus lint/type rules enough? - -## On acceptance - -When this RFC moves from `active/` to `accepted`, also: - -- update `docs/architecture/` with the accepted CLI inbound-adapter invariants; -- write an implementation plan under `docs/superpowers/plans/`; -- add architecture tests for the invariants listed above; -- decide whether the first PR stack should include the no-direct-DB-access - move or leave that as the second stack after mechanical package cleanup. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-cli-hygiene-cleanup.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-cli-hygiene-cleanup.md deleted file mode 100644 index d52e70d2f..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-cli-hygiene-cleanup.md +++ /dev/null @@ -1,300 +0,0 @@ -# PR 00: CLI Hygiene Cleanup - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Delete obvious dead/stale CLI code and make unsupported workflow -authoring fail clearly before larger refactors start. - -**Architecture:** This is a guardrail PR. It changes the minimum amount of code -needed to stop preserving no-ops, ignored flags, stale discovery slugs, and weak -typing artifacts. - -**Tech Stack:** argparse, pytest, ruff, small helper modules. - ---- - -## Goal - -Delete obvious dead code and fix stale or misleading CLI behavior before any -large folder migration. - -This PR should be small and behavior-preserving except where the current -behavior is plainly wrong or falsely advertised. - -## Source PRDs - -- `../prds/00-cli-hygiene-audit.md` -- `../01-smell-remediation-map.md` - -## Scope - -## Current State - -- `main.py` imports and calls no-op bootstrap/default-registration helpers. -- `doctor --verbose` is parsed but unused. -- On the PR #91 base, `workflow task-tree` uses task identity - (`--parent-task-id`). Keep a focused regression test so the parser and - handler do not drift again. -- Workflow parser exposes mutation actions whose live behavior is missing or - intentionally rejected by core. -- CLI discovery rows include stale builtins slugs. -- `commands/benchmark.py` contains brittle sandbox template path arithmetic. -- `BuildLog(Protocol)` models only `str(x)`. - -## Target State For This PR - -- No executable no-op compatibility helpers remain. -- Every audited flag is either implemented or removed. -- Live dynamic task authoring is removed from the human CLI surface; PR 01 - deletes `workflow manage` entirely. -- Static discovery rows are corrected while still marked temporary. -- Benchmark template lookup is isolated behind a helper. - -## Tasks - -### Delete no-op compatibility code - -Modify: - -- `ergon_cli/ergon_cli/main.py` -- `ergon_cli/ergon_cli/bootstrap.py` - -Work: - -- [ ] **Step 1: Remove default component no-op** - - Delete this function from `ergon_cli/ergon_cli/main.py`: - - ```python - def register_default_components() -> None: - return None - ``` - -- [ ] **Step 2: Remove builtin bootstrap no-op** - - Delete `ergon_cli/ergon_cli/bootstrap.py` if its only production symbol is: - - ```python - def register_and_publish_builtins() -> None: - return None - ``` - -- [ ] **Step 3: Remove calls/imports** - - Remove imports and conditional calls for both symbols from `main.py`. - -- [ ] **Step 4: Verify no references remain** - - ```bash - rg "register_and_publish_builtins|register_default_components" ergon - ``` - - Expected: no production references. - -Acceptance: - -- `rg "register_and_publish_builtins|register_default_components" ergon` has - no production references. -- CLI startup behavior is unchanged. - -### Fix or remove unused command flags - -Modify: - -- `ergon_cli/ergon_cli/main.py` -- `ergon_cli/ergon_cli/commands/doctor.py` -- `ergon_cli/ergon_cli/commands/workflow.py` - -Work: - -- [ ] **Step 1: Remove `doctor --verbose`** - - Remove parser registration: - - ```python - doctor.add_argument("--verbose", action="store_true", help="Show detailed output") - ``` - - Do not add a replacement until a real detail mode is specified. - -- [ ] **Step 2: Guard `task-tree --parent-task-id`** - - On the PR #91 base, the handler should read: - - ```python - parent = UUID(args.parent_task_id) if args.parent_task_id else None - ``` - - Do not reintroduce `--parent-node-id`; PR #91 moved the CLI surface to task - identity. - -- [ ] **Step 3: Remove `resource-list --explain`** - - Delete the parser flag unless implementing real explanation output in the same - commit. - -- [ ] **Step 4: Make unsupported workflow mutations explicit** - - For non-dry-run unsupported mutations, return a `WorkflowCommandOutput` with: - - ```text - dynamic workflow mutation from CLI is unsupported; use WorkerContext.spawn_task(Task(...)) - ``` - - Keep dry-run only where tests prove the behavior is useful. - -- [ ] **Step 5: Remove ignored dependency slugs with manage deletion** - - Treat `--depends-on-task-slug` as part of the workflow management surface that - PR 01 deletes from `ergon_cli`. Do not preserve it as a human CLI flag. - -Acceptance: - -- No audited flag is parsed and ignored. -- `workflow task-tree --parent-task-id` has a focused unit test. - -### Correct stale discovery rows - -Modify: - -- `ergon_cli/ergon_cli/discovery/__init__.py` -- `ergon_cli/ergon_cli/commands/worker.py` -- `ergon_cli/ergon_cli/commands/evaluator.py` - -Work: - -- [ ] **Step 1: Correct known stale slugs** - - Replace stale worker rows such as: - - ```python - ("react-worker", ...) - ("training-stub-worker", ...) - ``` - - with current builtins slugs. - -- [ ] **Step 2: Add discovery drift tests** - - Add tests that compare CLI list output to known builtins slugs available in - process. Keep static rows only as a temporary bridge. - -Acceptance: - -- `ergon worker list` exposes valid builtins worker slugs. -- Static discovery rows are marked temporary if they remain. - -### Extract benchmark template lookup - -Modify: - -- `ergon_cli/ergon_cli/commands/benchmark.py` - -Create: - -- `ergon_cli/ergon_cli/commands/benchmark_templates.py` - -Work: - -- [ ] **Step 1: Create helper module** - - Create `benchmark_templates.py` with: - - ```python - from pathlib import Path - - SANDBOX_TEMPLATES: dict[str, Path] = {...} - - def sandbox_template_for(slug: str) -> Path: - return SANDBOX_TEMPLATES[slug] - ``` - -- [ ] **Step 2: Use helper from command handler** - - `commands/benchmark.py` should import `sandbox_template_for` and no longer - contain `Path(__file__).parents[...]`. - -- [ ] **Step 3: Add helper tests** - - Test valid template slugs and unknown slug behavior. - -Acceptance: - -- Benchmark setup behavior remains unchanged. -- Generic command code does not contain brittle repo-relative path arithmetic. - -### Remove weak protocol type - -Modify: - -- `ergon_cli/ergon_cli/commands/benchmark.py` - -Work: - -- [ ] **Step 1: Delete protocol** - - Remove: - - ```python - class BuildLog(Protocol): - def __str__(self) -> str: ... - ``` - -- [ ] **Step 2: Keep boundary stringification** - - Type the callback input as `object` unless a real E2B type is available: - - ```python - def _print_build_log(log: object) -> None: - print(str(log)) - ``` - -Acceptance: - -- No one-method protocol exists only to model `str(x)`. - -## Tests - -Add or update: - -- `ergon_cli/tests/unit/cli/test_workflow_cli.py` -- `ergon_cli/tests/unit/cli/test_doctor_cli.py` -- `ergon_cli/tests/unit/cli/test_discovery_cli.py` -- `ergon_cli/tests/unit/cli/test_benchmark_setup.py` - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli -uv run pytest ergon_cli/tests/unit/cli -``` - -## PR Ledger - -- **Invariant landed:** no known no-op bootstrap, ignored audited flags, or stale - discovery slugs remain. -- **Bridge code introduced:** `benchmark_templates.py` as a temporary local - helper before builtins metadata ownership. -- **Old path still intentionally alive:** static discovery rows until PR 05/06. -- **Deletion gate:** PR 06 deletes compatibility command/discovery packages. -- **Tests added or updated:** workflow CLI, doctor parser, discovery, benchmark - setup. -- **Modules owned by this PR:** `main.py`, `commands/workflow.py`, - `commands/doctor.py`, `commands/benchmark.py`, `discovery`. - -## Out Of Scope - -- Moving command modules into final `domains/`. -- Removing all direct DB access from CLI. -- Replacing static discovery with builtins metadata. -- Implementing real dynamic task authoring. - -## Depends On - -- External base: - [DeepFlow-research/ergon#91](https://github.com/DeepFlow-research/ergon/pull/91) - or a later mainline commit containing PR #91. -- No earlier CLI refactor PR. This is the first PR in the CLI stack. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-program.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-program.md deleted file mode 100644 index f81ed1812..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/00-program.md +++ /dev/null @@ -1,165 +0,0 @@ -# CLI Domain Standardization Program Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move `ergon_cli` from a flat command bag into a clean inbound adapter -with typed domain boundaries, while relocating agent-facing dynamic authoring to -`ergon_builtins`. - -**Architecture:** Land guardrail and ownership fixes first, then split parser -composition, introduce shared command/result contracts, move persistence-backed -commands behind services, move benchmark/onboarding metadata ownership, and -finish by deleting compatibility packages. Every PR must be runnable after -merge. - -**Tech Stack:** Python 3.11+, argparse, Pydantic v2, SQLModel, pytest, ruff, -`ergon_core.api`, `ergon_builtins` toolkits. - ---- - -## Required Base - -This CLI refactor stack must branch from the landed contents of -[DeepFlow-research/ergon#91](https://github.com/DeepFlow-research/ergon/pull/91) -(`codex/core-refactor-pr12-final-architecture-gates`) or a later mainline commit -that contains that PR. - -That base matters because PR #91 finalizes the core package shape and -architecture guards: - -- `ergon_core.core.application.read_models` is retired. -- dashboard/API read DTOs and read services live under `ergon_core.core.views`. -- runtime graph/task/run services live under `ergon_core.core.application.runtime`. -- job contract DTOs are re-export-only boundaries; new CLI work must not depend - on job-local `contract.py` files for application behavior. - -If an implementation worktree is based on an older detached core-refactor branch, -rebase the CLI stack onto PR #91 before opening PR 00. Do not update this stack -against the older `application/read_models`, `application/graph`, -`application/tasks`, or `application/workflows` package layout. - -## Why This Program Exists - -The current CLI works, but it mixes unrelated roles: - -- human/operator lifecycle commands -- stale static discovery catalogues -- direct persistence reads -- benchmark setup path knowledge -- onboarding dependency metadata -- agent-facing workflow tooling -- a half-retired dynamic task authoring surface - -The target architecture treats `ergon_cli` as a terminal inbound adapter. -Authoring remains Python-first through `ergon_core.api`; agent-facing tools live -in `ergon_builtins`. - -## Churn Budget - -| Area | Likely churn | -| --- | ---: | -| CLI hygiene and tests | 500-1.2k changed lines | -| Builtins workflow toolkit ownership | 800-1.8k changed lines | -| Parser/domain folder split | 1k-2k changed lines | -| Shared output/errors/models | 800-1.5k changed lines | -| Runs/experiments boundary | 800-1.5k changed lines | -| Benchmarks/onboarding metadata | 1k-2k changed lines | -| Final migration/deletion guards | 500-1k changed lines | - -PRs above 2.5k non-generated changed lines should split unless the excess is -mechanical file moves or deletion. - -## Ownership Lanes - -| Lane | Owns | Must Not Own | -| --- | --- | --- | -| `ergon_core.api` | Public authoring models, `WorkerContext.spawn_task`, runtime facades | CLI grammar, builtins UX | -| `ergon_builtins` | Benchmark/toolkit metadata, agent-facing tools, toy integration fixtures | Human/operator CLI implementation | -| `ergon_cli` | Terminal parsing, typed command translation, rendering, operator lifecycle commands | Dynamic task authoring, direct DB ownership, benchmark truth | -| Core application services | Runtime reads/mutations, graph rules, persistence boundaries | Argparse, printing | -| Tests | Architecture guards, CLI parser/output tests, builtins integration tests | Production adapters | - -## Bridge Ledger - -| Bridge | Introduced | New Default | Deleted | -| --- | --- | --- | --- | -| Static CLI discovery rows | Existing | PR 05 metadata catalogue | PR 06 | -| `ergon_cli.commands.*` compatibility imports | Existing | PR 02 domain parsers, PR 03+ domain commands | PR 06 | -| Builtins importing `ergon_cli.commands.workflow` | Existing | PR 01 builtins adapter | PR 01 | -| CLI workflow live authoring facade | Existing | PR 01 rejects/relocates | PR 01 | -| Direct persistence imports in CLI run/experiment commands | Existing | PR 04 CLI services/core views/runtime services | PR 04 | -| Hardcoded benchmark template paths in CLI | Existing | PR 05 builtins metadata | PR 05/06 | - -## PR Sequence - -| PR | Name | Runnable After Merge | Primary Invariant | -| ---: | --- | --- | --- | -| 00 | CLI hygiene cleanup | yes | No obvious no-op/stale CLI behavior remains | -| 01 | Dynamic subtask toolkit ownership | yes | Agent dynamic authoring is builtins over `WorkerContext`, not CLI | -| 02 | CLI composition root and parser split | yes | `main.py` is a boring entrypoint | -| 03 | Shared output/errors/command models | yes | Converted domains stop leaking `Namespace` below commands | -| 04 | Runs/experiments service boundary | yes | CLI no longer owns direct persistence reads for run/experiment commands | -| 05 | Benchmarks/onboarding metadata ownership | yes | Benchmark truth lives with builtins/domain metadata, not generic CLI code | -| 06 | Final domain folder migration | yes | Compatibility packages and old import paths are gone | - -## PR Description Template - -Every PR must include: - -```markdown -### CLI Refactor Slice Ledger - -Invariant landed: - -Bridge code introduced: - -Old path still intentionally alive: - -Deletion gate: - -Tests added or updated: - -Modules owned by this PR: -``` - -## Parallelization Rules - -Safe in parallel: - -- Draft PR 01 tests while PR 00 hygiene lands. -- Draft PR 03 shared helpers after PR 02 parser module names are agreed. -- Draft PR 05 metadata catalogue while PR 04 works on runs/experiments. - -Do not parallelize changes to: - -- `ergon_cli/ergon_cli/main.py` across PR 00 and PR 02. -- `ergon_cli/ergon_cli/commands/workflow.py` across PR 00 and PR 01. -- `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` across PR 01 and - other builtins toolkit changes. - -## Final Deleted Or Forbidden Paths - -By PR 06: - -- `ergon_cli/ergon_cli/bootstrap.py` -- production imports of `ergon_cli.commands.*` -- production imports of `ergon_cli.discovery` -- production imports of `ergon_cli.rendering` -- production imports of `ergon_builtins -> ergon_cli` -- the human CLI `ergon workflow manage ...` surface -- generic CLI hardcoded benchmark sandbox template paths -- direct persistence imports in converted CLI domains - -## Final Architecture Guards - -PR 06 must land tests asserting: - -- `argparse.Namespace` does not pass below domain `commands.py`. -- Domain services do not import `argparse`. -- CLI domains do not import `ergon_core.core.persistence`. -- `ergon_builtins` production code does not import `ergon_cli`. -- Generic CLI code does not hardcode benchmark-specific metadata. -- Output printing lives at command/rendering boundaries, not service internals. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/01-dynamic-subtask-toolkit-ownership.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/01-dynamic-subtask-toolkit-ownership.md deleted file mode 100644 index 47f7c9a2f..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/01-dynamic-subtask-toolkit-ownership.md +++ /dev/null @@ -1,304 +0,0 @@ -# PR 01: Dynamic Subtask Toolkit Ownership - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move agent-facing dynamic subtask authoring out of `ergon_cli` and -prove builtins can spawn object-bound dynamic tasks through -`WorkerContext.spawn_task(Task(...))`. - -**Architecture:** `ergon_core.api` owns the runtime facade; -`ergon_builtins.tools` owns agent-facing tools; `ergon_cli` remains a -human/operator adapter. - -**Tech Stack:** Pydantic v2, pytest integration tests, WorkerContext, -TaskManagementService, builtins tool factories. - ---- - -## Goal - -Move agent-facing dynamic subtask authoring out of `ergon_cli` and into -`ergon_builtins`, backed by the canonical -`WorkerContext.spawn_task(Task(...))` API. - -## Source PRDs - -- `../prds/01-dynamic-subtask-authoring-toolkit.md` - -## Scope - -## Current State - -- `ergon_builtins.tools.workflow_cli_tool` imports - `ergon_cli.commands.workflow`. -- The CLI command `workflow manage add-task` looks like dynamic authoring but - cannot create object-bound tasks. -- `SubtaskLifecycleToolkit` already delegates `add_subtask` to - `WorkerContext.spawn_task`, but the single-command workflow tool does not. - -## Target State For This PR - -- Builtins production code has no import dependency on `ergon_cli`. -- A builtins-owned `workflow(command: str)` adapter can spawn a toy dynamic - child task. -- The child is persisted as a dynamic graph node with full object-bound - `task_json`. -- The human CLI no longer exposes `workflow manage`. - -## Tasks - -### Add reusable toy workflow fixtures - -Create: - -- `ergon_builtins/tests/fixtures/toy_workflow.py` - -Work: - -- [ ] **Step 1: Create toy public API objects** - - Add test-local Pydantic-compatible classes: - - ```python - class ToySandbox(Sandbox): - type_slug: ClassVar[str] = "toy-sandbox" - - async def provision(self) -> None: - return None - - - class ToyWorker(Worker): - type_slug: ClassVar[str] = "toy-worker" - - async def execute( - self, - task: Task, - *, - context: WorkerContext, - ) -> AsyncGenerator[WorkerStreamItem, None]: - yield WorkerOutput(output=f"completed {task.task_slug}", success=True) - ``` - -- [ ] **Step 2: Add frozen Pydantic harness** - - ```python - class ToyWorkflowHarness(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True) - - run_id: UUID - parent_task_id: UUID - parent_task: Task - context: WorkerContext - - def child_task(self, *, task_slug: str, description: str) -> Task: ... - def nodes(self) -> list[RunGraphNode]: ... - def edges(self) -> list[RunGraphEdge]: ... - def definition_tasks(self) -> list[ExperimentDefinitionTask]: ... - ``` - -- [ ] **Step 3: Build without external infrastructure** - - The harness must use in-memory/test DB helpers only. It must not require E2B, - Docker, provider SDKs, or external benchmark assets. - -Acceptance: - -- The toy task can round-trip through `Task.from_definition`. -- The harness exposes enough row inspection for negative tests. - -### Move workflow command adapter into builtins - -Create: - -- `ergon_builtins/ergon_builtins/tools/workflow_command_adapter.py` -- `ergon_builtins/ergon_builtins/tools/dynamic_task_factory.py` - -Modify: - -- `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` - -Work: - -- [ ] **Step 1: Add dynamic task factory** - - Add `dynamic_task_factory.py` with: - - ```python - class DynamicTaskFactory(Protocol): - def child_task( - self, - *, - parent: Task, - task_slug: str, - description: str, - ) -> Task: ... - ``` - - Add a default factory that copies the parent task with updated slug, - description, and `parent_task_slug`. - -- [ ] **Step 2: Add builtins command adapter** - - Add `workflow_command_adapter.py` that parses: - - ```text - inspect resource-list - inspect resource-content - inspect task-tree - inspect task-dependencies - inspect next-actions - manage add-subtask --task-slug <slug> --description <description> [--depends-on <node-id> ...] - ``` - -- [ ] **Step 3: Inject identity from WorkerContext** - - The model may provide only the command string. The adapter reads run/task - identity from `WorkerContext`, matching the existing safety property. - -- [ ] **Step 4: Reject context escape flags** - - Reject `--run-id`, `--node-id`, `--execution-id`, `--sandbox-task-key`, and - `--benchmark-type` before any service calls. - -- [ ] **Step 5: Update workflow_cli_tool** - - Replace imports from `ergon_cli.commands.workflow` with the builtins adapter. - -Acceptance: - -- `rg "ergon_cli" ergon_builtins/ergon_builtins/tools` returns no production - imports. -- The builtins adapter has no direct SQLModel insert path for dynamic nodes or - edges. - -### Delete workflow management from human CLI - -Modify: - -- `ergon_cli/ergon_cli/commands/workflow.py` - -Work: - -- [ ] **Step 1: Delete the `manage` parser branch** - - Remove this command family from the human CLI parser: - - ```text - workflow manage add-task - workflow manage add-edge - workflow manage restart-task - workflow manage abandon-task - workflow manage materialize-resource - ``` - -- [ ] **Step 2: Delete management dispatch** - - Remove `_handle_manage` and the manage branch from workflow command dispatch. - If resource materialization is still needed by agents, implement it through - the builtins workflow adapter or a typed builtins tool, not `ergon_cli`. - -- [ ] **Step 3: Remove mutation service imports** - - `ergon_cli.commands.workflow` should not import workflow mutation services - purely to support graph management. Keep only the dependencies needed for - operator inspection if inspection remains. - -- [ ] **Step 4: Add parser rejection test** - - CLI tests must assert `workflow manage ...` is not a valid human CLI command. - -Acceptance: - -- `ergon_cli` no longer registers `workflow manage`. -- `ergon_cli.commands.workflow` no longer implements `_handle_manage`. -- CLI unit tests prove `workflow manage ...` cannot be parsed/executed through - the human CLI. -- Builtins integration tests prove `workflow("manage add-subtask ...")` remains - available to agents. - -### Keep typed toolkit path preferred - -Modify: - -- `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` - -Work: - -- [ ] **Step 1: Verify typed tool delegation** - - Ensure `SubtaskLifecycleToolkit.add_subtask` calls: - - ```python - await context.spawn_task(task, depends_on=deps) - ``` - -- [ ] **Step 2: Fix stale containment comments** - - Remove comments claiming service-layer containment gaps where - `WorkerContext` now enforces the rule. - -Acceptance: - -- Typed lifecycle toolkit and string command adapter both use the same core - dynamic spawn path. - -## Tests - -Add: - -- `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` -- `ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` - -Update: - -- `ergon_cli/tests/unit/cli/test_workflow_cli.py` - -Required scenarios: - -- builtins workflow command spawns a toy dynamic child -- spawned child is `is_dynamic=True` -- spawned child has object-bound `task_json` -- no `ExperimentDefinitionTask` row is written -- context escape flags are rejected -- slug-only dynamic authoring is rejected -- dependency edges are written through core -- cycle-creating dependencies are rejected without partial writes -- non-descendant lifecycle mutation is blocked -- human CLI parser rejects `workflow manage ...` - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli ergon_builtins/ergon_builtins -uv run pytest ergon_cli/tests/unit/cli/test_workflow_cli.py -uv run pytest ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py -uv run pytest ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py -uv run pytest ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py -``` - -## PR Ledger - -- **Invariant landed:** dynamic subtask authoring is builtins over - `WorkerContext`, not CLI. -- **Bridge code introduced:** builtins-owned `workflow(command)` adapter. -- **Old path still intentionally alive:** CLI workflow inspection commands. -- **Deletion gate:** PR 06 removes old `ergon_cli.commands.workflow` - compatibility references. -- **Tests added or updated:** builtins workflow adapter integration tests, - subtask lifecycle toolkit tests, CLI parser deletion tests. -- **Modules owned by this PR:** `ergon_builtins.tools.workflow_cli_tool`, - `workflow_command_adapter.py`, `dynamic_task_factory.py`, - `ergon_cli.commands.workflow`. - -## Out Of Scope - -- Full workflow inspection rewrite. -- Human CLI domain folder migration. -- Generic command language in core. - -## Depends On - -- PR 00 is recommended first to reduce workflow CLI churn. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/02-cli-composition-root-and-parser-split.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/02-cli-composition-root-and-parser-split.md deleted file mode 100644 index 7c88d9486..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/02-cli-composition-root-and-parser-split.md +++ /dev/null @@ -1,199 +0,0 @@ -# PR 02: CLI Composition Root And Parser Split - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `main.py` a console-script shim and move parser registration -into domain-local modules without changing command semantics. - -**Architecture:** Introduce the final domain package names while leaving old -command implementation modules in place. This is a composition-only PR. - -**Tech Stack:** argparse, pytest parser tests, existing command handlers. - ---- - -## Goal - -Make `ergon_cli.main` a thin composition root by moving parser registration into -domain-local modules without changing command behavior. - -## Source PRDs - -- `../README.md` -- `../02-target-folder-shape.md` - -## Scope - -## Current State - -`ergon_cli.main.build_parser()` defines all command groups, all flags, and the -top-level dispatch maps in one file. Every CLI change touches the central -entrypoint. - -## Target State For This PR - -`main.py` delegates to `app.py`; `app.py` calls one parser registration function -per domain. Existing command handlers still live under `ergon_cli.commands`. - -## Tasks - -### Add app composition module - -Create: - -- `ergon_cli/ergon_cli/app.py` - -Modify: - -- `ergon_cli/ergon_cli/main.py` - -Work: - -- [ ] **Step 1: Create `app.py`** - - Add: - - ```python - def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(...) - subparsers = parser.add_subparsers(dest="command") - register_run_parser(subparsers) - register_experiment_parser(subparsers) - ... - return parser - ``` - -- [ ] **Step 2: Shrink `main.py`** - - `main.py` should import `build_parser` from `app.py` and keep the console - entrypoint plus a small dispatch helper. - -- [ ] **Step 3: Preserve behavior** - - Do not change flag names, defaults, or command handlers in this PR. - -Acceptance: - -- `main.py` contains no domain-specific flag definitions after this PR. - -### Create domain parser modules - -Create: - -```text -ergon_cli/ergon_cli/domains/ - __init__.py - benchmarks/parser.py - doctor/parser.py - eval/parser.py - evaluators/parser.py - experiments/parser.py - ingestion/parser.py - onboarding/parser.py - runs/parser.py - stack/parser.py - training/parser.py - workers/parser.py - workflow/parser.py -``` - -Work: - -- [ ] **Step 1: Create domain package skeleton** - - Create the listed `domains/*/parser.py` files with one - `register_<domain>_parser(subparsers)` function per domain. - -- [ ] **Step 2: Move parser code verbatim** - - Move the existing argparse setup from `main.py` into matching parser modules. - Keep imports pointed at old `ergon_cli.commands.*` handlers. - -- [ ] **Step 3: Add parser smoke tests** - - For each command group, parse at least one representative command to ensure - `command` and action fields match the old behavior. - -Acceptance: - -- Adding a flag for one domain does not require editing `main.py`. -- Existing CLI commands parse the same arguments as before. - -### Simplify dispatch - -Modify: - -- `ergon_cli/ergon_cli/app.py` -- `ergon_cli/ergon_cli/main.py` - -Work: - -- [ ] **Step 1: Bind handlers at parse time** - - Prefer: - - ```python - parser.set_defaults(handler=handle_run) - ``` - - on each command group, or keep one central registry if handler binding creates - too much churn. - -- [ ] **Step 2: Add one async-aware dispatch helper** - - ```python - async def dispatch(args: argparse.Namespace) -> int: - result = args.handler(args) - if inspect.isawaitable(result): - return await result - return result - ``` - -- [ ] **Step 3: Preserve help behavior** - - Missing command/help cases should behave as before. - -Acceptance: - -- There is one obvious command execution path. -- Unknown or missing commands still print help and return the expected exit - code. - -## Tests - -Add or update: - -- `ergon_cli/tests/unit/cli/test_parser_registration.py` -- existing command parser tests - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli -uv run pytest ergon_cli/tests/unit/cli -``` - -## PR Ledger - -- **Invariant landed:** `main.py` contains no domain-specific parser - registration. -- **Bridge code introduced:** domain parser modules importing old command - handlers. -- **Old path still intentionally alive:** `ergon_cli.commands.*`. -- **Deletion gate:** PR 06 removes old command package references. -- **Tests added or updated:** parser registration smoke tests. -- **Modules owned by this PR:** `app.py`, `main.py`, `domains/*/parser.py`. - -## Out Of Scope - -- Moving command implementation modules. -- Introducing typed command models. -- Rewriting output/error handling. - -## Depends On - -- PR 00 -- PR 01 if workflow parser churn overlaps diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/03-shared-output-errors-command-models.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/03-shared-output-errors-command-models.md deleted file mode 100644 index 3a065c280..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/03-shared-output-errors-command-models.md +++ /dev/null @@ -1,188 +0,0 @@ -# PR 03: Shared Output, Errors, Exit Codes, And Command Models - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Introduce shared CLI contracts and convert low-risk domains away from -raw `argparse.Namespace` implementation flow. - -**Architecture:** Commands translate argparse into typed Pydantic models; -services operate on typed inputs/results; rendering happens through shared -output helpers. - -**Tech Stack:** Pydantic v2, argparse boundary modules, pytest. - ---- - -## Goal - -Stop pushing `argparse.Namespace` into implementation code by adding shared CLI -contracts for typed commands, output rendering, and error handling. - -## Source PRDs - -- `../README.md` -- `../01-smell-remediation-map.md` -- `../02-target-folder-shape.md` - -## Scope - -## Current State - -Command modules accept `argparse.Namespace` directly, print from helper -functions, raise `SystemExit` ad hoc, and mix user output with domain checks. - -## Target State For This PR - -Converted domains follow: - -```text -argparse.Namespace -> Pydantic command model -> service -> result model -> shared.output -``` - -## Tasks - -### Add shared CLI primitives - -Create: - -```text -ergon_cli/ergon_cli/shared/ - __init__.py - errors.py - exit_codes.py - output.py - parsing.py -``` - -Work: - -- [ ] **Step 1: Add exit codes** - - In `exit_codes.py`: - - ```python - OK = 0 - USAGE = 2 - NOT_FOUND = 3 - RUNTIME_ERROR = 1 - ``` - -- [ ] **Step 2: Add CLI errors** - - In `errors.py`, define `CliError(message: str, exit_code: int)` plus focused - subclasses for usage, not-found, and dependency errors. - -- [ ] **Step 3: Add output helpers** - - In `output.py`, add helpers for table/text/json rendering. Keep printing at - command boundary: - - ```python - def render_json(payload: BaseModel | Mapping[str, object]) -> str: ... - def render_table(headers: Sequence[str], rows: Sequence[Sequence[object]]) -> str: ... - ``` - -- [ ] **Step 4: Add parsing helpers** - - In `parsing.py`, add UUID parsing that raises `CliUsageError`, not raw - `ValueError`. - -Acceptance: - -- Command handlers can return result objects or raise CLI errors without - directly calling `SystemExit`. - -### Convert low-risk domains first - -Modify: - -```text -ergon_cli/ergon_cli/domains/doctor/ -ergon_cli/ergon_cli/domains/stack/ -ergon_cli/ergon_cli/domains/workers/ -ergon_cli/ergon_cli/domains/evaluators/ -``` - -Create per domain as needed: - -```text -commands.py -models.py -service.py -``` - -Work: - -- [ ] **Step 1: Convert worker/evaluator list** - - Add tiny command/result models and route output through `shared.output`. - -- [ ] **Step 2: Convert doctor** - - Add `DoctorCommand` and `DoctorReport`. Check helpers should return data, not - print directly. - -- [ ] **Step 3: Convert stack** - - Add `StackCommand(action: Literal["start", "stop"])` and keep subprocess - execution in a service. - -- [ ] **Step 4: Add boundary assertions** - - Tests should fail if converted services import `argparse`. - -Acceptance: - -- `argparse.Namespace` does not pass below each converted domain's - `commands.py`. -- Converted domains have predictable output behavior. - -### Leave high-coupling domains for later PRs - -Do not convert yet: - -- runs -- experiments -- benchmarks/onboarding metadata ownership -- training optional dependency behavior beyond small error handling cleanup - -## Tests - -Add or update: - -- `ergon_cli/tests/unit/cli/test_shared_output.py` -- `ergon_cli/tests/unit/cli/test_shared_errors.py` -- domain tests for doctor, stack, worker list, evaluator list - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli -uv run pytest ergon_cli/tests/unit/cli -``` - -## PR Ledger - -- **Invariant landed:** converted domains no longer pass `argparse.Namespace` - below `commands.py`. -- **Bridge code introduced:** shared CLI primitives used by only low-risk - domains at first. -- **Old path still intentionally alive:** runs, experiments, benchmarks, - onboarding, training remain partially unconverted. -- **Deletion gate:** PR 06 final architecture tests enforce this globally. -- **Tests added or updated:** shared output/error tests and converted domain - tests. -- **Modules owned by this PR:** `shared/*`, doctor, stack, workers, evaluators. - -## Out Of Scope - -- Direct persistence cleanup in runs/experiments. -- Final folder migration. -- Builtins metadata catalogue. - -## Depends On - -- PR 02 diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/04-runs-experiments-service-boundary.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/04-runs-experiments-service-boundary.md deleted file mode 100644 index 0a4ff43f5..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/04-runs-experiments-service-boundary.md +++ /dev/null @@ -1,218 +0,0 @@ -# PR 04: Runs And Experiments Service Boundary - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove direct SQLModel/session ownership from CLI run and experiment -handlers. - -**Architecture:** CLI domains own parsing, command models, and rendering. -Core views/runtime services own persisted state access. - -**Tech Stack:** Pydantic v2 command/result models, core views/runtime services, -pytest, architecture import guards. - ---- - -## Goal - -Remove direct persistence/session ownership from CLI run and experiment command -handlers. - -## Source PRDs - -- `../README.md` -- `../01-smell-remediation-map.md` -- `../02-target-folder-shape.md` - -## Scope - -## Base Assumption - -This PR must be implemented on top of -[DeepFlow-research/ergon#91](https://github.com/DeepFlow-research/ergon/pull/91) -or later. On that base, `ergon_core.core.application.read_models` is a retired -package. Use `ergon_core.core.views.runs`, `ergon_core.core.views.experiments`, -and narrow `ergon_core.core.application.runtime` APIs instead. - -## Current State - -`commands/run.py` and `commands/experiment.py` import core persistence models, -sessions, and `ensure_db()` directly. They also parse UUIDs and render output -inside the same functions that perform reads. - -## Target State For This PR - -Run and experiment command handlers call CLI services backed by core -views/runtime services. Direct persistence imports disappear from these CLI -domains. - -## Tasks - -### Move runs into a typed CLI domain - -Create: - -```text -ergon_cli/ergon_cli/domains/runs/ - __init__.py - parser.py - commands.py - models.py - service.py -``` - -Modify: - -- `ergon_cli/ergon_cli/commands/run.py` - -Work: - -- [ ] **Step 1: Add run command models** - - ```python - class ListRunsCommand(BaseModel): - limit: int = 20 - status: str | None = None - definition_id: UUID | None = None - experiment: str | None = None - - class RunStatusCommand(BaseModel): - run_id: UUID - - class CancelRunCommand(BaseModel): - run_id: UUID - ``` - -- [ ] **Step 2: Add run result models** - - Add display-focused models such as `RunSummaryView` and `RunStatusView` in - `domains/runs/models.py`. - -- [ ] **Step 3: Add run service** - - `domains/runs/service.py` should call `ergon_core.core.views.runs.service` - for read/display queries and `ergon_core.core.application.runtime.run_records` - for cancellation/runtime lifecycle operations. If no core service exists for - a valid CLI query, add the narrow method to `core/views/runs/service.py` or - `core/application/runtime/run_records.py` instead of importing persistence in - CLI. - -- [ ] **Step 4: Remove direct persistence imports** - - `domains/runs/*` must not import `ergon_core.core.persistence`. - -Acceptance: - -- Runs command handlers do not import SQLModel sessions or persistence models. -- Invalid UUIDs become CLI errors, not tracebacks. - -### Move experiments into a typed CLI domain - -Create: - -```text -ergon_cli/ergon_cli/domains/experiments/ - __init__.py - parser.py - commands.py - models.py - service.py -``` - -Modify: - -- `ergon_cli/ergon_cli/commands/experiment.py` - -Work: - -- [ ] **Step 1: Add experiment command models** - - Add `ListExperimentsCommand`, `ShowExperimentCommand`, `ListTagsCommand`, and - `ListByTagCommand`. - -- [ ] **Step 2: Add experiment result models** - - Use result/view models that match terminal output and JSON rendering needs. - -- [ ] **Step 3: Add experiment service** - - `ShowExperimentCommand` should use `definition_id`, matching the PR #91 CLI - parser and `ExperimentDefinition` identity language. Delegate to - `ergon_core.core.views.experiments.service.ExperimentReadService`. Add a - narrow views method only if the CLI need is valid and not already present. Do - not recreate the retired - `ergon_core.core.application.read_models` package. - -- [ ] **Step 4: Normalize output** - - Replace logging-as-output with `shared.output`. - -Acceptance: - -- Experiment command handlers do not import persistence models directly. -- Logging is not used as the user-facing output channel. - -### Add architecture checks - -Add: - -- `ergon_cli/tests/unit/architecture/test_cli_persistence_boundaries.py` - -Work: - -- [ ] **Step 1: Add import-boundary test** - - Scan converted domain files for forbidden imports: - - ```python - FORBIDDEN = ("ergon_core.core.persistence", "sqlmodel.Session") - ``` - -- [ ] **Step 2: Add temporary exemption list** - - If an unconverted domain still violates the rule, list it explicitly with the - PR that removes the exemption. - -Acceptance: - -- Boundary tests prevent reintroducing direct persistence imports. - -## Tests - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli ergon_core/ergon_core -uv run pytest ergon_cli/tests/unit/cli -uv run pytest ergon_cli/tests/unit/architecture/test_cli_persistence_boundaries.py -``` - -If core services are added, also run their focused tests. - -## PR Ledger - -- **Invariant landed:** run/experiment CLI domains no longer own direct - persistence access. -- **Bridge code introduced:** narrow core views/runtime service methods if - missing. -- **Old path still intentionally alive:** other unconverted domains may still - have direct imports until later PRs. -- **Deletion gate:** PR 06 removes exemptions and old command modules. -- **Tests added or updated:** run/experiment CLI tests, architecture import - boundary tests, focused core service tests if needed. -- **Modules owned by this PR:** runs domain, experiments domain, relevant core - views/runtime services. - -## Out Of Scope - -- Dashboard/views redesign. -- Deleting legacy experiment/cohort compatibility unless already covered by core - refactor PRDs. -- Benchmark/onboarding metadata ownership. - -## Depends On - -- PR 03 diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/05-benchmarks-onboarding-metadata-ownership.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/05-benchmarks-onboarding-metadata-ownership.md deleted file mode 100644 index d5186a1be..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/05-benchmarks-onboarding-metadata-ownership.md +++ /dev/null @@ -1,223 +0,0 @@ -# PR 05: Benchmarks And Onboarding Metadata Ownership - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move benchmark setup/dependency/onboarding truth out of generic CLI -code and toward builtins-owned metadata. - -**Architecture:** Builtins owns benchmark metadata. CLI consumes that metadata -through typed benchmark/onboarding domain services. - -**Tech Stack:** Pydantic v2 metadata models, CLI domain services, pytest. - ---- - -## Goal - -Stop duplicating benchmark template, dependency, and onboarding metadata inside -generic CLI command code. - -## Source PRDs - -- `../README.md` -- `../01-smell-remediation-map.md` -- `../02-target-folder-shape.md` -- `../prds/00-cli-hygiene-audit.md` - -## Scope - -## Current State - -Benchmark setup paths, benchmark dependencies, and onboarding env-key sections -are hardcoded in CLI modules. That creates drift between builtins, onboarding, -and benchmark setup commands. - -## Target State For This PR - -Builtins exposes benchmark metadata; CLI benchmark/onboarding domains consume -that metadata. Every env key has an owner category. - -## Tasks - -### Move benchmark metadata toward builtins ownership - -Create or modify: - -```text -ergon_builtins/ergon_builtins/benchmarks/catalog.py -ergon_builtins/ergon_builtins/benchmarks/<slug>/metadata.py -``` - -Modify: - -```text -ergon_cli/ergon_cli/domains/benchmarks/ -ergon_cli/ergon_cli/commands/benchmark.py -``` - -Work: - -- [ ] **Step 1: Add builtins metadata model** - - Define a Pydantic model similar to: - - ```python - class BenchmarkCliMetadata(BaseModel): - model_config = ConfigDict(frozen=True) - - slug: str - sandbox_template: Path | None = None - required_packages: tuple[str, ...] = () - env_keys: tuple[str, ...] = () - supports_setup: bool = False - ``` - -- [ ] **Step 2: Add catalogue access** - - Add a function such as: - - ```python - def benchmark_cli_metadata() -> Mapping[str, BenchmarkCliMetadata]: ... - ``` - -- [ ] **Step 3: Update CLI benchmark setup** - - Replace local template dictionaries with calls into the metadata catalogue. - -Acceptance: - -- Adding a benchmark sandbox template does not require editing generic CLI - command handlers. -- CLI no longer needs benchmark-specific path arithmetic. - -### Reconcile onboarding dependency metadata - -Move or modify: - -```text -ergon_cli/ergon_cli/onboarding/profile.py -ergon_cli/ergon_cli/domains/onboarding/profile.py -ergon_cli/ergon_cli/domains/onboarding/env_writer.py -``` - -Work: - -- [ ] **Step 1: Verify package extras** - - Compare onboarding recommendations against actual package metadata. - -- [ ] **Step 2: Classify env keys** - - Introduce owner categories: - - ```python - EnvKeyOwner = Literal[ - "core-runtime", - "model-provider-runtime", - "benchmark-evaluation-runtime", - "infra-training-runtime", - "local-networking-dev", - ] - ``` - -- [ ] **Step 3: Decide Tailscale keys** - - Keep Tailscale keys only if they are explicitly classified as - `local-networking-dev`; otherwise delete them. - -- [ ] **Step 4: Update tests** - - Assert every written env key has an owner category. - -Acceptance: - -- Every env key written by onboarding has an owner category. -- Recommended extras install from the repo. -- Onboarding profile tests cover at least one infra/training selection. - -### Move benchmark/onboarding domains into final shape - -Create: - -```text -ergon_cli/ergon_cli/domains/benchmarks/ - __init__.py - parser.py - commands.py - models.py - service.py - templates.py - -ergon_cli/ergon_cli/domains/onboarding/ - __init__.py - parser.py - commands.py - service.py - profile.py - env_writer.py - installer.py - prompts.py -``` - -Work: - -- [ ] **Step 1: Move files into domain package** - - Move onboarding implementation into `domains/onboarding/` and benchmark setup - into `domains/benchmarks/`. - -- [ ] **Step 2: Preserve command UX** - - Existing command names and prompts should continue to work. - -- [ ] **Step 3: Remove redundant compatibility imports** - - Delete old top-level modules only when no production imports remain. - -Acceptance: - -- Benchmark and onboarding command code follows the typed command model pattern - introduced in PR 03. - -## Tests - -Add or update: - -- benchmark metadata/catalog tests -- benchmark setup tests -- onboarding profile tests -- env writer tests - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli ergon_builtins/ergon_builtins -uv run pytest ergon_cli/tests/unit/cli -uv run pytest ergon_builtins/tests/unit -``` - -## PR Ledger - -- **Invariant landed:** benchmark setup/onboarding metadata is no longer owned - by generic CLI command code. -- **Bridge code introduced:** builtins metadata catalogue. -- **Old path still intentionally alive:** any remaining compatibility import - shims until PR 06. -- **Deletion gate:** PR 06 removes old onboarding/benchmark command paths. -- **Tests added or updated:** metadata catalogue, benchmark setup, onboarding - profile/env writer tests. -- **Modules owned by this PR:** builtins benchmark metadata, CLI benchmark - domain, CLI onboarding domain. - -## Out Of Scope - -- Full final deletion of all old compatibility packages. -- Runs/experiments service boundary. - -## Depends On - -- PR 03 -- PR 04 optional, but recommended first if shared patterns are needed diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/06-final-domain-folder-migration.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/06-final-domain-folder-migration.md deleted file mode 100644 index ca26d9992..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/pr-plans/06-final-domain-folder-migration.md +++ /dev/null @@ -1,198 +0,0 @@ -# PR 06: Final Domain Folder Migration And Compatibility Deletion - -> **For agentic workers:** REQUIRED SUB-SKILL: Use -> `superpowers:subagent-driven-development` or -> `superpowers:executing-plans` to implement this plan task-by-task. Steps use -> checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the CLI domain migration and delete transitional import paths. - -**Architecture:** This is the cleanup gate PR. It should mostly move/delete -files and land architecture guards that prevent regression to the old shape. - -**Tech Stack:** pytest architecture tests, ruff, import-boundary scans. - ---- - -## Goal - -Complete the CLI domain migration and delete transitional compatibility modules. - -## Source PRDs - -- `../README.md` -- `../02-target-folder-shape.md` - -## Scope - -## Current State - -Earlier PRs introduced domain modules and shared helpers while preserving some -old import paths for review safety. - -## Target State For This PR - -The final folder shape matches `../02-target-folder-shape.md`; old -compatibility packages are deleted or reduced to documented shims with explicit -deletion gates. - -### Move remaining domains into final folder shape - -Ensure final source tree matches: - -```text -ergon_cli/ergon_cli/ - main.py - app.py - shared/ - domains/ - benchmarks/ - doctor/ - eval/ - evaluators/ - experiments/ - ingestion/ - onboarding/ - runs/ - stack/ - training/ - workers/ - workflow/ -``` - -Work: - -- [ ] **Step 1: Inventory remaining old imports** - - ```bash - rg "ergon_cli\\.(commands|discovery|rendering|onboarding)" ergon - ``` - -- [ ] **Step 2: Move remaining implementation** - - Move remaining code into the final `domains/` or `shared/` owners. - -- [ ] **Step 3: Delete empty packages** - - Delete old packages once all imports are gone. - -- [ ] **Step 4: Preserve console behavior** - - Run parser and command tests before and after file moves. - -Acceptance: - -- Old compatibility packages are deleted or contain only temporary import - shims with deletion comments. - -### Add final architecture tests - -Create or extend: - -```text -ergon_cli/tests/unit/architecture/ - test_cli_domain_boundaries.py - test_cli_import_boundaries.py -``` - -Rules: - -- `argparse.Namespace` does not pass below domain `commands.py`. -- Domain services do not import `argparse`. -- CLI domains do not import `ergon_core.core.persistence`. -- `ergon_builtins` production code does not import `ergon_cli`. -- Generic CLI code does not hardcode benchmark-specific metadata. -- Output printing lives at command/rendering boundaries, not service internals. - -Work: - -- [ ] **Step 1: Add Namespace boundary guard** - - Assert `argparse.Namespace` appears only in parser/commands boundary modules. - -- [ ] **Step 2: Add import guards** - - Assert: - - ```text - ergon_cli domains -> no ergon_core.core.persistence - ergon_builtins production -> no ergon_cli - domain services -> no argparse - ``` - -- [ ] **Step 3: Add benchmark metadata guard** - - Assert generic CLI code does not hardcode benchmark-specific template paths or - dependency tables. - -- [ ] **Step 4: Add output boundary guard** - - Audit converted services for direct `print(` calls and forbid them where the - domain has shared output models. - -Acceptance: - -- Architecture tests fail for the main old smells. - -### Remove temporary shims - -Work: - -- [ ] **Step 1: Delete old shims** - - Remove old modules after architecture tests pass. - -- [ ] **Step 2: Update docs references** - - Replace references to old command module paths in RFC/docs where needed. - -- [ ] **Step 3: Run final search** - - ```bash - rg "ergon_cli.commands|ergon_cli.discovery|ergon_cli.rendering" ergon - ``` - - Expected: no production references, or documented compatibility comments only. - -Acceptance: - -- `rg "ergon_cli.commands" ergon` has no production references unless an - intentional backward-compatibility shim is still documented. -- `rg "ergon_cli.discovery|ergon_cli.rendering" ergon` has no production - references after deletion. - -## Tests - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli ergon_builtins/ergon_builtins ergon_core/ergon_core -uv run pytest ergon_cli/tests/unit -uv run pytest ergon_builtins/tests/unit -uv run pytest ergon_builtins/tests/integration/tools -uv run pytest ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py -``` - -## PR Ledger - -- **Invariant landed:** final CLI domain folder structure is enforced. -- **Bridge code introduced:** none. -- **Old path still intentionally alive:** none unless explicitly documented. -- **Deletion gate:** this PR is the deletion gate. -- **Tests added or updated:** CLI architecture boundaries and import guards. -- **Modules owned by this PR:** all CLI package compatibility/deletion paths. - -## Out Of Scope - -- New CLI features. -- Further workflow product design. -- Core architecture refactors already covered by core RFCs. - -## Depends On - -- PR 00 -- PR 01 -- PR 02 -- PR 03 -- PR 04 -- PR 05 diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/00-cli-hygiene-audit.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/00-cli-hygiene-audit.md deleted file mode 100644 index 2c840ca31..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/00-cli-hygiene-audit.md +++ /dev/null @@ -1,363 +0,0 @@ -# PRD 00: CLI Hygiene Audit And Dead Code Cleanup - -## Goal - -Remove obvious dead code, stale command surface, and accidental drift from -`ergon_cli` before the larger domain refactor begins. - -This PRD is intentionally a pre-refactor cleanup. Its purpose is to avoid -moving dead compatibility helpers, ignored flags, stale static catalogues, or -hardcoded local knowledge into the new domain structure. - -## Problem - -The CLI package is lint-clean, but several files contain code that is dead, -misleading, stale, or too weakly owned to preserve during refactor. - -The highest-risk examples are: - -- no-op bootstrap functions that are still wired into `main.py` -- parsed flags that are never used -- workflow actions exposed by argparse but not implemented for real execution -- static discovery rows that have drifted from builtins slugs -- benchmark sandbox template paths hardcoded directly in command code -- type-only wrappers that add ceremony without improving safety -- direct database/session imports inside command handlers - -These issues make the CLI feel chaotic because the command surface promises -more than the implementation owns, and because operational details are mixed -with domain-facing command behavior. - -## Non-Goals - -This PRD does not perform the full CLI domain restructure. - -It should not: - -- move every command into the target `domains/` tree -- introduce the final shared rendering/error framework -- rewrite persistence-backed commands onto core views/runtime services -- redesign onboarding or benchmark metadata ownership in one pass - -Those belong in later PRDs. This pass should make the current package cleaner -and prevent known-bad code from being carried forward. - -## Proposed Cleanup Items - -### 1. Delete no-op bootstrap compatibility code - -Problem: - -`ergon_cli.bootstrap.register_and_publish_builtins()` is a no-op retained after -object-bound builtins stopped needing explicit CLI publication. `main.py` also -defines `register_default_components()` as a second no-op. - -Solution: - -Delete both no-op functions and remove their conditional calls from `main.py`. -If a compatibility note is still useful, keep it in docs rather than executable -code. - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Delete no-op builtin publication helper | `ergon_cli/bootstrap.py` | remove file if no imports remain | -| Delete no-op default component helper | `ergon_cli/main.py` | remove function | -| Remove conditional bootstrap calls | `ergon_cli/main.py` | direct dispatch only | - -Acceptance: - -- `rg "register_and_publish_builtins|register_default_components"` returns no - production references. -- CLI startup behavior is unchanged. - -### 2. Fix or remove unused command flags - -Problem: - -Several argparse flags are exposed but ignored. This creates false affordances -and makes tests less trustworthy. - -Known cases: - -| Flag | Current Location | Problem | -| --- | --- | --- | -| `doctor --verbose` | `ergon_cli/main.py`, `commands/doctor.py` | parsed but unused | -| `workflow resource-list --explain` | `ergon_cli/main.py`, `commands/workflow.py` | parsed but unused | -| `workflow task-tree --parent-task-id` | `commands/workflow.py` | PR #91 standardizes on task identity; add a regression test so parser/handler naming does not drift again | -| `workflow add-task --depends-on-task-slug` | older `commands/workflow.py` surfaces | removed by the PR #91 base; keep it out of the human CLI | - -Solution: - -For each flag, either implement the promised behavior or remove the flag. The -default choice should be removal unless the behavior is clearly part of the -near-term workflow UX. - -Proposed code and location: - -| Domain | Proposed Change | -| --- | --- | -| doctor | Remove `--verbose`, or thread it into a typed `DoctorCommand(verbose: bool)` and produce extra check details. | -| workflow | Keep `parent_task_id` argument handling covered if `task-tree` remains. | -| workflow | Remove `--explain`, or implement explanation output for resource listing. | -| workflow | Either pass dependency slugs into the workflow service or remove `--depends-on-task-slug` from non-dry-run command surface. | - -Acceptance: - -- No argparse flag is parsed without a production use or an explicit test that - documents no-op compatibility behavior. -- Tests cover `workflow task-tree --parent-task-id`. -- Tests cover the chosen behavior for `--depends-on-task-slug`. - -### 3. Align workflow parser actions with implementation - -Problem: - -`workflow` exposes actions including `add-edge`, `restart-task`, and -`abandon-task`, but non-dry-run execution only implements `add-task`. - -Solution: - -Do one of: - -- remove unsupported actions from the live parser until implemented -- keep them only under `--dry-run`, with clear validation -- implement the corresponding service calls - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Validate supported action set | `ergon_cli/commands/workflow.py` | short-term local guard | -| Later typed command models | `ergon_cli/domains/workflow/models.py` | one command model per action | -| Later executor split | `ergon_cli/domains/workflow/executor.py` | no parser-only actions | - -Acceptance: - -- A user cannot invoke a non-dry-run workflow action that silently does nothing - or raises an implementation surprise. -- Tests cover each action exposed by argparse. - -### 4. Replace stale static discovery rows - -Problem: - -`ergon_cli.discovery` hardcodes benchmark, worker, and evaluator rows. At least -the worker slugs appear stale relative to builtins naming. - -Examples: - -- CLI discovery lists `react-worker` -- builtins docs/code use `react-v1` -- CLI discovery lists `training-stub-worker` -- builtins docs/code use `training-stub` - -Solution: - -Short term, correct the rows and add tests that compare known builtins slugs. -Long term, replace static CLI discovery with metadata from `ergon_builtins`. - -Proposed code and location: - -| Phase | Proposed Location | -| --- | --- | -| Short-term corrected rows | `ergon_cli/discovery/__init__.py` | -| Long-term CLI catalogue service | `ergon_cli/domains/discovery/service.py` or domain-local list commands | -| Source of truth | `ergon_builtins` benchmark/worker/evaluator metadata | - -Acceptance: - -- `ergon worker list` and `ergon evaluator list` show slugs that are valid for - the current builtins package. -- Tests fail if static rows drift from builtins metadata that is available in - process. - -### 5. Move benchmark sandbox template knowledge out of command code - -Problem: - -`commands/benchmark.py` embeds sandbox template paths using -`Path(__file__).parents[3]`. This is brittle and makes the CLI own knowledge -that belongs to benchmark definitions or a benchmark-specific adapter. - -Solution: - -Short term, move template lookup into a small benchmark-local helper so command -handlers do not contain path arithmetic. Long term, expose sandbox template -metadata from `ergon_builtins` and have the CLI consume that. - -Proposed code and location: - -| Phase | Proposed Location | -| --- | --- | -| Short-term helper | `ergon_cli/commands/benchmark_templates.py` or `ergon_cli/domains/benchmarks/templates.py` during refactor | -| Long-term source | `ergon_builtins` benchmark metadata | -| CLI command | `ergon_cli/domains/benchmarks/commands.py` | - -Acceptance: - -- No direct `Path(__file__).parents[...]` sandbox template lookup remains in a - command handler. -- Adding a benchmark sandbox template does not require editing generic command - dispatch code. - -### 6. Remove type theater around build logs - -Problem: - -`BuildLog(Protocol)` in `commands/benchmark.py` only requires `__str__`. It -does not document meaningful behavior and gives a false sense of type safety. - -Solution: - -Replace with `object` and stringify at the boundary, or import/use the real E2B -log type if available without coupling the CLI too tightly. - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Remove `BuildLog` protocol | `ergon_cli/commands/benchmark.py` | inline `object` callback | -| Later typed E2B adapter | `ergon_cli/domains/benchmarks/service.py` | hide SDK-specific callback detail | - -Acceptance: - -- No one-off protocol exists solely to model `str(x)`. -- Build log printing behavior remains unchanged. - -### 7. Decide ownership for CLI-local environment keys - -Problem: - -`onboarding/env_writer.py` includes operational keys such as Tailscale settings. -They may be valid, but they currently appear as unowned static `.env` sections. - -Solution: - -Classify each env key as one of: - -- core runtime -- model/provider runtime -- benchmark/evaluation runtime -- infra/training runtime -- local networking/dev convenience - -Keep the keys that are still valid, but move the ownership explanation into a -typed profile or onboarding metadata layer. - -Proposed code and location: - -| Phase | Proposed Location | -| --- | --- | -| Short-term comments/section cleanup | `ergon_cli/onboarding/env_writer.py` | -| Later typed env key catalogue | `ergon_cli/domains/onboarding/profile.py` | -| Later writer | `ergon_cli/domains/onboarding/env_writer.py` | - -Acceptance: - -- Every env key written by onboarding has a named owner category. -- Local-only convenience keys are either documented as such or removed. - -### 8. Reconcile onboarding dependency drift - -Problem: - -`onboarding/profile.py` duplicates benchmark dependency requirements and refers -to infra extras that may no longer match package metadata. - -Solution: - -Short term, verify the extras against package metadata and correct stale values. -Long term, source benchmark and infrastructure dependency recommendations from -the package/domain that owns them. - -Proposed code and location: - -| Dependency Knowledge | Current Location | Long-Term Owner | -| --- | --- | --- | -| benchmark Python packages | `ergon_cli/onboarding/profile.py` | `ergon_builtins` benchmark metadata | -| infra/training extras | `ergon_cli/onboarding/profile.py` | package metadata or infra domain metadata | - -Acceptance: - -- Recommended extras install successfully from the repo. -- Onboarding tests cover at least one profile that selects infra/training - dependencies. - -### 9. Mark persistence-backed command handlers for later service extraction - -Problem: - -`commands/run.py` and `commands/experiment.py` directly import database/session -internals. This is larger than a hygiene patch, but it should be explicitly -marked so the refactor does not preserve the current boundary. - -Solution: - -Do not rewrite these in the hygiene PR unless tiny. Add the service extraction -to the later CLI domain PRDs and avoid worsening the direct DB coupling. - -Proposed code and location: - -| Command Area | Current Location | Later Target | -| --- | --- | --- | -| runs | `ergon_cli/commands/run.py` | `ergon_cli/domains/runs/service.py` delegating to core views/runtime services | -| experiments | `ergon_cli/commands/experiment.py` | `ergon_cli/domains/experiments/service.py` delegating to core views services | - -Acceptance: - -- Hygiene PR does not add new direct DB/session imports. -- Later refactor PR has a specific task to replace direct persistence reads. - -## Test Plan - -Add or update focused CLI unit tests for: - -- no bootstrap compatibility calls during startup -- `doctor` parser behavior after `--verbose` decision -- `workflow task-tree --parent-task-id` -- workflow unsupported action behavior -- workflow dependency slug behavior -- worker/evaluator list slugs -- benchmark setup template lookup helper - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli -uv run pytest ergon_cli/tests/unit/cli -``` - -## Proposed PR Shape - -This should be one small PR before the larger CLI domain migration: - -1. Delete no-op bootstrap/default component code. -2. Fix or remove ignored flags. -3. Align workflow parser actions with implementation. -4. Correct stale discovery slugs. -5. Extract benchmark template lookup out of command code. -6. Remove `BuildLog(Protocol)`. -7. Add focused tests for the above. - -## Open Questions - -- Should `workflow add-edge`, `restart-task`, and `abandon-task` be implemented - now, or removed until the workflow domain refactor? -- Should `doctor --verbose` be kept as a real detail mode, or deleted for now? -- Should static discovery survive as a compatibility shim, or should this PR - immediately read from builtins metadata? -- Are Tailscale env keys still part of supported local development, or are they - local operational residue? - -## Acceptance Criteria - -- Obvious no-op compatibility code is deleted. -- No CLI flag in the audited set is silently ignored. -- Workflow parser actions match executable behavior. -- Discovery rows no longer advertise stale builtins slugs. -- Benchmark sandbox template lookup no longer lives inline in the command - handler. -- The cleanup is covered by focused unit tests and does not change unrelated CLI - behavior. diff --git a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/01-dynamic-subtask-authoring-toolkit.md b/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/01-dynamic-subtask-authoring-toolkit.md deleted file mode 100644 index 35fc358da..000000000 --- a/docs/rfcs/active/2026-05-19-cli-domain-structure-standardization/prds/01-dynamic-subtask-authoring-toolkit.md +++ /dev/null @@ -1,602 +0,0 @@ -# PRD 01: Move Dynamic Subtask Authoring Out Of CLI - -## Goal - -Make dynamic subtask authoring a real object-bound runtime capability exposed to -agents through builtins toolkits, not through `ergon_cli` command handlers. - -The end state is: - -- `ergon_core.api.WorkerContext.spawn_task(Task(...))` remains the canonical - authoring API. -- `ergon_builtins` owns agent-facing tools that wrap that API. -- `ergon_cli` no longer exposes or implements live dynamic task creation. -- A toy integration test proves an agent/tool can spawn a real dynamic child - task under a benchmark sample. - -## Problem - -The current CLI command surface suggests that dynamic task authoring can happen -through: - -```bash -ergon workflow ... manage add-task --task-slug ... --description ... --worker ... -``` - -That shape is now wrong for the v2 object-bound architecture. It only carries -strings and slugs, while the real runtime contract requires a full -`Task(...)` object with inline worker, sandbox, and evaluator configuration. - -Core already enforces this direction: - -- `WorkflowService.add_task(..., dry_run=False)` raises with the message: - `add-task requires an object-bound Task in the final v2 schema; use WorkerContext.spawn_task(Task(...)) for dynamic tasks.` -- `TaskManagementService.add_subtask()` and `plan_subtasks()` reject the retired - slug-based API. -- `WorkerContext.spawn_task(Task(...))` is the sanctioned path and writes the - full task snapshot into `run_graph_nodes.task_json` with `is_dynamic=True`. - -The current package split also creates an inverted dependency: - -`ergon_builtins.tools.workflow_cli_tool` imports `ergon_cli.commands.workflow` -to expose a single agent-facing `workflow(command)` tool. - -That means a builtins agent toolkit depends on the human/operator CLI package. -The ownership should be reversed conceptually: builtins can expose a CLI-like -agent tool, but the implementation must wrap core APIs directly or use a -builtins-owned parser, not import `ergon_cli`. - -## Non-Goals - -This PRD does not redesign every workflow inspection command. - -It should not: - -- remove useful workflow inspection from human CLI if it remains operator-facing -- rewrite the full workflow resource browser -- redesign all builtins toolkits -- create a generic string-command language in core -- allow workers to mutate arbitrary graph nodes outside their containment scope - -## Architecture Decision - -Dynamic subtask creation is an authoring/runtime API, not a CLI domain. - -The layering should become: - -```text -ergon_core.api - WorkerContext.spawn_task(Task(...)) - WorkerContext.cancel_task(...) - WorkerContext.refine_task(...) - WorkerContext.restart_task(...) - WorkerContext.get_task(...) - -ergon_builtins.tools - SubtaskLifecycleToolkit - typed agent tools over WorkerContext - - WorkflowCliToolkit - optional single string-command tool for agents - implemented in builtins, wrapping core/builtins capabilities - -ergon_cli - human/operator commands only - no live dynamic task authoring command -``` - -The agent-facing CLI-like tool is allowed to exist, but it is a toolkit -implementation detail in `ergon_builtins`, not the public CLI package's -responsibility. - -## Proposed Changes - -### 1. Delete workflow management from `ergon_cli` - -Problem: - -`ergon_cli.commands.workflow` exposes `manage add-task`, `add-edge`, -`restart-task`, and `abandon-task`. Only dry-run behavior is supported for most -actions, and live `add-task` delegates to a core service that intentionally -rejects slug-based creation. - -Solution: - -Delete the whole `workflow manage` branch from the human CLI. The command-shaped -workflow management grammar belongs to the builtins agent toolkit, where it can -wrap `WorkerContext` and construct object-bound tasks. `ergon_cli` may keep -workflow inspection only if it is useful for operator debugging, but it must not -own graph mutation or sandbox/task management commands. - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Delete `manage` parser branch | `ergon_cli/commands/workflow.py` | no human CLI workflow management surface | -| Delete `_handle_manage` and manage dispatch | `ergon_cli/commands/workflow.py` | builtins owns command-shaped management | -| Move/replace resource materialization if agent-facing | `ergon_cli/commands/workflow.py` | builtins workflow toolkit or typed builtins tool | -| Keep inspect commands if useful | `ergon_cli/commands/workflow.py` now; later `ergon_cli/domains/workflow/` | operator/debug surface only | - -Acceptance: - -- `ergon_cli` no longer registers `workflow manage`. -- `ergon_cli.commands.workflow` no longer implements `_handle_manage`. -- `ergon_cli` no longer imports workflow mutation services for graph management. -- CLI parser tests assert `ergon workflow manage ...` is not a valid human CLI - command. -- Builtins integration tests assert `workflow("manage add-subtask ...")` still - works for agents. - -### 2. Move the single-command agent workflow tool into builtins ownership - -Problem: - -`ergon_builtins.tools.workflow_cli_tool` imports -`ergon_cli.commands.workflow.execute_workflow_command`. That makes a builtins -agent tool depend on the CLI package. - -Solution: - -Introduce a builtins-owned workflow command adapter. It may keep the -`workflow(command: str)` UX for agents, but its implementation should live under -`ergon_builtins.tools` and call core/public runtime facades directly. - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Stop importing `ergon_cli.commands.workflow` | `ergon_builtins/tools/workflow_cli_tool.py` | remove CLI dependency | -| Add builtins parser/dispatcher for agent command string | new `ergon_builtins/tools/workflow_command_adapter.py` | parses agent workflow commands | -| Keep factory name or provide compatibility wrapper | `ergon_builtins/tools/workflow_cli_tool.py` | delegates to builtins adapter | - -Initial command scope: - -```text -inspect resource-list -inspect resource-content -inspect task-tree -inspect task-dependencies -inspect next-actions -manage add-subtask -``` - -The `manage add-subtask` command must create an object-bound `Task`, not a -slug-only mutation. - -Acceptance: - -- `rg "ergon_cli" ergon_builtins/ergon_builtins/tools` returns no production - imports. -- The agent-facing `workflow(command)` tool still injects run/task/execution - identity from `WorkerContext`. -- The command adapter rejects context flags supplied by the model. - -### 3. Add reusable toy object-bound fixtures for integration tests - -Problem: - -To prove dynamic subtask spawning works end-to-end, tests need a small -serializable task/worker/sandbox combination that is not tied to an external -benchmark service. The same fixture should also be reused for negative tests so -the builtins adapter proves it respects core's authoring laws instead of only -passing the happy path. - -Solution: - -Add reusable test fixtures that build a complete object-bound `Task`, a parent -run graph node, and a `WorkerContext` wired through the normal runtime facade. -Use those fixtures for both successful dynamic spawning and prohibited mutation -tests. - -Proposed toy classes: - -```python -class ToySandbox(Sandbox): - type_slug: ClassVar[str] = "toy-sandbox" - - async def provision(self) -> None: - return None - - -class ToyWorker(Worker): - type_slug: ClassVar[str] = "toy-worker" - - async def execute( - self, - task: Task, - *, - context: WorkerContext, - ) -> AsyncGenerator[WorkerStreamItem, None]: - yield WorkerOutput(output=f"completed {task.task_slug}", success=True) -``` - -Proposed code and location: - -| Test Asset | Location | -| --- | --- | -| Toy task, worker, sandbox, and graph fixtures | `ergon_builtins/tests/fixtures/toy_workflow.py` | -| Adapter tests | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | -| Typed lifecycle toolkit tests | `ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` | -| Core spawn assertions can reuse existing helpers | `ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py` | - -Acceptance: - -- The toy spawned task persists as a `RunGraphNode`. -- The node has `is_dynamic=True`. -- The node's `task_json` contains a serializable `_type` discriminator for the - task, worker, and sandbox. -- No `ExperimentDefinitionTask` row is written for the dynamic child. -- The same fixture is reused by tests that assert prohibited authoring paths are - blocked. -- The fixture does not require E2B, Docker, provider SDKs, or external benchmark - assets. - -Required fixture API: - -```python -class ToyWorkflowHarness(BaseModel): - model_config = ConfigDict( - arbitrary_types_allowed=True, - frozen=True, - ) - - run_id: UUID - parent_task_id: UUID - parent_task: Task - context: WorkerContext - - def child_task(self, *, task_slug: str, description: str) -> Task: ... - def nodes(self) -> list[RunGraphNode]: ... - def edges(self) -> list[RunGraphEdge]: ... - def definition_tasks(self) -> list[ExperimentDefinitionTask]: ... -``` - -The harness should hide session setup details from tests while still letting -assertions inspect graph rows directly. - -### 3a. Reuse the toy fixture for authoring-law tests - -Problem: - -A happy-path spawn test can pass even if the builtins adapter accidentally -creates side doors for graph mutation. The integration suite should verify that -agent-facing tools cannot bypass core's runtime rules. - -Solution: - -Use the toy harness to assert the builtins workflow adapter delegates to -`WorkerContext` and `WorkflowGraphRepository` rather than performing its own -graph writes. - -Required test cases: - -| Test | Location | Expected Behavior | -| --- | --- | --- | -| Happy-path child spawn | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | `manage add-subtask` writes one `RunGraphNode` with `is_dynamic=True` and object-bound `task_json`. | -| No definition-row write | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | Spawning a child leaves `ExperimentDefinitionTask` count unchanged. | -| Context escape blocked | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | Agent-supplied `--run-id`, `--node-id`, `--execution-id`, or `--sandbox-task-key` is rejected before service calls. | -| Slug-only task creation blocked | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | Commands that try to specify only `--worker`/`--task-slug` without building a `Task` return a clear object-bound error. | -| Dependency edge is written through core | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | `manage add-subtask --depends-on <node-id>` creates a graph edge through `WorkerContext.spawn_task(..., depends_on=...)`. | -| Cycle-creating dependency is rejected | `ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` | A dependency request that would create a graph cycle raises/returns the core `CycleError` message and writes no new edge. | -| Non-descendant lifecycle mutation blocked | `ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` | `cancel_task`, `refine_task`, `restart_task`, or `get_task` against a non-descendant returns `ContainmentViolation`. | -| Typed toolkit spawn uses same path | `ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` | `SubtaskLifecycleToolkit.add_subtask(task)` writes the same dynamic graph shape as the CLI-like adapter. | - -Acceptance: - -- Negative tests assert row counts before and after the rejected call. -- Rejected calls do not leave partial `RunGraphNode` or `RunGraphEdge` rows. -- Tests compare failure messages to core errors where the core error is already - part of the public/runtime contract. -- The builtins adapter has no direct SQLModel insert path for dynamic nodes or - edges. - -### 4. Implement real dynamic spawning in the builtins toolkit - -Problem: - -The current single-command workflow tool can only dry-run `add-task` through the -CLI path. That does not exercise the real object-bound dynamic authoring API. - -Solution: - -Add a builtins-owned command that maps an agent request to an object-bound -`Task`. For the first implementation, keep the supported live-spawn shape small -and explicit. - -Recommended first command: - -```text -manage add-subtask --task-slug <slug> --description <description> [--depends-on <node-id> ...] -``` - -The adapter should construct the child task by copying the current task's -runtime contract: - -- `instance_key` from the current task -- `worker` from the current task, or a benchmark/toolkit-provided child worker -- `sandbox` from the current task, if the sandbox object is reusable -- `evaluators` from the current task, unless the benchmark toolkit provides a - narrower evaluator set - -If copying the current task's sandbox/worker/evaluators is not valid for a -benchmark, that benchmark should provide a task factory to the toolkit. - -Proposed code and location: - -| Responsibility | Location | -| --- | --- | -| Agent command parsing | `ergon_builtins/tools/workflow_command_adapter.py` | -| Object-bound child task construction | `ergon_builtins/tools/dynamic_task_factory.py` | -| Existing `workflow(command)` factory | `ergon_builtins/tools/workflow_cli_tool.py` | -| Typed lifecycle tools | `ergon_builtins/tools/subtask_lifecycle_toolkit.py` | - -Proposed interface: - -```python -class DynamicTaskFactory(Protocol): - def child_task( - self, - *, - parent: Task, - task_slug: str, - description: str, - ) -> Task: ... -``` - -Default behavior: - -```python -def default_child_task_factory( - *, - parent: Task, - task_slug: str, - description: str, -) -> Task: - return parent.model_copy( - update={ - "task_slug": task_slug, - "description": description, - "parent_task_slug": parent.task_slug, - } - ) -``` - -Acceptance: - -- The builtins workflow tool can spawn a real dynamic child task in an - integration test. -- The child task is object-bound and round-trippable through `Task.from_definition`. -- Benchmarks can override child task construction without editing core. - -### 5. Keep typed subtask lifecycle tools as the preferred agent API - -Problem: - -A single string-command tool is attractive for prompt simplicity, but typed -tools are safer and easier to validate. - -Solution: - -Treat `SubtaskLifecycleToolkit` as the preferred structured API. The CLI-like -`workflow(command)` tool is an alternate UX for agents that benefit from a -shell-style command vocabulary. - -Proposed code and location: - -| Action | Current Location | Target | -| --- | --- | --- | -| Ensure `add_subtask(task: Task, depends_on: list[str] | None)` remains object-bound | `ergon_builtins/tools/subtask_lifecycle_toolkit.py` | keep as preferred typed tool | -| Fix stale comments about missing containment if core now enforces it | `ergon_builtins/tools/subtask_lifecycle_toolkit.py` | docs match `WorkerContext` behavior | -| Add tests for typed toolkit spawning | `ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` | proves typed path still works | - -Acceptance: - -- `SubtaskLifecycleToolkit.add_subtask` delegates to `WorkerContext.spawn_task`. -- Cancellation/refinement/restart/get operations go through `WorkerContext`. -- Comments do not claim service-layer containment gaps that are now handled by - the facade. - -## Integration Test Plan - -### Test 1: builtins workflow command spawns a toy dynamic child - -File: - -`ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` - -Scenario: - -1. Create a test run and a parent `RunGraphNode` with an object-bound toy task - snapshot. -2. Construct a `WorkerContext` for the parent node using the normal runtime - service injection path. -3. Build `workflow(command)` from the builtins toolkit. -4. Call: - - ```python - await workflow('manage add-subtask --task-slug child --description "child work"') - ``` - -5. Query `RunGraphNode` rows. -6. Assert exactly one dynamic child exists. -7. Assert no `ExperimentDefinitionTask` row was inserted. -8. Assert `Task.from_definition(child.task_json)` reconstructs a task whose - `task_slug == "child"`. - -Expected result: - -- The test passes without importing `ergon_cli`. -- The dynamic child is persisted through `WorkerContext.spawn_task`. - -### Test 2: builtins workflow command rejects context escape flags - -File: - -`ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` - -Scenario: - -Call: - -```python -await workflow('inspect task-tree --run-id 00000000-0000-0000-0000-000000000000') -``` - -Expected result: - -- The tool returns an error explaining that context flags are injected by the - runtime and cannot be supplied by the agent. -- No query or mutation is attempted for the supplied fake ID. - -### Test 3: human CLI has no workflow management surface - -File: - -`ergon_cli/tests/unit/cli/test_workflow_cli.py` - -Scenario: - -Call the CLI parser or entrypoint with: - -```text -workflow manage add-task --task-slug child --description "child work" --worker toy-worker -``` - -Expected result: - -- The command is rejected by argparse because `manage` is not a registered - human CLI workflow subcommand. -- No graph node is inserted. - -### Test 4: builtins workflow command writes dependency edges through core - -File: - -`ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` - -Scenario: - -1. Use the toy harness to create a parent node and an existing sibling/child - dependency source node. -2. Call: - - ```python - await workflow( - f'manage add-subtask --task-slug child --description "child work" ' - f'--depends-on {dependency_source_id}' - ) - ``` - -3. Query `RunGraphEdge` rows. - -Expected result: - -- Exactly one edge is created from `dependency_source_id` to the spawned child. -- The edge is created by core graph APIs, not by adapter-local SQLModel writes. -- The spawned task is not dispatched immediately if it has unsatisfied - dependencies. - -### Test 5: builtins workflow command cannot create cyclic dependencies - -File: - -`ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py` - -Scenario: - -1. Use the toy harness to create a graph where adding the requested dependency - would create a cycle. -2. Call the workflow command that would request that dependency. - -Expected result: - -- The tool returns/raises the core `CycleError` message. -- `RunGraphEdge` count is unchanged. -- `RunGraphNode` count is unchanged unless the implementation validates the - dependency before spawning; if the implementation creates the node first, that - behavior is rejected and the test should fail. - -### Test 6: typed subtask lifecycle toolkit enforces containment - -File: - -`ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py` - -Scenario: - -1. Use the toy harness to create two unrelated task subtrees in the same run. -2. Build `SubtaskLifecycleToolkit(context=parent_context)`. -3. Attempt `cancel_task`, `refine_task`, `restart_task`, and `get_subtask` - against a node outside `parent_context`'s descendant tree. - -Expected result: - -- Each operation returns a failure containing `ContainmentViolation` or raises - that exception before mutation. -- Node status, description, and graph rows remain unchanged. - -## Proposed PR Shape - -This PR should come after PRD 00's basic hygiene cleanup or include only the -minimal overlapping workflow flag cleanup needed to avoid conflicts. - -Recommended order: - -1. Add integration tests that demonstrate the desired builtins-owned dynamic - spawning behavior and initially fail. -2. Add `ergon_builtins.tools.dynamic_task_factory`. -3. Add `ergon_builtins.tools.workflow_command_adapter`. -4. Update `ergon_builtins.tools.workflow_cli_tool` to use the builtins adapter - instead of importing `ergon_cli`. -5. Delete `workflow manage` from `ergon_cli.commands.workflow`; keep any - agent-facing management grammar only in builtins. -6. Update tests and docs references. - -## Verification Commands - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli ergon_builtins/ergon_builtins -uv run pytest ergon_cli/tests/unit/cli/test_workflow_cli.py -uv run pytest ergon_builtins/tests/integration/tools/test_workflow_command_adapter.py -uv run pytest ergon_builtins/tests/integration/tools/test_subtask_lifecycle_toolkit.py -uv run pytest ergon_core/tests/unit/runtime/test_spawn_dynamic_task.py -``` - -## Open Decisions - -### Should the CLI-like builtins tool support live `add-subtask`? - -Recommended decision: - -Yes, but only through object-bound task construction. It must never accept a -worker slug and synthesize an incomplete graph node. - -### Should the builtins workflow command copy the parent task by default? - -Recommended decision: - -Yes for the first toy/integration path, because it proves the API without -inventing benchmark-specific factories too early. Benchmarks with special child -task requirements can provide a `DynamicTaskFactory`. - -### Should human CLI keep workflow inspect commands? - -Recommended decision: - -Yes, if useful for debugging. Inspect commands are operator/debug behavior and -do not conflict with authoring ownership. Mutation commands should either be -dry-run-only or removed until they have an operator-grade use case. - -## Acceptance Criteria - -- `ergon_builtins` no longer imports `ergon_cli` in production code. -- `ergon_cli` no longer owns real dynamic subtask authoring. -- A builtins-owned agent tool can spawn a toy dynamic child task using - `WorkerContext.spawn_task(Task(...))`. -- The integration test proves the child is persisted as `is_dynamic=True` with - full object-bound `task_json`. -- No dynamic child creation path writes `experiment_definition_tasks`. -- The docs and error messages point authors toward - `WorkerContext.spawn_task(Task(...))`, not CLI graph mutation. diff --git a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/01-logic-smell-audit.md b/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/01-logic-smell-audit.md deleted file mode 100644 index 8d028fde7..000000000 --- a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/01-logic-smell-audit.md +++ /dev/null @@ -1,885 +0,0 @@ -# Logic Smell Audit - -This audit looks at `ergon_builtins` in `ergon-core-refactor-pr01` as example -code for public authors. The bar is higher than "works": these modules should -model the style we want external benchmark authors to copy. - -## Evaluation criteria - -For each smell, ask: - -1. Is this logic necessary in built-ins at all? -2. If necessary, does it live at the right domain boundary? -3. Could it be expressed with fewer concepts, less inheritance, fewer - compatibility paths, or simpler Python? -4. Does it depend only on the public API unless it is explicitly an adapter? -5. Would this be a good example for an external benchmark author? - -## High-priority smells - -### 1. `_manager_backed.py` should become an explicit E2B runtime + E2B sandbox base - -File: - -- `ergon_builtins/ergon_builtins/sandbox/_manager_backed.py` - -Current shape: - -- imports `ergon_core.core.shared.settings` -- imports `ergon_core.core.infrastructure.sandbox.manager.BaseSandboxManager` -- defines e2b SDK protocols -- defines two runtime adapters -- creates and binds e2b sandboxes -- owns a workspace bootstrap command -- shells out for direct `list_files` - -Necessary parts: - -- an E2B-backed implementation of the public `SandboxRuntime` protocol -- optional E2B SDK import handling -- creating a fresh E2B sandbox for `Sandbox.provision()` -- reconnecting to an existing E2B sandbox for `Sandbox.from_definition(..., sandbox_id=...)` -- shared workspace bootstrap - -Smells: - -- Built-ins reaches into core infrastructure instead of using only public - sandbox interfaces. -- `_ManagerBackedSandboxRuntime` and `_DirectSandboxRuntime` duplicate - `run_command`, `write_file`, `read_file`, `sandbox_id`, and `close_local`. -- `cmd: Sequence[str]` is rendered with `" ".join(cmd)`, which is not shell - safe and gives users the false impression that list-form commands are - argument-safe. -- `list_files()` in `_DirectSandboxRuntime` interpolates `path` into - `find {path}` without shell quoting. -- `close()` semantics differ sharply between the two runtime classes: - manager-backed terminate vs direct kill. The docstring explains it, but the - type surface does not make lifecycle ownership obvious. -- The module name starts private but is effectively a public built-ins adapter - used by benchmark sandboxes. -- The real benchmark sandboxes only call `provision_e2b_runtime()` and - `bind_e2b_runtime()`, both of which return `_DirectSandboxRuntime`. - `_ManagerBackedSandboxRuntime` is not the production path for built-in E2B - benchmarks; it appears to support smoke fixtures. -- Every E2B-backed benchmark sandbox repeats the same `provision()` and - `_bind_runtime()` implementation. - -Cleaner expression: - -```text -sandbox/ - e2b_runtime.py # E2BSandboxRuntime, optional import, SDK protocol - e2b_sandbox.py # E2BSandbox base class for benchmark sandbox specs -``` - -The runtime should be one concrete implementation of `SandboxRuntime`: - -```python -class E2BSandboxRuntime: - def __init__(self, sandbox: E2BSandboxHandle) -> None: - self._sandbox = sandbox - self.sandbox_id = sandbox.sandbox_id - - @classmethod - async def create( - cls, - *, - template: str | None, - envs: dict[str, str] | None, - timeout_seconds: int | None, - ) -> "E2BSandboxRuntime": - sandbox = await create_e2b_sandbox( - template=template, - envs=envs, - timeout_seconds=timeout_seconds, - ) - await bootstrap_workspace(sandbox) - return cls(sandbox) - - @classmethod - async def connect(cls, sandbox_id: str) -> "E2BSandboxRuntime": - return cls(await connect_e2b_sandbox(sandbox_id)) - - async def run_command(...): ... - async def write_file(...): ... - async def read_file(...): ... - async def list_files(...): ... - async def close(self) -> None: ... - async def close_local(self) -> None: ... -``` - -Benchmark-specific E2B sandboxes should inherit a shared base class: - -```python -class E2BSandbox(Sandbox): - template: str | None = None - - async def provision(self) -> None: - object.__setattr__(self, "_runtime", await self._runtime_create()) - - async def _bind_runtime(self, sandbox_id: str) -> None: - object.__setattr__(self, "_runtime", await self._runtime_connect(sandbox_id)) - - async def _runtime_create(self) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.create( - template=self.template, - envs=self.env or None, - timeout_seconds=self.timeout_seconds, - ) - - async def _runtime_connect(self, sandbox_id: str) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.connect(sandbox_id) -``` - -Then benchmark sandboxes become config-only in the common case: - -```python -class LeanSandbox(E2BSandbox): - template: str = "ergon-minif2f-v1" - requires_network: bool = False - output_path: str = "/workspace/final_output/" - lean_version: str = "4.7.0" -``` - -This removes the misleading manager-backed naming, removes duplicated -provision/bind boilerplate from every benchmark sandbox, and keeps built-ins -from importing `BaseSandboxManager`. - -### 2. `cloud_passthrough.py` is probably unnecessary - -File: - -- `ergon_builtins/ergon_builtins/models/cloud_passthrough.py` - -Current shape: - -```python -def resolve_cloud( - target: str, - *, - model_name: str | None = None, - policy_version: str | None = None, - api_key: str | None = None, -) -> ResolvedModel: - return ResolvedModel(model=target, supports_logprobs=False) -``` - -Necessary parts: - -- The behavior is already the fallback behavior of `resolve_model_target()`. - -Smells: - -- `model_name`, `policy_version`, and `api_key` are unused. -- The docstring says "PydanticAI's infer_model", but the function does not call - anything; it just returns the string target. -- Keeping this resolver encourages a fake abstraction: it has the same effect - as doing nothing unless registered elsewhere. - -Cleaner expression: - -- Delete this file unless there is a real backend registration path that needs - a callable. -- If a callable is needed for registry symmetry, name it - `resolve_passthrough_target`, accept only `target`, and document that the - resolver intentionally returns a string for PydanticAI to infer later. -- If `api_key` support matters, this should not be passthrough; use concrete - provider adapters such as OpenRouter or OpenAI Responses. - -### 3. Model resolution mixes routing, policy, and capture settings - -File: - -- `ergon_builtins/ergon_builtins/models/resolution.py` - -Necessary parts: - -- prefix-based resolution is useful -- capture settings are useful for richer transcripts and logprobs - -Smells: - -- Model policy is embedded in string-prefix checks: - `claude-opus-4.7`, `anthropic/claude-opus-4`, `openai/gpt-5`, - `google/gemini-3`, `moonshotai/kimi-k2`. -- `capture_model_settings_for()` knows provider-specific PydanticAI setting - names for Anthropic, OpenRouter, OpenAI Responses, and Gemini. -- Prefix routing and transcript-capture policy change for different reasons - but live in one module. - -Cleaner expression: - -```text -models/ - resolution.py # resolve target -> ResolvedModel - capture_settings.py # target -> model settings - policies.py # provider/model policy table -``` - -Prefer a data table over a chain of prefix checks: - -```python -CAPTURE_POLICIES = ( - CapturePolicy(prefix="vllm", requires_logprobs=True, settings=OPENAI_LOGPROBS), - CapturePolicy(prefix="openai-responses", settings=OPENAI_RESPONSES_REASONING), - CapturePolicy(prefix="google", settings=GEMINI_THINKING), -) -``` - -Provider-specific exceptional cases can still be functions, but they should be -named as policy, not hidden inside generic model resolution. - -### 4. ResearchRubrics judge criterion crosses the public API boundary - -File: - -- `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py` - -Current shape: - -- imports `RunResourceView` -- imports `RunResourceRepository` -- imports `get_session` -- imports `RunResourceKind` -- reads resource files from local paths -- builds prompts -- calls the LLM judge -- formats `CriterionEvidence` - -Necessary parts: - -- ResearchRubrics needs final report evidence. -- The criterion needs prompt construction and LLM judgment. - -Smells: - -- Benchmark-specific criterion imports core application and persistence - internals directly. -- Evidence loading, resource classification, file IO, prompt construction, - model calling, and outcome construction are all in one class. -- The criterion is hard to unit-test without patching DB/session/file access. -- `rubric_text` mirrors `rubric.criterion`, which is useful for snapshots, but - currently looks like denormalized state with unclear source of truth. - -Cleaner expression: - -```text -benchmarks/researchrubrics/criteria/ - judge.py # Criterion subclass only - evidence.py # ResearchRubricsEvidenceLoader / DTOs - prompts.py # prompt rendering -``` - -The criterion should depend on a public context method or an injected loader: - -```python -class ResearchRubricsJudgeCriterion(Criterion): - evidence_loader: ResearchRubricsEvidenceLoader = Field(default_factory=...) - - async def evaluate(self, context: CriterionContext) -> CriterionOutcome: - evidence = await self.evidence_loader.load(context) - verdict = await judge_report(...) - return build_outcome(...) -``` - -Longer-term, `CriterionContext` should expose a public resource-reading -capability so benchmark criteria do not need repositories or sessions. - -### 5. `graph_toolkit.py` is useful behavior, but not library-tier as written - -File: - -- `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` - -Necessary parts: - -- agents need run-scoped resource discovery -- graph/resource inspection tools are useful shared built-ins - -Smells: - -- The toolkit imports repositories, ORM rows, and `get_session` directly. -- Each nested tool repeats budget checking and session access. -- `list_child_resources()` and `list_descendant_resources()` open a new DB - session for every child/parent iteration. -- Tool construction and graph/resource query logic are coupled in one class. -- It converts ORM rows directly to LLM-facing DTOs, so persistence concepts - leak into the tool surface. - -Cleaner expression: - -```text -tools/ - graph_toolkit.py # pydantic-ai tool builder - graph_queries.py # pure-ish query service over public context/port - graph_tool_responses.py # LLM-facing DTOs -``` - -The public tool builder should delegate to a query port: - -```python -class ResourceGraphReader(Protocol): - async def list_resources_for_execution(self, execution_id: UUID) -> list[ResourceRef]: ... - async def list_descendant_resources(self, root_execution_id: UUID, max_depth: int) -> list[ResourceRef]: ... -``` - -Then the runtime can provide a repository-backed implementation, and built-ins -can stay library-tier. - -### 6. `SubtaskLifecycleToolkit` exposes removed and not-fully-contained tools - -File: - -- `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` - -Necessary parts: - -- manager agents need safe subtask lifecycle tools -- using `WorkerContext` as the capability boundary is the right direction - -Smells: - -- `plan_subtasks` is still exposed but always returns failure saying it was - removed. -- The class docstring says containment is not fully enforced for some tools. - This is not example-quality for manager-agent tools. -- Several success DTOs return placeholders such as `old_status="unknown"`, - `old_description=""`, `invalidated_node_ids=[]`, and `cascaded_count=0`. -- Broad `except Exception` wraps every tool, which is acceptable at an LLM tool - boundary, but should be normalized through one helper rather than repeated. - -Cleaner expression: - -- Delete `plan_subtasks` from the built tool list if it is removed. -- Do not expose cancel/refine/restart/get unless `WorkerContext` guarantees - subtree containment. -- Return accurate operation results from `WorkerContext` instead of placeholder - fields, or simplify response models to only fields the tool truly knows. -- Centralize error wrapping: - -```python -async def _run_tool(operation: Awaitable[T]) -> T | ToolFailure: - try: - return await operation - except ValueError as exc: - return ToolFailure(error=str(exc)) -``` - -### 7. `ReActWorker` has stale imports and a too-magical final-output path - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -Necessary parts: - -- A reusable ReAct worker is a good shared built-in. -- Transcript adapter usage is needed for generation capture. - -Smells: - -- Imports `ContextEventService`, `Session`, `Field`, and `UUID` but does not use - them. -- Imports context part models from `ergon_core.core.domain...`, not public API. -- `_run_agent()` mixes model resolution, Logfire configuration, agent - construction, iteration, transcript flushing, iteration-limit handling, and - final output extraction. -- `_latest_final_result_message()` depends on the pydantic-ai structured output - appearing as a tool call named `final_result`; this is subtle and not obvious - from the worker API. - -Cleaner expression: - -```text -workers/ - react_worker.py - react_agent.py # build_agent(...) - react_transcript.py # run and stream transcript chunks - react_output.py # final output extraction -``` - -At minimum: - -- remove unused imports -- move final-output extraction into a named helper module with tests -- prefer public API imports for transcript chunk types, or add a public core - export if none exists - -### 8. `_tools.py` modules are the clearest repeated code smell - -Files: - -- `benchmarks/minif2f/_tools.py` -- `benchmarks/gdpeval/_tools.py` -- `benchmarks/swebench_verified/_tools.py` -- `benchmarks/researchrubrics/_tools.py` - -Necessary parts: - -- runtime tool builders should remain lazy so serializable toolkit configs do - not import heavy runtime dependencies too early - -Smells: - -- each file mixes response models, sandbox operations, parsing helpers, and - pydantic-ai `Tool` construction -- nested functions make the actual domain operations hard to test directly -- many operations catch broad exceptions and return typed failure responses - without a shared convention -- some command paths interpolate user-provided file paths or commands -- MiniF2F has an explicit TODO saying this should be broken down - -Cleaner expression: - -For each benchmark: - -```text -tools/ - response_models.py - operations.py - tool_builder.py -``` - -The tool builder should only adapt operation functions into pydantic-ai tools: - -```python -def build_tools(toolkit: MiniF2FToolkit, *, sandbox: Sandbox, task: Task) -> list[Tool]: - ops = LeanOperations(sandbox=sandbox, workspace=toolkit.lean_workspace) - return [ - Tool(function=ops.write_file, takes_ctx=False), - Tool(function=ops.check_file, takes_ctx=False), - ] -``` - -### 9. `workers/research_rubrics/_run_skill.py` appears obsolete or mislabeled - -File: - -- `ergon_builtins/ergon_builtins/workers/research_rubrics/_run_skill.py` - -Current shape: - -- docstring says this is a stub skill runner that asks the model to produce - plausible tool responses instead of calling real tools -- imports benchmark toolkit response models -- constructs a fresh pydantic-ai agent per skill call - -Smells: - -- If object-bound ResearchRubrics workers now use benchmark-local toolkits, - this old worker package may not be necessary. -- A stub runner that simulates tool calls with another LLM is a surprising - library-tier example unless it is explicitly a test fixture. -- It lives under `workers/research_rubrics`, while the RFC target says - benchmark-specific worker/tool behavior belongs under - `benchmarks/researchrubrics`. - -Cleaner expression: - -- Delete it if no active imports need it. -- If still needed for tests, move to `tests/fixtures` or a clearly named - `benchmarks/researchrubrics/testing.py`. -- If needed as a real fallback mode, rename it to make the behavior explicit, - e.g. `synthetic_skill_runner.py`, and document why simulated tools are - acceptable. - -### 10. Compatibility import surfaces should be retired aggressively - -Files/packages: - -- `ergon_builtins/shared/criteria/*` -- `ergon_builtins/shared/workers/*` -- `ergon_builtins/shared/models/*` -- `ergon_builtins/evaluators/rubrics/swebench_rubric.py` -- `ergon_builtins/evaluators/criteria/*` -- `ergon_builtins/workers/baselines/*` - -Smells: - -- Some are one-line re-exports. -- Some benchmark code still imports through old locations. -- The package offers multiple plausible import paths for the same concept. - -Cleaner expression: - -- Pick canonical concept packages: - `workers`, `models`, `tools`, `sandbox`, `common`, `observability`, - and `benchmarks/<slug>`. -- Update imports in one PR. -- Add architecture tests that reject imports from compatibility packages. - -### 11. `ReActWorker` stores runtime tools on mutable private state - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -Current shape: - -```python -if self.toolkit is not None and task.sandbox is not None: - self._tools = list(self.toolkit.tools(task.sandbox, task)) -async for chunk in self._run_agent(task, context): - yield chunk -``` - -Necessary parts: - -- `ReActWorker` needs benchmark-specific runtime tools. -- Toolkit config needs to round-trip in task JSON. - -Smells: - -- Runtime tools are written onto `self._tools` before execution. That makes a - Pydantic model instance carry per-run mutable state. -- If the same worker instance is ever reused concurrently, tools from one task - can bleed into another task. -- `_run_agent()` reads `self._tools` implicitly instead of receiving the tools - it should use. -- `_seed_messages` has the same hidden runtime-state smell. - -Cleaner expression: - -Pass runtime-only values as local variables: - -```python -async def execute(self, task: Task, *, context: WorkerContext) -> AsyncGenerator[WorkerStreamItem, None]: - tools = self._build_tools(task) - seed_messages = self._build_seed_messages(task, context) - async for chunk in self._run_agent(task, context, tools=tools, seed_messages=seed_messages): - yield chunk -``` - -Then `_run_agent()` becomes a pure execution helper for one run rather than a -method that depends on mutable private attributes. - -### 12. `ReActWorker` has no explicit behavior for missing toolkit sandbox - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -Current behavior: - -- if `toolkit is not None` but `task.sandbox is None`, the worker silently runs - with no tools - -Smell: - -- A benchmark-specific ReAct worker with a toolkit almost certainly requires a - live sandbox. Running without tools produces confusing model behavior instead - of an immediate authoring/runtime error. - -Cleaner expression: - -```python -def _build_tools(self, task: Task) -> list[AgentTool]: - if self.toolkit is None: - return [] - if task.sandbox is None: - raise ValueError(f"{self.name} has a toolkit but task {task.task_slug!r} has no sandbox") - return list(self.toolkit.tools(task.sandbox, task)) -``` - -This makes the failure mode crisp and teaches external authors the expected -contract. - -### 13. `ReActWorker` imports core internals and stale dependencies - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -Smells: - -- imports `AssistantTextPart`, `ContextPartChunk`, and `ToolCallPart` from - `ergon_core.core.domain.generation.context_parts` -- imports `ContextEventService`, `Field`, `Session`, and `UUID` but does not use - them - -Cleaner expression: - -- remove stale imports immediately -- expose generation chunk types through `ergon_core.api` or - `ergon_core.api.worker` if built-ins workers are expected to yield them -- update built-ins to import only the public generation/worker API - -This may require a small core API export before built-ins can fully stop -importing `ergon_core.core.domain.generation.context_parts`. - -### 14. `ReActWorker` final-output extraction is too implicit - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -Current shape: - -- pydantic-ai structured output appears as a `final_result` tool call -- `_latest_final_result_message()` finds that tool call by name and pulls - `final_assistant_message` -- if not found, worker falls back to the last assistant text chunk - -Smells: - -- The worker's success result depends on a magic tool-call name. -- The fallback can mark success from ordinary assistant text even if structured - output failed. -- The extraction behavior is important enough to deserve direct tests and a - named module. - -Cleaner expression: - -```text -workers/ - react_output.py -``` - -```python -def worker_output_from_chunks(chunks: Sequence[ContextPartChunk]) -> WorkerOutput: - structured = latest_structured_final_result(chunks) - if structured is not None: - return WorkerOutput(output=structured.final_assistant_message, success=True) - assistant_text = latest_assistant_text(chunks) - if assistant_text is not None: - return WorkerOutput(output=assistant_text, success=True, metadata={"source": "assistant_text_fallback"}) - return WorkerOutput(output="", success=False, metadata={"source": "missing_final_output"}) -``` - -The fallback source should be visible in metadata. - -### 15. `TrainingStubWorker` is random, which makes it weaker as a test fixture - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py` - -Necessary parts: - -- A synthetic worker that emits multi-turn chunks and logprobs is useful for - RL and generation-extraction tests. - -Smells: - -- It uses global `random` directly. -- Number of turns, number of tokens, and logprobs vary on every run. -- This makes snapshots and regression tests noisier than necessary. -- It imports generation chunk types from core internals. - -Cleaner expression: - -Add explicit config: - -```python -class TrainingStubWorker(Worker): - type_slug: ClassVar[str] = "training-stub" - name: str = "training-stub" - seed: int = 0 - min_turns: int = 2 - max_turns: int = 3 - min_tokens: int = 8 - max_tokens: int = 16 -``` - -Then use a local RNG: - -```python -rng = random.Random(f"{self.seed}:{task.task_slug}") -chunks = _build_synthetic_chunks(task.task_slug, rng=rng, ...) -``` - -This keeps the worker synthetic while making it deterministic per task. - -### 16. Benchmark worker factories hard-code default models inconsistently - -Files: - -- `benchmarks/minif2f/workers.py` -- `benchmarks/swebench_verified/workers.py` -- `benchmarks/gdpeval/workers.py` -- `benchmarks/researchrubrics/workers.py` - -Current shape: - -- MiniF2F hard-codes `openai:gpt-4o-mini` with no override argument. -- SWE-Bench, GDPEval, and ResearchRubrics accept `model` and - `max_iterations`. -- worker names vary (`solver`, `swebench-solver`, `gdpeval-runner`, - `research-runner`). -- prompts for MiniF2F/SWE live in shared `react_prompts`; prompts for GDP and - ResearchRubrics live inline in worker factory modules. - -Smells: - -- Examples teach inconsistent factory signatures. -- The default model choice is repeated instead of named once. -- Prompt location is inconsistent. -- `workers.py` also exports rubric factories, so "worker factory" modules are - really "component factory" modules. - -Cleaner expression: - -```python -DEFAULT_WORKER_MODEL = "openai:gpt-4o-mini" - -def make_minif2f_worker( - *, - model: str = DEFAULT_WORKER_MODEL, - max_iterations: int = 30, -) -> ReActWorker: ... -``` - -Move prompts to benchmark-local `prompts.py`: - -```text -benchmarks/minif2f/prompts.py -benchmarks/swebench_verified/prompts.py -benchmarks/gdpeval/prompts.py -benchmarks/researchrubrics/prompts.py -``` - -And decide whether rubric factories belong in: - -- `worker_factory.py` only if the file becomes `factories.py`, or -- `rubric.py` / `rubric_factory.py` if we want names to stay precise - -### 17. `Toolkit` depends on core private serialization helpers - -File: - -- `ergon_builtins/ergon_builtins/workers/baselines/toolkit.py` - -Necessary parts: - -- Toolkit configs need to serialize with a concrete type discriminator so - object-bound workers round-trip. - -Smells: - -- Imports `TaskDefinitionJson` and `import_component_subclass` from - `ergon_core.api._serialization`, which is a private API module by name. -- `tools(self, sandbox: Any, task: Any) -> list` loses type information at the - key worker/tool boundary. - -Cleaner expression: - -- promote the serialization helper needed by external authoring components to a - public API location, or create a small built-ins-local helper rather than - importing `_serialization` -- type the tool boundary with public protocols: - -```python -class Toolkit(BaseModel, ABC): - @abstractmethod - def tools(self, sandbox: Sandbox, task: Task) -> Sequence[AgentTool]: ... -``` - -If importing `Task` causes cycles, define a tiny public protocol for the fields -toolkits use. - -## Lower-priority cleanups - -### Generated/source-trash artifacts - -The branch contains generated artifacts and stray files: - -- `__pycache__` directories -- `.pyc` files -- `benchmarks/gdpeval/Untitled` - -These should not exist in a library-tier example package. Delete them and add a -test or repository hygiene check that rejects them. - -### Optional dependency handling - -Optional imports are fine, but each optional boundary should have one clean -pattern: - -- import lazily inside the function that needs the optional package, or -- define a dedicated adapter module with a single helpful installation error - -`_manager_backed.py` currently mixes optional import fallback classes, dynamic -module import, and runtime configuration in the same file. Prefer a small -`e2b_sdk.py` adapter. - -### Broad exceptions at tool boundaries - -Broad exception handling is often reasonable for LLM-facing tools: a tool -should return a typed failure instead of crashing the whole agent loop. But it -should be deliberate and consistent. - -Recommendation: - -- broad exceptions are allowed only inside LLM tool call boundaries -- internal operation functions should raise typed exceptions or return typed - results -- a shared helper should convert exceptions to failure responses - -## Suggested cleanup sequence - -### PR 1: Hygiene and dead surface audit - -- delete generated artifacts and `Untitled` -- remove unused imports in `ReActWorker` -- delete `cloud_passthrough.py` if no active imports require it -- add architecture tests for generated artifacts and compatibility imports - -### PR 2: Canonical import paths - -- move `workers/baselines/*` to `workers/*` -- move benchmark `workers.py` files to `worker_factory.py` -- update imports -- delete `shared/workers` and `shared/models` - -### PR 3: Benchmark-local tools - -- split one benchmark `_tools.py` into `tools/response_models.py`, - `tools/operations.py`, and `tools/tool_builder.py` -- use that as the template for the other benchmarks -- start with MiniF2F because the TODO already marks the issue and the domain - is compact - -### PR 4: Criteria boundary cleanup - -- move MiniF2F `rules/proof_verification.py` into - `criteria/proof_verification.py` -- move SWE-Bench `criterion.py` into `criteria/test_resolution.py` -- split ResearchRubrics judge into `criteria/judge.py`, - `criteria/evidence.py`, and `criteria/prompts.py` - -### PR 5: Core boundary adapters - -- replace direct repository/session imports in benchmark criteria and shared - tools with public context capabilities or injected ports -- decide whether graph/resource tools belong in built-ins or core - -### PR 6: Sandbox adapter simplification - -- replace `_manager_backed.py` with `sandbox/e2b_runtime.py` and - `sandbox/e2b_sandbox.py` -- introduce `E2BSandboxRuntime` -- introduce `E2BSandbox` -- make MiniF2F, SWE-Bench, ResearchRubrics, and GDPEval sandboxes inherit from - `E2BSandbox` -- remove duplicated benchmark `provision()` and `_bind_runtime()` methods -- quote shell paths -- remove built-ins imports of `BaseSandboxManager` - -## Proposed architecture tests - -Add tests that fail when: - -- `ergon_builtins` contains `__pycache__`, `.pyc`, or files named `Untitled` -- benchmark packages import `ergon_core.core.persistence` -- benchmark packages import `ergon_core.core.application` except through - explicitly approved public facade adapters -- benchmark criteria import repositories or `get_session` -- source imports from `ergon_builtins.shared` -- source imports from `ergon_builtins.evaluators` -- benchmark packages import `ergon_builtins.workers.baselines` -- source imports from `ergon_builtins.sandbox._manager_backed` -- benchmark E2B sandbox subclasses override `provision()` or `_bind_runtime()` - without an explicit justification comment - -These tests are cheap and will keep the example package from drifting back -into migration-mode code. diff --git a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/02-implementation-stack.md b/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/02-implementation-stack.md deleted file mode 100644 index cf379d9e5..000000000 --- a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/02-implementation-stack.md +++ /dev/null @@ -1,378 +0,0 @@ -# Implementation Stack - -This stack refactors `ergon_builtins` toward library-tier example code while -keeping each PR reviewable on its own. Each PR should branch from `dev` or from -the previous PR in the stack, depending on how tightly the changes depend on -earlier renames. - -## Stack goals - -- make the active built-ins package shape obvious -- remove migration-era compatibility import surfaces -- keep benchmark authoring examples clean and copyable -- keep E2B mechanics behind a small infrastructure boundary -- make worker/tool/runtime code simpler to test - -## PR 1: Hygiene, Guardrails, and Dead Surface Inventory - -Branch: - -```text -codex/builtins-pr01-hygiene-guardrails -``` - -Goal: - -Create cheap safety rails before moving code. This PR should avoid broad -behavioral refactors. - -Scope: - -- delete generated artifacts from `ergon_builtins` -- delete stray files such as `benchmarks/gdpeval/Untitled` -- remove obvious unused imports in worker modules -- add architecture tests that prevent the same clutter from returning -- confirm whether `cloud_passthrough.py` is imported or registered anywhere - -Files to touch: - -- `ergon_builtins/ergon_builtins/**/__pycache__/` -- `ergon_builtins/ergon_builtins/**/*.pyc` -- `ergon_builtins/ergon_builtins/benchmarks/gdpeval/Untitled` -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- `ergon_builtins/tests/unit/architecture/test_builtin_package_hygiene.py` -- possibly `ergon_builtins/ergon_builtins/models/cloud_passthrough.py` - -Tests: - -```bash -pytest ergon_builtins/tests/unit/architecture -q -pytest ergon_builtins/tests/unit/workers -q -``` - -Acceptance criteria: - -- no `__pycache__`, `.pyc`, or `Untitled` files under - `ergon_builtins/ergon_builtins` -- architecture test fails if generated artifacts reappear -- `react_worker.py` no longer imports unused `UUID`, `Field`, `Session`, or - `ContextEventService` -- decision recorded for `cloud_passthrough.py`: deleted if unused, otherwise - renamed/documented as intentional passthrough - -Review risk: - -- low -- mostly filesystem hygiene and import cleanup - -## PR 2: Canonical Worker Package and Factory Shape - -Branch: - -```text -codex/builtins-pr02-workers-canonical -``` - -Goal: - -Make worker imports and benchmark worker factories boringly consistent. - -Scope: - -- move `workers/baselines/react_worker.py` to `workers/react_worker.py` -- move `workers/baselines/training_stub_worker.py` to - `workers/training_stub_worker.py` -- move `workers/baselines/toolkit.py` to `workers/toolkit.py` -- move `workers/baselines/tool_budget.py` to `workers/tool_budget.py` -- move `workers/baselines/react_prompts.py` into benchmark-local - `prompts.py` modules where prompts are benchmark-specific -- rename benchmark `workers.py` modules to `worker_factory.py` -- make factory signatures consistent across benchmarks -- remove `shared/workers` -- remove or clearly quarantine `workers/research_rubrics/_run_skill.py` - -Target structure: - -```text -ergon_builtins/ergon_builtins/workers/ - __init__.py - react_worker.py - react_output.py - toolkit.py - tool_budget.py - training_stub_worker.py - -ergon_builtins/ergon_builtins/benchmarks/<slug>/ - prompts.py - worker_factory.py -``` - -Factory convention: - -```python -DEFAULT_WORKER_MODEL = "openai:gpt-4o-mini" - -def make_<slug>_worker( - *, - model: str = DEFAULT_WORKER_MODEL, - max_iterations: int = <benchmark_default>, -) -> ReActWorker: - ... -``` - -ReAct worker cleanup: - -- replace mutable `_tools` run state with local `tools` passed into - `_run_agent` -- add explicit error when `toolkit` is set but `task.sandbox` is missing -- move final output extraction to `workers/react_output.py` -- add metadata showing whether final output came from structured output, - assistant-text fallback, or missing output - -Training stub cleanup: - -- add deterministic config: - -```python -seed: int = 0 -min_turns: int = 2 -max_turns: int = 3 -min_tokens: int = 8 -max_tokens: int = 16 -``` - -- use a local RNG seeded by `seed` and `task.task_slug` - -Tests: - -```bash -pytest ergon_builtins/tests/unit/workers -q -pytest ergon_builtins/tests/unit/test_*_v2_definition.py -q -pytest ergon_builtins/tests/unit/benchmarks -q -``` - -Acceptance criteria: - -- no source imports from `ergon_builtins.shared.workers` -- no source imports from `ergon_builtins.workers.baselines` -- benchmark worker factories live in `worker_factory.py` -- benchmark prompts live in benchmark packages -- `TrainingStubWorker` output is deterministic for the same seed and task slug -- `ReActWorker` no longer stores per-run tool lists on the worker instance - -Review risk: - -- medium -- many imports change, but behavior should remain intentionally equivalent - except deterministic training stub output and clearer missing-sandbox errors - -## PR 3: E2B Sandbox Infrastructure Base - -Branch: - -```text -codex/builtins-pr03-e2b-sandbox-runtime -``` - -Goal: - -Replace `_manager_backed.py` with an explicit E2B runtime implementation and -shared E2B sandbox base class. - -Scope: - -- create `sandbox/e2b_runtime.py` -- create `sandbox/e2b_sandbox.py` -- delete `sandbox/_manager_backed.py` -- update MiniF2F, SWE-Bench, ResearchRubrics, and GDPEval sandboxes to inherit - from `E2BSandbox` -- remove repeated `provision()` and `_bind_runtime()` implementations from - benchmark sandboxes -- update smoke fixtures to avoid importing private built-ins runtime classes -- update stale core sandbox docs that still name `ManagerBackedSandboxRuntime` - -Target runtime: - -```python -class E2BSandboxRuntime: - sandbox_id: str - - @classmethod - async def create( - cls, - *, - template: str | None, - envs: dict[str, str] | None, - timeout_seconds: int | None, - ) -> "E2BSandboxRuntime": ... - - @classmethod - async def connect(cls, sandbox_id: str) -> "E2BSandboxRuntime": ... - - async def run_command(...): ... - async def write_file(...): ... - async def read_file(...): ... - async def list_files(...): ... - async def close(self) -> None: ... - async def close_local(self) -> None: ... -``` - -Target base: - -```python -class E2BSandbox(Sandbox): - template: str | None = None - - async def provision(self) -> None: - object.__setattr__(self, "_runtime", await self._runtime_create()) - - async def _bind_runtime(self, sandbox_id: str) -> None: - object.__setattr__(self, "_runtime", await self._runtime_connect(sandbox_id)) - - async def _runtime_create(self) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.create( - template=self.template, - envs=self.env or None, - timeout_seconds=self.timeout_seconds, - ) - - async def _runtime_connect(self, sandbox_id: str) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.connect(sandbox_id) -``` - -Tests: - -```bash -pytest ergon_builtins/tests/unit/state/test_criteria_do_not_spawn_sandboxes.py -q -pytest ergon_builtins/tests/unit/benchmarks -q -pytest tests/fixtures/smoke_components -q -``` - -Acceptance criteria: - -- no source imports from `ergon_builtins.sandbox._manager_backed` -- benchmark E2B sandboxes do not override `provision()` or `_bind_runtime()` - unless a justification comment explains benchmark-specific behavior -- shell paths inside E2B `list_files()` are quoted -- built-ins no longer imports `BaseSandboxManager` -- core public sandbox docs no longer mention `ManagerBackedSandboxRuntime` as - the concrete production implementation - -Review risk: - -- medium-high -- this touches runtime lifecycle, so keep behavior equivalent and rely on - existing sandbox/reconnect tests - -## PR 4: Benchmark-Local Tools and Criteria Boundaries - -Branch: - -```text -codex/builtins-pr04-benchmark-domains -``` - -Goal: - -Finish the domain cleanup by moving benchmark-specific tools and criteria into -benchmark-owned packages and removing compatibility evaluator/shared surfaces. - -Scope: - -- split each benchmark `_tools.py` into: - -```text -tools/ - __init__.py - response_models.py - operations.py - tool_builder.py -``` - -- move MiniF2F `rules/proof_verification.py` to - `criteria/proof_verification.py` -- move SWE-Bench `criterion.py` to `criteria/test_resolution.py`, with helper - modules for patch extraction/grading if useful -- split ResearchRubrics judge logic into: - -```text -criteria/ - judge.py - evidence.py - prompts.py -``` - -- split GDPEval staged rubric into: - -```text -rubric/ - __init__.py - aggregation.py - stage.py - staged_rubric.py -``` - -- remove `shared/criteria`, `shared/models`, and `evaluators` after imports are - migrated -- add architecture tests for domain boundaries - -Tests: - -```bash -pytest ergon_builtins/tests/unit/benchmarks -q -pytest ergon_builtins/tests/unit/state -q -pytest ergon_builtins/tests/unit/test_*_v2_definition.py -q -pytest ergon_builtins/tests/unit/architecture -q -``` - -Acceptance criteria: - -- no source imports from `ergon_builtins.shared` -- no source imports from `ergon_builtins.evaluators` -- no benchmark criteria import `ergon_core.core.persistence` -- no benchmark criteria import repositories or `get_session` -- benchmark-specific tools live under `benchmarks/<slug>/tools` -- benchmark-specific criteria live under `benchmarks/<slug>/criteria` -- existing task definition round-trip tests still pass - -Review risk: - -- high -- this PR has the largest move set; keep it mechanical where possible and - avoid behavior changes beyond cleaner module boundaries - -## Stack order recommendation - -Preferred order: - -1. PR 1: Hygiene and guardrails -2. PR 2: Worker canonicalization -3. PR 3: E2B sandbox infrastructure -4. PR 4: Benchmark tools and criteria domains - -If PR 4 feels too large, split it by benchmark: - -```text -PR 4a: MiniF2F tools/criteria -PR 4b: SWE-Bench tools/criteria -PR 4c: ResearchRubrics criteria/evidence -PR 4d: GDPEval rubric/tools -``` - -That would make a cleaner review experience, but the conceptual stack remains -the same. - -## Final stack acceptance - -The stack is complete when: - -- `ergon_builtins` has one obvious canonical import path per concept -- generated artifacts are rejected by tests -- workers do not carry per-run mutable tool state -- training stub output is deterministic -- E2B setup lives behind `E2BSandbox` and `E2BSandboxRuntime` -- benchmark sandboxes are mostly declarative config -- benchmark tools and criteria live under benchmark packages -- built-ins no longer imports core persistence/application internals from - benchmark criteria -- compatibility packages are deleted or explicitly quarantined as temporary diff --git a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/README.md b/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/README.md deleted file mode 100644 index 13af93294..000000000 --- a/docs/rfcs/active/2026-05-19-ergon-builtins-domain-refactor/README.md +++ /dev/null @@ -1,482 +0,0 @@ ---- -status: active -opened: 2026-05-19 -author: codex -architecture_refs: - - docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/00-readme.md - - ergon_builtins/ergon_builtins/benchmarks/README.md -supersedes: [] -superseded_by: null ---- - -# RFC: Ergon Built-ins Domain Refactor - -## Problem - -`ergon_builtins` has mostly moved to object-bound, Python-authored benchmarks, -but its package layout still carries several migration-era shapes: - -- benchmark-owned implementations live beside compatibility packages such as - `evaluators/`, `shared/criteria/`, and `shared/workers/` -- files named for one concept contain several concepts, especially rubric - state, aggregation logic, evidence loading, prompt construction, and tool - response models -- some benchmark criteria and tools import core application or persistence - internals directly -- stale `__pycache__` directories and compatibility import paths obscure which - modules are active source and which are historical leftovers - -This makes it harder to tell where new benchmark code should go, which import -paths are canonical, and where the domain boundary is between built-in -benchmark authoring and core runtime services. - -## Proposal - -Keep the current object-bound authoring model: each benchmark package owns its -payload schema, dataset loading, sandbox, toolkit, worker factories, criteria, -and rubric. Do not restore a central registry or CLI authoring path as part of -this refactor. - -The refactor should make `ergon_builtins` read as a set of benchmark domains -plus a small set of genuinely shared runtime primitives. - -See [01-logic-smell-audit.md](01-logic-smell-audit.md) for the current -library-tier audit that motivates the target package shape and migration order. -See [02-implementation-stack.md](02-implementation-stack.md) for the proposed -4-PR stack off `dev`. - -## Intended folder structure - -```text -ergon_builtins/ - ergon_builtins/ - __init__.py - - benchmarks/ - README.md - __init__.py - - minif2f/ - __init__.py - benchmark.py - task_schemas.py - sandbox.py - sandbox_template/ - Dockerfile - README.md - e2b.toml - e2b.toml.template - utils.py - worker_factory.py - prompts.py - toolkit.py - tools/ - __init__.py - lean_operations.py - response_models.py - tool_builder.py - criteria/ - __init__.py - proof_verification.py - rubric.py - constants.py - - swebench_verified/ - __init__.py - benchmark.py - task_schemas.py - sandbox.py - sandbox_template/ - Dockerfile - e2b.toml.template - utils.py - worker_factory.py - prompts.py - toolkit.py - tools/ - __init__.py - git_operations.py - response_models.py - tool_builder.py - criteria/ - __init__.py - test_resolution.py - grading.py - patch_extraction.py - rubric.py - - researchrubrics/ - __init__.py - benchmark.py - vanilla.py - task_schemas.py - sandbox.py - worker_factory.py - prompts.py - toolkit.py - tools/ - __init__.py - report_operations.py - research_operations.py - response_models.py - tool_builder.py - criteria/ - __init__.py - judge.py - evidence.py - prompts.py - rubric.py - - gdpeval/ - __init__.py - benchmark.py - task_schemas.py - loader.py - sandbox.py - worker_factory.py - prompts.py - toolkit.py - tools/ - __init__.py - document_operations.py - response_models.py - tool_builder.py - criteria/ - __init__.py - code_check_builders.py - llm_judge_builders.py - rubric/ - __init__.py - aggregation.py - stage.py - staged_rubric.py - - common/ - __init__.py - llm/ - __init__.py - structured_judge.py - llm_context/ - __init__.py - adapters/ - __init__.py - base.py - pydantic_ai.py - - models/ - __init__.py - cloud_passthrough.py - openrouter_backend.py - openrouter_responses_backend.py - resolution.py - transformers_backend.py - vllm_backend.py - - observability/ - __init__.py - pydantic_ai_logfire.py - - sandbox/ - __init__.py - e2b_runtime.py - e2b_sandbox.py - - tools/ - __init__.py - bash_sandbox_tool.py - graph_toolkit.py - graph_toolkit_types.py - subtask_lifecycle_toolkit.py - workflow_cli_tool.py - - workers/ - __init__.py - react_prompts.py - react_worker.py - toolkit.py - tool_budget.py - training_stub_worker.py -``` - -## Package boundary rules - -### Benchmark packages - -Benchmark packages are the primary domain boundary. - -Each benchmark package owns: - -- its `Benchmark` subclass and `Task[...]` subclass -- its task payload models and dataset row conversion -- dataset loading and optional data dependency handling -- benchmark-specific sandbox setup -- benchmark-specific toolkit config -- benchmark-specific runtime tools -- benchmark-specific worker factories -- benchmark-specific criteria -- benchmark-specific rubric aggregation - -Benchmark packages may import from: - -- `ergon_core.api` -- `ergon_builtins.common` -- `ergon_builtins.models` -- `ergon_builtins.observability` -- `ergon_builtins.sandbox` -- `ergon_builtins.tools` -- `ergon_builtins.workers` -- sibling modules in the same benchmark package - -Benchmark criteria should not import core persistence repositories, SQLModel -sessions, or application service implementations directly. If a criterion needs -run resources, logs, or graph data, that data should arrive through -`CriterionContext` public capabilities or a small injected adapter owned by the -runtime boundary. - -### Shared workers - -`ergon_builtins.workers` contains generic worker machinery only. - -It may contain: - -- `ReActWorker` -- `TrainingStubWorker` -- the serializable `Toolkit` base class -- ReAct prompt fragments that are genuinely shared -- tool budget primitives - -It should not contain benchmark-owned worker factories. A function like -`make_minif2f_worker()` belongs in -`ergon_builtins.benchmarks.minif2f.worker_factory`. - -### Shared tools - -`ergon_builtins.tools` contains reusable tools that are not specific to one -benchmark domain. - -Good fits: - -- sandbox bash command wrapper -- workflow CLI tool -- graph/resource inspection tools -- subtask lifecycle tools - -Poor fits: - -- Lean proof tools -- SWE-Bench git/apply helpers -- ResearchRubrics report-writing helpers -- GDPEval document transformation helpers - -Benchmark-specific tools should live under -`ergon_builtins.benchmarks.<slug>.tools`. - -### Criteria and rubrics - -There should be no top-level `evaluators/` package after the refactor unless a -criterion or rubric is truly independent of all benchmark payloads and evidence -models. - -Canonical locations: - -- MiniF2F proof checking: - `benchmarks/minif2f/criteria/proof_verification.py` -- SWE-Bench test resolution: - `benchmarks/swebench_verified/criteria/test_resolution.py` -- ResearchRubrics dynamic LLM judging: - `benchmarks/researchrubrics/criteria/judge.py` -- GDPEval staged rubric: - `benchmarks/gdpeval/rubric/staged_rubric.py` - -If a generic `LLMJudgeCriterion` remains useful, put it under either: - -- `ergon_builtins.common.llm` if it is primarily a judging utility, or -- `ergon_builtins.tools` only if it is exposed as a worker tool - -Do not keep both `shared/criteria/*` and `evaluators/criteria/*` as competing -import surfaces. - -### Models - -`ergon_builtins.models` remains the canonical home for model target resolution -and concrete backend adapters. The current `shared/models` compatibility layer -should be removed once imports are updated. - -### Sandbox infrastructure - -`ergon_builtins.sandbox` remains singular and contains cross-cutting sandbox -adapters only. Concrete benchmark sandboxes stay in benchmark packages: - -- `benchmarks/minif2f/sandbox.py` -- `benchmarks/swebench_verified/sandbox.py` -- `benchmarks/researchrubrics/sandbox.py` -- `benchmarks/gdpeval/sandbox.py` - -There should be no source `sandboxes/` package. Stale `sandboxes/__pycache__` -artifacts should be removed. - -The E2B sandbox layer should be expressed as two concepts: - -- `E2BSandboxRuntime`: the concrete implementation of the public - `SandboxRuntime` protocol using the E2B SDK under the hood -- `E2BSandbox`: a reusable `Sandbox` subclass that implements `provision()` and - `_bind_runtime()` once by attaching an `E2BSandboxRuntime` - -Benchmark E2B sandboxes should usually be config-only subclasses of -`E2BSandbox`. They should not each repeat the same provision/bind boilerplate. - -Target shape: - -```python -class E2BSandboxRuntime: - @classmethod - async def create( - cls, - *, - template: str | None, - envs: dict[str, str] | None, - timeout_seconds: int | None, - ) -> "E2BSandboxRuntime": ... - - @classmethod - async def connect(cls, sandbox_id: str) -> "E2BSandboxRuntime": ... - - async def run_command(...): ... - async def write_file(...): ... - async def read_file(...): ... - async def list_files(...): ... - async def close(self) -> None: ... - async def close_local(self) -> None: ... -``` - -```python -class E2BSandbox(Sandbox): - template: str | None = None - - async def provision(self) -> None: - object.__setattr__(self, "_runtime", await self._runtime_create()) - - async def _bind_runtime(self, sandbox_id: str) -> None: - object.__setattr__(self, "_runtime", await self._runtime_connect(sandbox_id)) - - async def _runtime_create(self) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.create( - template=self.template, - envs=self.env or None, - timeout_seconds=self.timeout_seconds, - ) - - async def _runtime_connect(self, sandbox_id: str) -> E2BSandboxRuntime: - return await E2BSandboxRuntime.connect(sandbox_id) -``` - -Benchmark sandboxes then collapse to declarative config: - -```python -class LeanSandbox(E2BSandbox): - template: str = "ergon-minif2f-v1" - requires_network: bool = False - output_path: str = "/workspace/final_output/" - lean_version: str = "4.7.0" -``` - -## Migration - -The refactor should happen in small slices. - -1. Remove generated artifacts from the source tree: - delete `__pycache__` directories, `.pyc` files, and stray files such as - `benchmarks/gdpeval/Untitled`. - -2. Establish canonical worker imports: - move `workers/baselines/*` to `workers/*`, update imports, and delete the - compatibility wrappers in `shared/workers`. - -3. Rename benchmark factory modules: - move each `benchmarks/<slug>/workers.py` to - `benchmarks/<slug>/worker_factory.py`, update docs and imports. - -4. Move benchmark-specific tool code: - replace each monolithic `_tools.py` with a package-level `tools/` folder - containing response models, operations, and a thin `tool_builder.py`. - -5. Move benchmark-specific criteria: - replace `rules/`, `criteria.py`, and detached top-level evaluator imports - with benchmark-local `criteria/` packages. - -6. Split heavy rubric logic: - especially `gdpeval/rubric.py` into `rubric/stage.py`, - `rubric/aggregation.py`, and `rubric/staged_rubric.py`. - -7. Replace `_manager_backed.py` with explicit E2B sandbox infrastructure: - introduce `sandbox/e2b_runtime.py` and `sandbox/e2b_sandbox.py`, then make - benchmark E2B sandboxes inherit from `E2BSandbox` instead of overriding - `provision()` and `_bind_runtime()` individually. - -8. Remove compatibility import surfaces: - delete `shared/criteria`, `shared/models`, and `evaluators` once no imports - depend on them. - -9. Add architecture tests: - enforce no source `__pycache__`, no top-level `evaluators`, no - `shared/criteria`, no direct persistence imports from benchmark criteria, - no `_manager_backed.py`, and one canonical benchmark package shape. - -## Invariants affected - -This RFC introduces these invariants: - -- Built-in benchmark authoring is benchmark-owned and object-bound. -- Benchmark-specific concepts live under - `ergon_builtins.benchmarks.<slug>`. -- Shared packages contain only code used across multiple benchmark domains. -- Benchmark criteria do not import core persistence or application service - implementations directly. -- E2B-backed benchmark sandboxes inherit common E2B provisioning and binding - behavior instead of each repeating E2B runtime setup. -- Compatibility import packages are temporary and should not be used as - canonical authoring locations. - -## Alternatives considered - -### Reintroduce central registries - -Rejected for this RFC. The current built-ins README explicitly says there is -no CLI authoring path and no core registry to edit. The refactor should clarify -the architecture that exists now rather than revive an older slug registry -design. - -### Keep `_tools.py` files but clean internals - -Rejected as the final structure. The `_tools.py` pattern keeps runtime tool -builders import-light, which is useful, but the files now mix response models, -sandbox operations, parsing helpers, and pydantic-ai tool construction. A -`tools/` subpackage preserves lazy imports while giving each concept a home. - -### Keep top-level `shared/` - -Partially rejected. Some shared code is real, but `shared/criteria`, -`shared/workers`, and `shared/models` are currently mostly compatibility import -surfaces. The clearer shape is canonical packages named by concept: -`workers`, `models`, `tools`, `common`, `sandbox`, and `observability`. - -## Open questions - -1. Should generic `CodeCheckCriterion` and `LLMJudgeCriterion` remain as - public built-in criteria, or should they become benchmark-local builders - only? -2. Should `ResearchRubricsJudgeCriterion` load run resources through new - `CriterionContext` public methods, or through an injected resource reader - adapter? -3. Should `workers/react_prompts.py` stay generic, or should benchmark-specific - prompts move entirely to `benchmarks/<slug>/prompts.py`? -4. Should `tools/graph_toolkit.py` remain in built-ins, or move closer to core - public worker context APIs? - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - -- update `ergon_builtins/ergon_builtins/benchmarks/README.md` -- add an implementation plan under `docs/superpowers/plans/` -- add architecture tests that enforce the accepted package boundaries diff --git a/docs/rfcs/active/architecture-refactor-audit/01-dependency-inversion.md b/docs/rfcs/active/architecture-refactor-audit/01-dependency-inversion.md deleted file mode 100644 index 7b0ccfced..000000000 --- a/docs/rfcs/active/architecture-refactor-audit/01-dependency-inversion.md +++ /dev/null @@ -1,418 +0,0 @@ ---- -status: active -opened: 2026-04-27 -author: GPT-5.5 -architecture_refs: - - docs/architecture/01_public_api.md - - docs/architecture/03_providers.md - - docs/architecture/06_builtins.md -supersedes: [] -superseded_by: null ---- - -# RFC: Dependency Inversion And Package Boundaries - -## Problem - -The declared package graph says `ergon_core` is the reusable runtime and public -API, `ergon_builtins` supplies default implementations, `ergon_cli` adapts user -commands, and `ergon_infra` handles training/provisioning helpers. The source -graph is messier. Core runtime code imports the builtins registry, builtins -tooling imports CLI command modules, and test harness paths pull CLI -composition back into core. - -These dependencies work in the workspace, but they blur ownership. A reader -cannot easily tell which package owns composition, which APIs are stable, or -how to add a new benchmark/worker without coupling to the current default -registry. - -## Current findings - -### Core runtime imports builtins registry - -Runtime paths resolve slugs by importing `ergon_builtins.registry` directly. -This appears in Inngest handlers and services such as worker execution, -benchmark-run startup, evaluator dispatch, sandbox setup, output persistence, -and workflow initialization. The practical result is that core is not only a -runtime contract package; it also knows about the default plugin bundle. - -### Builtins registry reaches into core internals - -`ergon_builtins.registry` implements public `ergon_core.api` contracts, but it -also imports provider internals for model backend registration and sandbox -manager types. Some of this may be unavoidable today, but it should be named as -an extension boundary rather than an incidental import path. - -### Builtins tooling imports CLI command code - -`ergon_builtins.tools.workflow_cli_tool` imports `WorkflowCommandContext`, -`WorkflowCommandOutput`, and `execute_workflow_command` from -`ergon_cli.commands.workflow`. That makes an agent-facing builtin tool depend -on the CLI command layer instead of a shared application/service API. - -### Core test harness imports CLI composition - -`ergon_core.core.api.test_harness` imports `ergon_cli.composition` when the -test harness is enabled. The flag keeps this out of production by default, but -the import direction is still surprising for a core package. - -### CLI composition contains example-specific branches - -`ergon_cli.composition.build_experiment` performs registry lookup and then -branches for smoke workers and `researchrubrics-workflow-cli-react`. Those -branches may encode real composition needs, but they live in the generic CLI -composition path rather than behind benchmark/worker-owned composition hooks. - -## Target shape - -The target dependency direction should be: - -```text -ergon_core.api <- implemented by builtins and custom packages -ergon_core.runtime <- depends on injected registries/services, not builtins -ergon_builtins <- default implementation bundle -ergon_cli <- adapter that wires a registry bundle into core services -ergon_infra <- training/provisioning adapter over public/core services -ergon-dashboard <- frontend over HTTP/event contracts -``` - -Core may define protocols and service interfaces. Builtins may implement them. -CLI and application startup may choose the default builtins registry. Runtime -code should receive a resolver or registry interface rather than importing the -default bundle. - -## Standards proposed - -- Public contracts belong under `ergon_core.api` or a deliberately named core - interface module. -- A package should not import an adapter layer that is higher-level than - itself. In particular, builtins should not import `ergon_cli.commands.*`. -- Runtime services should depend on protocols such as `WorkerResolver`, - `BenchmarkResolver`, `EvaluatorResolver`, `SandboxManagerResolver`, or one - combined `RuntimeRegistry`. -- Example-specific composition should be owned by the benchmark/worker bundle - that requires it, or represented as data on the public API. -- Test-only composition should enter through explicit startup/plugin hooks, not - direct core-to-cli imports. - -## Candidate fixes - -Each candidate below should be treated as a small implementation plan, not an -idea bucket. A follow-up implementation plan may split these into separate PRs, -but each candidate already names the files, steps, tests, and acceptance gate -expected before the work is considered real. - -### DI-1: Add a runtime registry protocol in core - -**Issue fixed:** Core runtime code cannot express "I need a worker/benchmark/evaluator -resolver" without importing the concrete builtins registry, so dependency -direction is encoded as an implementation detail instead of a contract. - -Create a small protocol owned by core that contains the lookup methods runtime -code actually needs: - -- `get_worker(slug)` -- `get_benchmark(slug)` -- `get_evaluator(slug)` -- `get_sandbox_manager(slug)` -- optional install-hint lookup for user-facing errors - -Candidate location: `ergon_core.api.registry` if this becomes public extension -surface, or `ergon_core.core.runtime.registry` if it stays internal. The first -implementation can be an adapter around `ergon_builtins.registry`, preserving -all current slug names and optional-extra behavior. - -Files: - -- Create: `ergon_core/ergon_core/api/registry.py` or - `ergon_core/ergon_core/core/runtime/registry.py`. -- Create: `ergon_builtins/ergon_builtins/runtime_registry.py`. -- Modify: `ergon_builtins/ergon_builtins/registry.py` only if the adapter needs - a stable export. -- Test: `tests/unit/runtime/test_runtime_registry_contract.py`. - -Sketch: - -```python -from typing import Protocol - -class RuntimeRegistry(Protocol): - def get_worker(self, slug: str): ... - def get_benchmark(self, slug: str): ... - def get_evaluator(self, slug: str): ... - def get_sandbox_manager(self, slug: str): ... - def install_hint_for(self, slug: str) -> str | None: ... -``` - -Steps: - -- [ ] Add the protocol and a typed missing-slug error or document that `KeyError` - remains the compatibility behavior. -- [ ] Add a builtins-backed adapter over the existing registry dictionaries. -- [ ] Preserve model backend registration side effects at builtins registry - import time. -- [ ] Add a fake in-memory registry for tests that should not import builtins. -- [ ] Keep existing public imports of `ergon_builtins.registry` working. - -Verification: - -- Unit tests for successful and missing slug lookup. -- Characterization test that CLI defaults still resolve the same worker, - benchmark, evaluator, and sandbox manager classes. -- `python -c "from ergon_builtins.registry import WORKERS, BENCHMARKS"` still - succeeds in the workspace environment. - -Acceptance gate: - -- [ ] Registry contract tests pass for both the fake registry and builtins - adapter. -- [ ] No runtime behavior changes: current benchmark, worker, evaluator, and - sandbox slugs resolve to the same objects. -- [ ] Architecture docs mention where registry protocols live. - -### DI-2: Stop importing `ergon_builtins.registry` from core runtime modules - -**Issue fixed:** `ergon_core` is declared as the reusable runtime package, but -runtime modules currently depend on the default builtins bundle at import time. -That makes builtins a hidden runtime prerequisite and prevents fake/custom -registries from being injected cleanly. - -Replace direct registry imports in core runtime paths with an injected resolver -or application-level registry object. Initial target modules include: - -- `core/runtime/inngest/benchmark_run_start.py` -- `core/runtime/inngest/worker_execute.py` -- `core/runtime/inngest/evaluate_task_run.py` -- `core/runtime/inngest/sandbox_setup.py` -- `core/runtime/inngest/persist_outputs.py` -- `core/runtime/services/workflow_initialization_service.py` -- `core/api/app.py` - -The first pass can use a default registry provider at process startup so -behavior stays identical while import direction improves. - -Files: - -- Modify: `ergon_core/ergon_core/core/runtime/inngest/benchmark_run_start.py`. -- Modify: `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py`. -- Modify: `ergon_core/ergon_core/core/runtime/inngest/evaluate_task_run.py`. -- Modify: `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py`. -- Modify: `ergon_core/ergon_core/core/runtime/inngest/persist_outputs.py`. -- Modify: - `ergon_core/ergon_core/core/runtime/services/workflow_initialization_service.py`. -- Modify: `ergon_core/ergon_core/core/api/app.py`. -- Test: `tests/unit/architecture/test_package_boundaries.py`. - -Steps: - -- [ ] Add a process-level registry provider or dependency accessor in core. -- [ ] Configure the builtins-backed registry from CLI/API startup. -- [ ] Convert each runtime module from `from ergon_builtins.registry import ...` - to the registry accessor. -- [ ] Keep error messages for unknown slugs at least as clear as today. -- [ ] Remove any import-time builtins dependency from core runtime modules. - -Verification: - -- Architecture test that `ergon_core.core.runtime` does not import - `ergon_builtins`. -- Existing benchmark/run tests continue to pass without slug changes. -- `rg "ergon_builtins.registry" ergon_core/ergon_core/core/runtime` returns no - matches. - -Acceptance gate: - -- [ ] Direct runtime imports of `ergon_builtins.registry` are gone. -- [ ] Unknown-slug behavior is characterized and preserved or deliberately - improved in a documented way. -- [ ] CLI/API startup still wires the default builtins registry. - -### DI-3: Move workflow command execution out of the CLI command module - -**Issue fixed:** Builtin agent tools reuse workflow behavior by importing -`ergon_cli.commands.workflow`, which makes a non-CLI package depend on CLI -command parsing/rendering code. - -Extract the command parsing/execution core from `ergon_cli.commands.workflow` -into a shared service module that has no CLI rendering dependency. The CLI -command should parse argv and render output; builtin tools should call the same -shared executor directly. - -Candidate owner: `ergon_core.core.runtime.services.workflow_command_service` if -the command surface is runtime-owned, or `ergon_cli.workflow_application` if it -is intentionally an application-layer adapter. The key rule is that -`ergon_builtins` should not import `ergon_cli.commands.*`. - -Verification: - -- Existing `tests/unit/cli/test_workflow_cli.py` still validates CLI behavior. -- New builtin-tool test imports the shared executor without importing the CLI - command module. -- Architecture test blocks `ergon_builtins -> ergon_cli.commands`. - -Files: - -- Create: - `ergon_core/ergon_core/core/runtime/services/workflow_command_service.py` - or a similarly named shared application module. -- Modify: `ergon_cli/ergon_cli/commands/workflow.py`. -- Modify: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py`. -- Test: `tests/unit/cli/test_workflow_cli.py`. -- Test: `tests/unit/state/test_workflow_cli_tool.py` or equivalent builtin - tool test. - -Steps: - -- [ ] Identify the current command parser/executor/renderer responsibilities in - `ergon_cli.commands.workflow`. -- [ ] Move parser and executor into the shared module without changing command - strings. -- [ ] Leave stdout/stderr formatting and argparse integration in CLI. -- [ ] Update the builtin workflow tool to call the shared executor. -- [ ] Add an import-boundary test that prevents future builtin imports from - `ergon_cli.commands`. - -Acceptance gate: - -- [ ] CLI workflow tests pass with unchanged expected output. -- [ ] Builtin workflow tool tests pass without importing CLI command modules. -- [ ] `rg "ergon_cli.commands" ergon_builtins/ergon_builtins/tools` returns no - matches, except an explicit migration allowlist if needed. - -### DI-4: Replace special-case CLI experiment branches with composition descriptors - -**Issue fixed:** Generic CLI experiment composition contains hard-coded -knowledge of specific worker families, so every new example with special -bindings risks adding another `if worker_slug == ...` branch. - -Move the smoke-worker and `researchrubrics-workflow-cli-react` branch knowledge -out of generic `build_experiment`. Candidate shape: - -- Workers or benchmarks may expose an optional composition descriptor. -- The descriptor declares extra worker bindings, evaluator bindings, and static - assignment strategy. -- `build_experiment` applies descriptors generically after registry lookup. - -This keeps current behavior while making future examples add data rather than a -new `if worker_slug == ...` branch. - -Verification: - -- Characterization tests for smoke worker composition. -- Characterization tests for research-rubrics workflow composition. -- A test that a synthetic descriptor can add an extra worker binding without - editing `ergon_cli.composition`. - -Files: - -- Modify: `ergon_cli/ergon_cli/composition/__init__.py`. -- Add: a composition descriptor type under `ergon_core.api` or - `ergon_cli.composition`. -- Modify smoke fixture registration under - `ergon_core/ergon_core/test_support/smoke_fixtures/`. -- Modify research-rubrics worker/benchmark registration under - `ergon_builtins/ergon_builtins/workers/research_rubrics/` or - `ergon_builtins/ergon_builtins/registry_data.py`. -- Test: `tests/unit/cli/test_build_experiment_composition.py`. - -Current branches to eliminate from generic composition: - -- `_is_smoke_worker(worker_slug)`. -- `worker_slug == "researchrubrics-workflow-cli-react"`. -- suffix parsing for `-smoke-worker` and `-sadpath-smoke-worker`. -- direct imports of smoke timing criteria from generic CLI composition. - -Sketch: - -```python -class ExperimentCompositionDescriptor(BaseModel): - extra_workers: dict[str, WorkerSpec] - extra_evaluators: dict[str, Evaluator] - static_assignments: dict[str, list[str]] -``` - -Steps: - -- [ ] Add the descriptor type and a no-op default descriptor. -- [ ] Teach `build_experiment` to ask the selected worker/benchmark registry - entry for a descriptor. -- [ ] Move smoke leaf/recursive/failing-leaf bindings into smoke fixture-owned - descriptor code. -- [ ] Move research-rubrics manager/researcher bindings into - research-rubrics-owned descriptor code. -- [ ] Add an architecture test that blocks new hard-coded worker slug branches - in `ergon_cli.composition`. - -Acceptance gate: - -- [ ] No generic composition branch checks a concrete worker slug. -- [ ] Existing smoke and research-rubrics composition behavior is unchanged. -- [ ] A synthetic descriptor test proves new special composition can be added - without editing `build_experiment`. - -### DI-5: Route smoke/test harness composition through startup plugins - -**Issue fixed:** Test harness and smoke-fixture setup rely on direct imports -that blur production startup, CLI composition, and test-support registration. - -Replace direct core-to-CLI composition imports in test-harness paths with the -same registry/composition extension point used by production startup. Smoke -fixtures can still be opt-in, but the opt-in should register providers through -a plugin hook rather than teaching core about CLI composition. - -Verification: - -- Test harness remains disabled by default. -- With `ENABLE_TEST_HARNESS=1`, smoke fixtures still register and run. -- Architecture test documents the only allowed test-support imports. - -Files: - -- Modify: `ergon_core/ergon_core/core/api/test_harness.py`. -- Modify: `ergon_core/ergon_core/core/api/app.py`. -- Modify or use existing startup plugin settings in - `ergon_core/ergon_core/core/settings.py`. -- Test: `tests/unit/architecture/test_smoke_fixture_package_boundary.py`. -- Test: harness tests that currently exercise `ENABLE_TEST_HARNESS`. - -Steps: - -- [ ] Inventory current `ENABLE_TEST_HARNESS` and `ENABLE_SMOKE_FIXTURES` - behavior. -- [ ] Define the plugin hook that can register smoke fixtures or experiment - builders. -- [ ] Move test-harness composition to the plugin path. -- [ ] Preserve disabled-by-default behavior. -- [ ] Add an architecture allowlist for the few remaining test-support imports, - if any. - -Acceptance gate: - -- [ ] Test harness smoke behavior still works under explicit opt-in. -- [ ] Core app startup no longer needs to know smoke fixture implementation - modules by name. -- [ ] Architecture tests fail if new production runtime modules import - `ergon_core.test_support`. - -## Migration / risk - -The risk is not algorithmic behavior; it is import-time behavior. The current -registry performs eager optional-capability imports and model backend -registration. Moving this behind protocols must preserve: - -- Existing CLI defaults and slug names. -- Optional extras behavior and install hints. -- Model backend registration side effects. -- Test harness smoke fixture behavior under explicit flags. - -The first implementation step should be characterization tests around registry -resolution and CLI experiment construction before import paths are changed. - -## Open questions - -- Should the registry protocol live in `ergon_core.api`, `ergon_core.core`, or - a new package such as `ergon_runtime_contracts`? -- Should CLI remain the primary composition root, or should FastAPI startup and - CLI share a new composition module? -- Do existing consumers import `ergon_builtins.registry` directly, and if so do - those imports need compatibility wrappers? diff --git a/docs/rfcs/active/architecture-refactor-audit/02-test-brittleness-and-gaps.md b/docs/rfcs/active/architecture-refactor-audit/02-test-brittleness-and-gaps.md deleted file mode 100644 index 1525b9b61..000000000 --- a/docs/rfcs/active/architecture-refactor-audit/02-test-brittleness-and-gaps.md +++ /dev/null @@ -1,441 +0,0 @@ ---- -status: active -opened: 2026-04-27 -author: GPT-5.5 -architecture_refs: - - docs/architecture/07_testing.md - - docs/architecture/02_runtime_lifecycle.md - - docs/architecture/04_persistence.md -supersedes: [] -superseded_by: null ---- - -# RFC: Test Brittleness And Confidence Gaps - -## Problem - -Behavior-preserving refactors need trustworthy tests. Ergon already has useful -unit, integration, e2e, state, and real-LLM tiers, but the test surface has -grown alongside the code. Some tests encode current implementation details, -some test-support concepts leak toward runtime code, and some important -package-boundary expectations are not yet expressed as contracts. - -The goal is to make tests better at preserving behavior while reducing their -ability to freeze accidental architecture. - -## Current findings - -### Test support has explicit gates, but the boundary is fragile - -Smoke fixtures and test harness paths are mostly gated behind environment -flags such as `ENABLE_TEST_HARNESS` and `ENABLE_SMOKE_FIXTURES`. This is -useful, but it means import discipline matters. A small number of direct -imports can turn test-only composition into runtime coupling. - -### Existing architecture tests are valuable but narrow - -There are tests that assert smoke fixtures do not move into old production -paths. That pattern should expand: import-boundary rules should cover core to -builtins, builtins to CLI, and core to CLI exceptions. - -### State tests exercise behavior but may mix concerns - -The `tests/unit/state` tier appears to group workflow/tool/research-rubric -state behavior rather than a dedicated state package. These tests are useful, -but they should make clear whether they are verifying public behavior, database -state transitions, or current helper implementation. - -### Real-LLM and e2e tests are opt-in - -Opt-in real-LLM rollout tests and dashboard/e2e tests are valuable for catching -integration failures, but they are not always part of the fast feedback loop. -The refactor program needs a smaller characterization layer for behavior that -must not change during architecture cleanup. - -### Fixtures can hide missing contracts - -When tests rely on broad fixtures or sentinel identities, they can keep passing -even though production composition boundaries are unclear. Refactors should -prefer explicit fake providers and public-contract setup over reaching into -runtime internals. - -## Target shape - -The test suite should have a clear contract for each tier: - -- **Architecture tests** enforce import direction, package ownership, and - allowed exceptions. -- **Unit tests** verify pure behavior and service logic without requiring the - default builtins registry unless that is the unit under test. -- **State/integration tests** verify persisted runtime transitions through - public service boundaries. -- **E2E tests** verify deployed surfaces and dashboard/API hydration. -- **Real-LLM tests** verify representative model-facing workflows and artifact - health, gated by explicit credentials. - -Each behavior-preserving refactor should start by identifying which tier locks -the behavior being preserved. - -## Standards proposed - -- Add architecture tests for dependency direction and allowed import - exceptions. Exceptions should be named and justified in one place. -- Prefer fake implementations of public protocols over sentinel strings that - runtime code must recognize. -- Keep smoke fixtures and real-LLM harnesses under test-support or tests, with - explicit opt-in registration. -- Avoid tests that assert line-by-line implementation detail unless the detail - is itself a contract. -- For every major refactor, add or identify characterization tests before - moving code. -- Keep slow/e2e/real-LLM tests useful but non-blocking for local refactor - loops; provide smaller contract tests for behavior that must always pass. - -## Candidate fixes - -Each candidate below should include enough detail for an implementation plan to -be written without rediscovering the audit. Tests are themselves part of the -architecture here: they define what future refactors are not allowed to break. - -### TB-1: Add import-boundary architecture tests - -**Issue fixed:** Package-boundary rules are currently mostly social -conventions, so new reverse imports or ad hoc slug branches can land without a -fast test failure. - -Create tests that parse imports and enforce the intended package graph. Start -with warnings/allowlists for current known violations, then tighten the rules -as dependency-inversion fixes land. - -Initial rules: - -- `ergon_core.core.runtime` should not import `ergon_builtins`. -- `ergon_core` should not import `ergon_cli` except explicitly allowed - test-harness paths. -- `ergon_builtins` should not import `ergon_cli.commands`. -- Production runtime modules should not import `ergon_core.test_support` or - `tests.*`. - -Candidate location: `tests/unit/architecture/test_package_boundaries.py`. - -Suggested helper shape: - -```python -def assert_no_imports(package_root: Path, forbidden: str, *, allowlist: set[str]) -> None: - offenders = scan_python_imports(package_root, forbidden) - unexpected = offenders - allowlist - assert unexpected == set() -``` - -Initial allowlist should include only named, reviewed exceptions. Avoid broad -directory-level exceptions unless the whole directory is intentionally an -adapter or test-support surface. - -Steps: - -- [ ] Implement a small AST-based import scanner, not a regex-only test. -- [ ] Add rules for core-to-builtins, core-to-cli, builtins-to-cli, and - production-to-test-support. -- [ ] Encode current known violations as explicit allowlist entries with a - linked candidate fix ID. -- [ ] Add a second test that fails on new concrete worker/benchmark slug - branches in generic composition modules. -- [ ] Document how to update the allowlist when a refactor removes a violation. - -Verification: - -- Test fails with a clear list of violating import edges. -- Current exceptions are named in one allowlist with comments. - -Acceptance gate: - -- [ ] Architecture test passes with only reviewed exceptions. -- [ ] Adding `from ergon_cli.commands...` to a builtin tool fails the test. -- [ ] Adding `worker_slug == "some-example"` to generic composition fails or is - caught by the branch-pattern test. - -### TB-2: Add CLI benchmark-run characterization tests - -**Issue fixed:** The benchmark-run path combines DB setup, experiment -composition, persistence, cohort creation, run creation, event dispatch, and -polling. Refactoring it without characterization tests risks changing behavior -while only moving imports around. - -Before changing composition or registry resolution, lock down the current -observable `ergon benchmark run` setup path without requiring a live Inngest -run: - -- `ensure_db()` is called before persistence. -- `build_experiment()` receives CLI args unchanged. -- `experiment.validate()` runs before `experiment.persist()`. -- cohort resolution uses the explicit cohort or benchmark slug. -- `create_run()` receives the persisted definition. -- `WorkflowStartedEvent` carries the run ID and definition ID. -- polling reads `RunRecord` until a terminal status. - -Candidate location: `tests/unit/cli/test_benchmark_run_flow.py`. - -Suggested cases: - -- `benchmark run` persists before dispatching. -- explicit `--cohort` is used when present. -- default cohort name falls back to benchmark slug. -- timeout returns a timeout handle without pretending the run completed. -- terminal failed/cancelled status exits non-zero. - -Test approach: - -- Monkeypatch `ensure_db`, `build_experiment`, `experiment_cohort_service`, - `create_run`, `inngest_client.send`, and `get_session`. -- Use a fake session whose `get(RunRecord, run.id)` returns a sequence of - statuses. -- Avoid real Postgres, real Inngest, and real builtins imports unless the test - is explicitly about registry wiring. - -Verification: - -- Tests use fakes/mocks at service boundaries, not real Postgres or real - Inngest. -- Refactors of composition/import paths keep this test green. - -Acceptance gate: - -- [ ] A future rewrite of `run_benchmark` can move code around but cannot skip - validate, persist, run creation, event dispatch, or terminal polling. -- [ ] The test names describe user-visible behavior, not private helper calls. - -### TB-3: Add registry protocol contract tests - -**Issue fixed:** Once registry lookup becomes injectable, there is no shared -contract proving that the builtins adapter and test fakes behave the same way. - -Once a registry/resolver protocol exists, test it independently from CLI and -runtime orchestration: - -- known worker/benchmark/evaluator slugs resolve; -- unknown slugs produce a typed error or clear `KeyError`; -- optional install hints remain available; -- model backend registration side effects still happen exactly once. - -Candidate location: `tests/unit/runtime/test_runtime_registry_contract.py` or -`tests/unit/api/test_registry_contract.py`, depending on ownership. - -Verification: - -- Same contract runs against the builtins-backed registry adapter and a small - fake registry used by tests. - -Files: - -- Test: `tests/unit/runtime/test_runtime_registry_contract.py`. -- Fixture/helper: a fake registry implementation near the test or under - `ergon_core.test_support`. -- Optional test: `tests/unit/architecture/test_registry_imports.py`. - -Steps: - -- [ ] Write the contract tests against a fixture parameter named `registry`. -- [ ] Run the same tests against the builtins adapter and fake registry. -- [ ] Assert missing-slug behavior explicitly. -- [ ] Assert install hints do not require importing data-heavy optional extras. -- [ ] Assert model backend registration remains idempotent. - -Acceptance gate: - -- [ ] Runtime services can be tested with fake registries. -- [ ] Builtins adapter passes the same contract as the fake implementation. -- [ ] Contract tests fail if a registry lookup imports CLI code. - -### TB-4: Reclassify `tests/unit/state` by contract type - -**Issue fixed:** The `state` test tier mixes workflow commands, persisted -runtime transitions, worker/tool behavior, benchmark composition, and fixture -behavior under one vague label. - -Add comments, module names, or a README that explains what the "state" tier -means. Then split or rename tests where the current grouping hides intent. - -Suggested categories: - -- workflow command behavior; -- persisted graph/task state transitions; -- worker/tool state interaction; -- research-rubrics benchmark/worker composition; -- fixture-only behavior. - -Verification: - -- A reader can tell why each state test exists without knowing the historical - branch that introduced it. -- No test loses coverage during renaming or movement. - -Files: - -- Add: `tests/unit/state/README.md` or rename/split tests into clearer - directories. -- Review: - `tests/unit/state/test_research_rubrics_workers.py`. -- Review: - `tests/unit/state/test_research_rubrics_benchmark.py`. -- Review workflow/tool state tests in the same directory. - -Steps: - -- [ ] Inventory each state test file and classify it as workflow command, - persisted graph/task transition, worker/tool behavior, benchmark - composition, or fixture behavior. -- [ ] Rename files only when the existing name hides the contract. -- [ ] Move fixture-only behavior under a fixture/test-support category if it is - not testing runtime state. -- [ ] Add README language that "state" is a test tier, not a production domain - package. - -Acceptance gate: - -- [ ] Every file in `tests/unit/state` has an obvious contract category. -- [ ] No test import path changes require production code changes. - -### TB-5: Add fast artifact-health tests for real-LLM assumptions - -**Issue fixed:** Some real-LLM artifact assumptions are only checked in opt-in -credentialed paths, so artifact schema or parser regressions can slip past the -fast local suite. - -The real-LLM artifact-health harness is opt-in, but some assumptions should be -validated without credentials: - -- rollout artifact directories are named and shaped consistently; -- required metadata fields are present; -- failed/incomplete runs produce diagnosable artifacts; -- fixture artifacts exercise the same reader/parser used by real runs. - -Candidate location: extend -`tests/unit/runtime/test_real_llm_rollout_artifact_health.py` or split a helper -contract test nearby. - -Verification: - -- Fast tests run without `ERGON_REAL_LLM`. -- Real-LLM tests remain opt-in but rely on the same artifact validation helper. - -Files: - -- Review/extend: - `tests/unit/runtime/test_real_llm_rollout_artifact_health.py`. -- Review: - `tests/real_llm/artifact_health.py`. -- Review: - `tests/real_llm/rollout.py`. - -Required cases: - -- artifact directory with complete healthy rollout passes; -- missing required metadata fails with actionable error; -- partial failed rollout still produces enough diagnostic fields; -- worker slug extraction handles both snake_case and camelCase shapes; -- fixture artifact parser is the same parser used by real-LLM checks. - -Acceptance gate: - -- [ ] Unit artifact-health tests pass without network credentials. -- [ ] Real-LLM path delegates to the same validation helper. -- [ ] Failure messages name the missing artifact or field. - -### TB-6: Replace sentinel-aware runtime tests with fake provider tests - -**Issue fixed:** Tests that rely on stub sandbox IDs or sentinel parsing -encourage production runtime code to understand test/provider implementation -details. - -Where runtime tests currently require stub or sentinel sandbox identities, -introduce fake provider implementations that satisfy public provider protocols. -The runtime should observe provider behavior, not parse provider-specific -sentinel strings. - -Verification: - -- Tests still cover skipped, failed, cancelled, and cleanup paths. -- Production runtime modules no longer need helpers such as - `is_stub_sandbox_id`. - -Files: - -- Review tests touching sandbox cleanup, cancellation, skipped tasks, and - propagation. -- Add fake provider helpers under `ergon_core/ergon_core/test_support/` only if - they are reusable across test tiers. -- Pair with code cleanup in `core/sandbox/manager.py` only after - characterization tests exist. - -Steps: - -- [ ] Inventory tests that assert or construct stub sandbox IDs. -- [ ] Define fake provider behavior in terms of public provider methods: - create, reconnect, terminate, publish resources. -- [ ] Replace tests that expect sentinel parsing with tests that assert provider - method calls and runtime state transitions. -- [ ] Add an architecture test blocking runtime imports of - `is_stub_sandbox_id`. - -Acceptance gate: - -- [ ] Runtime behavior for skipped/failed/cancelled cleanup is still covered. -- [ ] Runtime code no longer branches on provider-specific sentinel strings. -- [ ] Test fakes live under test support, not production provider modules. - -## Phase gates for the test stream - -### Phase T1 — Boundary tests first - -Scope: - -- `tests/unit/architecture/test_package_boundaries.py`. -- Allowlist current violations with links to `DI-*` / `CQ-*`. - -Acceptance: - -- [ ] Boundary tests pass and fail when a deliberate forbidden import is added - locally. - -### Phase T2 — Characterization before refactor - -Scope: - -- CLI benchmark-run characterization. -- Registry contract tests. -- Artifact-health fast contracts. - -Acceptance: - -- [ ] Refactor candidates have tests that describe the behavior they preserve. -- [ ] No new test requires real Postgres, real Inngest, or real LLM credentials. - -### Phase T3 — Ratchet allowlists down - -Scope: - -- After dependency-inversion and code-quality refactors land, remove resolved - allowlist entries. - -Acceptance: - -- [ ] Import-boundary allowlist shrinks over time. -- [ ] New exceptions require an RFC or explicit architecture-doc note. - -## Migration / risk - -The main risk is over-constraining architecture too early. The first pass -should allow existing known exceptions with comments, then ratchet them down as -refactors land. - -The second risk is test churn without confidence gain. New tests should be -written around observable behavior and import contracts, not around temporary -helper names introduced during the refactor. - -## Open questions - -- Should architecture tests live under `tests/unit/architecture`, or should - there be a dedicated `tests/architecture` tier? -- Which tests should be required before accepting dependency-inversion work? -- Should real-LLM artifact-health checks define a small golden contract that - can run without external model credentials? diff --git a/docs/rfcs/active/architecture-refactor-audit/03-code-quality.md b/docs/rfcs/active/architecture-refactor-audit/03-code-quality.md deleted file mode 100644 index 864c88a45..000000000 --- a/docs/rfcs/active/architecture-refactor-audit/03-code-quality.md +++ /dev/null @@ -1,642 +0,0 @@ ---- -status: active -opened: 2026-04-27 -author: GPT-5.5 -architecture_refs: - - docs/architecture/README.md - - docs/architecture/01_public_api.md - - docs/architecture/02_runtime_lifecycle.md - - docs/architecture/06_builtins.md -supersedes: [] -superseded_by: null ---- - -# RFC: Code Quality, Duplication, And Complexity - -## Problem - -Fast iteration has left parts of Ergon with high-complexity functions, -branch-heavy example paths, duplicated orchestration logic, and names that no -longer communicate precise ownership. The project already uses Ruff, ty, -slopcop, xenon, and radon-related tooling, but current configuration mostly -documents pre-existing debt rather than defining a refactor target. - -This audit defines the code-quality lens for behavior-preserving cleanup. - -## Current findings - -### Known complexity debt is already listed - -The root `pyproject.toml` has explicit complexity ignores for files such as -experiment persistence, experiment validation, RL rollout/extraction, MiniF2F -loading, file evidence collection, transformer message formatting, and scripts. -Those comments are useful because they identify areas where orchestration has -grown large enough to need ownership review. - -### Generic paths contain example-specific branches - -`ergon_cli.composition.build_experiment` has special branches for smoke workers -and `researchrubrics-workflow-cli-react`. These branches preserve necessary -behavior today, but the pattern does not scale. Generic composition code should -not need to know every benchmark or worker family that requires extra bindings. - -### Tool and workflow code can duplicate service behavior - -CLI command modules, builtin tools, and runtime services all touch workflow -semantics. Without a shared application service boundary, the same concept can -be parsed, validated, or executed in multiple places. - -### Names sometimes encode historical implementation - -Names such as "stub" can mean test double, development default, or lightweight -implementation depending on context. Ambiguous names make it harder to enforce -production/test boundaries and public/private API rules. - -### Deep nesting often reflects missing concepts - -When functions perform lookup, construction, validation, persistence, event -dispatch, and rendering in one flow, nesting and branch count increase. The -answer is not mechanical extraction; it is naming the concepts that already -exist and moving them to the owner that can enforce their invariants. - -## Target shape - -Code quality should be judged against architecture, not only metrics: - -- A module should have one clear owner and one reason to change. -- Public APIs should describe stable concepts, not current storage or CLI - mechanics. -- Composition should be declarative where possible and isolated where it must - branch. -- Runtime orchestration should read as a sequence of named domain operations. -- Tests should cover behavior before complexity-reducing rewrites. - -## Standards proposed - -- Treat new high-complexity ignores as design review triggers, not routine - lint suppressions. -- Prefer small domain objects or command/result types when a function is - passing many loosely related parameters across package boundaries. -- Keep branch-heavy compatibility paths local to adapters or composition - modules, not inside core runtime services. -- Deduplicate only after confirming the duplicated code represents the same - concept. Similar code in different domains may deserve different names. -- Rename "stub", "smoke", and "test" concepts when they are production - defaults or examples rather than test doubles. -- Use architecture docs to record anti-patterns and accepted exceptions so - refactors do not rely on tribal memory. - -## Candidate fixes - -Each candidate below should be concrete enough to become a scoped PR or a -section in an implementation plan. The intent is not generic "clean code"; the -intent is to find where the project encoded missing domain concepts as -duplicated services, private helpers, slug branches, or lint suppressions. - -### CQ-1: Create a complexity ledger from current ignores - -**Issue fixed:** Complexity suppressions are documented inline in -`pyproject.toml`, but there is no owner, smell classification, priority, or -exit criterion for paying the debt down. - -Turn the existing `pyproject.toml` complexity-ignore comments into an explicit -ledger that ranks each offender by risk, ownership, and likely refactor path. - -Initial entries should include: - -- `ExperimentPersistenceService.persist_definition` -- `Experiment.validate` -- RL rollout/extraction helpers -- MiniF2F problem loading -- file evidence collection -- transformer message formatting -- standalone scripts ignored for CLI/script reasons - -Candidate output: a section in this RFC, or a separate -`complexity-ledger.md` in this folder if the list gets long. - -Verification: - -- Every current C901 ignore has an owner, reason, and intended disposition: - keep, split, move, rename, or delete. -- New C901 ignores require adding an entry to the ledger. - -Ledger fields: - -```markdown -| Item | File | Current reason | Domain owner | Smell | Candidate fix | Gate | -|---|---|---|---|---|---|---| -``` - -Smell taxonomy: - -- orchestration doing persistence work; -- validation rules hidden in one large method; -- example-specific branch in generic path; -- private helper cluster that wants a domain object; -- duplicate service responsibility; -- optional dependency/test fallback mixed into production flow. - -Steps: - -- [ ] Convert each current C901 ignore into a ledger row. -- [ ] Run `rg "^def _|^ def _|class .*Service" ergon_core/ergon_core/core/runtime/services` - and add obvious private-helper clusters to the ledger even if not C901. -- [ ] Rank rows by "blocks dependency inversion", "blocks test confidence", - and "local cleanup only". -- [ ] Add a policy that any new C901 ignore must cite a ledger row or RFC. - -Acceptance gate: - -- [ ] The ledger exists and covers every current complexity ignore. -- [ ] The ledger includes at least the large service/private-helper clusters in - `task_management_service.py`, `workflow_service.py`, - `graph_repository.py`, `task_execution_service.py`, and - `experiment_persistence_service.py`. - -### CQ-2: Split experiment composition into generic pipeline plus descriptors - -**Issue fixed:** Generic experiment composition currently knows about concrete -worker families and fixture behavior, which turns every special example into a -potential new branch in shared CLI code. - -Refactor `ergon_cli.composition.build_experiment` so the generic path performs -only these steps: - -1. load registry; -2. construct benchmark/evaluator; -3. ask the selected benchmark/worker for any composition descriptor; -4. build the `Experiment` from descriptors and defaults. - -Current smoke and research-rubrics branches become descriptor providers. This -preserves behavior but removes the pattern where each special worker adds a new -generic CLI branch. - -Verification: - -- Existing smoke and research-rubrics composition tests pass. -- A new fake descriptor test proves a worker can request extra bindings without - changing `build_experiment`. - -Files: - -- Modify: `ergon_cli/ergon_cli/composition/__init__.py`. -- Add descriptor type where selected by `DI-4`. -- Modify smoke fixture registration and research-rubrics registration to - provide descriptors. -- Test: `tests/unit/cli/test_build_experiment_composition.py`. - -Implementation steps: - -- [ ] Write tests that fail on current hard-coded branches being required for - smoke and research-rubrics composition. -- [ ] Add descriptor support with a no-op default. -- [ ] Move smoke branch logic into smoke-owned descriptor provider. -- [ ] Move research-rubrics branch logic into research-rubrics-owned descriptor - provider. -- [ ] Delete `_is_smoke_worker`, `_build_smoke_experiment`, and - `_build_researchrubrics_workflow_experiment` from generic composition - once descriptors cover them. -- [ ] Add an architecture test that blocks new `if worker_slug ==` branches in - generic composition code. - -Acceptance gate: - -- [ ] `ergon_cli.composition` no longer contains concrete worker slug checks. -- [ ] Existing smoke and research-rubrics unit tests pass. -- [ ] New descriptor test demonstrates extension without modifying CLI - composition. - -### CQ-3: Split workflow command execution from CLI rendering - -**Issue fixed:** Workflow parsing, execution, and CLI rendering are coupled -together, causing non-CLI callers to import CLI command modules and making -workflow behavior harder to test independently. - -Separate workflow command concerns into three layers: - -- parser: command string/argv to typed command; -- executor: typed command plus context/session/service to result; -- renderer: result to CLI stdout/stderr text. - -The CLI owns rendering. Builtin agent tools call parser/executor and format -tool-friendly strings. Runtime services own state changes. - -Verification: - -- CLI tests assert the same stdout/stderr behavior. -- Builtin workflow tool tests no longer import `ergon_cli.commands.workflow`. -- Parser/executor tests cover invalid commands, missing context, dry-run paths, - and successful resource/topology operations. - -Files: - -- Add shared parser/executor module selected by `DI-3`. -- Modify: `ergon_cli/ergon_cli/commands/workflow.py`. -- Modify: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py`. -- Test: `tests/unit/cli/test_workflow_cli.py`. -- Test: builtin workflow tool test under `tests/unit/state` or a clearer - renamed location. - -Acceptance gate: - -- [ ] CLI rendering remains byte-for-byte compatible where tests already assert - output. -- [ ] Builtin tools no longer import CLI command modules. -- [ ] Shared executor accepts typed context rather than raw argparse namespace. - -### CQ-4: Audit and rename ambiguous "stub" concepts - -**Issue fixed:** The word "stub" is used across test doubles, development -defaults, smoke fixtures, and lightweight implementations, making it unclear -which code is production behavior and which code is test support. - -Classify every "stub" usage into one of four buckets: - -- test double; -- smoke fixture; -- development default; -- lightweight production implementation. - -Then rename where the current name lies about ownership. For example, a -production default should not be named like a test double, while a test fake -should live under test support and use fake/test naming consistently. - -Verification: - -- `rg "stub|smoke|test_harness|test_support"` has an reviewed allowlist for - production packages. -- User-facing CLI defaults do not imply test-only implementations unless they - really are test-only. - -Files: - -- Review: `ergon_core/ergon_core/core/sandbox/manager.py`. -- Review: `ergon_core/ergon_core/core/runtime/inngest/benchmark_run_start.py`. -- Review: `ergon_core/ergon_core/core/rl/eval_runner.py`. -- Review: `ergon_core/ergon_core/test_support/smoke_fixtures/`. -- Review user-facing CLI defaults in `ergon_cli/ergon_cli/main.py`. - -Steps: - -- [ ] Produce a `stub-smoke-test-naming` section in the complexity ledger or a - small adjacent audit file. -- [ ] Rename test doubles to `Fake*` or `Test*` and move them under - test-support when possible. -- [ ] Rename lightweight production defaults to names that describe their - behavior, not their historical test role. -- [ ] Make production request contracts require explicit worker/evaluator - choices where defaulting to a stub hides behavior. -- [ ] Add tests for any compatibility aliases that must remain. - -Acceptance gate: - -- [ ] Production runtime modules do not branch on "stub" identity. -- [ ] User-facing docs/defaults no longer imply that test doubles are production - defaults. - -### CQ-5: Refactor `persist_definition` behind smaller persistence writers - -**Issue fixed:** Experiment definition persistence is concentrated in one -high-complexity method, so table-writing mechanics and experiment invariants -are hard to review independently. - -`ExperimentPersistenceService.persist_definition` is allowed to be complex -today because it writes a full experiment graph. Keep the transaction boundary, -but split the implementation into named private writer methods or helper -objects: - -- definition row writer; -- worker/evaluator writer; -- instance/task/dependency writer; -- assignment writer; -- task-evaluator link writer. - -The goal is not to change schema or behavior; it is to make persistence -invariants reviewable in smaller units. - -Verification: - -- Existing persistence tests pass. -- Add a focused test for multi-worker assignments if one does not already - cover the branch that motivated CLI special cases. -- Transaction rollback behavior remains unchanged. - -Files: - -- Modify: - `ergon_core/ergon_core/core/runtime/services/experiment_persistence_service.py`. -- Potential new helpers under - `ergon_core/ergon_core/core/persistence/definitions/` if the extracted code - is persistence-model-specific rather than runtime-service-specific. -- Test existing experiment persistence tests, plus add focused tests if missing. - -Implementation steps: - -- [ ] Add characterization tests for single-worker, multi-worker, dependency, - assignment, and evaluator-link persistence. -- [ ] Extract private writer methods without changing transaction boundaries. -- [ ] Name each writer by domain concept, not table name only. -- [ ] Keep `Experiment.persist()` public behavior unchanged. -- [ ] Remove or reduce the C901 ignore only if the extracted shape makes that - honest. - -Acceptance gate: - -- [ ] The service reads as orchestration over named writer steps. -- [ ] Rollback behavior remains a single transaction. -- [ ] Multi-worker assignment behavior is covered by tests. - -### CQ-6: Refactor `Experiment.validate` into rule objects or named validators - -**Issue fixed:** Experiment validation rules are concentrated in one -high-complexity public method, which makes it hard to tell which invariant -failed and hard to add tests for individual rule families. - -Split validation by invariant category while preserving the public -`Experiment.validate()` entrypoint: - -- task uniqueness and dependency validity; -- worker assignment validity; -- evaluator requirement coverage; -- multi-worker/subtask binding validity. - -This makes future public API changes easier to reason about without changing -the caller contract. - -Verification: - -- Existing validation tests pass. -- Each validator has at least one direct test for its failure mode. -- Error messages stay at least as actionable as current messages. - -Files: - -- Modify: `ergon_core/ergon_core/api/experiment.py`. -- Potential create: `ergon_core/ergon_core/api/experiment_validation.py`. -- Test: existing experiment API tests or new - `tests/unit/api/test_experiment_validation.py`. - -Implementation steps: - -- [ ] Snapshot current validation failure messages for representative invalid - experiments. -- [ ] Extract validators for task graph, assignments, evaluator coverage, and - worker bindings. -- [ ] Keep `Experiment.validate()` as the single public entrypoint. -- [ ] Avoid introducing a new public validation framework unless tests show it - pays for itself. - -Acceptance gate: - -- [ ] Public caller behavior is unchanged. -- [ ] Validation rules are testable independently. -- [ ] The original C901 ignore can be removed or justified with a smaller - remaining scope. - -### CQ-7: Establish a "no new branch-if example path" rule - -**Issue fixed:** The codebase has no enforceable guardrail preventing new -example-specific slug checks from being added to generic composition or runtime -paths. - -Add code review guidance and, where possible, tests that reject new generic -composition branches keyed to a specific benchmark or worker slug. The standard -should be: if an example needs special composition, it must declare that need -through a descriptor/hook owned by the example package. - -Verification: - -- Architecture or lint-style test detects new `if worker_slug ==` branches in - generic composition modules, with an allowlist during migration. -- Architecture docs record the accepted extension point. - -Files: - -- Test: `tests/unit/architecture/test_no_ad_hoc_slug_branching.py`. -- Update: `docs/architecture/06_builtins.md` after descriptor/composition - extension point is accepted. - -Rules to enforce: - -- No concrete benchmark/worker/evaluator slug comparisons in generic CLI - composition. -- No suffix parsing for a worker family in generic composition. -- No test-support imports from generic composition unless behind an approved - plugin/harness boundary. -- Slug checks are allowed inside the package that owns the slug family. - -Suggested test inputs: - -- Scan `ergon_cli/ergon_cli/composition`. -- Scan generic runtime services after registry injection is introduced. -- Allowlist current branches only until `CQ-2` lands. - -Acceptance gate: - -- [ ] Adding a new concrete slug branch to generic composition fails tests. -- [ ] Approved extension point is documented. - -### CQ-8: Add module ownership headers only where boundaries are unclear - -**Issue fixed:** Some modules repeatedly attract code from neighboring domains -because their ownership boundary is implicit and only understood by recent -contributors. - -For modules that repeatedly attract misplaced code, add a short top-level -docstring stating what the module owns and what does not belong there. Good -targets are composition, workflow command execution, registry adapters, and -test-support bootstrap modules. - -Verification: - -- Headers are short and enforceable, not narrative. -- Any new ownership statement points to the relevant architecture doc or RFC. - -Candidate modules: - -- `ergon_cli/ergon_cli/composition/__init__.py`. -- Shared workflow command executor introduced by `CQ-3`. -- Registry protocol/adapter modules introduced by `DI-1`. -- Smoke fixture bootstrap modules. -- Runtime services that remain broad after the DDD audit. - -Acceptance gate: - -- [ ] Header says what belongs and what does not belong. -- [ ] Header does not duplicate implementation details. -- [ ] Reviewers can use it to reject misplaced future code. - -### CQ-9: Audit runtime services using DDD-style boundaries - -**Issue fixed:** The runtime services folder contains many service-shaped -modules, but it is not clear which are true domain/application services and -which are duplicated lifecycle fragments or repositories wearing service names. - -The services folder currently contains many service-shaped modules. Some may be -right-sized; others may be procedural clusters that hide duplicate domain -concepts. Audit the folder using domain-driven ownership questions before -moving code: - -- What aggregate or lifecycle does this service own? -- What invariant does it enforce? -- What repositories/providers does it depend on? -- Which other services duplicate the same decision? -- Which private helpers are really domain policies? - -Initial service map to audit: - -```text -ergon_core/ergon_core/core/runtime/services/ - task_management_service.py - task_execution_service.py - workflow_service.py - workflow_initialization_service.py - workflow_finalization_service.py - graph_repository.py - task_cleanup_service.py - task_propagation_service.py - subtask_cancellation_service.py - subtask_blocking_service.py - task_inspection_service.py - experiment_persistence_service.py - evaluator_dispatch_service.py - evaluation_persistence_service.py - rubric_evaluation_service.py - run_service.py - run_read_service.py - cohort_service.py - cohort_stats_service.py - communication_service.py -``` - -Likely duplicate/overlap questions: - -- Do `task_management_service`, `subtask_cancellation_service`, - `subtask_blocking_service`, `task_cleanup_service`, and - `task_propagation_service` encode one task-lifecycle domain or genuinely - separate use cases? -- Does `workflow_service` duplicate graph/resource lookup logic that belongs in - a graph/resource application service? -- Is `graph_repository` both persistence repository and mutation-domain - service? -- Are evaluation dispatch, rubric evaluation, and evaluation persistence cleanly - separated by responsibility? - -Deliverable: - -- Add `04-runtime-service-domain-audit.md` to this RFC folder, or add a - detailed section here if the audit stays short. - -Acceptance gate: - -- [ ] Every service module has a one-sentence responsibility statement. -- [ ] Duplicate responsibilities are listed with candidate merge/split actions. -- [ ] No code moves happen until characterization tests cover the affected - lifecycle. - -### CQ-10: Audit private helpers as design-smell signals - -**Issue fixed:** Large clusters of private helpers can hide missing domain -policies, query objects, DTO mappers, or misplaced responsibilities, but today -they are not audited as architecture signals. - -Private `_` functions are not inherently bad, but clusters of private helpers -often mean the code is compensating for a missing domain object, policy, or -repository. Audit helpers before extracting them mechanically. - -Initial findings to inspect: - -- `task_management_service.py` has validation, invalidation, edge reset, - execution lookup, and dispatch helpers. -- `workflow_service.py` has sandbox manager lookup, task/resource references, - node scope resolution, descendant traversal, producer lookup, and copy - destination helpers. -- `graph_repository.py` has row lookup, sequence allocation, mutation logging, - cycle checks, DTO conversion, and snapshot helpers. -- `task_execution_service.py` has graph-native preparation, definition - preparation, attempt numbering, and status emission. - -Classification: - -- **Keep private helper:** local readability helper with no independent - invariant. -- **Promote to domain policy:** helper encodes a rule that needs tests and a - name. -- **Move to repository/query:** helper is mostly persistence lookup. -- **Move to DTO/mapper:** helper converts persistence rows to transport/domain - objects. -- **Delete after boundary change:** helper exists only because current package - layering is wrong. - -Acceptance gate: - -- [ ] Helper audit identifies at least five helpers to promote/move/delete. -- [ ] Each promoted helper gets a direct test or is covered by an existing - characterization test. -- [ ] No helper is extracted merely to reduce line count without a better name - or owner. - -## Phase gates for the code-quality stream - -### Phase Q1 — Audit before movement - -Scope: - -- Complexity ledger. -- Runtime service domain audit. -- Private-helper audit. -- Ad hoc branch architecture tests with current allowlist. - -Acceptance: - -- [ ] Audits identify concrete files and candidate actions. -- [ ] Tests prevent new ad hoc slug branches. -- [ ] No production behavior changes. - -### Phase Q2 — Composition and workflow cleanup - -Scope: - -- Descriptor-based experiment composition. -- Workflow parser/executor/renderer split. - -Acceptance: - -- [ ] Generic composition has no concrete example slug branches. -- [ ] Builtin tools no longer import CLI command modules. -- [ ] Characterization tests pass. - -### Phase Q3 — Service/domain refactors - -Scope: - -- One lifecycle cluster at a time, chosen from the service domain audit. -- Start with the cluster that blocks dependency inversion or test clarity most. - -Acceptance: - -- [ ] Behavior is locked by characterization tests before moving code. -- [ ] Each extracted domain policy has a named owner and test. -- [ ] Complexity ignores shrink or have updated ledger justification. - -## Migration / risk - -The main risk is aesthetic refactoring that changes behavior or creates more -abstractions without reducing coupling. Refactors should be small enough to -review and should preserve public behavior unless a separate RFC says -otherwise. - -The second risk is over-indexing on cyclomatic complexity. Some orchestration -is inherently sequential and readable. A lower branch count is only a win if -the resulting names clarify invariants and failure modes. - -## Open questions - -- Which complexity metric should become a hard CI gate after the first cleanup - pass: Ruff C901, xenon rank, radon score, or a smaller custom import/size - check? -- Should `ergon_cli.composition` remain one module after descriptors are - introduced, or should it become a package with separate composition owners? -- Which naming changes are worth compatibility wrappers, and which can be - changed directly because they are branch-local implementation details? diff --git a/docs/rfcs/active/architecture-refactor-audit/README.md b/docs/rfcs/active/architecture-refactor-audit/README.md deleted file mode 100644 index 6b7eb5c31..000000000 --- a/docs/rfcs/active/architecture-refactor-audit/README.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -status: active -opened: 2026-04-27 -author: GPT-5.5 -architecture_refs: - - docs/architecture/README.md - - docs/architecture/01_public_api.md - - docs/architecture/02_runtime_lifecycle.md - - docs/architecture/04_persistence.md - - docs/architecture/06_builtins.md - - docs/architecture/07_testing.md -supersedes: [] -superseded_by: null ---- - -# RFC: Architecture Refactor Audit - -## Problem - -Ergon has moved quickly enough that useful behavior now lives beside accidental -structure: direct package coupling, special-case composition branches, -duplicated setup logic, test-support leakage, and high-complexity orchestration -code. The immediate goal is not to redesign product behavior. It is to make the -existing behavior easier to understand, test, extend, and preserve. - -This RFC folder starts an audit-driven refactor program. It separates the work -into three lenses so each proposal can stay concrete: - -- [`01-dependency-inversion.md`](01-dependency-inversion.md) covers package - boundaries, public API shape, registry resolution, and cross-package imports. -- [`02-test-brittleness-and-gaps.md`](02-test-brittleness-and-gaps.md) covers - brittle tests, fixture boundaries, missing contract tests, and real-LLM/e2e - confidence gaps. -- [`03-code-quality.md`](03-code-quality.md) covers duplication, branch-heavy - example paths, excessive nesting, cyclomatic complexity, naming drift, and - file ownership. - -## Refactor rule - -Behavior stays the same unless a follow-up RFC explicitly changes it. The -program should first extract boundaries, name concepts, move code to better -owners, and add characterization tests around risky flows. Any behavioral -change discovered during cleanup should be split into a separate bug or RFC. - -## Target architecture principles - -1. **Core owns contracts, not default implementations.** `ergon_core` should - expose stable interfaces and runtime services; concrete benchmark, worker, - evaluator, model, and sandbox registrations should be injected through an - explicit composition boundary. -2. **Builtins are plugins, not runtime prerequisites.** `ergon_builtins` should - implement public contracts and provide a default registry bundle without - requiring core runtime imports to know about that bundle. -3. **CLI is an adapter.** `ergon_cli` should parse user input and call shared - application services. Agent tools and core runtime code should not depend on - CLI command modules. -4. **Tests are consumers of public contracts.** Test support may provide - fixtures, fake providers, and smoke registrations, but core code should not - branch on test identities or sentinel values. -5. **Complexity should be paid down near ownership boundaries.** Large - orchestration functions should be split by responsibility only when the - split clarifies invariants or makes behavior easier to test. - -## Proposal - -Adopt this RFC folder as the tracking document for an architecture audit. Each -child document should collect concrete findings, define the target shape, and -list candidate refactors in dependency order. Accepted follow-up RFCs and -implementation plans can then pull from these findings without turning this -folder into a single mega-plan. - -The initial work should prioritize: - -1. Dependency inversion and composition boundaries, because package coupling - makes every later cleanup harder. -2. Test brittleness and missing contract coverage, because behavior-preserving - refactors need confidence. -3. Code quality and complexity cleanup, because it benefits most after the - owning modules and contracts are clearer. - -## Invariants affected - -This audit does not change runtime invariants by itself. It may produce -follow-up RFCs that update: - -- `docs/architecture/01_public_api.md` if public API ownership changes. -- `docs/architecture/02_runtime_lifecycle.md` if runtime composition or task - orchestration boundaries change. -- `docs/architecture/06_builtins.md` if registry/plugin semantics change. -- `docs/architecture/07_testing.md` if test tier responsibilities change. - -## Migration - -No code migration is proposed in this folder directly. Migration guidance lives -inside each child audit document and should be converted into implementation -plans only after the target architecture is accepted. - -Before implementation, each refactor should have: - -- A characterization test or existing test reference for the behavior being - preserved. -- A clear package-boundary statement: what module owns the new abstraction and - which packages may import it. -- A rollback path if the refactor uncovers behavior that differs from the docs. - -## Alternatives considered - -### One giant architecture RFC - -This would be easy to create, but it would encourage broad, vague findings and -make acceptance difficult. Dependency inversion, tests, and code quality have -different audiences and different risk profiles. - -### Three unrelated top-level RFCs - -This would make each stream independently acceptable, but it would hide the -shared refactor goal. The folder keeps the audit cohesive while preserving -focused documents. - -### Immediate code cleanup without an audit - -This risks preserving the current accidental architecture under new names. -Because the goal is behavior-preserving refactor, the first deliverable should -be shared understanding and standards. - -## Open questions - -- Which package boundary should own registry resolution: core, a new - composition package, or the CLI/application layer? -- How much backward compatibility is required for current import paths inside - the repo? -- Should complexity thresholds become CI-enforced once the first cleanup pass - lands, or should they remain advisory until the major offenders are reduced? - -## On acceptance - -When this RFC folder is accepted: - -- Move the folder or accepted child docs under `docs/rfcs/accepted/`. -- Link the first implementation plan in `docs/superpowers/plans/`. -- Update affected architecture docs with any new import-boundary or testing - invariants. diff --git a/docs/rfcs/active/final-worker-output-source-of-truth.md b/docs/rfcs/active/final-worker-output-source-of-truth.md deleted file mode 100644 index 09495a56e..000000000 --- a/docs/rfcs/active/final-worker-output-source-of-truth.md +++ /dev/null @@ -1,177 +0,0 @@ -# Final Worker Output Source of Truth - -_Sketch for treating `WorkerOutput` as the semantic final answer, rather than inferring it from context transcript events._ - ---- - -## Problem - -`ReActWorker.get_output()` currently reconstructs the worker's final output by reading persisted `RunContextEvent` rows and taking the last `assistant_text`, with a fallback that searches for a `final_result` tool call. That works, but it conflates three different concepts: - -- `assistant_text`: model text emitted during a generation turn -- `tool_call(final_result)`: PydanticAI's structured-output protocol -- `WorkerOutput`: the worker's final semantic result for the task execution - -The final answer should not be inferred from transcript shape. It should be the explicit output returned by the worker and persisted by the runtime. - -## Current State - -The codebase already has most of the right destination: - -- `WorkerOutput(output=..., success=..., metadata=...)` is the worker API's semantic final result. -- `worker_execute_fn()` receives the worker's `WorkerOutput` after `worker.get_output(worker_context)`. -- `WorkerExecuteResult.final_assistant_message` carries that value from `worker-execute` back to `task-execute`. -- `execute_task_fn()` passes `worker_result.final_assistant_message` into `FinalizeTaskExecutionCommand`. -- `TaskExecutionService.finalize_success()` persists it to `RunTaskExecution.final_assistant_message`. -- `RunTaskExecution` also has `output_json` for structured execution output metadata. - -So the persistence model already has a first-class execution-level field for the final assistant message. The weak part is upstream: `ReActWorker.get_output()` still computes that value by re-reading the context-event transcript. - -## Desired Shape - -The runtime should treat final worker output as execution-level data, not as another transcript event. - -```text -worker.execute() yields GenerationTurn events - | - v -ContextEventRepository persists transcript evidence - | - v -worker.get_output() returns WorkerOutput - | - v -TaskExecutionService.finalize_success() persists execution result - | - v -RunTaskExecution.final_assistant_message / output_json are the source of truth -``` - -In this model: - -- `RunContextEvent` remains the append-only transcript log. -- `RunTaskExecution.final_assistant_message` is the final human-readable answer. -- `RunTaskExecution.output_json` can hold structured metadata from `WorkerOutput.metadata`. -- Rollout-card export reads both: context events for the trace, task execution fields for final execution outputs. - -## Proposed Contract - -`WorkerOutput` should be the only object that defines a worker's final semantic output. - -```python -class WorkerOutput(BaseModel): - output: str - success: bool = True - metadata: dict[str, Any] = Field(default_factory=dict) -``` - -The runtime should persist it as: - -```text -RunTaskExecution.final_assistant_message = WorkerOutput.output -RunTaskExecution.output_json = { - "worker_output": { - "success": WorkerOutput.success, - "metadata": WorkerOutput.metadata, - }, - "resource_ids": [...] -} -``` - -If we want the full `WorkerOutput` object available in exports, use `output_json["worker_output"]` rather than adding a new `RunContextEvent` type. - -## ReActWorker Implication - -`ReActWorker` should stop deriving output by querying `ContextEventRepository`. - -Instead, it should capture the structured final result while running the PydanticAI agent. The worker already configures: - -```python -agent: Agent[None, _AgentOutput] = Agent( - model=resolved.model, - instructions=self.system_prompt or None, - tools=self.tools, - output_type=_AgentOutput, -) -``` - -The final `_AgentOutput.final_assistant_message` should be stored on the worker instance during `execute()`, then returned directly from `get_output()`. - -Conceptually: - -```python -class ReActWorker(Worker): - def __init__(...): - ... - self._final_output: _AgentOutput | None = None - self._turn_count = 0 - - async def _run_agent(...): - async with agent.iter(...) as run: - ... - self._final_output = run.result.output - - def get_output(self, context: WorkerContext) -> WorkerOutput: - if self._final_output is None: - return WorkerOutput(output="", success=False) - return WorkerOutput( - output=self._final_output.final_assistant_message, - success=True, - metadata={ - "reasoning": self._final_output.reasoning, - "turn_count": self._turn_count, - }, - ) -``` - -The exact PydanticAI result access may differ, but the ownership is the important part: the worker returns the structured final result it received from the agent, rather than reconstructing it from persisted context events. - -## Why Not `final_agent_message` Context Events? - -A new context event type would make the transcript easier to query, but it blurs the abstraction boundary. - -`RunContextEvent` should answer: "What happened during the model/tool interaction?" - -`RunTaskExecution` should answer: "What did this worker execution finally produce?" - -The final output belongs to the second question. Mirroring it into a rollout-card export is useful; storing it as another transcript event is optional and should not be the source of truth. - -## Implementation Sketch - -1. Keep `ContextEventRepository` unchanged as the transcript serializer. -2. Update `WorkerExecuteResult` only if needed to carry `WorkerOutput.metadata`. -3. Update `FinalizeTaskExecutionCommand` to carry `worker_output_metadata` or a full `worker_output_json`. -4. Update `TaskExecutionService.finalize_success()` to persist: - - `final_assistant_message` - - `output_json["worker_output"]` - - existing `resource_ids` if present -5. Update `ReActWorker` to capture its PydanticAI structured result during execution. -6. Replace `ReActWorker._base_output()` with a simple read of the captured structured output. -7. Remove `_latest_final_result_message()` if no other worker needs it. -8. Update rollout-card export to include task execution final outputs from `RunTaskExecution`, not by scanning `RunContextEvent`. - -## Migration / Compatibility - -Existing completed runs may only have context events, so readers should remain tolerant: - -- Prefer `RunTaskExecution.final_assistant_message`. -- If absent, optionally fall back to the old transcript inference for legacy runs. -- Do not use the fallback in new execution paths. - -This preserves old data while making new runs explicit. - -## Tests - -Add focused tests for: - -- `ReActWorker.get_output()` returns the captured structured `_AgentOutput`, not the last `assistant_text`. -- A run with intermediate `assistant_text` plus final structured output persists the structured final output. -- `TaskExecutionService.finalize_success()` writes `final_assistant_message` and `output_json["worker_output"]`. -- Context event replay still reconstructs transcript messages without needing final-output semantics. -- Legacy read helpers fall back to transcript inference only when `RunTaskExecution.final_assistant_message` is missing. - -## Open Questions - -1. Should `WorkerExecuteResult` carry the full `WorkerOutput.metadata`, or should `worker_execute_fn()` persist it directly before returning? -2. Should `RunTaskExecution.output_json` store the full `WorkerOutput` shape, or only `metadata` plus resource references? -3. Should rollout-card export call this field `worker_output`, `execution_output`, or `final_worker_output`? diff --git a/docs/rfcs/paper-nice-to-haves/2026-04-21-agent-framework-adapter-layer.md b/docs/rfcs/paper-nice-to-haves/2026-04-21-agent-framework-adapter-layer.md deleted file mode 100644 index c7687c9e3..000000000 --- a/docs/rfcs/paper-nice-to-haves/2026-04-21-agent-framework-adapter-layer.md +++ /dev/null @@ -1,205 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: paper-parity -architecture_refs: [docs/architecture/02_runtime_lifecycle.md, docs/architecture/03_providers.md, docs/architecture/06_builtins.md#extension-points] -supersedes: [] -superseded_by: null -paper_blocking: false -paper_blocking_reason: >- - Not paper-blocking. The paper (Appendix E) honestly describes the five - non-PydanticAI agent frameworks as `planned (RFC-...)` rows. Landing - this RFC flips rows to `integrated`, but the paper's claims are - well-formed under either state. The Appendix C preservation table does - not depend on any of these adapters existing; it is about projection - operators over already-recorded trajectories. ---- - -# RFC: Agent-framework adapter layer - -> **Not paper-blocking.** Appendix~E of the NeurIPS 2026 Ergon paper -> lists the five non-PydanticAI agent frameworks (LangGraph, CrewAI, -> AutoGen, Google ADK, Claude Code) as `planned (RFC-...)` rows, and -> the paper is honest about which integrations are shipped today versus -> scoped-but-not-yet-written. This RFC may land before, during, or after -> paper submission without altering any paper claim. The preservation -> table in Appendix~C is about projections over recorded trajectories, -> not about which runtimes produce those trajectories; no cell in that -> table changes based on which adapters exist. Contrast with the three -> projection RFCs in this directory, which *are* paper-blocking. - -## Problem - -The paper (Appendix E, "Tech-Stack Integrations") describes Ergon as hosting -six agent frameworks as inner runtimes: Pydantic AI, LangGraph, CrewAI, -AutoGen, Google ADK, and Claude Code. The codebase today integrates **one** -of those six. - -Concretely, the only adapter that wraps a third-party agent loop and emits -mutations into Ergon's `RunRecord` / `RunGraph` lives in -`ergon_builtins/ergon_builtins/workers/baselines/react_worker.py:L28-L105`, -which wraps `pydantic_ai.Agent.iter()`. The supporting machinery is: - -- `ergon_core/ergon_core/core/persistence/context/assembly.py:L94-L130` — - rebuilds `pydantic_ai.messages.ModelRequest` / `ModelResponse` from stored - `RunContextEvent` rows so a worker can resume mid-trajectory. -- `ergon_core/ergon_core/core/providers/generation/pydantic_ai_format.py` — - pulls text, tool calls, and logprobs out of serialized PydanticAI - responses for downstream extraction. -- `ergon_builtins/ergon_builtins/tools/research_rubrics_toolkit.py:L63` — - wraps Ergon-side tools as `pydantic_ai.tools.Tool` instances. - -For LangGraph, CrewAI, AutoGen, Google ADK, and Claude Code, there are zero -imports, zero adapter files, and zero declared dependencies anywhere in -`ergon_core`, `ergon_builtins`, `ergon_infra`, or `tests`. - -The PydanticAI integration emerged organically — there is no documented -adapter contract, no abstract base class, and no test that asserts "this -is what an agent-framework adapter must do". The pattern is implicit, lives -in one place, and has not been generalized. Adding a second framework today -means duplicating undocumented assumptions about message replay, tool -serialization, logprob extraction, and mutation emission. - -## Proposal - -Promote the implicit Pydantic AI integration to an explicit -**`AgentRuntimeAdapter`** contract that any inner-loop framework can -implement. Then ship a stub adapter per planned framework that demonstrates -the shape but defers full integration to follow-on RFCs. - -### Contract sketch - -```python -# ergon_core/ergon_core/api/agent_runtime_adapter.py (new) - -from typing import Protocol -from ergon_core.api.run_context import RunContextEvent -from ergon_core.api.tool import ToolDescriptor - -class AgentRuntimeAdapter(Protocol): - """Wraps a third-party agent loop so it can be driven by Ergon. - - Implementations MUST: - - serialize each turn as a sequence of RunContextEvents - (system_prompt | user_message | assistant_text | thinking | - tool_call | tool_result) committed to the WAL before returning; - - emit GenerationTurn rows with logprobs and env_mask when the - underlying framework exposes them; - - rebuild framework-native message state from stored events - (i.e. support resumption from any sequence boundary); - - expose Ergon-side tools as the framework's native tool type. - """ - - framework_id: str # e.g. "pydantic_ai", "langgraph", "crewai" - - def run_turn(self, *, input_event: RunContextEvent, - tools: list[ToolDescriptor]) -> list[RunContextEvent]: ... - - def replay_to(self, *, events: list[RunContextEvent]) -> object: - """Rebuild framework-native message state. Returns native object.""" - ... -``` - -Refactor `ReActWorker` to be the concrete `pydantic_ai` -`AgentRuntimeAdapter`. Add stub adapters in -`ergon_infra/ergon_infra/agent_adapters/` (new package) for each planned -framework: `langgraph_adapter.py`, `crewai_adapter.py`, -`autogen_adapter.py`, `google_adk_adapter.py`, `claude_code_adapter.py`. -Each stub raises `NotImplementedError` with a one-line pointer to the -follow-on RFC, but registers `framework_id` and a `requirements` block so -downstream tooling can discover what's planned. - -### Per-framework follow-on RFCs - -Each of the five planned adapters gets its own follow-on RFC after this -contract lands. The follow-on RFCs are not blocked on this one being -implemented — they can be written speculatively against the contract sketch -above. Suggested order (easiest to hardest): - -1. **Claude Code** — agent loop is `claude-code-sdk`; trajectory shape is - already close to Ergon's event log. -2. **LangGraph** — agent loop is the compiled `StateGraph`; tool calls map - 1:1; multi-node walks need a flattening rule onto Ergon's `RunGraphNode` - parent_node_id. -3. **Google ADK** — analogous to LangGraph (state-machine-shaped). -4. **AutoGen** — multi-agent by default; needs a per-agent `worker_binding_key` - convention. -5. **CrewAI** — task delegation is opaque; will likely require shimming - into Ergon's existing `spawn_subtask` action. - -## Invariants affected - -This RFC introduces a new public-API invariant in -`docs/architecture/01_public_api.md`: - -> **Invariant (agent runtime adapters).** Every framework integrated as an -> Ergon inner runtime ships an `AgentRuntimeAdapter` implementation that -> commits one `RunContextEvent` per turn-boundary observable to the -> framework, before returning control. Adapters that cannot meet this -> contract MUST NOT be merged. - -Touches `docs/architecture/03_providers.md` (provider/adapter boundary) and -`docs/architecture/06_builtins.md#extension-points` (adds a new extension -point: "Add an agent runtime"). - -## Migration - -- `ReActWorker` becomes the canonical example implementation. No behavior - change to existing benchmarks; the class gains an explicit - `framework_id = "pydantic_ai"` attribute and is documented as the - reference adapter. -- Stub adapters in `ergon_infra/agent_adapters/` add 5 new files, no edits - to existing call paths. -- No DB / Alembic migration. Schema is unchanged; this RFC formalizes - what the schema already supports. -- Tests: add `tests/state/test_agent_adapter_contract.py` with one passing - case (PydanticAI via ReActWorker) and five `pytest.skip` cases naming - the follow-on RFCs. - -## Alternatives considered - -- **Leave the contract implicit.** Rejected: the paper claims six - integrations and we have one. A new contributor adding LangGraph today - has no reference; the result will be a second snowflake. -- **Adapt frameworks behind a single mega-class.** Rejected: each framework - has different replay semantics; single-class polymorphism becomes a - long if-elif on `framework_id`. The Protocol approach lets each adapter - live in its own file. -- **Cut the planned adapters from the paper and only describe Pydantic - AI.** Rejected by the paper authors — Appendix E is framed as a - descriptive integrations *map*, including planned rows; this RFC is the - parity work. - -## Open questions - -- Should `AgentRuntimeAdapter` be a `Protocol` (structural) or a - `class(ABC)` (nominal)? `Protocol` is cheaper to adopt; `ABC` enforces - presence at class-definition time (cf. the `Benchmark` ABC pattern in - RFC `2026-04-18-onboarding-deps-on-benchmark-abc.md`). Lean toward - `ABC` for consistency. -- The `replay_to(...)` return type is `object` because each framework - returns its own native message-list type. Worth a `TypeVar` parametrization? - Probably not — the adapter is the only consumer of its own return type. -- Where does the framework-id registry live? Suggest - `ergon_infra/agent_adapters/registry.py` mapping `framework_id` → - adapter class, mirroring the trainer-adapter registry proposed in - RFC `2026-04-21-rl-trainer-adapter-expansion.md`. - -## Paper parity - -This RFC closes one of three parity gaps identified during the Appendix E -audit (sibling RFCs: `2026-04-21-rl-trainer-adapter-expansion.md`, -`2026-04-21-projection-operator-*.md`). Until this RFC is accepted and the -five follow-on adapters land, Appendix E describes those rows as -`planned (RFC-2026-04-21-agent-framework-adapter-layer)` rather than -`integrated`. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Add `AgentRuntimeAdapter` to `docs/architecture/01_public_api.md` core - abstractions. - - Add the new extension point ("Add an agent runtime") to - `docs/architecture/06_builtins.md#extension-points`. - - Open the five follow-on RFCs (one per planned framework). - - Update Appendix E in the paper repo to reference accepted state. diff --git a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-call-tree.md b/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-call-tree.md deleted file mode 100644 index be1db166d..000000000 --- a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-call-tree.md +++ /dev/null @@ -1,191 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: paper-parity -architecture_refs: [docs/architecture/08_rl_loop.md, docs/architecture/04_persistence.md] -supersedes: [] -superseded_by: null -paper_blocking: true -paper_blocking_reason: >- - Preservation claim in Appendix~C (Table~\ref{tab:preservation}) requires - π_call-tree to be an implementable projection that preserves call-depth - structure. Charlie commits to landing this RFC and its implementation - before the NeurIPS 2026 paper submission goes live. ---- - -# RFC: Projection operator — nested sub-LM call tree - -> **Paper-blocking.** This RFC is a hard pre-submission dependency for the -> NeurIPS 2026 Ergon paper. Appendix~C's preservation table claims -> $\pi_{\text{call-tree}}$ preserves call-depth structure (diagonal ✓ -> cell under the *Call-dep* column); that claim requires the projection -> function specified in this RFC to exist in code by submission time. In -> addition, the "spawn-ordering invariant" in the Proposal section is a -> substrate guarantee the paper implicitly relies on; the state-invariant -> test that confirms it must land alongside the projection. The -> surrounding RFC text remains written in scoping voice, but the decision -> to ship has already been made — this is no longer a "nice to have." -> Target: implementation landed and tested before paper submission. - -## Problem - -The paper claims the Ergon substrate emits a "nested sub-LM call tree" -projection: a recursive structure where each node is a turn (or a tool -invocation) and edges are parent → child sub-call relationships. This -projection feeds reward-model training pipelines that score reasoning -*structure* (not just final answers) and downstream visualization tooling -that renders rollouts as collapsible trees. - -The substrate records all the inputs: - -- `RunGraphNode.parent_node_id` at - `ergon_core/core/persistence/graph/models.py:L44-L89` — sub-task - containment. -- `RunContextEvent` at - `ergon_core/core/persistence/context/models.py:L25-L49` — the per-turn - events (`system_prompt`, `user_message`, `assistant_text`, `thinking`, - `tool_call`, `tool_result`). -- `GenerationTurn` at - `ergon_core/core/persistence/telemetry/models.py:L383-L464` — the - generation metadata (model, logprobs, env_mask) bound to each - assistant turn. - -But `extract_agent_trajectories` flattens all of this into a single token -stream per agent. Tool calls are inlined as tokens; sub-task boundaries -are erased. There is no recursive tree builder. - -## Proposal - -Add a new projection in `ergon_core/core/rl/projections/call_tree.py`. It -walks the `(RunGraphNode, RunContextEvent)` pair to produce a tree: - -```python -# ergon_core/core/rl/projections/call_tree.py (new) - -from dataclasses import dataclass, field -from uuid import UUID -from ergon_core.core.persistence.context.models import RunContextEventKind - -@dataclass -class CallTreeTurn: - """One assistant turn within a single agent's execution.""" - sequence: int - role: str # "assistant" | "user" | "system" - content: str | None # text payload (assistant_text, etc.) - thinking: str | None # if event_kind == "thinking" - tool_calls: list["ToolInvocation"] = field(default_factory=list) - children: list["CallTreeTurn"] = field(default_factory=list) - # children are non-empty when this turn spawned sub-tasks - -@dataclass -class ToolInvocation: - tool_name: str - arguments: dict - result: str | None # populated by the matching tool_result - spawned_node_id: UUID | None # set if this tool call spawned a sub-task - -@dataclass -class CallTree: - run_id: UUID - root_node_id: UUID - turns: list[CallTreeTurn] # the root node's turns; each turn's - # children list points at sub-task subtrees - -def project_call_tree(*, run_id: UUID) -> CallTree: - """Recursively walk RunGraphNode by parent_node_id; for each node, - fold its RunContextEvent stream into CallTreeTurn objects, then attach - sub-task subtrees as children of the spawning turn.""" - ... -``` - -The recursion is depth-first on `parent_node_id`. Spawning is detected by -matching a `tool_call` event whose `tool_name == "spawn_subtask"` to the -`RunGraphNode` it created (the link is the `node.added` mutation that -follows the tool call within ε of its sequence number). - -### Output formats - -`CallTree` itself is the canonical in-memory representation. Two -serialization helpers ship with the projection: - -- `to_jsonl(call_tree, fp)` — each line is one `CallTreeTurn` with a - `path: list[int]` indexing its position in the tree. Suitable for - `pandas.read_json(lines=True)`. -- `to_dict(call_tree) -> dict` — nested dict mirroring the in-memory - shape. Suitable for direct ingestion into reward-model training - pipelines that expect tree-shaped JSON (DPO-tree, RLAIF-tree, etc.). - -## Invariants affected - -Adds an invariant to `docs/architecture/08_rl_loop.md`: - -> **Invariant (call-tree projection).** The call-tree projection -> reconstructs sub-task spawning by matching `tool_call(spawn_subtask)` -> events to `node.added` mutations within the same run, ordered by -> mutation sequence. The substrate guarantees these two events appear in -> immediate sequence (the spawning is synchronous within the worker's -> turn). - -If this guarantee is *not* in fact provided by the existing runtime, -that is a substrate bug to fix before this projection ships — captured as -an open question below. - -## Migration - -- New file `ergon_core/core/rl/projections/call_tree.py`. -- No DB migration. No API change. -- Tests: add `tests/rl/test_projection_call_tree.py` with cases for - (a) zero-spawn run (flat list of turns), (b) one-spawn run (tree of - depth 2), (c) multi-spawn run (tree of depth 3+, sibling sub-tasks), - (d) tool-call without spawn (no `children` populated, `tool_calls` - populated). - -## Alternatives considered - -- **Fold call-tree information into `extract_agent_trajectories`** - (return both flat and tree shape from one function). Rejected: the - consumers are different (RL trainers want flat; reward models want - tree); coupling them means every trainer pays the tree-walk cost. -- **Materialize the call tree in a new `RunCallTree` table.** Rejected: - derivable in O(N + M) from existing tables; materialization is a - caching choice deferrable until the projection becomes hot. -- **Use Mermaid/Graphviz emission as the canonical format.** Rejected: - reward-model training pipelines want JSON; visualization tooling can - render JSON via a thin adapter. - -## Open questions - -- Does the runtime in fact emit `tool_call(spawn_subtask)` immediately - before its `node.added` mutation in mutation-sequence order? If not, - the matching rule above breaks. Action item: add a `tests/state/` - test verifying this invariant before this RFC's implementation lands. -- How do we handle a `tool_call` that *fails* before spawning the - sub-task? Suggest: `spawned_node_id` stays `None`; the failure is - recoverable from the tool_result's payload. -- Should `CallTreeTurn` carry `logprobs` for downstream scoring, or do - we keep the projection token-shape-free and have consumers join - back to `GenerationTurn`? Lean keep it token-free. - -## Paper parity - -**Paper-blocking.** Sibling paper-blocking RFCs: -`2026-04-21-projection-operator-option-tagged.md`, -`2026-04-21-projection-operator-mcts.md`. Appendix~C of the NeurIPS 2026 -paper claims $\pi_{\text{call-tree}}$ preserves call-depth structure in -Table~\ref{tab:preservation}; that claim requires the projection function -specified here to exist in code by submission. It also depends on the -spawn-ordering invariant being confirmed (see "Open questions") — -resolving that open question is paper-blocking alongside the projection -implementation, because if the substrate doesn't in fact order spawns -synchronously with the originating turn, the projection is not -well-defined and Table~\ref{tab:preservation}'s ✓ would have to soften -to ~. Until landed, Appendix~E still describes the call-tree row as -`planned (RFC-2026-04-21-projection-operator-call-tree)`; landing flips -that row to `integrated` and satisfies the Appendix~C claim. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Add the spawn-ordering invariant to - `docs/architecture/08_rl_loop.md` (and confirm or fix runtime). - - Update Appendix E (paper repo) to cite accepted state. diff --git a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-mcts.md b/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-mcts.md deleted file mode 100644 index 53526e4ab..000000000 --- a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-mcts.md +++ /dev/null @@ -1,219 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: paper-parity -architecture_refs: [docs/architecture/08_rl_loop.md, docs/architecture/04_persistence.md] -supersedes: [] -superseded_by: null -paper_blocking: true -paper_blocking_reason: >- - Preservation claim in Appendix~C (Table~\ref{tab:preservation}) makes - two load-bearing MCTS commitments: (a) π_mcts preserves MCTS-entropy - structure (diagonal ✓), and (b) π_json-log × MCTS-ent is ✓ (the raw - substrate already captures everything an MCTS workload records). Both - cells assume the mcts.* annotation namespace from this RFC's Option A - is live by submission. Charlie commits to landing this RFC before the - NeurIPS 2026 paper submission goes live. ---- - -# RFC: Projection operator — MCTS search-tree records - -> **Paper-blocking.** This RFC is a hard pre-submission dependency for the -> NeurIPS 2026 Ergon paper, and carries **two** distinct load-bearing -> claims in Appendix~C's Table~\ref{tab:preservation}: -> -> 1. The diagonal ✓ under the *MCTS-ent* column in the $\pi_{\text{mcts}}$ -> row — requires the projection function in this RFC to exist in code. -> 2. The ✓ under the *MCTS-ent* column in the $\pi_{\text{json-log}}$ -> row — requires the `mcts.*` annotation namespace (Option A) to be -> reserved and in the substrate, even if no MCTS driver has yet been -> written. The claim is that Ergon's raw substrate *can* carry these -> statistics without schema change; landing Option A as a reserved -> namespace (with the default-filled projection behaviour) makes that -> claim true by construction. -> -> The surrounding RFC text remains written in scoping voice, and the -> "Open questions" about whether a concrete MCTS driver ships alongside -> are *still* open (a driver is not paper-blocking; the namespace and -> projection are). Target: Option A landed — namespace reserved, -> projection function shipped with default-filled behaviour on -> zero-annotation trees, tests passing — before paper submission. - -## Problem - -The paper claims the Ergon substrate can emit a fifth trajectory -projection — **MCTS-style search-tree records** — suitable for training -tree-search policies (AlphaZero-style value/prior networks, ToT-fine-tune, -rStar-Math, OmegaPRM). - -The shape consumers expect is a per-node record with: - -- visit count `N(s, a)`; -- accumulated reward / Q-value `Q(s, a)`; -- prior probability `P(s, a)` (from the policy network at expansion time); -- parent / child pointers; -- terminal flag and backup-propagated value. - -None of this exists in the Ergon schema today. `RunGraphMutation` is an -append-only audit log, not a search-statistics store. `RunGraphNode` -represents a subtask, not a search-tree node. There is no notion of -"visit count" or "backup" anywhere in the codebase. - -Unlike the option-tagged and call-tree projections (sibling RFCs), the -MCTS projection **cannot be implemented as a pure read over the existing -schema** — the required statistics are never recorded. - -## Proposal - -Two options; this RFC recommends Option A. - -### Option A (recommended): annotation-namespace for search statistics - -Use the existing `RunGraphAnnotation` table (see -`ergon_core/core/persistence/graph/models.py:L123-L165`) with a reserved -namespace `mcts.*` to record per-node search statistics. Annotations are -already append-only and per-node-keyed, so this is zero schema change. - -Namespace conventions: - -``` -mcts.visits → {"n": int} -mcts.q_value → {"q": float, "samples": int} -mcts.prior → {"p": float, "model": str} -mcts.terminal → {"terminal": bool, "value": float | null} -``` - -One annotation per `node.added` for `mcts.prior` (set by the expanding -policy). One annotation per backup for `mcts.visits` and `mcts.q_value` -(incremented by the MCTS driver). - -The projection walks `RunGraphNode` + their `mcts.*` annotations and -emits a flat list of search-tree records: - -```python -# ergon_core/core/rl/projections/mcts.py (new) - -from dataclasses import dataclass -from uuid import UUID - -@dataclass(frozen=True) -class SearchTreeRecord: - node_id: UUID - parent_node_id: UUID | None - state_token_ids: list[int] # prompt_ids at expansion - action_token_ids: list[int] # completion_ids chosen - visits: int - q_value: float - prior: float - terminal: bool - backup_value: float | None - -def project_mcts(*, run_id: UUID) -> list[SearchTreeRecord]: - ... -``` - -### Option B: dedicated `SearchTreeNode` table - -Add a new table `search_tree_nodes` with the statistics columns inline. -Faster lookups; stricter typing; but requires an Alembic migration, a -write path in the MCTS driver, and a new invariant about consistency -between `RunGraphNode` and `SearchTreeNode`. - -**Why Option A:** annotations are append-only like the rest of the WAL, -so they inherit replay / resumption semantics for free. The read-side -cost is one JOIN; acceptable until an MCTS-heavy workload emerges. -Option B is a follow-on if Option A becomes a bottleneck. - -### MCTS-driver-side contract - -A minimal MCTS driver (not in scope of this RFC) must: - -1. At expansion time, write `mcts.prior` annotations for each child. -2. At rollout completion, backup values and write updated `mcts.visits` - / `mcts.q_value` annotations up the tree. -3. At terminal states, write `mcts.terminal`. - -The substrate does not enforce this; the driver is responsible. The -projection tolerates missing annotations (defaults: visits=0, q=0.0, -prior=uniform) so partially-annotated trees still project. - -## Invariants affected - -Adds an invariant to `docs/architecture/08_rl_loop.md`: - -> **Invariant (MCTS search-tree projection).** Search statistics are -> stored in the `mcts.*` `RunGraphAnnotation` namespace. The projection -> defaults to visits=0, q=0.0, prior=uniform-over-siblings when -> annotations are missing, so a partially-annotated tree still projects -> without error. - -Does not change any existing invariant. Reserves the `mcts.*` annotation -namespace — a new sub-invariant of the existing annotation tombstone -semantics. - -## Migration - -- New file `ergon_core/core/rl/projections/mcts.py`. -- No DB migration (Option A). A new Alembic revision if Option B is - ultimately chosen. -- Tests: add `tests/rl/test_projection_mcts.py` with (a) fully-annotated - tree, (b) partially-annotated tree (defaults filled), (c) zero- - annotation tree (degenerate case — flat list of records with - visits=0 / q=0.0 / prior=1/n). -- Reserve `mcts.*` namespace in a comment at - `ergon_core/core/persistence/graph/models.py` near the annotation - model. - -## Alternatives considered - -- **Option B: dedicated `SearchTreeNode` table** (discussed above). Deferred. -- **Reuse `GenerationTurn.metadata` JSON field for search stats.** - Rejected: `GenerationTurn` is per-turn, not per-node. Search-tree - statistics are per-node and evolve with backups — the annotations table - is the right venue. -- **Skip this projection entirely and remove it from the paper.** - Rejected by authors; tree-search RL is an increasingly active research - thread and Ergon's append-only WAL is genuinely well-suited to it. - -## Open questions - -- Does any current workload actually *need* MCTS today, or is this - purely a roadmap item? Resolved by the paper-blocking decision above: - a concrete MCTS driver is **not** paper-blocking (it remains a roadmap - item listed in Appendix~E), but the `mcts.*` namespace reservation and - the default-filled projection function **are**. Accept Option A, - ship the projection with tests, defer the first concrete driver PR - to a follow-on RFC. -- Should `mcts.prior` be stored as a single annotation per parent (with - all children's priors as one payload), or one per child? Lean per - child for symmetry with visits / q_value. -- Does the projection need to reconstruct a DAG or only a tree? MCTS - search-trees are trees by construction, but if a position is reached - via multiple paths (transposition tables), the structure becomes a - DAG. Suggest: tree only for v1; DAG support is a follow-on RFC. - -## Paper parity - -**Paper-blocking (two claims).** Sibling paper-blocking RFCs: -`2026-04-21-projection-operator-option-tagged.md`, -`2026-04-21-projection-operator-call-tree.md`. Unlike those, this RFC -is load-bearing for **two** cells in Appendix~C's -Table~\ref{tab:preservation}: the diagonal $\pi_{\text{mcts}}$ × -*MCTS-ent* = ✓, and the raw-substrate-alias $\pi_{\text{json-log}}$ × -*MCTS-ent* = ✓. The second claim is the reason the `mcts.*` annotation -namespace (Option A) must be reserved in the substrate before submission -even if no MCTS driver exists yet — the paper's position is that raw -$\tau_E$ dominates the preservation poset, which requires the substrate -to *be capable of* recording MCTS statistics without schema change. -Until landed, Appendix~E still describes the MCTS row as -`planned (RFC-2026-04-21-projection-operator-mcts)`; landing Option A -flips that row to `integrated` and satisfies both Appendix~C claims. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Add the `mcts.*` annotation-namespace reservation to - `docs/architecture/08_rl_loop.md`. - - Update Appendix E (paper repo) to cite accepted state. - - Open a follow-on RFC for the first MCTS driver implementation - (alpha-zero-style, ToT-style, or rStar-Math-style — pick one). diff --git a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-option-tagged.md b/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-option-tagged.md deleted file mode 100644 index d974ac0c0..000000000 --- a/docs/rfcs/paper-nice-to-haves/2026-04-21-projection-operator-option-tagged.md +++ /dev/null @@ -1,172 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: paper-parity -architecture_refs: [docs/architecture/08_rl_loop.md, docs/architecture/04_persistence.md] -supersedes: [] -superseded_by: null -paper_blocking: true -paper_blocking_reason: >- - Preservation claim in Appendix~C (Table~\ref{tab:preservation}) requires - π_macro to be an implementable projection that preserves option-termination - structure. Charlie commits to landing this RFC and its implementation - before the NeurIPS 2026 paper submission goes live. ---- - -# RFC: Projection operator — option-tagged transitions - -> **Paper-blocking.** This RFC is a hard pre-submission dependency for the -> NeurIPS 2026 Ergon paper. Appendix~C's preservation table claims -> $\pi_{\text{macro}}$ preserves option-termination structure (diagonal -> ✓ cell under the *Opt-term* column); that claim requires the projection -> function specified in this RFC to exist in code by submission time. The -> surrounding RFC text remains written in scoping voice, but the decision -> to ship has already been made — this is no longer a "nice to have." -> Target: implementation landed and tested before paper submission. - -## Problem - -The paper (§3.2 and Appendix E) lists five canonical trajectory projections -the Ergon substrate can emit. Two are implemented today (step-indexed -tuples, per-agent streams — both via `extract_agent_trajectories` at -`ergon_core/core/rl/extraction.py:L49-L117`). Three are not. This RFC -covers the **option-tagged transitions** projection (hierarchical / -semi-MDP framing). - -The projection's purpose is to feed hierarchical RL trainers (option-critic, -HIRO, FuN-style) that consume `(s, option, sub-policy-rewards, s')` tuples -rather than flat `(s, a, r, s')`. The substrate already records every input -the projection needs: - -- `RunGraphNode` (parent_node_id) at - `ergon_core/core/persistence/graph/models.py:L44-L89` — the call-tree - needed to identify option boundaries. -- `RunGraphMutation` events at - `ergon_core/core/persistence/graph/models.py:L172-L186` — give a totally - ordered timeline of `node.added` / `node.status_changed` events. -- `GenerationTurn` rows at - `ergon_core/core/persistence/telemetry/models.py:L383-L464` — provide the - `(prompt_ids, completion_ids, logprobs)` that bind to each option. - -But there is no projection function. `extract_agent_trajectories` flattens -the call tree into per-agent token streams; option boundaries are -discarded. - -## Proposal - -Add a new projection in `ergon_core/core/rl/projections/option_tagged.py` -(new package). API: - -```python -# ergon_core/core/rl/projections/option_tagged.py (new) - -from dataclasses import dataclass -from ergon_core.core.persistence.graph.models import RunGraphNode - -@dataclass(frozen=True) -class OptionBoundary: - option_id: str # parent_node_id whose lifetime defines this option - sub_policy_id: str # the worker / agent that executes inside - enter_sequence: int # mutation sequence at node.added - exit_sequence: int # mutation sequence at terminal status - return_value: float # accumulated reward inside the option - -@dataclass(frozen=True) -class OptionTaggedTransition: - s: list[int] # prompt_ids at option entry - option: str # option_id - rewards: list[float] # per-step rewards inside this option - s_prime: list[int] # prompt_ids at option exit (next-state) - -def project_option_tagged(*, run_id: UUID) -> list[OptionTaggedTransition]: - """Walk RunGraphMutation timeline; identify option boundaries via - parent_node_id transitions; aggregate sub-policy rewards within each - option. Returns one transition per option entry.""" - ... -``` - -The projection consumes the same `(MutationLog, GenerationTurns, -TaskEvaluations)` triple that `extract_agent_trajectories` consumes — no -new data sources required. The walk is O(N + M) in nodes + mutations. - -### Option-boundary rule - -An *option* is the lifetime of a `RunGraphNode` from `node.added` to its -first terminal `node.status_changed` (in `{COMPLETED, FAILED, CANCELLED}`). -Sub-policy rewards are the rewards assigned to `GenerationTurn` rows whose -`worker_binding_key` matches the option's executing worker. - -This rule is consistent with the existing notion of a "task" in -`docs/architecture/02_runtime_lifecycle.md` and does not require any new -schema fields. - -## Invariants affected - -Adds an invariant to `docs/architecture/08_rl_loop.md`: - -> **Invariant (option-tagged projection).** The option-tagged projection -> defines an option as the lifetime of a single `RunGraphNode` from -> `node.added` to first terminal `node.status_changed`. Hierarchical -> trainers consuming this projection get one transition per option entry. - -No existing invariants change. The `RunGraphNode` parent/child structure, -already documented in `04_persistence.md`, is the load-bearing source of -truth. - -## Migration - -- New package `ergon_core/core/rl/projections/` with `__init__.py` and - `option_tagged.py`. Empty `__init__.py` until additional projections - land via the sibling RFCs. -- No DB migration. No API change to `/rollouts/{batch_id}` (this - projection is consumed directly by trainer adapters that opt in). -- Tests: add `tests/rl/test_projection_option_tagged.py` with three - cases — single-task run (one option), parent-with-two-children run - (three options), and a deeper nested run (test the recursive case). - -## Alternatives considered - -- **Define an option as a sequence of `GenerationTurn` rows up to a - tool-call boundary.** Rejected: tool calls are intra-option events, not - option boundaries. The call-tree projection (separate RFC) covers the - intra-option tool-call structure. -- **Defer until a hierarchical trainer is integrated and let the adapter - invent its own boundary rule.** Rejected: would create two competing - notions of "option" between, e.g., a future SkyRL hierarchical adapter - and a future option-critic adapter. The boundary rule is substrate - policy, not trainer policy. -- **Materialize options as a new DB table.** Rejected: derivable in O(N) - from existing tables. Materialization is a caching choice that can - follow if the projection becomes hot. - -## Open questions - -- Should `OptionBoundary` be exposed as a public API type or stay internal - to the projection? Lean public (downstream tooling will want to inspect - it), but only after a real consumer exists. -- How are nested options (option-within-option) represented in the - flat list returned by `project_option_tagged`? Suggest: tag each - transition with `parent_option: str | None` and let the consumer build - the tree. - -## Paper parity - -**Paper-blocking.** This RFC is one of three paper-blocking projection -RFCs (siblings: `2026-04-21-projection-operator-call-tree.md`, -`2026-04-21-projection-operator-mcts.md`). Appendix~C of the NeurIPS -2026 paper claims a specific preservation pattern for $\pi_{\text{macro}}$ -in Table~\ref{tab:preservation}; that claim is load-bearing for §2.2 -("different communities record different things") and for the paper's -central thesis that Ergon's raw substrate $\tau_E$ uniquely dominates the -poset of community-native projections. The claim requires the projection -function specified here to exist in code by submission. Until landed, -Appendix~E still describes the option-tagged row as -`planned (RFC-2026-04-21-projection-operator-option-tagged)`; landing -flips that row to `integrated` and satisfies the Appendix~C claim. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Add the option-boundary invariant to `docs/architecture/08_rl_loop.md`. - - Update Appendix E (paper repo) to cite accepted state. - - Link the implementation plan in `docs/superpowers/plans/`. diff --git a/docs/rfcs/paper-nice-to-haves/2026-04-21-rl-trainer-adapter-expansion.md b/docs/rfcs/paper-nice-to-haves/2026-04-21-rl-trainer-adapter-expansion.md deleted file mode 100644 index d4b2d3d01..000000000 --- a/docs/rfcs/paper-nice-to-haves/2026-04-21-rl-trainer-adapter-expansion.md +++ /dev/null @@ -1,210 +0,0 @@ ---- -status: active -opened: 2026-04-21 -author: paper-parity -architecture_refs: [docs/architecture/08_rl_loop.md] -supersedes: [] -superseded_by: null -paper_blocking: false -paper_blocking_reason: >- - Not paper-blocking. The paper (Appendix E) honestly describes the six - non-shipped RL trainers as `planned (RFC-...)` rows. Landing this RFC - flips rows to `integrated`, but the paper's claims are well-formed - under either state. The Appendix C preservation table is about - projection operators over already-recorded trajectories, not about - downstream trainer consumption. ---- - -# RFC: RL trainer adapter expansion - -> **Not paper-blocking.** Appendix~E of the NeurIPS 2026 Ergon paper -> lists the six non-shipped RL trainers (SkyRL, ProRL Agent, AReaL, -> AgentGym-RL, RAGEN, MARTI) as `planned (RFC-...)` rows, and the paper -> is honest about which adapters ship today versus which are scoped. -> This RFC may land before, during, or after paper submission without -> altering any paper claim. The preservation table in Appendix~C is -> about projections over recorded trajectories, not about downstream -> trainer consumption; no cell in that table changes based on how many -> trainers consume `/rollouts/{batch_id}`. Contrast with the three -> projection RFCs in this directory, which *are* paper-blocking. - -## Problem - -The paper (Appendix E) describes Ergon as feeding **nine** RL trainers via -thin HTTP adapters: TRL, VERL, OpenRLHF, SkyRL, ProRL Agent, AReaL, -AgentGym-RL, RAGEN, and MARTI. The codebase today ships **three**: - -- `ergon_infra/ergon_infra/adapters/trl_http.py` (L1-L92) — TRL GRPO -- `ergon_infra/ergon_infra/adapters/verl_http.py` (L1-L83) — VERL agent loop - (via `@register("ergon")`) -- `ergon_infra/ergon_infra/adapters/openrlhf_http.py` (L1-L85) — OpenRLHF - agent-func pattern - -All three follow the same handshake (documented in -`docs/architecture/08_rl_loop.md:L132-L135`): - -1. `POST /rollouts/submit` with `{"definition_id": UUID, "num_episodes": int}` - → `{"batch_id": UUID}` (HTTP 202). -2. Poll `GET /rollouts/{batch_id}` until `status in {"complete", "failed"}`. -3. Receive `{"status": "complete", "trajectories": [Trajectory, ...]}`. -4. Map each `Trajectory` into the trainer's native batch type. - -For SkyRL, ProRL Agent, AReaL, AgentGym-RL, RAGEN, and MARTI there are -zero adapter files, zero imports, and zero `pyproject.toml` optional-deps -entries. There is no `TrainerBackend` enum, registry, or factory — each -adapter is discovered today by "grep for `*_http.py` in -`ergon_infra/adapters/`". - -The server side (`ergon_core/core/api/rollouts.py`, -`ergon_core/core/rl/rollout_service.py`, -`ergon_core/core/rl/extraction.py`) is trainer-agnostic. The gap is -entirely on the adapter-side: six missing files. - -## Proposal - -Two things, sequenced as two PRs. - -### Part 1 — formalize the contract - -Elevate the handshake from documentation prose to a public -`TrainerHttpAdapter` Protocol in `ergon_infra/ergon_infra/adapters/base.py` -(new file) and add a `TrainerBackend` enum mapping each supported backend -to its adapter module. Shape: - -```python -# ergon_infra/ergon_infra/adapters/base.py (new) - -from enum import Enum -from typing import Protocol -from ergon_core.core.rl.rollout_types import Trajectory - -class TrainerBackend(str, Enum): - TRL = "trl" - VERL = "verl" - OPENRLHF = "openrlhf" - SKYRL = "skyrl" # planned - PRORL_AGENT = "prorl_agent" # planned - AREAL = "areal" # planned - AGENTGYM_RL = "agentgym_rl" # planned - RAGEN = "ragen" # planned - MARTI = "marti" # planned - -class TrainerHttpAdapter(Protocol): - backend: TrainerBackend - def submit(self, *, definition_id: str, num_episodes: int) -> str: - """POST /rollouts/submit; returns batch_id.""" - ... - def poll(self, batch_id: str) -> list[Trajectory]: - """GET /rollouts/{batch_id} until terminal; returns trajectories.""" - ... - def to_native(self, trajectories: list[Trajectory]) -> object: - """Map to trainer's native batch type (trainer-specific).""" - ... -``` - -Refactor the three existing adapters to expose `backend: TrainerBackend` -and the `submit` / `poll` / `to_native` methods explicitly (they -already implement the logic — this is a rename + Protocol conformance). - -### Part 2 — ship six stub adapters, one per planned trainer - -Each stub is ~40 lines: a module exposing a `make_{trainer}_rollout_func()` -factory that raises `NotImplementedError("see RFC-...")` and documents the -trainer-native batch type the adapter must ultimately emit. - -File | Trainer | Key open item ----|---|--- -`skyrl_http.py` | SkyRL | Confirm SkyRL's native batch contract; expected: `dict` mirroring OpenRLHF -`prorl_agent_http.py` | ProRL Agent | Confirm trajectory-level vs transition-level consumption -`areal_http.py` | AReaL | Confirm async vs sync preference -`agentgym_rl_http.py` | AgentGym-RL | Confirm whether tool-call tokens are masked or inlined -`ragen_http.py` | RAGEN | Confirm multi-turn reward aggregation -`marti_http.py` | MARTI | Confirm whether MARTI's multi-agent shape needs a per-agent fan-out beyond `extract_agent_trajectories` - -For each stub, declare an optional-deps group in -`ergon_infra/pyproject.toml` under `[project.optional-dependencies]`: - -```toml -training-skyrl = ["skyrl>=0.1"] -training-prorl = ["prorl-agent>=0.1"] -training-areal = ["areal>=0.1"] -training-agentgym = ["agentgym-rl>=0.1"] -training-ragen = ["ragen>=0.1"] -training-marti = ["marti>=0.1"] -``` - -(Version pins are placeholders pending the follow-on RFC for each trainer -that resolves actual PyPI availability.) - -## Invariants affected - -Adds an invariant to `docs/architecture/08_rl_loop.md`: - -> **Invariant (trainer adapter contract).** All HTTP trainer adapters MUST -> consume `Trajectory` (see `rollout_types.py:L38-L51`) and expose -> `submit` / `poll` / `to_native` per the `TrainerHttpAdapter` Protocol. -> Trainer-specific logic (field renaming, batch shaping) lives in -> `to_native`; never in the server. - -No change to existing invariants; this promotes prose-level guidance -("follow the pattern of `trl_http.py`") to an enforced Protocol. - -## Migration - -- Part 1 is source-only refactor on three existing adapters. They already - conform to the Protocol semantically; this adds nominal conformance - + enum. -- Part 2 adds six new files; no existing file is modified. -- No DB migration. No API change to `/rollouts/submit` or - `/rollouts/{batch_id}`. -- Tests: add `tests/rl/test_trainer_adapter_contract.py` asserting all - three real adapters are `isinstance(TrainerHttpAdapter)` and the six - stubs raise `NotImplementedError` citing their follow-on RFCs. - -## Alternatives considered - -- **One RFC per trainer (six RFCs).** Rejected: each would say "write - `{trainer}_http.py` matching the existing pattern". Grouping into one - RFC reflects the fact that the design decision is the same in all six - cases. Per-trainer follow-on RFCs resolve trainer-specific quirks - (native batch type, sync/async, reward aggregation) on a case-by-case - basis. -- **Skip the Protocol, just write the six stubs.** Rejected: without a - Protocol, the next contributor has the same ambient-contract problem we - have today. Formalizing the contract is cheap (one new file). -- **Cut five trainers from the paper.** Rejected by authors — Appendix E - is descriptive and includes planned rows. The paper parity path is to - ship stubs + RFCs, not retreat. - -## Open questions - -- Should the `TrainerBackend` enum values drive discovery (e.g., an - entry-points-based registry), or is manual import-site registration - sufficient? Suggest deferring to a follow-on RFC once a third consumer - emerges. -- Do we want a `training-all` extras group bundling all six planned - trainers, or only the three real ones? Suggest: `training` bundles - the three real trainers; planned ones are opt-in by name. -- Version pin strategy for six packages that may not all be on PyPI under - the expected name. Open-question resolution happens in the per-trainer - follow-on RFCs. - -## Paper parity - -This RFC is the parity work for row 2 of the paper's Appendix E -integrations map. Sibling RFCs: -`2026-04-21-agent-framework-adapter-layer.md` (row 1, agents), -`2026-04-21-projection-operator-*.md` (row 3, projections). Until this -RFC's Part 2 lands, Appendix E describes SkyRL / ProRL / AReaL / -AgentGym-RL / RAGEN / MARTI as `planned (RFC-2026-04-21-rl-trainer-adapter-expansion)`. - -## On acceptance - -When this RFC moves from `active/` to `accepted/`, also: - - Add the `TrainerHttpAdapter` invariant to - `docs/architecture/08_rl_loop.md`. - - Open six follow-on RFCs (one per planned trainer) or, for trainers - with no PyPI distribution yet, mark them `blocked` pending upstream - release. - - Update the paper's Appendix E to cite accepted state for the three - real adapters and planned state for the six new stubs. diff --git a/docs/rfcs/paper-nice-to-haves/README.md b/docs/rfcs/paper-nice-to-haves/README.md deleted file mode 100644 index 8a9fef6ff..000000000 --- a/docs/rfcs/paper-nice-to-haves/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Paper-tied RFCs - -RFCs scoped specifically to back claims made in the Ergon NeurIPS 2026 -paper. Originally framed as "nice-to-haves" (claims the paper makes about -Appendix~E integrations that the codebase doesn't yet ship); as of -2026-04-21 this directory also holds a distinct second class — RFCs that -are **paper-blocking** because Appendix~C's preservation table makes -formal claims about projection operators that require those projections -to exist in code by submission. Charlie has committed to landing the -paper-blocking RFCs before the paper goes live. - -The directory name is kept for continuity (it's referenced from Appendix -E), but the `paper_blocking` front-matter field on each RFC is the source -of truth for which class it belongs to. - -## Paper-blocking RFCs (must land before submission) - -These RFCs back **Appendix~C preservation claims**. Each diagonal ✓ -in Table~\ref{tab:preservation} under a community-native behavioural -quantity depends on the matching projection operator being shipped. The -claim that raw $\tau_E$ (via $\pi_{\text{json-log}}$) dominates the -preservation poset additionally depends on the `mcts.*` annotation -namespace being reserved. - -| RFC | Paper claim backed | Effort estimate | -|---|---|---| -| `2026-04-21-projection-operator-option-tagged.md` | Appendix~C: $\pi_{\text{macro}}$ × *Opt-term* = ✓ | S (pure read over existing schema) | -| `2026-04-21-projection-operator-call-tree.md` | Appendix~C: $\pi_{\text{call-tree}}$ × *Call-dep* = ✓ (+ spawn-ordering invariant) | S (pure read; one runtime invariant to confirm) | -| `2026-04-21-projection-operator-mcts.md` | Appendix~C: $\pi_{\text{mcts}}$ × *MCTS-ent* = ✓ **and** $\pi_{\text{json-log}}$ × *MCTS-ent* = ✓ | S (Option A: reserve `mcts.*` annotation namespace + default-filled projection) | - -Landing checklist (each RFC): -- Implementation PR merged on `main`. -- Tests passing (see each RFC's "Migration" section for the test matrix). -- Corresponding invariant added to `docs/architecture/08_rl_loop.md`. -- RFC moved from `paper-nice-to-haves/` to `accepted/` with a stub left - here pointing at its accepted-state path. -- Appendix~E row in the paper repo flipped from `planned (RFC-...)` to - `integrated`. - -## Roadmap RFCs (not paper-blocking) - -These back **Appendix~E integration-map rows** only. The paper honestly -describes each target row as `planned (RFC-...)`; landing the RFC flips -the row to `integrated` but does not alter any paper claim. They can -land before, during, or after paper submission. - -| RFC | Paper-parity row | Effort estimate | -|---|---|---| -| `2026-04-21-agent-framework-adapter-layer.md` | Agents row: 5 frameworks (LangGraph, CrewAI, AutoGen, Google ADK, Claude Code) | M (contract + 5 stub adapters) | -| `2026-04-21-rl-trainer-adapter-expansion.md` | RL row: 6 trainers (SkyRL, ProRL Agent, AReaL, AgentGym-RL, RAGEN, MARTI) | M (Protocol + 6 stub adapters) | - -## Conventions - -- Filename and front-matter format follow `docs/rfcs/TEMPLATE.md`, plus - two paper-specific front-matter fields: `paper_blocking: bool` and - `paper_blocking_reason: str` (the justification for the flag). -- The front-matter `status` field is set to `active` (these are open - scoping documents); promotion to `accepted` follows the same rules as - the core RFC stream. -- When one of these moves to `accepted/`, leave a stub here pointing to - its accepted-state path so the paper-parity audit trail remains - intact. -- A paper-blocking RFC that has not yet landed on the submission date - is a blocker for submission, not a reviewer risk to accept — - `paper_blocking: true` is a commitment, not a forecast. - -## Out of scope - -Anything required for the existing experiments in §5 of the paper, or -for the production runtime, or for any benchmark in `ergon_builtins/`, -belongs in `docs/rfcs/active/` (core stream), not here. The distinction -between this directory and `docs/rfcs/active/` is *not* "core work vs -paper work" — it is "RFCs tied specifically to paper-parity claims vs -RFCs tied to the production runtime and experiments." Some items in -this directory (the three paper-blocking projection RFCs) are arguably -core enough to move to `active/`; that's an organizational decision -pending. diff --git a/docs/superpowers/brainstorms/2026-04-21-real-llm-debug-harness.md b/docs/superpowers/brainstorms/2026-04-21-real-llm-debug-harness.md deleted file mode 100644 index 745b6efbd..000000000 --- a/docs/superpowers/brainstorms/2026-04-21-real-llm-debug-harness.md +++ /dev/null @@ -1,156 +0,0 @@ -# Brainstorm: Real-LLM debug harness - -**Date:** 2026-04-21 -**Status:** Converged → RFC at `docs/rfcs/active/2026-04-21-real-llm-debug-harness.md` -**Participants:** @cm2435 + agent - ---- - -## Premise - -Our existing test tiers (`tests/unit/`, `tests/integration/`, `tests/e2e/`) all -run against stubbed or deterministic workers. That means we've never actually -validated, end-to-end, that: - -1. The three benchmark sandboxes (`researchrubrics`, `minif2f`, - `swebench-verified`) are correctly provisioned and usable by a real LLM. -2. The ReAct worker correctly uses the tools it's given (`add_subtask`, - `cancel_task`, `bash`, env-specific) under a real LLM's decision-making. -3. The verifiers/criteria correctly read the outputs a real LLM would produce. - -Ad-hoc local runs have surfaced bugs that stub-tier tests miss. The goal is to -codify the "run a real experiment locally and poke it" workflow as a first-class -pytest tier — then use that tier as a bug-hunting instrument during an overnight -autonomous loop. - -## Converged decisions - -### Location — A (`tests/real_llm/` as a pytest tier) - -Rejected: `examples/`, sibling repo, hybrid. Rationale: this is **debugging -infrastructure**, not shipped examples. Pytest gives us fixtures, conftest -reuse, parametrize, and skip semantics for free. Examples can be added later. - -### Worker shape — generic ReAct + per-benchmark DI'd toolkit - -Rejected: per-benchmark specialized workers. The shipped ReAct worker -(`ergon_builtins/workers/baselines/react_worker.py`) takes `tools=[...]` at -construction. We build a `benchmark_toolkit_composer` module that, given a -benchmark slug + `WorkerContext`, returns the right union of toolkits: - -- `researchrubrics` → `SubtaskLifecycleToolkit(8)` ∪ `ResearchRubricsToolkit(6)` - ∪ `ResearchGraphToolkit(6)` -- `minif2f` → `SubtaskLifecycleToolkit(8)` ∪ `MiniF2FToolkit(Lean)` -- `swebench-verified` → `SubtaskLifecycleToolkit(8)` ∪ `SWEBenchToolkit(bash + - str-replace editor)` - -All three per-env toolkits already exist in the repo; the composer is a small -DI factory, not new tool code. **`edit_task_topology` as discussed is not a new -tool — existing `refine_task`/`plan_subtasks`/`restart_task` cover it.** - -### Execution topology — C (hybrid stack) - -Rejected: in-process runtime (doesn't test what users run); compose-up-only -(slow dev iteration). - -pytest session fixture runs `docker compose -f docker-compose.real-llm.yml up --d` unless `--assume-stack-up` flag is passed. Per-test: spawns `ergon -benchmark run` as subprocess, polls -`/api/test/read/run/{id}/state` from the smoke-shared-infra harness (PR #25) -until the run reaches a terminal state, then: - -- DB assertions via in-process `get_session()` -- Playwright assertions (must-have) via `playwright.async_api` against the live - dashboard — screenshots on every run - -Best of both: CI reproducibility via the default fixture, fast dev iteration -via `--assume-stack-up` when a developer already has `pnpm dev:test` running. - -### LLM provider / model — Sonnet 4.6 via OpenRouter - -Rejected: per-benchmark model; direct Anthropic. Single knob -(`ERGON_REAL_LLM_MODEL`, default -`openrouter/anthropic/claude-sonnet-4.6`), so matrix expansion is a future -trivial change. - -### Budget gate — OpenRouter `/api/v1/auth/key` polling - -OpenRouter exposes `GET /api/v1/auth/key` returning -`{data: {usage, limit, limit_remaining, ...}}`. We snapshot `usage` at session -start and poll between tests; if `usage - baseline > ERGON_REAL_LLM_BUDGET_USD` -(default `5.0`), remaining tests skip with a clear reason. This plus per-test -wall-clock and max-turn caps from the ReAct agent itself prevents overnight -surprises. - -### "Three examples per benchmark" (PR 2) — A (three random distinct instances) - -Rejected: same-task × 3 seeds (doesn't test diversity); hand-picked tiered -(too much curation upfront; interesting for PR 3 if this pattern sticks). Seed -pinned in the config so re-runs pick the same instances — reproducibility -without the curation overhead. - -### PR 1 acceptance — A (infra-only, stub-worker smoke canary) - -Rejected: canary-with-real-LLM in PR 1 (couples infra merge to LLM -availability + cost). The real-LLM agent runs happen in the **bug-hunt phase -between PR 1 and PR 2**, and each bug becomes its own -`docs/bugs/open/YYYY-MM-DD-<slug>.md` + fix PR. PR 2 doesn't freeze until the -bug-hunt phase has converged. - -### PR 2 assertion philosophy — Tiered (infra hard, result soft) - -Rejected: strict (too red; LLMs are nondeterministic); permissive-only -(misses the "agent isn't using its tools at all" failure mode). - -Hard gates (test fails if violated): - -- Run reaches a terminal status within budget -- Sandbox set up correctly (no connect/provisioning error) -- At least one tool call made -- Postgres row exists with the right relationships (graph nodes, mutations) -- Playwright finds the run in the dashboard and renders the run detail page - -Soft gate (per-benchmark, all instances together): - -- At least 1 of 3 instances per benchmark produced a non-zero criterion score - -Raw scores are reported, not asserted. - -## Key phrases (for RFC consistency) - -- **`tests/real_llm/`** — the pytest tier -- **`@pytest.mark.real_llm`** — the opt-in marker -- **`benchmark_toolkit_composer`** — the DI factory -- **`docker-compose.real-llm.yml`** — the stack overlay -- **`openrouter_budget`** — the budget gate module -- **`--assume-stack-up`** — the dev-iteration fixture flag -- **`ERGON_REAL_LLM=1`** + **`OPENROUTER_API_KEY`** — the two env vars that - activate the tier -- **`ERGON_REAL_LLM_MODEL`** — model override -- **`ERGON_REAL_LLM_BUDGET_USD`** — spend cap - -## Explicit non-goals - -- Not a CI default. This tier never runs on every PR; only on a manual - dispatch or the `real-llm` label. -- Not a deterministic regression gate. LLM outputs drift; the infra gate is - the pass/fail line, scores are artifacts. -- Not a replacement for `tests/e2e/` (Playwright + stubs). That tier stays - for fast end-to-end frontend coverage. -- Not a new tool. `edit_task_topology` does not become a thing — existing - tools cover the capability. - -## Open questions deferred to the RFC - -(none major; all of Q1–Q5 converged) - -## Dependencies on other in-flight work - -- **smoke-shared-infra PR #25** (`feature/smoke-shared-infra`) lands the - `/api/test/*` harness endpoints we poll. Must merge before real-LLM PR 1 - starts. Real-LLM PR 1 branches off `main` after #25 merges. -- **testing-posture-reset** (`docs/rfcs/active/2026-04-18-testing-posture-reset.md`) - PRs 2–4 land the Docker caching, integration-test fixtures, and tests/e2e/ - deletion. Real-LLM PR 1 can proceed without these (falls back to - `--assume-stack-up` on dev machines), but CI reproducibility is cleaner once - they land. diff --git a/docs/superpowers/brainstorms/2026-05-13-unit-test-audit.md b/docs/superpowers/brainstorms/2026-05-13-unit-test-audit.md deleted file mode 100644 index 9e4211ae8..000000000 --- a/docs/superpowers/brainstorms/2026-05-13-unit-test-audit.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: Unit test audit and prune -date: 2026-05-13 -status: approved ---- - -# Unit test audit and prune - -## Context - -`pnpm run test:be:fast` runs 619 unit tests in 23.86s wall / 106.84s CPU -(`-n auto` across 8 workers, this machine). Time is uniformly distributed: -the top 50 slowest tests account for ~16s of CPU; the remaining ~570 tests -account for ~91s. There is no concentrated set of slow offenders to kill; -the suite is large and the cost per test averages ~170ms. - -The user's hypothesis — old/brittle tests that no longer earn their keep, -plus tests that could be consolidated — is consistent with the data. - -## Goal - -Cut 25–30% of unit tests without losing meaningful coverage. Wall-clock -improvement follows from count reduction. - -## Deletion rubric - -A test is a deletion candidate if any of: - -1. **Type-system shadowing** — asserts what Pydantic/ty already enforces - (field exists, type is `str`, default is `None`). -2. **Test-double dominance** — every meaningful collaborator is mocked; - the test verifies the mock setup, not the code under test. -3. **Trivial delegation** — exercises a one-line wrapper or pass-through. -4. **Redundant architecture check** — duplicates another architecture - test's invariant on the same code paths. -5. **Dead-code coverage** — covers code no production caller reaches - (often left over from deleted features). -6. **Parametrize bloat** — N parametrized cases that all hit the same - branch (vs. genuinely covering different ones). - -## Consolidation rubric - -Multiple tests share setup and assert facets of one operation that a -single well-named test could cover. - -## Must-keep rubric - -A test MUST be kept if: - -- It pins behavior derived from the May 2026 authoring API redesign - postmortem (per CLAUDE.md "Agent regression guardrails"). -- It is the only test exercising a public-API contract. -- It exercises a non-trivial branch or invariant the codebase relies on. - -## Domain ordering (highest-ROI first) - -1. **`ergon_core/tests/unit/architecture/`** (9 files) — highest redundancy - suspected; tests do FS/AST walks of the source tree. -2. **Contract tests** (`test_*_contract.py` across packages) — likely - Pydantic/ty overlap. -3. **State/schema tests** (`tests/unit/state/` in each pkg) — same. -4. **Registry/catalog tests** — moderate value; check for redundancy. -5. **Runtime tests** (`ergon_core/tests/unit/runtime/`) — highest signal, - touch last; cut here only if obviously bloated. -6. **CLI tests** — keep most; prune duplicate command-shape assertions. -7. **`smoke_base/` and postmortem-linked tests** — audit ONLY with - explicit per-file approval. - -## Per-domain process - -1. List every test file in the domain with a one-line "what does this - prove" summary per test. -2. Classify each test: **KEEP / CONSOLIDATE / DELETE** with one-line - reasoning. -3. Present the classification table; wait for user approval before any - deletion. -4. Implement changes; run that domain's tests + the full unit suite. -5. Commit on `feature/copy-authoring-api-redesign-v2-rfcs` (current - branch) with message `audit: prune <domain> (-N tests, -Ms wall)`. - -## Safety rules - -- Each domain is its own commit (revertable individually). -- After each domain: run `pnpm run test:be:fast` AND - `pnpm run test:be:integration` to surface coverage gaps. -- Each deletion's commit message names the surviving test (or invariant - enforced elsewhere) that still covers the same property — or - explicitly states "no replacement; accepted gap because X." -- Postmortem-linked tests excluded unless user OKs each one. -- Stage only audit files. Do not touch user's WIP (CLAUDE.md, RFC docs, - untracked AGENTS.md/CODEX.md/etc.). - -## Measurement - -- Baseline: 619 tests / 23.86s wall / 106.84s CPU (this machine, - `-n auto`). -- Each domain commit records new count + wall + CPU in the message body. -- Final summary lives in the PR description. - -## Stop criteria - -Whichever first: -- <5% prunable in the next domain. -- Wall-clock under 15s. -- User calls it. - -## Out of scope - -- Folder restructure (top-level `/tests` consolidation, type-vs-domain - split). Discussed and parked. -- Integration / e2e / real_llm audits — different cost structure. -- Strategy A (session-scoped FS-walk fixture for architecture tests) — - may or may not be needed after domain 1; revisit at that point. diff --git a/docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md b/docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md deleted file mode 100644 index 1b261b155..000000000 --- a/docs/superpowers/brainstorms/2026-05-15-kill-experiment-class.md +++ /dev/null @@ -1,417 +0,0 @@ -# Killing the `Experiment` Class: Public API Rethink - -**Date:** 2026-05-15 -**Status:** Proposal / brainstorm — not yet folded into a PR plan. -**Context:** Surfaced while workshopping PR 6.5 (domain colocation in `ergon_builtins`). The discussion started with "where should per-benchmark `Sandbox`/`Toolkit` live?", moved through "what's the composition layer in v2?", and landed at "does `Experiment` actually earn its keep as a public API concept?" - -## TL;DR - -In v2 as it stands, `Experiment` is a Pydantic struct that holds `(benchmark, name, description, metadata)` — three pieces of metadata wrapped around a Benchmark. All the composition (worker, sandbox, evaluators) already lives **inside** the Benchmark via inline `Task.worker` / `Task.sandbox` / `Task.evaluators`. So `Experiment` is pure accidental abstraction: no fields that need to be class fields rather than function parameters, no behaviour that's not just persistence ceremony. - -**Proposal:** delete the `Experiment` *class*, but keep the word "experiment" as a **string identifier** that groups related runs. Concretely: - -``` -Benchmark — the configured unit of work (already exists) -DefinitionHandle — identity + metadata for one persisted Benchmark -persist_benchmark(b, *, name, experiment=None, metadata=None) -> DefinitionHandle -launch_run(definition_id) -> RunHandle (already exists) -``` - -An **experiment** becomes a `str | None` column on the persisted definition row. Two benchmarks persisted with the same `experiment="strategy-ablation-2026-05-14"` belong to the same experiment. No `Experiment` class, no `Cohort` class — just a string. - -This naming is deliberate: the word "experiment" stays in the user's vocabulary (it's the natural research term for "I ran a benchmark with two configurations to compare them"), but the *abstraction* drops from "class holding composition" to "string tag grouping definitions." Users keep saying "I ran an experiment"; the code stops carrying a class that pretended to be composition. - -The **CLI** becomes a thin convenience over the Python API. Per-benchmark `experiment.py` files (currently in PR 8's plan) are deleted — the CLI dispatches via two flat dicts: `BENCHMARK_CLASSES[slug] -> type[Benchmark]` and `WORKER_FACTORIES[slug] -> dict[name, factory]`. - -## The observation - -`Experiment` does exactly one thing today: wrap a Benchmark with a name and metadata for persistence. Walk through what's in the class: - -| `Experiment` field | What it actually is | -|---|---| -| `benchmark` | The thing being persisted; doesn't need wrapping | -| `name` | A label, could be a function parameter | -| `description` | Same | -| `metadata` | Same | - -There is no field on `Experiment` that needs to be a class field rather than a function parameter to a `persist_benchmark(benchmark, *, name, ...)` call. The class offers no behaviour beyond construction and persistence routing. - -Worse, the name `Experiment` carries the v1 connotation of "composition tuple" (benchmark + worker_spec + evaluator_spec), which it explicitly is **not** in v2. Every time we explain the architecture, someone has to be told "no, in v2 the Experiment is just a labeled persistable Benchmark; composition lives inside the Benchmark." That's a smell. - -## The proposal - -### Public API surface - -```python -# 1. Configure a Benchmark (per PR 6.5: parameterised constructor) -benchmark = MiniF2FBenchmark( - worker_factory=make_minif2f_react_worker, - sandbox_factory=LeanSandbox, # default — usually omit - evaluator_factory=make_minif2f_rubric, - limit=10, -) - -# 2. Persist with a label + optional experiment grouping -handle = persist_benchmark( - benchmark, - name="minif2f-react-2026-05-14", - experiment="strategy-ablation-2026-05-14", # optional string tag - metadata={"created_by": "charlie", "notes": "..."}, -) - -# 3. Launch -await launch_run(handle.definition_id) -``` - -That's the entire authoring API. Three calls. Composable, version-controllable, importable from a notebook. - -### What dies (classes / DTOs / files) - -- `ergon_core.api.experiment.Experiment` — **class deleted**. The word "experiment" survives as a string-typed concept, not a class. -- `ergon_core.core.application.experiments.service.ExperimentService.define_benchmark_experiment` — replaced by `persist_benchmark` -- `ExperimentDefineRequest` DTO — no longer needed; arguments go directly to `persist_benchmark` -- `BUILTIN_EXPERIMENT_FACTORIES` (PR 8) — replaced with `BENCHMARK_CLASSES` + `WORKER_FACTORIES` dicts -- Per-benchmark `experiment.py` files (PR 8's current plan) — never created; replaced by simpler CLI dispatch -- The "experiment-as-composition" framing in the v2 RFC — replaced with "Benchmark is the configured unit; an experiment is just a string tag grouping related definitions" - -### What churns (renames, signature changes) - -- `ExperimentRecord` SQLAlchemy model → renamed to `BenchmarkDefinitionRecord`. The table itself can keep its existing physical name (`experiments`?) to avoid a destructive Alembic migration — only the Python model class renames. *Open question: is the existing table name something other than `experiments`? If so, no churn.* -- `persist_definition(experiment)` → `persist_benchmark(benchmark, *, name, experiment=None, metadata=None) -> DefinitionHandle`. Signature change: takes a `Benchmark`, not an `Experiment` wrapper. The `experiment=` kwarg here is the **string tag**, not the deleted class. -- Inngest event payloads that name `experiment_id` semantically — re-check whether they mean "the definition_id" (just a name change to `definition_id`) or whether they referenced the wrapper-class field surface (need rewiring). -- Dashboard TypeScript that queries `ExperimentRecord` by name — adapt to the renamed model / column-renamed shape. - -### What lives (unchanged) - -- `Benchmark` ABC + `build_instances()` returning `Task` objects with inline worker/sandbox/evaluators -- `MiniF2FBenchmark(worker_factory=..., sandbox_factory=...)` parameterised constructor (PR 6.5 Task 5) -- `Task.worker` / `Task.sandbox` / `Task.evaluators` inline fields (PR 5) -- `launch_run(definition_id)` — already correct shape -- `DefinitionHandle(definition_id, benchmark_type, ...)` — already exists -- The user-facing word **"experiment"** — survives as a string column, a CLI flag, and a verb in the docs. No vocabulary change for users. - -### What's new - -- `persist_benchmark(benchmark, *, name, experiment=None, metadata=None) -> DefinitionHandle` — replaces `persist_definition(experiment_obj)` -- `experiment: str | None` column on the persisted `BenchmarkDefinitionRecord` -- `ergon_core.api.experiments` query helper — minimal surface, just `list_definitions_in_experiment(name: str) -> list[DefinitionHandle]` (or skip the helper and use a `select` directly; YAGNI is fine). -- CLI commands: - - `ergon experiment show <name>` — lists definitions in an experiment with run status - - `--experiment=<name>` flag on `ergon run <benchmark>` to tag a new definition into an experiment - -## Worked examples - -### Two workers, same benchmark, one experiment — Python - -```python -from ergon_builtins.benchmarks.minif2f import ( - MiniF2FBenchmark, - make_minif2f_react_worker, - make_minif2f_cot_worker, -) -from ergon_core.api import persist_benchmark, launch_run - -EXPERIMENT = "minif2f-strategy-ablation-2026-05-15" - -for label, worker_factory in [ - ("react", make_minif2f_react_worker), - ("cot", make_minif2f_cot_worker), -]: - benchmark = MiniF2FBenchmark(worker_factory=worker_factory, limit=10) - handle = persist_benchmark( - benchmark, - name=f"minif2f-{label}", - experiment=EXPERIMENT, - metadata={"strategy": label}, - ) - await launch_run(handle.definition_id) -``` - -4 imports, 11 lines of authoring code. Two definitions persisted into the same experiment. No `Experiment` class anywhere — `EXPERIMENT` is just a string the user picked. - -### Observing the runs — CLI - -The CLI does not start the runs (that was Python above). It observes them: - -```bash -ergon experiment show ablation-2026-05-15 -# NAME DEFINITION_ID RUNS STATUS -# minif2f-react abc-123-... 1 running -# minif2f-cot def-456-... 1 pending - -ergon run status <run-id> -ergon run cancel <run-id> -``` - -See the next section for the full rationale. - -## CLI role, made explicit - -The CLI has **one honest role**: lifecycle and observation of persisted state. It is **not** an authoring interface, not a convenience wrapper, and not a parallel dispatch path. There is exactly one way to start a run: write Python that calls `persist_benchmark(...)` + `launch_run(...)`. - -### What the CLI keeps - -- `ergon run status <run-id>` — check status of a running run -- `ergon run cancel <run-id>` — cancel a running run -- `ergon run list [--experiment=<name>]` — list runs, optionally filtered -- `ergon experiment show <name>` — list definitions in an experiment, with run status -- `ergon experiment list` — list known experiment names -- Future: `ergon dashboard` (open the local dashboard), `ergon logs <run-id>`, etc. - -All of these operate on state that **already exists in the database**. The CLI's job is to inspect and steer; not to create. - -### What the CLI drops - -- ❌ `ergon experiment define --benchmark <slug> ...` — gone. No CLI-side authoring. -- ❌ `ergon experiment run <definition-id>` — gone. Use Python; `launch_run` is one line. -- ❌ `ergon run <benchmark> --worker <name> ...` — gone. No stock-benchmark dispatch from the CLI. -- ❌ `BENCHMARK_CLASSES` + `WORKER_FACTORIES` dicts — gone. No CLI registry of benchmarks/workers. -- ❌ `BUILTIN_EXPERIMENT_FACTORIES` (PR 8's design) — gone. Never built. -- ❌ Per-benchmark `experiment.py` / `_cli_factory.py` files — gone. Never built. - -### Why kill the CLI authoring route entirely? - -The PR 8 plan introduces a *second* authoring route: argparse → factory dict → benchmark constructor → `Experiment` → persist. That's a parallel surface that: - -- **Must be kept in sync** with the Python API (every benchmark constructor kwarg needs a CLI flag; every worker factory needs a registry entry). -- **Misleads new contributors** about where the "real" authoring happens. -- **Constrains the Python API** to only expose what argparse can pass (no callables, no objects, no inline lambdas). -- **Has its own test surface** doubling the assertions for every code path. - -By having exactly **one** authoring path (Python), all of those costs disappear. Reproducibility is "the script that built the experiment is checked into your repo." No drift between two surfaces because there's only one. - -The argument for keeping a CLI authoring route is "users who don't want to write Python." The counter-argument is: those users were going to write a multi-line argparse invocation anyway (`--limit 10 --worker react --experiment=ablation-2026-05-15 --name="run-1"`), which is *more* characters than the equivalent Python and not version-controlled. A 5-line Python script is strictly better for that user too — they just need to know it exists. - -### Authoring example (Python only) - -```python -# kick_off.py — checked into the user's repo, version-controlled -import asyncio -from ergon_builtins.benchmarks.minif2f import ( - MiniF2FBenchmark, make_minif2f_react_worker, -) -from ergon_core.api import persist_benchmark, launch_run - -async def main(): - benchmark = MiniF2FBenchmark(worker_factory=make_minif2f_react_worker, limit=10) - handle = persist_benchmark(benchmark, name="minif2f-react", experiment="ablation-2026-05-15") - await launch_run(handle.definition_id) - print(f"DEFINITION_ID={handle.definition_id}") - -asyncio.run(main()) -``` - -```bash -$ uv run python kick_off.py -DEFINITION_ID=abc-123-... - -$ ergon experiment show ablation-2026-05-15 -NAME DEFINITION_ID RUNS STATUS -minif2f-react abc-123-... 1 running - -$ ergon run status <run-id> -status: running, tasks_completed: 3/10 -``` - -The CLI never appears in the authoring step. It appears only when observing. - -### Implementation shape (CLI side) - -The lifecycle commands read from the database via existing repositories. No new factory dicts. No per-benchmark imports. The full CLI source under this proposal: - -```python -# ergon_cli/commands/run.py — observation only - -def handle_run_status(args: Namespace) -> int: - state = RunRepository().get_state(UUID(args.run_id)) - print(f"status: {state.status}, tasks_completed: {state.completed}/{state.total}") - return 0 - -def handle_run_cancel(args: Namespace) -> int: - cancel_run(UUID(args.run_id), reason=args.reason or "cli-cancel") - return 0 - -def handle_run_list(args: Namespace) -> int: - runs = RunRepository().list(experiment=args.experiment) # filter optional - for r in runs: - print(f"{r.run_id} {r.experiment or '-':30s} {r.status}") - return 0 -``` - -```python -# ergon_cli/commands/experiment.py — observation only - -def handle_experiment_show(args: Namespace) -> int: - defs = DefinitionRepository().list(experiment=args.experiment_name) - for d in defs: - latest = RunRepository().latest_for_definition(d.definition_id) - print(f"{d.name:30s} {d.definition_id} {latest.status if latest else 'no runs'}") - return 0 - -def handle_experiment_list(args: Namespace) -> int: - names = DefinitionRepository().distinct_experiments() - for n in names: - print(n) - return 0 -``` - -That's the entire CLI command surface for the experiment / run lifecycle. No `BENCHMARK_CLASSES`. No `WORKER_FACTORIES`. No `BUILTIN_EXPERIMENT_FACTORIES`. No per-benchmark CLI registration burden. Adding a new benchmark requires zero CLI changes — users just import it from Python. - -## Naming choice: why we keep the word "experiment" - -Two alternatives were considered for the grouping-string concept: - -- **`cohort`** — borrowed from clinical-trials / observational-study terminology. Precise, but jargon-y for a Python API. -- **`experiment`** — the word every researcher already uses for "I ran this benchmark with two configurations to compare them". Carries some baggage from the deleted class, but that baggage is what we *want* the user to keep in their vocabulary. - -**Decision: keep `experiment` as the user-facing name.** The class dies, but the *word* survives as a string label. Users say "I ran experiment X with two definitions"; the code stores `experiment: str | None` on the definition row. Vocabulary continuity for users, no class-level abstraction. - -This is the same move as collapsing `WorkerSpec` into "just a slug + name" in earlier PRs — keep the word, drop the class. - -## Experiment string vs structured class - -**Decision: string column, not a class.** YAGNI for now. - -What you need to do with experiments in practice: -- Tag related definitions with a common label -- Query "all definitions in experiment X" -- Compare their results in the dashboard - -All of that works with a single string column. If we later need structured experiment metadata (shared parameters, ablation axes, comparison hypotheses, parent/child relationships), add an `Experiments` table that definitions FK into — **as a separate later RFC**. Not before. - -The CLI gets one `--experiment=<name>` flag. Python users pass `experiment="..."` to `persist_benchmark`. Done. - -## What's the migration cost? - -Real but bounded. Sketch: - -### Code changes -- **Hard delete** `ergon_core/api/experiment.py` (the `Experiment` class). No deprecation alias. -- Delete `ergon_core/core/application/experiments/service.py::define_benchmark_experiment` -- **Rename in place**: `persist_definition(experiment)` → `persist_benchmark(benchmark, *, name, experiment=None, metadata=None) -> DefinitionHandle`. Signature change. No "add new + deprecate old" — one rename, all callers updated in the same PR. -- Rename `ExperimentRecord` SQLAlchemy model → `BenchmarkDefinitionRecord` in `ergon_core/core/persistence/telemetry/models.py`. Rename the **physical table too** — there's no production data to preserve, and the Alembic chain is being dropped/regenerated wholesale later anyway. -- Add `experiment: str | None` column on the renamed `BenchmarkDefinitionRecord` (just a SQLModel field; no fiddly Alembic dance since tables get dropped/recreated). -- Update every test that references `Experiment` (~20 files based on rough grep). -- Update Inngest event payloads that mention "experiment" (probably 5-10 places). -- Update dashboard reads — the dashboard queries `ExperimentRecord` directly, so TypeScript needs updating. -- **Delete the CLI authoring route**: `ergon_cli/commands/experiment.py::handle_experiment_define`, `handle_experiment_run`, and any tests around them. -- **Hard break on CLI** — no deprecation warnings on removed commands. Ergon is pre-1.0; clean break is acceptable. - -### Doc changes -- v2 RFC: rewrite `00-readme.md`, `01-api-surface.md` to reflect the cleaner model (no `Experiment` class; `experiment` is a string tag). -- `docs/architecture/01_public_api.md`: drop `Experiment` from the type list; add `persist_benchmark`; note Python-only authoring. -- `docs/architecture/06_builtins.md`: update to reflect lifecycle-only CLI; add a "Discovery" section pointing at the new `README.md` catalogue. -- **New `ergon_builtins/benchmarks/README.md`** — discovery doc listing every builtin benchmark + the workers it ships with, plus the import path. Replaces the CLI registry as the catalogue. - -### What we DON'T need -- No Alembic migration dance. Tables are dropped/recreated; we just adjust the SQLModel. -- No deprecation aliases for either `Experiment` or the removed CLI commands. -- No `launch_run_sync(...)` wrapper. Users write `asyncio.run(launch_run(...))` in their scripts. -- No `ergon list benchmarks` static-catalogue command. Catalogue lives in the README. - -### Effort estimate -- Pure code + tests: ~1 PR worth of focused work. -- Plus the dashboard TypeScript update. -- Plus the catalogue README. - -Not trivial. Not huge. Probably 1-2 days of focused work. - -## Where in the PR sequence? - -### Decision: fold everything into PR 6.5; PR 8 becomes the lifecycle-CLI cleanup PR. - -PR 6.5's scope expands from "domain colocation" to "domain colocation + API surgery." Concretely PR 6.5 now does: - -- **Original scope (kept):** - - File moves (`sandboxes/lean.py` → `benchmarks/minif2f/sandbox.py`, etc.) - - Rename `worker_factory.py` → `workers.py` + split out legacy - - Add `sandbox/` top-level dir stub - - Parameterise `MiniF2FBenchmark` with `worker_factory` / `sandbox_factory` / `evaluator_factory` kwargs - - Architecture doc updates (cardinality matrix, anti-patterns) - - v2 RFC framing rewrite - -- **New scope (added):** - - **Drop Task 4** (the `experiment.py` placeholder) — never create the file. - - **Hard delete** `ergon_core.api.experiment.Experiment` class. No alias. - - **Rename in place** `persist_definition(experiment)` → `persist_benchmark(benchmark, *, name, experiment=None, ...)`. - - **Rename** `ExperimentRecord` SQLModel → `BenchmarkDefinitionRecord` (table too — no data to preserve). - - **Add** `experiment: str | None` column on the renamed model. - - **Delete** CLI authoring commands: `ergon experiment define`, `ergon experiment run`, the entire `BUILTIN_EXPERIMENT_FACTORIES` design. - - **Update** every test that constructs `Experiment(...)` to use the new shape. - - **Update** Inngest event payloads that name `experiment` semantically. - - **Update** the dashboard's TypeScript reads of `ExperimentRecord`. - - **Add** `ergon_builtins/benchmarks/README.md` catalogue. - -PR 8 (the existing `09-pr-08-cli-composition.md`) becomes a **slim cleanup PR** with one job: the lifecycle / observation commands. - -- Add `ergon run status <run-id>`, `ergon run cancel <run-id>`, `ergon run list [--experiment=<name>]` -- Add `ergon experiment show <name>`, `ergon experiment list` -- Delete the now-stale parts of its existing plan (factory dict task, per-benchmark `experiment.py` task, etc.). - -### Honest pushback on the sequencing - -**I'd push back on this once before you commit.** Folding the `Experiment` kill into PR 6.5 makes PR 6.5 a substantially bigger and riskier PR than originally scoped. Concretely: - -- **Original PR 6.5:** ~6 file moves, 1 ctor change, ~15 doc/import updates. Reviewable in 30 minutes. Low risk — it's mostly mechanical. -- **PR 6.5 with the kill folded in:** all of the above PLUS deletion of a public API class, signature change on the persistence function, SQLModel + table rename, new column, CLI command deletion, dashboard TypeScript update, ~20-30 test files touched, Inngest payload changes. Closer to a 2–4 hour focused review. Higher risk because it crosses layers (API / persistence / CLI / dashboard). - -**Two reasons you might still want to fold it in:** - -1. **No intermediate state** — landing the parameterised benchmark with `Experiment` still alive creates a window where the architecture is half-migrated. Folding avoids that. -2. **Velocity** — one big PR is sometimes faster than two sequential PRs through review and CI. - -**Two reasons you might want to split it:** - -1. **Reviewability** — PR 6.5 is meant to be the "easy win" reorganization. Folding makes it harder to spot bugs. -2. **Bisectability** — if something breaks after PR 6.5 lands, "was it the file moves or the Experiment kill?" becomes a harder bisect. - -**Middle-ground option not yet considered:** ship PR 6.5 as two commits in one branch, but as one PR — first commit = file moves + parameterise (the "easy win"), second commit = Experiment kill + persist_benchmark + CLI deletion (the surgery). Same PR, but the diff is mentally splittable for review. Then PR 8 is the lifecycle CLI cleanup as planned. - -**My recommendation: do the middle-ground.** One PR titled "PR 6.5: domain colocation + kill Experiment class" with two clearly labelled commits. Get the benefits of one-shot cutover without making the diff incomprehensible. PR 8 = lifecycle CLI cleanup, as you suggested. - -If you prefer the split-into-two-PRs version (PR 6.5 = original scope, PR 6.6 = kill), say so and I'll write the plans that way. - -## Decisions taken - -All ten questions raised during the brainstorm are now resolved. Recorded here for the plan author: - -| # | Decision | Choice | -|---|---|---| -| 1 | Kill `Experiment` class? | ✅ **Yes — hard delete, no alias** | -| 2 | Keep `experiment` as a `str \| None` column? | ✅ **Yes** | -| 3 | Kill the CLI authoring route entirely? | ✅ **Yes — no `ergon run <bench>`, no `experiment define/run`** | -| 4 | `persist_definition` → `persist_benchmark`: rename in place vs add+deprecate? | ✅ **Rename in place** (signature change; all callers updated in one PR) | -| 5 | `ExperimentRecord` SQLModel rename: keep table name, or rename table too? | ✅ **Rename the table too** — no production data; Alembic chain gets dropped/regenerated wholesale | -| 6 | CLI removed commands: deprecation warnings, or clean break? | ✅ **Clean break** — Ergon is pre-1.0 | -| 7 | PR 6.5 Task 4 (create `experiment.py` stub): keep or drop? | ✅ **Drop entirely** — file never gets created | -| 8 | PR sequencing | ✅ **Fold into PR 6.5; PR 8 becomes lifecycle-CLI cleanup** (with my pushback noted above re: PR-6.5 size) | -| 9 | `ergon list benchmarks` static-catalogue command? | ✅ **Skip** — `ergon_builtins/benchmarks/README.md` is the catalogue | -| 10 | Sync `launch_run_sync` wrapper? | ✅ **Skip** — users write `asyncio.run(launch_run(...))` | - -Once these are recorded, the next step is to turn this brainstorm into a concrete plan: `docs/superpowers/plans/2026-05-15-kill-experiment-class.md` (the work) plus updates to `docs/rfcs/active/2026-05-11-authoring-api-redesign-v2/09-implementation-plan/07b-pr-6-5-domain-colocation.md` (expand PR 6.5's scope) and `09-pr-08-cli-composition.md` (slim PR 8 down to lifecycle commands). - -## What this is NOT proposing - -- **Not changing `Benchmark`'s shape.** PR 5/6's design (inline worker/sandbox/evaluators on each Task) stands. -- **Not changing `Task`.** The object-bound authoring shape is correct. -- **Not changing the worker / sandbox / toolkit abstractions.** Those earn their keep. -- **Not killing the CLI binary.** `ergon` still exists; it just stops being an authoring interface. Lifecycle / observation commands (`status`, `cancel`, `experiment show`, etc.) stay. -- **Not changing how runs / evaluations work.** Run lifecycle is unchanged; only the persistence-time class and the CLI authoring path disappear. -- **Not removing the word "experiment" from the user's vocabulary.** It becomes a string label (`experiment="..."`) rather than a class (`Experiment(...)`). - -The change is targeted: (1) one class (`Experiment`) + its persistence ceremony, (2) the CLI authoring route (`define` / `run <benchmark>` commands and the factory-dict layer that powers them), and (3) the misleading framing around both. Everything else stays. - -## Summary of decisions taken in discussion - -The brainstorm grew over several rounds. Recorded here for future readers: - -1. **`Experiment` class dies.** Hard delete, no alias. It's a wrapper around `(benchmark, name, description, metadata)` with no behaviour beyond persistence routing. Function parameters do the same job with less abstraction. -2. **The word "experiment" survives as a `str | None` column.** Two definitions with the same `experiment="x"` tag are part of the same experiment. No `Cohort` or `Experiment` class. -3. **The CLI authoring route dies entirely.** Clean break, no deprecation. No `ergon experiment define`, no `ergon run <benchmark> --worker <name>`, no `BENCHMARK_CLASSES` / `WORKER_FACTORIES` dicts, no per-benchmark `experiment.py` files. One authoring path: Python. -4. **The CLI keeps lifecycle / observation commands.** `run status`, `run cancel`, `run list`, `experiment show`, `experiment list`. All read-only against persisted state. No static `list benchmarks` command — the catalogue is a README in `ergon_builtins/benchmarks/`. -5. **`persist_definition` → `persist_benchmark`** — rename in place, signature change, all callers updated in one PR. -6. **`ExperimentRecord` SQLModel + physical table both rename** to `BenchmarkDefinitionRecord`. No production data; Alembic chain gets dropped/regenerated wholesale, so no migration dance. -7. **Async authoring stays async.** No `launch_run_sync` wrapper; users write `asyncio.run(launch_run(...))`. -8. **PR sequencing: fold into PR 6.5; PR 8 becomes slim lifecycle-CLI cleanup.** My pushback on size is recorded above — the middle-ground recommendation is "one PR, two clearly-labelled commits." Final call is the user's. - -These eight decisions form the spine of the eventual PR 6.5 (expanded) + slimmed-PR-8 plans. diff --git a/docs/superpowers/plans/2026-04-16-restart-and-downstream-invalidation.md b/docs/superpowers/plans/2026-04-16-restart-and-downstream-invalidation.md deleted file mode 100644 index c4d2b5f81..000000000 --- a/docs/superpowers/plans/2026-04-16-restart-and-downstream-invalidation.md +++ /dev/null @@ -1,232 +0,0 @@ -# Restart Task & Downstream Invalidation - -> **Depends on:** `2026-04-16-subtask-lifecycle.md` (containment model, cancel cascade, dep edges). -> **Branch:** `feature/type-tightening-prep` - -## Problem - -A manager agent cannot re-run a completed or failed subtask. Today the only recovery path is "cancel the old node, create a new one" — but that orphans existing dependency edges. The manager has to rebuild the entire subgraph. - -More critically: if D is COMPLETED and its downstream E is in-progress, and the manager decides D's output is wrong and needs re-running, there is no way to: -1. Reset D back to PENDING -2. Cancel E (it's running against stale input) -3. Reset E to PENDING so it re-runs after D completes again - -This is the "downstream invalidation on re-run" pattern (see Fractal OS task propagation). - -## Test Scenario (E2E integration test) - -This is the acceptance test — the north star that task propagation logic doesn't get broken. It runs in CI as a state test (SQLite, no Docker, no sleeps). - -### Graph topology - -``` -Manager (L1, RUNNING) -├── Graph 1 — Diamond with fan-in: -│ A ──→ B ──→ F -│ A ──→ C ──→ F (F has TWO incoming edges: from B and from C) -│ -├── Graph 2 — Chain (3-deep, for recursive invalidation): -│ D ──→ E ──→ G -│ -└── H (independent leaf, no deps — sanity baseline) -``` - -8 child nodes. Created via three `plan_subtasks` calls (Graph 1, Graph 2, H standalone via `add_subtask`). - -### Step-by-step (15 steps) - -| Step | Action | Assert | Tests | -|------|--------|--------|-------| -| 1 | Manager spawns Graph 1 (A→B, A→C, B→F, C→F), Graph 2 (D→E→G), and H | 8 child nodes. A, D, H are roots (PENDING, no deps). B, C, E, F, G blocked. | Creation, edge wiring | -| 2 | A completes | B and C become READY. F stays PENDING (not all deps met). | **Fan-out**: completion unblocks multiple targets | -| 3 | B completes | F stays PENDING (C not done yet). | **Fan-in**: partial dep satisfaction does NOT unblock | -| 4 | C completes | F becomes READY (all deps met). | **Fan-in**: full dep satisfaction unblocks | -| 5 | F completes | Graph 1 fully done. | Normal completion | -| 6 | D completes → E becomes READY | E is READY. G stays PENDING. | Chain propagation | -| 7 | E starts running | E is RUNNING. | Status transition | -| 8 | Manager cancels E (while RUNNING) | E is CANCELLED. G stays PENDING (managed subtask, not auto-cancelled). Edge E→G INVALIDATED. | **Cancel running node**, managed subtask no-cascade | -| 9 | Manager restarts E (from CANCELLED) | E is PENDING. Edge E→G reset to EDGE_PENDING. task/ready emitted for E. | **Restart from CANCELLED**, edge reset | -| 10 | E completes → G becomes READY | G is READY. | Re-propagation after restart | -| 11 | G completes | Graph 2 fully done. | Normal completion | -| 12 | Manager restarts B (B was COMPLETED, F was COMPLETED) | B is PENDING. **F is invalidated** (stale input): F goes CANCELLED, edges B→F and C→F reset to EDGE_PENDING. G unaffected (different graph). | **Restart with completed downstream**, deep invalidation, cross-graph isolation | -| 13 | B completes again | **F re-activates**: C is still COMPLETED, B now COMPLETED → all deps satisfied → F becomes READY. | **Fan-in re-activation**: CANCELLED target re-activates when all deps re-satisfied | -| 14 | F completes again | Graph 1 re-done. | Re-propagation after invalidation | -| 15 | H completes, manager completes | All 8 children + manager terminal. Zero FAILED. Workflow COMPLETED. | **Workflow terminal detection** | - -### What this covers - -| Pattern | Steps | Why it matters | -|---------|-------|----------------| -| Fan-out (one source → multiple targets) | 2 | Most basic propagation | -| Fan-in (multiple sources → one target) | 3, 4 | #1 source of propagation bugs — must wait for ALL deps | -| Diamond (fan-out + fan-in) | 2–5 | Where edge-counting bugs hide | -| Chain propagation (A→B→C) | 6, 10 | Multi-hop dependency | -| Cancel while RUNNING | 8 | Real-world manager behavior (not just cancelling PENDING) | -| Managed subtask no-cascade | 8 | Downstream stays PENDING, not auto-cancelled | -| Restart from CANCELLED | 9 | Terminal → PENDING, edge reset | -| Restart with completed downstream (deep invalidation) | 12 | The hard case: stale output must cascade | -| Fan-in re-activation | 13 | CANCELLED target re-activates when all deps re-satisfied | -| Cross-graph isolation | 12 | Restart in Graph 1 doesn't touch Graph 2 | -| Independent leaf | 15 | Node with no deps, no downstream — sanity | -| Workflow terminal | 15 | All terminal + zero FAILED = COMPLETED | - -## Current State — What Works - -| Operation | Works? | Code path | -|-----------|--------|-----------| -| `plan_subtasks` with dep edges | Yes | `TaskManagementService.plan_subtasks` | -| Cancel D, E stays PENDING | Yes | `on_task_completed_or_failed` skips managed subtasks | -| D completes → E becomes READY | Yes | `on_task_completed_or_failed(COMPLETED)` satisfies edges | -| Recursive cancel cascade | Yes | `SubtaskCancellationService.cancel_orphans` BFS | -| First-writer-wins guard | Yes | `update_node_status(only_if_not_terminal=True)` | -| Fan-in propagation | Yes | `on_task_completed_or_failed` checks ALL incoming source nodes | - -## Gaps - -### Gap 1: `restart_task` service method + tool - -**No way to reset a terminal node back to PENDING.** The manager can cancel, refine, and add — but cannot restart. - -Needed: -- `RestartTaskCommand(run_id, node_id)` DTO -- `RestartTaskResult(node_id, old_status)` DTO -- `TaskManagementService.restart_task()`: - - Validate node is in TERMINAL_STATUSES (raise `TaskNotTerminalError` if not) - - Call `_invalidate_downstream()` first (see Gap 2) - - Reset node status to PENDING (unguarded `update_node_status`, `only_if_not_terminal=False`) - - Reset all outgoing edges from this node to EDGE_PENDING - - Emit `task/ready` event so the scheduler picks it up - - Return result -- `SubtaskLifecycleToolkit._make_restart_task()` closure → 8th tool -- Tool signature: `restart_task(node_id: str) -> dict` - -### Gap 2: Downstream invalidation on restart - -**When a node is restarted, its downstream targets are running against stale input.** They must be cancelled and re-queued. - -Needed — `_invalidate_downstream()` called by `restart_task` before resetting the node's own edges: - -``` -_invalidate_downstream(session, run_id, node_id, graph_repo): - for each outgoing edge from node_id: - reset edge to EDGE_PENDING - target = edge.target_node - if target is non-terminal (PENDING, READY, RUNNING): - cancel target (it may be running against stale input) - emit task/cancelled for cleanup - if target is COMPLETED: - # target's output is stale — cancel it too, then - # reset its outgoing edges and recurse - cancel target - emit task/cancelled for cleanup - recurse into target's outgoing edges -``` - -The recursion terminates because: -- The graph is a DAG (no cycles, enforced by Kahn's algorithm in `plan_subtasks`) -- We only recurse into COMPLETED targets (CANCELLED/FAILED have no stale output) - -**Key semantic:** Restarting B resets B→F edge to PENDING, cancels F, and resets F's outgoing edges too. When B completes again, normal propagation re-satisfies the B→F edge. If C is still COMPLETED (the other fan-in source), F becomes READY. - -### Gap 3: Re-activating a CANCELLED node via propagation - -Currently `on_task_completed_or_failed(COMPLETED)` only unblocks PENDING targets: - -```python -if candidate_node.status != TaskExecutionStatus.PENDING: - continue -``` - -After step 12, F is CANCELLED (invalidated by B's restart). When B completes again in step 13, propagation needs to re-activate F. The check must also allow CANCELLED targets to become READY when ALL incoming edges are satisfied. - -Change: when a source completes and the target is CANCELLED with all deps now satisfied, reset the target to PENDING and add it to `newly_ready`. - -**Re-activation policy (decided):** Any CANCELLED managed subtask (parent_node_id set) re-activates when all deps are satisfied. If the manager explicitly cancelled a node and doesn't want it re-activated, it can re-cancel. This avoids needing a `cancel_cause` column on the node and keeps the propagation logic simple. - -Static workflow nodes (parent_node_id=None) do NOT re-activate — they have no supervisor to adapt. - -### Gap 4: Widen `refine_task` to work on terminal nodes - -Currently `refine_task` rejects any non-PENDING node: - -```python -if node.status != PENDING: - raise TaskNotPendingError(command.node_id, node.status) -``` - -The manager should be able to edit a task's description or properties before restarting it. The composition is: `refine_task(D, new_description)` → `restart_task(D)`. - -Change: allow `refine_task` on any status **except RUNNING** (the worker is actively using the description). PENDING, COMPLETED, FAILED, CANCELLED are all refinable. - -This is a one-line change: `if node.status == RUNNING: raise TaskRunningError(...)`. - -## File Map - -### New files - -| Path | Purpose | -|------|---------| -| `tests/state/test_restart_and_invalidation.py` | Unit tests for restart_task + downstream invalidation | -| `tests/state/test_propagation_reactivation.py` | Unit tests for CANCELLED → PENDING re-activation on dep satisfaction | -| `tests/state/test_manager_dag_scenario.py` | Full 15-step E2E scenario | - -### Modified files - -| Path | Change | -|------|--------| -| `ergon_core/.../services/task_management_dto.py` | Add `RestartTaskCommand`, `RestartTaskResult` | -| `ergon_core/.../services/task_management_service.py` | Add `restart_task()`, `_invalidate_downstream()`. Widen `refine_task` status check. | -| `ergon_core/.../execution/propagation.py` | `on_task_completed_or_failed`: allow CANCELLED managed subtasks to re-activate when all deps satisfied | -| `ergon_core/.../runtime/errors/delegation_errors.py` | Add `TaskNotTerminalError`, `TaskRunningError` | -| `ergon_builtins/.../tools/subtask_lifecycle_toolkit.py` | Add `_make_restart_task()`, update `get_tools()` to return 8 tools | - -## Implementation Order - -### Phase 1: Widen `refine_task` + restart_task (no downstream invalidation yet) - -- [ ] Widen `refine_task`: change guard from `status != PENDING` to `status == RUNNING` -- [ ] Add `TaskNotTerminalError`, `TaskRunningError` to delegation_errors -- [ ] Add `RestartTaskCommand` / `RestartTaskResult` DTOs -- [ ] Implement `TaskManagementService.restart_task()`: validate terminal, reset node to PENDING, reset outgoing edges to EDGE_PENDING, emit task/ready -- [ ] Add `_make_restart_task()` to toolkit (8th tool) -- [ ] Unit test: restart from COMPLETED, FAILED, CANCELLED; reject restart of PENDING/RUNNING -- [ ] Unit test: outgoing edges reset to EDGE_PENDING -- [ ] Unit test: `refine_task` now works on COMPLETED/FAILED/CANCELLED, still rejects RUNNING - -### Phase 2: Downstream invalidation - -- [ ] Implement `_invalidate_downstream()` in task_management_service: walks outgoing edges, cancels non-terminal targets, recurses into completed targets -- [ ] Wire into `restart_task()` (called before edge reset) -- [ ] Unit test: restart D while E is RUNNING → E cancelled, edge D→E reset -- [ ] Unit test: restart D while E is COMPLETED → E cancelled, E's outgoing edges reset -- [ ] Unit test: deep chain D→E→G, restart D → E and G both invalidated -- [ ] Unit test: fan-in — restart B, F invalidated, but C→F edge also reset - -### Phase 3: Re-activation of CANCELLED targets on dep satisfaction - -- [ ] Modify `on_task_completed_or_failed(COMPLETED)`: for managed subtasks (parent_node_id set), check CANCELLED targets alongside PENDING -- [ ] When all incoming source nodes are COMPLETED for a CANCELLED managed subtask, reset to PENDING + add to newly_ready -- [ ] Unit test: D restarts, completes again → E (CANCELLED) re-activates -- [ ] Unit test: fan-in re-activation — B restarts, completes → F re-activates (C still COMPLETED) -- [ ] Unit test: static workflow CANCELLED node does NOT re-activate (no parent_node_id) -- [ ] Unit test: user-cancelled node with all deps satisfied — re-activates (manager can re-cancel) - -### Phase 4: E2E integration test - -- [ ] Write `test_manager_dag_scenario.py` with the full 15-step scenario -- [ ] Uses `TaskManagementService` + `WorkflowGraphRepository` directly (no Inngest, no stubs sleeping) -- [ ] Simulates worker completion via `graph_repo.update_node_status()` + `on_task_completed_or_failed()` -- [ ] All assertions from the 15-step scenario table -- [ ] Runs in `tests/state/` (SQLite, per-test transaction rollback) -- [ ] Wire into CI via existing `pnpm run test:be:state` - -## Open Questions - -1. **Concurrent restart + completion race:** If the manager calls `restart_task(D)` at the same moment D's worker reports completion, who wins? The restart should win (manager's intent is explicit), but we need to think about the edge reset happening after propagation already satisfied the edges. Likely solved by: `restart_task` runs in the manager's session, propagation runs in the Inngest step's session. The DB transaction that commits first wins. If propagation committed first (edges satisfied), restart will re-reset them. If restart committed first, propagation sees the node as PENDING and skips. - -2. **Event semantics for restart:** Should `restart_task` emit a new event type (`task/restarted`) or reuse `task/ready`? A new event type gives better observability in Inngest dashboard but adds another function to register. Recommendation: reuse `task/ready` — the scheduler doesn't need to distinguish "first run" from "re-run". - -3. **Sandbox lifecycle on restart:** When a node is restarted, its downstream targets get `task/cancelled` events which trigger `cleanup_cancelled_task_fn` (sandbox teardown). The restarted node itself goes PENDING→RUNNING and gets a fresh sandbox via normal `execute_task_fn` dispatch. The old execution's sandbox for the restarted node is NOT explicitly cleaned up — it will be orphaned. We may want to emit `task/cancelled` for the restarted node too, just for sandbox cleanup, even though it's not really "cancelled". Or add a `task/restarted` event that triggers cleanup. - -4. **`refine_task` on RUNNING — edge case:** A manager might want to refine a running task's description for the _next_ run (if it plans to restart it). Current plan blocks RUNNING. Alternative: allow refine on RUNNING but document that the current execution won't see the change. Keeping it blocked is safer for now. diff --git a/docs/superpowers/plans/2026-04-16-subtask-lifecycle.md b/docs/superpowers/plans/2026-04-16-subtask-lifecycle.md deleted file mode 100644 index 431786c2e..000000000 --- a/docs/superpowers/plans/2026-04-16-subtask-lifecycle.md +++ /dev/null @@ -1,2094 +0,0 @@ -# Subtask Lifecycle Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add structured subtask containment, cancellation cascading, and dependency edges to Ergon's dynamic delegation system, enabling DAG-shaped subtask plans and bounded compute. - -**Architecture:** Containment is stored on `RunGraphNode` (`parent_node_id` + `level`). Edges become pure dependencies. `ABANDONED` is replaced by `CANCELLED` everywhere. When a parent reaches terminal, an Inngest-driven cascade cancels all live descendants. New toolkit provides `add_subtask`, `plan_subtasks`, `cancel_task`, `refine_task`, `list_subtasks`, `get_subtask`, and sandboxed `bash`. - -**Tech Stack:** Python 3.13, SQLModel, Alembic, Inngest (>=0.3.0), Pydantic v2, pydantic-ai, PostgreSQL - -**Decisions (from RFC open questions):** -- Q6 (in-flight runs): Drain runs before deploy. Migration is offline-only. -- Q8 (toolkit rename): Delete `task_management_toolkit.py` entirely. No shim/re-export. -- Q9 (multi-trigger): Keep three separate Inngest functions (SDK 0.3 doesn't support `triggers=[]` list cleanly). - -**Coding standard (§4.7):** Every new class and public method must include a docstring explaining *why* the design choice was made. Class docstrings state why the class exists as a separate unit. Method docstrings explain non-obvious design choices. Copy the docstrings from the RFC code blocks verbatim — they were reviewed and approved. - -**Reference:** Full spec at `ergon_paper_plans/roadmap/code/C2_dynamic-delegation/SUBTASK_LIFECYCLE_RFC.md` (branch `feature/subtask-lifecycle-rfc`). Extract to /tmp if needed: `git -C /path/to/ergon_paper_plans show feature/subtask-lifecycle-rfc:roadmap/code/C2_dynamic-delegation/SUBTASK_LIFECYCLE_RFC.md > /tmp/RFC.md` - -**Branch:** `feature/type-tightening-prep` on `ergon` repo (4 prep commits already landed: UUID NewType aliases in `shared/types.py`, NodeStatus/EdgeStatus Literals in `status_conventions.py`, keyword-only UUID args in `graph_repository.py`, typed aliases on graph/task DTOs). - ---- - -## File Map - -### New files - -| Path | Purpose | Phase | -|------|---------|-------| -| `ergon_core/migrations/versions/<rev>_add_containment_and_cancelled.py` | Alembic migration: add `parent_node_id` + `level`, backfill, delete delegation edges, rename ABANDONED→CANCELLED | 0 | -| `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py` | Cascade-cancel orphaned children (§7.2) | 2 | -| `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_dto.py` | `CancelOrphansResult` | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_cleanup_service.py` | Sandbox/execution/context teardown (§7.3) | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_cleanup_dto.py` | `CleanupResult` | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_inspection_service.py` | Read-only listing/snapshotting (§7.4) | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py` | `SubtaskInfo`, `SubtaskStatus` | 2 | -| `ergon_core/ergon_core/core/runtime/inngest/cancel_orphan_subtasks.py` | Three `cancel_orphans_on_*_fn` Inngest functions (§9.1) | 3 | -| `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` | `cleanup_cancelled_task_fn` (§9.2) | 3 | -| `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` | New toolkit replacing `TaskManagementToolkit` (§8) | 4 | -| `ergon_builtins/ergon_builtins/tools/bash_sandbox_tool.py` | Sandboxed `bash` tool (§8) | 4 | -| `tests/state/test_subtask_cancellation_service.py` | Unit: cascade cancel | 2 | -| `tests/state/test_task_inspection_service.py` | Unit: listing, ordering, excerpt | 2 | -| `tests/state/test_plan_subtasks.py` | Unit: plan_subtasks validation + execution | 2 | -| `tests/state/test_conditional_status_writes.py` | Unit: only_if_not_terminal guard | 1 | -| `tests/state/test_task_cleanup_service.py` | Unit: idempotent cleanup, `execution_id=None` path | 2 | - -### Modified files - -| Path | Change | Phase | -|------|--------|-------| -| `ergon_core/ergon_core/core/persistence/graph/models.py` | Add `parent_node_id` + `level` to `RunGraphNode` | 0 | -| `ergon_core/ergon_core/core/persistence/graph/status_conventions.py` | `ABANDONED`→`CANCELLED`, `EDGE_ACTIVE`→remove, add `EDGE_INVALIDATED` | 0 | -| `ergon_core/ergon_core/core/persistence/shared/enums.py` | Add `TaskExecutionStatus.CANCELLED` | 0 | -| `ergon_core/ergon_core/core/runtime/events/task_events.py` | Add `TaskCancelledEvent` + `CancelCause` | 0 | -| `ergon_core/ergon_core/core/runtime/inngest_client.py` | Add `TASK_CANCEL` export | 1 | -| `ergon_core/ergon_core/core/runtime/services/graph_repository.py` | `update_node_status` returns bool + `only_if_not_terminal`; `add_node` grows `parent_node_id`/`level` | 1 | -| `ergon_core/ergon_core/core/runtime/errors/delegation_errors.py` | Add `CycleDetectedError`, `DuplicateLocalKeyError`, `UnknownLocalKeyError` | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_management_service.py` | Rewrite: `add_task`→`add_subtask`, `abandon_task`→`cancel_task`, add `plan_subtasks` | 2 | -| `ergon_core/ergon_core/core/runtime/services/task_management_dto.py` | Replace all DTOs with RFC §6.1 versions | 2 | -| `ergon_core/ergon_core/core/runtime/execution/propagation.py` | Relaxed `is_workflow_complete`; propagation returns `(ready, invalidated)` | 3 | -| `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` | Add `TASK_CANCEL` to cancel list | 3 | -| `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` | Emit `task/cancelled(cause="dep_invalidated")` for invalidated targets | 3 | -| `ergon_core/ergon_core/core/api/runs.py` | Rewrite `_build_task_map` for stored containment | 3 | -| `ergon_core/ergon_core/core/persistence/queries.py` | `list_children_of` uses `parent_node_id` instead of edge traversal | 3 | -| `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` | `list_child_resources`/`list_descendant_resources` use `parent_node_id` query | 4 | -| `ergon_builtins/ergon_builtins/workers/research_rubrics/manager_worker.py` | Switch to `SubtaskLifecycleToolkit`, update system prompt | 4 | -| `ergon_builtins/ergon_builtins/workers/baselines/manager_researcher_worker.py` | Use `build_subtask_lifecycle_tools` | 4 | -| `ergon_builtins/ergon_builtins/benchmarks/delegation_smoke/benchmark.py` | Drive new toolkit, cover cancel-cascade in smoke | 4 | -| `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py` | `PropagateTaskCompletionCommand` grows `terminal_status` + return grows `invalidated_targets` | 3 | -| `ergon_core/ergon_core/core/dashboard/emitter.py` | Handle `TaskCancelledEvent` — emit `node.cancelled` dashboard mutation | 5 | - -### Deleted files - -| Path | Reason | Phase | -|------|--------|-------| -| `ergon_builtins/ergon_builtins/tools/task_management_toolkit.py` | Replaced by `subtask_lifecycle_toolkit.py`. No backwards compat. | 4 | - ---- - -## Phase 0: Schema + Status Vocabulary - -### Task 0.1: Add containment columns to RunGraphNode model - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/graph/models.py` -- Test: `pnpm run check:be:lint` - -- [ ] **Step 1: Read the current RunGraphNode model** - -```bash -cat -n ergon_core/ergon_core/core/persistence/graph/models.py -``` - -- [ ] **Step 2: Add parent_node_id and level fields** - -Add after the `assigned_worker_key` field (before `created_at`): - -```python - # Containment: self-referential FK to the spawning node. - # NULL for definition-seeded roots; set for every dynamic subtask. - # Stored (not derived) so a single SELECT on run_graph_nodes gives - # a fully legible hierarchy without joins or edge traversal. - parent_node_id: UUID | None = Field( - default=None, - foreign_key="run_graph_nodes.id", - index=True, - ) - - # Depth in the containment tree. 0 for roots, parent.level + 1 - # for dynamic subtasks. Stored for debuggability and to avoid - # N+1 level computation at query/rendering time. - level: int = Field(default=0) -``` - -- [ ] **Step 3: Run lint** - -```bash -pnpm run check:be:lint -``` - -Expected: All checks passed - -- [ ] **Step 4: Run tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All tests pass (new fields have defaults, no schema enforcement in SQLite tests) - -- [ ] **Step 5: Commit** - -```bash -git add ergon_core/ergon_core/core/persistence/graph/models.py -git commit -m "feat(schema): add parent_node_id and level to RunGraphNode - -Containment columns for the subtask hierarchy. parent_node_id is a -self-referential FK (NULL for roots). level is the depth in the tree -(0 for roots). Both are stored, not derived, for debuggability and -to avoid N+1 queries on the FE rendering path." -``` - -### Task 0.2: Replace ABANDONED with CANCELLED in status vocabulary - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/graph/status_conventions.py` -- Modify: `ergon_core/ergon_core/core/persistence/shared/enums.py` -- Verify: all callers of `ABANDONED` constant - -- [ ] **Step 1: Update status_conventions.py** - -Replace the full file content: - -```python -"""Conventional status values for RunGraphNode and RunGraphEdge. - -The graph layer accepts any string at the DB level -- these are not -enforced by the schema. They are the values used by the core runtime, -propagation, and dynamic delegation logic. Experiment layers may add -domain-specific statuses without changing core code. - -The Literal type aliases below are for use in service signatures, DTOs, -and function annotations. They catch typos at type-check time without -constraining the DB column. -""" - -from typing import Literal - -# ── Node status ─────────────────────────────────────────────────── -PENDING = "pending" -READY = "ready" -RUNNING = "running" -COMPLETED = "completed" -FAILED = "failed" -CANCELLED = "cancelled" - -TERMINAL_STATUSES = frozenset({COMPLETED, FAILED, CANCELLED}) - -NodeStatus = Literal["pending", "ready", "running", "completed", "failed", "cancelled"] - -# ── Edge status ─────────────────────────────────────────────────── -# Edges are pure dependency relations (containment lives on the node). -# "active" is removed — delegation edges no longer exist. -EDGE_PENDING = "pending" -EDGE_SATISFIED = "satisfied" -EDGE_INVALIDATED = "invalidated" - -EdgeStatus = Literal["pending", "satisfied", "invalidated"] -``` - -- [ ] **Step 2: Add CANCELLED to TaskExecutionStatus enum** - -In `ergon_core/ergon_core/core/persistence/shared/enums.py`, add to the `TaskExecutionStatus` StrEnum: - -```python - CANCELLED = "cancelled" -``` - -- [ ] **Step 3: Find and fix all references to ABANDONED** - -```bash -rg -n "ABANDONED\b|\"abandoned\"|'abandoned'" ergon_core/ ergon_builtins/ tests/ -``` - -For each hit: -- Replace `ABANDONED` symbol with `CANCELLED` -- Replace `"abandoned"` string literal with `"cancelled"` -- Replace `EDGE_ACTIVE` with `EDGE_SATISFIED` where it's used for dependency edges, or remove if it was delegation-only -- Replace `EDGE_ABANDONED` with `EDGE_INVALIDATED` - -Key files to check: -- `ergon_core/ergon_core/core/runtime/execution/propagation.py` -- `ergon_core/ergon_core/core/runtime/services/task_management_service.py` -- `ergon_core/ergon_core/core/runtime/services/task_management_dto.py` (AbandonTaskCommand, AbandonTaskResult — leave renaming to Phase 2) -- `tests/state/*.py` - -**Important:** In this phase, only rename the status values. Do NOT rename methods/DTOs yet (that's Phase 2). For `abandon_task` and `AbandonTaskCommand`, change internal status writes from `ABANDONED` to `CANCELLED` but keep the method/class names. - -- [ ] **Step 4: Run tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All pass. If any test asserts `"abandoned"`, update assertion to `"cancelled"`. - -- [ ] **Step 5: Commit** - -```bash -git add -A -git commit -m "feat(status): replace ABANDONED with CANCELLED across codebase - -Single non-success terminal status. ABANDONED is gone from all code -and test fixtures. Edge vocabulary narrowed: EDGE_ACTIVE removed -(delegation edges eliminated), EDGE_INVALIDATED added for dep-failure -path. TaskExecutionStatus gains CANCELLED value." -``` - -### Task 0.3: Add TaskCancelledEvent - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/events/task_events.py` -- Test: lint + type check - -- [ ] **Step 1: Read current task_events.py** - -```bash -cat -n ergon_core/ergon_core/core/runtime/events/task_events.py -``` - -- [ ] **Step 2: Add CancelCause literal and TaskCancelledEvent class** - -Append to the file (after existing event classes): - -```python -# ── Cancel cause ────────────────────────────────────────────────── - -CancelCause = Literal[ - "manager_decision", - "parent_terminal", - "dep_invalidated", - "run_cancelled", -] - - -class TaskCancelledEvent(BaseModel): - """Emitted whenever a node transitions from non-terminal into CANCELLED. - - Consumers: - - cancel_orphan_subtasks_fn (recurse cascade to descendants) - - cleanup_cancelled_task_fn (release sandbox, mark execution row) - - execute_task_fn (via TASK_CANCEL matcher — drops queued / terminates running) - - dashboard_emitter - """ - - name: ClassVar[str] = "task/cancelled" - - run_id: UUID - definition_id: UUID - node_id: UUID - execution_id: UUID | None - cause: CancelCause - - model_config = {"frozen": True} -``` - -Add `Literal` to the typing import if not already present. Add `ClassVar` import if not already present. Check existing events for the import pattern and match it. - -- [ ] **Step 3: Run lint + tests** - -```bash -pnpm run check:be:lint && pnpm run test:be:fast -``` - -Expected: All pass - -- [ ] **Step 4: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/events/task_events.py -git commit -m "feat(events): add TaskCancelledEvent and CancelCause - -New event emitted whenever a node transitions to CANCELLED. Carries -cause (manager_decision, parent_terminal, dep_invalidated, run_cancelled) -for audit logging and trajectory analysis." -``` - -### Task 0.4: Create Alembic migration - -**Files:** -- Create: `ergon_core/migrations/versions/<auto>_add_containment_and_cancelled.py` - -- [ ] **Step 1: Generate migration skeleton** - -```bash -cd ergon_core && uv run alembic revision --autogenerate -m "add_containment_and_cancelled" -``` - -- [ ] **Step 2: Edit the migration** - -Replace the upgrade function body with the migration from RFC §5.1.2: - -```python -def upgrade() -> None: - # 1. Add parent_node_id column + FK + index. - op.add_column( - "run_graph_nodes", - sa.Column("parent_node_id", sa.Uuid(), nullable=True), - ) - op.create_foreign_key( - "fk_run_graph_nodes_parent", - "run_graph_nodes", "run_graph_nodes", - ["parent_node_id"], ["id"], - ondelete="SET NULL", - ) - op.create_index( - "ix_run_graph_nodes_parent_node_id", - "run_graph_nodes", ["parent_node_id"], - ) - - # 2. Add level column (default 0 = root). - op.add_column( - "run_graph_nodes", - sa.Column("level", sa.Integer(), server_default="0", nullable=False), - ) - - # 3. Backfill: old delegation edges (status='active') become parent_node_id - # on their target node. - op.execute(""" - UPDATE run_graph_nodes tgt - SET parent_node_id = e.source_node_id - FROM run_graph_edges e - WHERE e.target_node_id = tgt.id - AND e.status = 'active' - """) - - # 4. Recursive level backfill. - op.execute(""" - WITH RECURSIVE tree AS ( - SELECT id, 0 AS depth - FROM run_graph_nodes - WHERE parent_node_id IS NULL - UNION ALL - SELECT n.id, t.depth + 1 - FROM run_graph_nodes n - JOIN tree t ON n.parent_node_id = t.id - ) - UPDATE run_graph_nodes - SET level = tree.depth - FROM tree - WHERE run_graph_nodes.id = tree.id - """) - - # 5. Delete delegation edges — containment now lives on the node. - op.execute("DELETE FROM run_graph_edges WHERE status = 'active'") - - # 6. Status vocabulary: ABANDONED -> CANCELLED. - op.execute("UPDATE run_graph_nodes SET status = 'cancelled' WHERE status = 'abandoned'") - op.execute("UPDATE run_graph_edges SET status = 'invalidated' WHERE status = 'abandoned'") - - -def downgrade() -> None: - # Downgrade is lossy: delegation edges cannot be reconstructed. - op.execute("UPDATE run_graph_nodes SET status = 'abandoned' WHERE status = 'cancelled'") - op.execute("UPDATE run_graph_edges SET status = 'abandoned' WHERE status = 'invalidated'") - op.drop_index("ix_run_graph_nodes_parent_node_id") - op.drop_constraint("fk_run_graph_nodes_parent", "run_graph_nodes") - op.drop_column("run_graph_nodes", "level") - op.drop_column("run_graph_nodes", "parent_node_id") -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_core/migrations/ -git commit -m "feat(migration): add containment columns and migrate ABANDONED→CANCELLED - -Adds parent_node_id (self-FK, indexed) and level (int) to -run_graph_nodes. Backfills from delegation edges via recursive CTE. -Deletes delegation edges. Renames abandoned→cancelled in both -nodes and edges." -``` - ---- - -## Phase 1: Conditional Status Writes - -### Task 1.1: Add only_if_not_terminal guard to update_node_status - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/graph_repository.py` -- Test: `tests/state/test_conditional_status_writes.py` (new) - -- [ ] **Step 1: Write the failing test** - -Create `tests/state/test_conditional_status_writes.py`: - -```python -"""Tests for the only_if_not_terminal conditional guard on update_node_status. - -The conditional write is the single invariant that closes all race conditions -in the cascade cancellation system (RFC §4.4). Every concurrent path — -cancel vs complete, cascade vs cascade, manager cancel vs engine cascade — -resolves to "first writer wins" via this guard. -""" - -from uuid import uuid4 - -import pytest -from sqlmodel import Session - -from ergon_core.core.persistence.graph.status_conventions import ( - CANCELLED, - COMPLETED, - FAILED, - PENDING, - RUNNING, - TERMINAL_STATUSES, -) -from ergon_core.core.runtime.services.graph_dto import MutationMeta -from ergon_core.core.runtime.services.graph_repository import WorkflowGraphRepository - -META = MutationMeta(actor="test", reason="test") - - -class TestConditionalStatusWrites: - def test_guard_blocks_write_on_completed_node(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=COMPLETED, meta=META, - ) - result = repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=CANCELLED, - meta=META, only_if_not_terminal=True, - ) - assert result is False - - def test_guard_blocks_write_on_failed_node(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=FAILED, meta=META, - ) - result = repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=CANCELLED, - meta=META, only_if_not_terminal=True, - ) - assert result is False - - def test_guard_blocks_write_on_cancelled_node(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=CANCELLED, meta=META, - ) - result = repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=COMPLETED, - meta=META, only_if_not_terminal=True, - ) - assert result is False - - def test_guard_allows_write_on_running_node(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=RUNNING, meta=META, - ) - result = repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=CANCELLED, - meta=META, only_if_not_terminal=True, - ) - assert result is True - refreshed = repo.get_node(db_session, run_id=run_id, node_id=node.id) - assert refreshed.status == CANCELLED - - def test_unconditional_write_still_works(self, db_session: Session) -> None: - """Without the guard, writes proceed even on terminal nodes.""" - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=COMPLETED, meta=META, - ) - # Default only_if_not_terminal=False: unconditional - result = repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=FAILED, - meta=META, - ) - assert result is True - - def test_guard_does_not_emit_mutation_on_blocked_write(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - run_id = uuid4() - - node = repo.add_node( - db_session, run_id, - task_key="t1", instance_key="i0", description="test", - status=COMPLETED, meta=META, - ) - mutations_before = repo.get_mutations(db_session, run_id) - count_before = len(mutations_before.mutations) - - repo.update_node_status( - db_session, - run_id=run_id, node_id=node.id, new_status=CANCELLED, - meta=META, only_if_not_terminal=True, - ) - mutations_after = repo.get_mutations(db_session, run_id) - assert len(mutations_after.mutations) == count_before -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/state/test_conditional_status_writes.py -v -``` - -Expected: FAIL — `update_node_status` doesn't accept `only_if_not_terminal` yet, and returns `GraphNodeDto` not `bool`. - -- [ ] **Step 3: Implement conditional guard** - -In `ergon_core/ergon_core/core/runtime/services/graph_repository.py`, modify `update_node_status`: - -```python - def update_node_status( - self, - session: Session, - *, - run_id: UUID, - node_id: UUID, - new_status: str, - meta: MutationMeta, - only_if_not_terminal: bool = False, - ) -> bool: - """Transition a node's status. Returns True if the write applied. - - When ``only_if_not_terminal`` is True, the write is skipped if the - node is already in a terminal status (COMPLETED, FAILED, CANCELLED). - This is the single invariant that closes all race conditions in the - cascade cancellation system — concurrent paths that both attempt to - write a terminal status resolve to "first writer wins" without - requiring distributed locks. - """ - node = self._get_node_row(session, run_id, node_id) - - if only_if_not_terminal and node.status in TERMINAL_STATUSES: - return False - - old_status = node.status - node.status = new_status - node.updated_at = utcnow() - session.add(node) - session.flush() - - self._log_mutation( - session, - run_id, - mutation_type="node.status_changed", - target_type="node", - target_id=node_id, - meta=meta, - old_value=NodeStatusChangedMutation(status=old_status), - new_value=NodeStatusChangedMutation(status=new_status), - ) - return True -``` - -Add `TERMINAL_STATUSES` to the imports from `status_conventions`. - -**Important:** This changes the return type from `GraphNodeDto` to `bool`. All existing callers must be updated. - -- [ ] **Step 4: Update all callers of update_node_status** - -Search for all callers: - -```bash -rg "update_node_status\(" ergon_core/ tests/ --files-with-matches -``` - -For each caller that was using the returned `GraphNodeDto`, update to handle `bool`. Most callers ignore the return value, so the change is usually just removing the assignment. Key files: -- `propagation.py` — `_update_task_status` and `mark_task_*` helpers -- `task_management_service.py` — `abandon_task` (still named that in this phase) -- `task_execution_service.py` -- Tests - -- [ ] **Step 5: Run tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All pass including new `test_conditional_status_writes.py` - -- [ ] **Step 6: Commit** - -```bash -git add -A -git commit -m "feat(graph_repo): add only_if_not_terminal guard to update_node_status - -Returns bool (True = applied, False = blocked). This is the single -invariant closing all race conditions in cascade cancellation — -concurrent terminal writes resolve to first-writer-wins without -distributed locks." -``` - -### Task 1.2: Extend add_node with parent_node_id and level - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/graph_repository.py` - -- [ ] **Step 1: Read current add_node signature** - -The current signature is: -```python -def add_node(self, session, run_id, *, task_key, instance_key, description, status, assigned_worker_key=None, meta) -``` - -- [ ] **Step 2: Add parent_node_id and level kwargs** - -```python - def add_node( - self, - session: Session, - run_id: UUID, - *, - task_key: str, - instance_key: str, - description: str, - status: str, - assigned_worker_key: str | None = None, - parent_node_id: UUID | None = None, - level: int = 0, - meta: MutationMeta, - ) -> GraphNodeDto: - """Create a graph node. Writes the containment columns directly. - - parent_node_id and level are set at creation time and never change. - The caller (TaskManagementService) computes level = parent.level + 1. - """ -``` - -In the body where `RunGraphNode(...)` is constructed, add: -```python - parent_node_id=parent_node_id, - level=level, -``` - -- [ ] **Step 3: Run tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All pass (new params have defaults matching old behavior) - -- [ ] **Step 4: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/services/graph_repository.py -git commit -m "feat(graph_repo): add parent_node_id and level to add_node - -Containment columns are set at creation time and never change. -Defaults (None, 0) match pre-existing root-node behavior." -``` - -### Task 1.3: Add TASK_CANCEL to inngest_client - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/inngest_client.py` - -- [ ] **Step 1: Add TASK_CANCEL export** - -```python -# Per-node cancel matcher. Fires on task/cancelled for this exact node_id. -# Used by execute_task_fn to drop queued or terminate in-flight invocations -# when a parent terminates or the manager explicitly cancels. -TASK_CANCEL = [ - inngest.Cancel( - event="task/cancelled", - if_exp="event.data.node_id == async.data.node_id", - ), -] -``` - -- [ ] **Step 2: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/inngest_client.py -git commit -m "feat(inngest): add TASK_CANCEL per-node cancel matcher - -Matches task/cancelled events by node_id. Used by execute_task_fn -to drop queued invocations or terminate in-flight workers when -their parent terminates." -``` - ---- - -## Phase 2: Services - -### Task 2.1: New error classes for plan validation - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/errors/delegation_errors.py` - -- [ ] **Step 1: Add new error classes** - -```python -class CycleDetectedError(DelegationError): - """Raised when plan_subtasks dependency graph contains a cycle.""" - - def __init__(self, remaining_keys: list[str]) -> None: - super().__init__(f"Cycle detected among keys: {remaining_keys}") - self.remaining_keys = remaining_keys - - -class DuplicateLocalKeyError(DelegationError): - """Raised when plan_subtasks has duplicate local_key values.""" - - def __init__(self, key: str) -> None: - super().__init__(f"Duplicate local_key: {key!r}") - self.key = key - - -class UnknownLocalKeyError(DelegationError): - """Raised when depends_on references a local_key not in the plan.""" - - def __init__(self, unknown: list[str]) -> None: - super().__init__(f"Unknown depends_on keys: {unknown}") - self.unknown = unknown -``` - -- [ ] **Step 2: Run lint + tests** - -```bash -pnpm run check:be:lint && pnpm run test:be:fast -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/errors/delegation_errors.py -git commit -m "feat(errors): add CycleDetectedError, DuplicateLocalKeyError, UnknownLocalKeyError - -Validation errors for plan_subtasks — cycle detection via Kahn's -algorithm, duplicate local_key guard, and unknown depends_on -reference guard." -``` - -### Task 2.2: New DTOs for subtask lifecycle - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/task_management_dto.py` -- Create: `ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py` -- Create: `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_dto.py` -- Create: `ergon_core/ergon_core/core/runtime/services/task_cleanup_dto.py` - -- [ ] **Step 1: Replace task_management_dto.py** - -Replace the entire file with the RFC §6.1 DTOs. Copy verbatim from the RFC (the `AddSubtaskCommand`, `AddSubtaskResult`, `PlanSubtasksCommand`, `PlanSubtasksResult`, `SubtaskSpec`, `CancelTaskCommand`, `CancelTaskResult`, `RefineTaskCommand`, `RefineTaskResult`). - -- [ ] **Step 2: Create task_inspection_dto.py** - -Copy RFC §6.2 verbatim — `SubtaskInfo` and `SubtaskStatus`. - -- [ ] **Step 3: Create subtask_cancellation_dto.py** - -Copy RFC §6.3 first half — `CancelOrphansResult`. - -- [ ] **Step 4: Create task_cleanup_dto.py** - -Copy RFC §6.3 second half — `CleanupResult`. - -- [ ] **Step 5: Run lint** - -```bash -pnpm run check:be:lint -``` - -Fix any import issues (old DTOs were imported elsewhere — tests will break, that's expected at this stage). - -- [ ] **Step 6: Commit** - -```bash -git add -A -git commit -m "feat(dto): replace task management DTOs with subtask lifecycle DTOs - -AddTaskCommand→AddSubtaskCommand, AbandonTaskCommand→CancelTaskCommand. -New: PlanSubtasksCommand/Result, SubtaskSpec, SubtaskInfo, CancelOrphansResult, -CleanupResult. All frozen Pydantic v2 models with typed UUID aliases." -``` - -### Task 2.3: Rewrite TaskManagementService - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/task_management_service.py` -- Test: `tests/state/test_task_management_service.py` (update) -- Test: `tests/state/test_plan_subtasks.py` (new) - -- [ ] **Step 1: Write plan_subtasks validation tests** - -Create `tests/state/test_plan_subtasks.py`: - -```python -"""Tests for plan_subtasks — atomic sub-DAG creation with validation. - -plan_subtasks is the primary way managers express structured delegation. -It creates all nodes and edges in a single transaction, rejects cycles -via Kahn's algorithm, and only dispatches roots (nodes with no deps). -""" - -from uuid import uuid4 - -import pytest -from sqlmodel import Session - -from ergon_core.core.persistence.graph.status_conventions import PENDING -from ergon_core.core.runtime.errors.delegation_errors import ( - CycleDetectedError, - DuplicateLocalKeyError, - UnknownLocalKeyError, -) -from ergon_core.core.runtime.services.graph_dto import MutationMeta -from ergon_core.core.runtime.services.graph_repository import WorkflowGraphRepository -from ergon_core.core.runtime.services.task_management_dto import ( - PlanSubtasksCommand, - SubtaskSpec, -) -from ergon_core.core.runtime.services.task_management_service import TaskManagementService - -META = MutationMeta(actor="test", reason="test") - - -def _make_parent(repo: WorkflowGraphRepository, session: Session, run_id): - """Create a root node to serve as parent for plan_subtasks.""" - return repo.add_node( - session, run_id, - task_key="root", instance_key="i0", description="root", - status="running", meta=META, - ) - - -class TestPlanSubtasksValidation: - def test_rejects_duplicate_local_keys(self, db_session: Session) -> None: - svc = TaskManagementService() - repo = WorkflowGraphRepository() - run_id = uuid4() - parent = _make_parent(repo, db_session, run_id) - - with pytest.raises(DuplicateLocalKeyError): - svc.plan_subtasks( - db_session, - PlanSubtasksCommand( - run_id=run_id, - parent_node_id=parent.id, - subtasks=[ - SubtaskSpec(local_key="a", description="A"), - SubtaskSpec(local_key="a", description="A2"), - ], - ), - ) - - def test_rejects_unknown_depends_on(self, db_session: Session) -> None: - svc = TaskManagementService() - repo = WorkflowGraphRepository() - run_id = uuid4() - parent = _make_parent(repo, db_session, run_id) - - with pytest.raises(UnknownLocalKeyError): - svc.plan_subtasks( - db_session, - PlanSubtasksCommand( - run_id=run_id, - parent_node_id=parent.id, - subtasks=[ - SubtaskSpec(local_key="a", description="A", depends_on=["nonexistent"]), - ], - ), - ) - - def test_rejects_cycle(self, db_session: Session) -> None: - svc = TaskManagementService() - repo = WorkflowGraphRepository() - run_id = uuid4() - parent = _make_parent(repo, db_session, run_id) - - with pytest.raises(CycleDetectedError): - svc.plan_subtasks( - db_session, - PlanSubtasksCommand( - run_id=run_id, - parent_node_id=parent.id, - subtasks=[ - SubtaskSpec(local_key="a", description="A", depends_on=["b"]), - SubtaskSpec(local_key="b", description="B", depends_on=["a"]), - ], - ), - ) - - def test_creates_nodes_and_edges_atomically(self, db_session: Session) -> None: - svc = TaskManagementService() - repo = WorkflowGraphRepository() - run_id = uuid4() - parent = _make_parent(repo, db_session, run_id) - - result = svc.plan_subtasks( - db_session, - PlanSubtasksCommand( - run_id=run_id, - parent_node_id=parent.id, - subtasks=[ - SubtaskSpec(local_key="t1", description="Task 1"), - SubtaskSpec(local_key="t2", description="Task 2", depends_on=["t1"]), - ], - ), - ) - - assert "t1" in result.nodes - assert "t2" in result.nodes - assert result.roots == ["t1"] - - # Verify nodes exist and have correct containment - t1 = repo.get_node(db_session, run_id=run_id, node_id=result.nodes["t1"]) - assert t1.status == PENDING - # parent_node_id check requires reading the raw model — use get_graph - graph = repo.get_graph(db_session, run_id) - t1_raw = next(n for n in graph.nodes if n.id == result.nodes["t1"]) - t2_raw = next(n for n in graph.nodes if n.id == result.nodes["t2"]) - - # Verify edge exists from t1 → t2 - edges = repo.get_outgoing_edges(db_session, run_id=run_id, node_id=result.nodes["t1"]) - assert len(edges) == 1 - assert edges[0].target_node_id == result.nodes["t2"] -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/state/test_plan_subtasks.py -v -``` - -Expected: FAIL — `plan_subtasks` doesn't exist yet, DTOs don't match. - -- [ ] **Step 3: Rewrite task_management_service.py** - -Replace the service with the RFC §7.1 implementation. Key changes: -- `add_task` → `add_subtask` (with `depends_on` parameter, writes `parent_node_id` + `level`) -- `abandon_task` → `cancel_task` (emits `TaskCancelledEvent`, uses `only_if_not_terminal`) -- Add `plan_subtasks` (Kahn's algorithm for cycle detection, atomic node+edge creation) -- Add `_resolve_definition_id` (replaces external `_lookup_definition_id`) -- Add `_dispatch_task_ready` helper -- Add `_validate_plan` static method -- Add `_count_non_terminal_descendants` module-level helper -- Add `_latest_execution_id` module-level helper - -Copy the implementation from RFC §7.1 verbatim (the docstrings are already reviewed). - -**Note:** `_count_non_terminal_descendants` uses a recursive CTE over `parent_node_id`. `_latest_execution_id` queries the task_executions table. Both need to work with the current DB schema. Check how the existing code queries executions and adapt. - -- [ ] **Step 4: Update test_task_management_service.py** - -Update all imports and assertions: -- `AddTaskCommand` → `AddSubtaskCommand` -- `AbandonTaskCommand` → `CancelTaskCommand` -- `AddTaskResult` → `AddSubtaskResult` -- `AbandonTaskResult` → `CancelTaskResult` -- `add_task` → `add_subtask` -- `abandon_task` → `cancel_task` -- `"abandoned"` → `"cancelled"` in all assertions -- Update result field accesses (e.g. `edge_id` no longer on result) - -- [ ] **Step 5: Run all tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All pass - -- [ ] **Step 6: Commit** - -```bash -git add -A -git commit -m "feat(service): rewrite TaskManagementService with subtask lifecycle - -add_task→add_subtask (with depends_on, parent_node_id+level), -abandon_task→cancel_task (emits TaskCancelledEvent, conditional write), -new plan_subtasks (atomic sub-DAG creation with Kahn's cycle detection). -definition_id resolved from run_id at dispatch time — not on command DTOs." -``` - -### Task 2.4: SubtaskCancellationService - -**Files:** -- Create: `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py` -- Test: `tests/state/test_subtask_cancellation_service.py` (new) - -- [ ] **Step 1: Write the failing tests** - -Create `tests/state/test_subtask_cancellation_service.py`: - -```python -"""Tests for SubtaskCancellationService — single-level cascade cancel. - -This service marks non-terminal children of a parent as CANCELLED and -returns events for the caller to emit. It does NOT recurse — cascade -to grandchildren is driven by Inngest re-delivering task/cancelled. -Separated from TaskCleanupService (different concerns: state vs resources) -and from TaskManagementService (different callers: engine vs agent). -""" - -from uuid import uuid4 - -import pytest -from sqlmodel import Session - -from ergon_core.core.persistence.graph.status_conventions import ( - CANCELLED, - COMPLETED, - PENDING, - RUNNING, -) -from ergon_core.core.runtime.services.graph_dto import MutationMeta -from ergon_core.core.runtime.services.graph_repository import WorkflowGraphRepository -from ergon_core.core.runtime.services.subtask_cancellation_service import ( - SubtaskCancellationService, -) - -META = MutationMeta(actor="test", reason="test") - - -class TestCancelOrphans: - def test_cancels_non_terminal_children(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - svc = SubtaskCancellationService(graph_repo=repo) - run_id = uuid4() - - parent = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="parent", status=COMPLETED, meta=META, - ) - child_running = repo.add_node( - db_session, run_id, task_key="c1", instance_key="i0", - description="running child", status=RUNNING, - parent_node_id=parent.id, level=1, meta=META, - ) - child_pending = repo.add_node( - db_session, run_id, task_key="c2", instance_key="i0", - description="pending child", status=PENDING, - parent_node_id=parent.id, level=1, meta=META, - ) - child_completed = repo.add_node( - db_session, run_id, task_key="c3", instance_key="i0", - description="completed child", status=COMPLETED, - parent_node_id=parent.id, level=1, meta=META, - ) - - result = svc.cancel_orphans( - db_session, - run_id=run_id, - definition_id=uuid4(), - parent_node_id=parent.id, - cause="parent_terminal", - ) - - assert len(result.cancelled_node_ids) == 2 - assert child_running.id in result.cancelled_node_ids - assert child_pending.id in result.cancelled_node_ids - assert child_completed.id not in result.cancelled_node_ids - assert len(result.events_to_emit) == 2 - - def test_empty_children_is_noop(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - svc = SubtaskCancellationService(graph_repo=repo) - run_id = uuid4() - - parent = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="leaf", status=COMPLETED, meta=META, - ) - - result = svc.cancel_orphans( - db_session, - run_id=run_id, - definition_id=uuid4(), - parent_node_id=parent.id, - cause="parent_terminal", - ) - - assert result.cancelled_node_ids == [] - assert result.events_to_emit == [] - - def test_only_cancels_direct_children_not_grandchildren(self, db_session: Session) -> None: - """Grandchildren are cancelled by Inngest re-delivering task/cancelled.""" - repo = WorkflowGraphRepository() - svc = SubtaskCancellationService(graph_repo=repo) - run_id = uuid4() - - root = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="root", status=COMPLETED, meta=META, - ) - child = repo.add_node( - db_session, run_id, task_key="c1", instance_key="i0", - description="child", status=RUNNING, - parent_node_id=root.id, level=1, meta=META, - ) - grandchild = repo.add_node( - db_session, run_id, task_key="gc1", instance_key="i0", - description="grandchild", status=RUNNING, - parent_node_id=child.id, level=2, meta=META, - ) - - result = svc.cancel_orphans( - db_session, - run_id=run_id, - definition_id=uuid4(), - parent_node_id=root.id, - cause="parent_terminal", - ) - - # Only direct child cancelled, not grandchild - assert result.cancelled_node_ids == [child.id] - # Grandchild still running (will be handled by next cascade) - gc = repo.get_node(db_session, run_id=run_id, node_id=grandchild.id) - assert gc.status == RUNNING -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/state/test_subtask_cancellation_service.py -v -``` - -- [ ] **Step 3: Implement SubtaskCancellationService** - -Create `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py`. Copy implementation from RFC §7.2 verbatim (includes the approved docstrings). - -**Adaptation needed:** The RFC uses `only_if_not_terminal=True` on `update_node_status`. Make sure the call uses keyword args matching our updated signature from Task 1.1. - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/state/test_subtask_cancellation_service.py -v -``` - -Expected: All pass - -- [ ] **Step 5: Commit** - -```bash -git add -A -git commit -m "feat(service): add SubtaskCancellationService - -Single-level cascade: marks non-terminal children of a parent as -CANCELLED, returns events for caller to emit. Does NOT recurse — -cascade to grandchildren is driven by Inngest re-delivering -task/cancelled events." -``` - -### Task 2.5: TaskInspectionService - -**Files:** -- Create: `ergon_core/ergon_core/core/runtime/services/task_inspection_service.py` -- Test: `tests/state/test_task_inspection_service.py` (new) - -- [ ] **Step 1: Write the failing tests** - -Create `tests/state/test_task_inspection_service.py`: - -```python -"""Tests for TaskInspectionService — read-only subtask queries. - -Separated from TaskManagementService because inspection has no side -effects. The toolkit injects this independently of the write services. -""" - -from uuid import uuid4 - -from sqlmodel import Session - -from ergon_core.core.persistence.graph.status_conventions import ( - COMPLETED, - FAILED, - PENDING, - RUNNING, -) -from ergon_core.core.runtime.services.graph_dto import MutationMeta -from ergon_core.core.runtime.services.graph_repository import WorkflowGraphRepository -from ergon_core.core.runtime.services.task_inspection_service import TaskInspectionService - -META = MutationMeta(actor="test", reason="test") - - -class TestListSubtasks: - def test_returns_direct_children_only(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - svc = TaskInspectionService() - run_id = uuid4() - - parent = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="parent", status=RUNNING, meta=META, - ) - child = repo.add_node( - db_session, run_id, task_key="c1", instance_key="i0", - description="child", status=PENDING, - parent_node_id=parent.id, level=1, meta=META, - ) - grandchild = repo.add_node( - db_session, run_id, task_key="gc1", instance_key="i0", - description="grandchild", status=PENDING, - parent_node_id=child.id, level=2, meta=META, - ) - - results = svc.list_subtasks(db_session, run_id=run_id, parent_node_id=parent.id) - - assert len(results) == 1 - assert results[0].node_id == child.id - - def test_returns_deterministic_order(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - svc = TaskInspectionService() - run_id = uuid4() - - parent = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="parent", status=RUNNING, meta=META, - ) - for key in ["c3", "c1", "c2"]: - repo.add_node( - db_session, run_id, task_key=key, instance_key="i0", - description=key, status=PENDING, - parent_node_id=parent.id, level=1, meta=META, - ) - - results = svc.list_subtasks(db_session, run_id=run_id, parent_node_id=parent.id) - - task_keys = [r.task_key for r in results] - assert task_keys == sorted(task_keys) - - def test_empty_children(self, db_session: Session) -> None: - repo = WorkflowGraphRepository() - svc = TaskInspectionService() - run_id = uuid4() - - parent = repo.add_node( - db_session, run_id, task_key="root", instance_key="i0", - description="leaf", status=RUNNING, meta=META, - ) - - results = svc.list_subtasks(db_session, run_id=run_id, parent_node_id=parent.id) - assert results == [] -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/state/test_task_inspection_service.py -v -``` - -- [ ] **Step 3: Implement TaskInspectionService** - -Create `ergon_core/ergon_core/core/runtime/services/task_inspection_service.py`. - -Copy from RFC §7.4, **with these adaptations** since the codebase doesn't have `TaskExecutionRepository.get_latest_for_node` or `ContextEventRepository.mark_stream_closed`: - -1. For `_latest_output` and `_latest_error`: query the task_executions table directly via SQLModel. Check how existing code (e.g. `task_execution_service.py`) queries execution records and follow that pattern. -2. For the `output` field: if the execution repo pattern doesn't support fetching output text, return `None` for now and add a `# TODO: wire output extraction` comment. The core listing/ordering functionality is what matters for this phase. - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/state/test_task_inspection_service.py -v -``` - -Expected: All pass - -- [ ] **Step 5: Commit** - -```bash -git add -A -git commit -m "feat(service): add TaskInspectionService for read-only subtask queries - -list_subtasks returns direct children ordered by task_key. -get_subtask returns a single SubtaskInfo with dep edges and status. -Separated from TaskManagementService — no side effects, no write deps." -``` - -### Task 2.6: TaskCleanupService (stub) - -**Files:** -- Create: `ergon_core/ergon_core/core/runtime/services/task_cleanup_service.py` - -- [ ] **Step 1: Create TaskCleanupService** - -The RFC's TaskCleanupService references `release_sandbox_for_execution` (doesn't exist) and `ContextEventRepository.mark_stream_closed` (doesn't exist). Create a working service with the parts that CAN work, and stub the rest: - -```python -"""TaskCleanupService — releases infrastructure for a CANCELLED task execution. - -Responsibilities: mark the execution row as CANCELLED, close the -context event stream so trajectory serializers stop tailing it, -and release the E2B sandbox (the main cost driver). - -Separated from SubtaskCancellationService because that service -operates on graph nodes (state transitions, fan-out) while this -one operates on execution resources (sandbox, telemetry rows, -context streams). They also have different failure characteristics: -a failed sandbox teardown should be retried for *this* node -without re-cancelling siblings. - -Idempotent: every mutating call checks current state before -writing, so Inngest retries=3 is safe. -""" - -import logging -from uuid import UUID - -from sqlmodel import Session, select - -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.shared.enums import TaskExecutionStatus -from ergon_core.core.persistence.telemetry.models import TaskExecution -from ergon_core.core.runtime.services.task_cleanup_dto import CleanupResult - -logger = logging.getLogger(__name__) - - -class TaskCleanupService: - """Releases infrastructure for a single CANCELLED task execution. - - See module docstring for design rationale. - """ - - async def cleanup( - self, - *, - run_id: UUID, - node_id: UUID, - execution_id: UUID | None, - ) -> CleanupResult: - if execution_id is None: - return CleanupResult( - run_id=run_id, node_id=node_id, execution_id=None, - sandbox_released=False, execution_row_updated=False, - ) - - with get_session() as session: - execution_updated = self._mark_execution_cancelled(session, execution_id) - session.commit() - - # TODO: sandbox teardown — ergon_core/core/runtime/sandbox/teardown.py - # does not exist yet. Wire release_sandbox_for_execution when sandbox - # management is implemented. - sandbox_released = False - - logger.info( - "task-cleanup node_id=%s execution_id=%s sandbox=%s", - node_id, execution_id, sandbox_released, - ) - return CleanupResult( - run_id=run_id, - node_id=node_id, - execution_id=execution_id, - sandbox_released=sandbox_released, - execution_row_updated=execution_updated, - ) - - def _mark_execution_cancelled(self, session: Session, execution_id: UUID) -> bool: - exe = session.exec( - select(TaskExecution).where(TaskExecution.id == execution_id) - ).first() - if exe is None or exe.status in { - TaskExecutionStatus.COMPLETED, - TaskExecutionStatus.FAILED, - TaskExecutionStatus.CANCELLED, - }: - return False - exe.status = TaskExecutionStatus.CANCELLED - session.add(exe) - return True -``` - -Check the actual `TaskExecution` model location and import path. The model might be at a different path — search for `class TaskExecution` in the codebase. - -- [ ] **Step 2: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 3: Commit** - -```bash -git add -A -git commit -m "feat(service): add TaskCleanupService for cancelled task resource cleanup - -Marks execution row as CANCELLED. Sandbox teardown stubbed — will -be wired when sandbox management module exists. Idempotent under -retry." -``` - ---- - -## Phase 3: Inngest Functions + Propagation - -### Task 3.1: Wire TASK_CANCEL into execute_task_fn - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` - -- [ ] **Step 1: Read the current decorator** - -```bash -head -50 ergon_core/ergon_core/core/runtime/inngest/execute_task.py -``` - -- [ ] **Step 2: Add TASK_CANCEL to the cancel list** - -Import `TASK_CANCEL` from `inngest_client`: - -```python -from ergon_core.core.runtime.inngest_client import RUN_CANCEL, TASK_CANCEL, inngest_client -``` - -Update the decorator: - -```python -@inngest_client.create_function( - fn_id="task-execute", - trigger=inngest.TriggerEvent(event="task/ready"), - cancel=[*RUN_CANCEL, *TASK_CANCEL], # ← added TASK_CANCEL - retries=0, - ... -) -``` - -- [ ] **Step 3: Run lint + tests** - -```bash -pnpm run check:be:lint && pnpm run test:be:fast -``` - -- [ ] **Step 4: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/inngest/execute_task.py -git commit -m "feat(inngest): wire TASK_CANCEL matcher into execute_task_fn - -task/cancelled events matching this node_id now drop queued or -terminate in-flight invocations. Zero-compute path for the common -race where parent finishes immediately after spawning." -``` - -### Task 3.2: Create cancel_orphan_subtasks Inngest functions - -**Files:** -- Create: `ergon_core/ergon_core/core/runtime/inngest/cancel_orphan_subtasks.py` - -- [ ] **Step 1: Create the file** - -Copy the implementation from RFC §9.1 verbatim (includes the two-step durable execution with `step.run` and approved docstrings). - -- [ ] **Step 2: Register in inngest __init__** - -Check how other Inngest functions are registered. If there's an `__init__.py` or a registration list, add the three new functions. - -```bash -cat ergon_core/ergon_core/core/runtime/inngest/__init__.py -``` - -Add imports for the three new functions. - -- [ ] **Step 3: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 4: Commit** - -```bash -git add -A -git commit -m "feat(inngest): add cancel_orphan_subtasks functions - -Three Inngest functions — one per trigger event (task/completed, -task/failed, task/cancelled). Each scans children of the triggering -node and cancels non-terminal ones. Uses step.run for durable -two-phase execution: scan-and-cancel then emit-events." -``` - -### Task 3.3: Create cleanup_cancelled_task Inngest function - -**Files:** -- Create: `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` - -- [ ] **Step 1: Create the file** - -Copy from RFC §9.2 (the step.run version with `_update_db_rows` and `_release_sandbox` steps). - -**Adaptation:** Since `release_sandbox_for_execution` doesn't exist, stub the sandbox step: - -```python - async def _release_sandbox() -> bool: - # TODO: wire when sandbox management module exists - return False - - sandbox_released = await ctx.step.run("release-sandbox", _release_sandbox) -``` - -- [ ] **Step 2: Register in inngest __init__** - -Add the import. - -- [ ] **Step 3: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 4: Commit** - -```bash -git add -A -git commit -m "feat(inngest): add cleanup_cancelled_task_fn - -Two durable steps: update-db-rows (mark execution CANCELLED) and -release-sandbox (stubbed — pending sandbox management module). -Each step independently retryable via Inngest retries=3." -``` - -### Task 3.4: Update propagation for dep-failure cascade and relaxed workflow terminal - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/execution/propagation.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py` — `PropagateTaskCompletionCommand` grows `terminal_status`; return type grows `invalidated_targets` - -- [ ] **Step 1: Read current propagation.py structure** - -```bash -rg "^def |^class |^async def " ergon_core/ergon_core/core/runtime/execution/propagation.py -``` - -Understand which flavour is active (definition-based vs graph-native). The graph-native path (`on_task_completed_by_node`, `mark_task_*_by_node`) is what we modify. - -- [ ] **Step 2: Update is_workflow_complete** - -The function should check that all nodes are terminal AND none are FAILED: - -```python -def is_workflow_complete(session: Session, run_id: UUID) -> bool: - """Every node terminal; zero FAILED. CANCELLED nodes are neutral — - a run with {COMPLETED, CANCELLED} is considered successful.""" - from ergon_core.core.persistence.graph.status_conventions import ( - FAILED, TERMINAL_STATUSES, - ) - statuses = list( - session.exec(select(RunGraphNode.status).where(RunGraphNode.run_id == run_id)).all() - ) - if not statuses: - return True - return ( - all(s in TERMINAL_STATUSES for s in statuses) - and not any(s == FAILED for s in statuses) - ) -``` - -- [ ] **Step 3: Update propagation to return invalidated targets** - -The graph-native `on_task_completed_by_node` (or equivalent) should return `(newly_ready, invalidated_targets)` when a dep source fails. For COMPLETED sources, outgoing edges become SATISFIED. For FAILED/CANCELLED sources, outgoing edges become INVALIDATED and targets are reported. - -Follow the RFC §9.4 pattern. - -- [ ] **Step 4: Update propagate_execution.py to emit task/cancelled for invalidated targets** - -In the Inngest wrapper, after calling the propagation service, emit `task/cancelled(cause="dep_invalidated")` events for each invalidated target. - -- [ ] **Step 5: Ensure all mark_task_* helpers use only_if_not_terminal for terminal writes** - -Any call to `update_node_status` that writes a terminal status should pass `only_if_not_terminal=True`. - -- [ ] **Step 6: Run tests** - -```bash -pnpm run test:be:fast -``` - -Fix any test failures from the return type change or new propagation behavior. - -- [ ] **Step 7: Commit** - -```bash -git add -A -git commit -m "feat(propagation): dep-failure cascade and relaxed workflow terminal - -FAILED/CANCELLED source invalidates outgoing edges. Invalidated -targets get task/cancelled(cause=dep_invalidated). Workflow terminal -detection tolerates CANCELLED nodes — only FAILED blocks success. -All terminal status writes use only_if_not_terminal=True." -``` - -### Task 3.5: Rewrite _build_task_map for stored containment - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/runs.py` - -- [ ] **Step 1: Read current _build_task_map** - -```bash -rg -n "_build_task_map" ergon_core/ergon_core/core/api/runs.py -``` - -- [ ] **Step 2: Replace with three-pass implementation** - -Copy the RFC §9.5 implementation: -- Pass 1: build DTOs reading `parent_node_id` and `level` from node columns -- Pass 2: derive `child_ids` and `is_leaf` via reverse lookup -- Pass 3: dependency edges → `depends_on_ids` - -- [ ] **Step 3: Update list_children_of in queries.py** - -Read the `list_children_of` function: -```bash -rg -n "list_children_of" ergon_core/ergon_core/core/persistence/queries.py -A 20 -``` - -Replace the edge-traversal implementation with a `parent_node_id` query: -```python -def list_children_of(self, session, run_id, parent_node_id): - """Direct children via containment column — no edge traversal.""" - # Resolves TODO(graph-edges) at the old implementation - ... -``` - -- [ ] **Step 4: Update run summary endpoint for cancelled counts (AC8)** - -Search for where the run summary/detail response is built: -```bash -rg "completed.*failed\|status.*count\|RunDetailResponse\|RunSummary" ergon_core/ergon_core/core/api/runs.py -n -``` - -Ensure the response includes `cancelled` count alongside `completed` and `failed`. The `_build_task_map` return tuple already counts by status — add `cancelled` to the counts. - -- [ ] **Step 5: Run tests** - -```bash -pnpm run test:be:fast -``` - -- [ ] **Step 6: Commit** - -```bash -git add -A -git commit -m "feat(api): rewrite _build_task_map for stored containment - -Three clean passes: node columns (parent_id, level), reverse lookup -(child_ids, is_leaf), dependency edges (depends_on_ids). Removes the -'first edge = parent' heuristic. Also rewrites list_children_of to -use parent_node_id query (resolves TODO(graph-edges)). Run summary -now reports cancelled count separately (AC8)." -``` - ---- - -## Phase 4: Tools + Worker Integration - -### Task 4.1: Create bash_sandbox_tool - -**Files:** -- Create: `ergon_builtins/ergon_builtins/tools/bash_sandbox_tool.py` - -- [ ] **Step 1: Create the file** - -```python -"""Sandboxed bash tool for manager agents. - -Provides a bash callable that runs commands inside the manager's E2B -sandbox. Primary use case: `sleep N` between subtask-status polls. -Also supports light inspection (cat, echo, grep). - -This is a separate module (not inline in the toolkit) because it has -no dependency on the subtask lifecycle services — it only needs the -sandbox_id. Other toolkits can reuse it independently. -""" - -from collections.abc import Callable -from typing import Any - -from ergon_core.core.runtime.sandbox.exec import sandbox_exec - - -def make_sandbox_bash_tool(*, sandbox_id: str) -> Callable[..., Any]: - """Produce a single bash callable bound to the given sandbox.""" - - async def bash(command: str, timeout_s: int = 30) -> dict[str, object]: - """Run a shell command inside the manager's sandbox. Useful for: - - `sleep 10` between subtask-status polls; - - `cat` / `echo` for light inspection; - - simple pipes (grep / awk). - No host-filesystem access; network policy is inherited from the sandbox.""" - try: - result = await sandbox_exec( - sandbox_id=sandbox_id, command=command, timeout_s=timeout_s, - ) - return { - "success": True, - "stdout": result.stdout, - "stderr": result.stderr, - "exit_code": result.exit_code, - } - except Exception as exc: # noqa: BLE001 - return {"success": False, "error": str(exc)} - - return bash -``` - -**Note:** Check if `sandbox_exec` exists. If not, search for the actual sandbox execution helper: -```bash -rg "sandbox_exec\|run_command\|execute_command" ergon_core/ --files-with-matches -``` - -Adapt the import to match the actual helper. If no sandbox exec helper exists at all, stub it: - -```python -async def _stub_sandbox_exec(*, sandbox_id: str, command: str, timeout_s: int): - """Stub until sandbox management module exists.""" - raise NotImplementedError("Sandbox exec not yet wired") -``` - -- [ ] **Step 2: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_builtins/ergon_builtins/tools/bash_sandbox_tool.py -git commit -m "feat(tools): add bash_sandbox_tool for manager agents - -Sandboxed bash callable for sleep-between-polls and light inspection. -Separate module with no subtask lifecycle dependencies — reusable -by other toolkits." -``` - -### Task 4.2: Create SubtaskLifecycleToolkit - -**Files:** -- Create: `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` - -- [ ] **Step 1: Create the file** - -Copy from RFC §8 verbatim (includes the approved docstrings). The toolkit produces seven tools: -1. `add_subtask` -2. `plan_subtasks` -3. `cancel_task` -4. `refine_task` -5. `list_subtasks` -6. `get_subtask` -7. `bash` (via `make_sandbox_bash_tool`) - -Plus the factory function: -```python -def build_subtask_lifecycle_tools( - *, - run_id: RunId, - parent_node_id: NodeId, - sandbox_id: str, -) -> list[Callable[..., Any]]: - return SubtaskLifecycleToolkit( - run_id=run_id, - parent_node_id=parent_node_id, sandbox_id=sandbox_id, - ).get_tools() -``` - -- [ ] **Step 2: Run lint** - -```bash -pnpm run check:be:lint -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py -git commit -m "feat(tools): add SubtaskLifecycleToolkit - -Closure factory producing seven manager-facing tool callables. -Captures run_id and parent_node_id from WorkerContext — agents -never see raw UUIDs. definition_id resolved by service at dispatch. -Replaces TaskManagementToolkit." -``` - -### Task 4.3: Delete TaskManagementToolkit and update workers - -**Files:** -- Delete: `ergon_builtins/ergon_builtins/tools/task_management_toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/workers/research_rubrics/manager_worker.py` -- Modify: any other file importing `TaskManagementToolkit` - -- [ ] **Step 1: Find all imports of the old toolkit** - -```bash -rg "TaskManagementToolkit\|task_management_toolkit" ergon_builtins/ ergon_core/ tests/ -``` - -- [ ] **Step 2: Delete task_management_toolkit.py** - -```bash -rm ergon_builtins/ergon_builtins/tools/task_management_toolkit.py -``` - -- [ ] **Step 3: Update manager_worker.py** - -Replace `TaskManagementToolkit` usage with `build_subtask_lifecycle_tools`. Update the system prompt to reference the new tool names: `add_subtask`, `plan_subtasks`, `cancel_task`, `refine_task`, `list_subtasks`, `get_subtask`, `bash`. - -Read the current manager_worker.py to understand the exact integration point (how tools are injected, how the system prompt references them). - -- [ ] **Step 4: Update other workers and benchmarks** - -Update these files to use `build_subtask_lifecycle_tools`: -- `ergon_builtins/ergon_builtins/workers/baselines/manager_researcher_worker.py` -- `ergon_builtins/ergon_builtins/benchmarks/delegation_smoke/benchmark.py` — drive new toolkit, add cancel-cascade path to smoke test - -Verify no `"abandoned"` references in: -- `ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` -- `ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py` - -- [ ] **Step 5: Update graph_toolkit.py** - -Update `list_child_resources` and `list_descendant_resources` to use `parent_node_id` query instead of edge traversal. - -```bash -rg "list_children_of\|list_child_resources\|list_descendant" ergon_builtins/ -n -``` - -- [ ] **Step 6: Update/delete old toolkit tests and worker tests** - -```bash -rg "task_management_toolkit\|TaskManagementToolkit" tests/ --files-with-matches -``` - -Delete or rename test files for the old toolkit. Key files: -- `tests/state/test_task_management_toolkit.py` → rename to `test_subtask_lifecycle_toolkit.py`, update for new tools -- `tests/state/test_graph_toolkit.py` — update assertions for `parent_node_id` query -- `tests/state/test_research_rubrics_workers.py` — update for new toolkit and tool names - -- [ ] **Step 7: Run all tests** - -```bash -pnpm run test:be:fast -``` - -- [ ] **Step 8: Commit** - -```bash -git add -A -git commit -m "feat(tools): replace TaskManagementToolkit with SubtaskLifecycleToolkit - -Deleted task_management_toolkit.py. Updated manager_worker.py to use -build_subtask_lifecycle_tools. Updated graph_toolkit to use -parent_node_id queries. Updated system prompts for new tool names." -``` - -### Task 4.4: Update remaining test fixtures - -**Files:** -- Modify: `tests/state/test_delegation_scenario.py` -- Modify: `tests/state/test_propagation_graph_native.py` -- Modify: any test still referencing old patterns - -- [ ] **Step 1: Find all remaining references to old patterns** - -```bash -rg "\"abandoned\"|ABANDONED|add_task|abandon_task|AbandonTask|AddTask[^S]|delegation.*edge\|edge.*delegation" tests/ -``` - -- [ ] **Step 2: Fix each reference** - -- `"abandoned"` → `"cancelled"` -- `ABANDONED` → `CANCELLED` -- `add_task` → `add_subtask` (in service calls) -- `abandon_task` → `cancel_task` (in service calls) -- `AbandonTaskCommand` → `CancelTaskCommand` -- `AddTaskCommand` → `AddSubtaskCommand` -- Remove test fixtures that create delegation edges (edges with `status='active'` as containment) - -- [ ] **Step 3: Run full test suite** - -```bash -pnpm run test:be:fast -``` - -- [ ] **Step 4: Run full lint + format** - -```bash -pnpm run check:be -``` - -- [ ] **Step 5: Commit** - -```bash -git add -A -git commit -m "chore(tests): update all fixtures for subtask lifecycle - -Replace abandoned→cancelled, add_task→add_subtask, abandon_task→cancel_task. -Remove delegation edge fixtures. All tests green." -``` - ---- - -## Phase 5: FE Alignment - -### Task 5.1: Update FE status vocabulary and edge types - -**Files:** -- Modify: `ergon-dashboard/src/lib/types.ts` -- Modify: `ergon-dashboard/src/generated/rest/contracts.ts` -- Modify: `ergon-dashboard/src/components/common/StatusBadge.tsx` -- Modify: `ergon-dashboard/src/components/dag/DAGCanvas.tsx` -- Modify: `ergon-dashboard/src/features/graph/state/graphMutationReducer.ts` -- Modify: `ergon-dashboard/src/components/dag/TaskGraphStatusIcon.tsx` -- Remove/modify: `ergon-dashboard/src/features/graph/components/GraphDelegationEdge.tsx` -- Modify: `ergon_core/ergon_core/core/dashboard/emitter.py` — handle `TaskCancelledEvent`, emit `node.cancelled` dashboard mutation - -- [ ] **Step 1: Update dashboard emitter for TaskCancelledEvent** - -Read the current emitter and add handling for `TaskCancelledEvent`: -```bash -rg "graph_mutation\|TaskCompleted\|TaskFailed" ergon_core/ergon_core/core/dashboard/emitter.py -n -``` - -Add a handler that emits a `node.cancelled` mutation when `TaskCancelledEvent` is received. Follow the same pattern as the existing `TaskCompletedEvent`/`TaskFailedEvent` handlers. - -- [ ] **Step 2: Find all FE references to "abandoned" and "graphDelegation"** - -```bash -rg "abandoned|graphDelegation|delegation" ergon-dashboard/src/ --type ts --type tsx -``` - -- [ ] **Step 3: Replace "abandoned" with "cancelled" in types** - -In `types.ts`: -- `TaskStatus` zod enum: replace `"abandoned"` with `"cancelled"` -- `TaskState.status`: same - -In `contracts.ts`: -- If auto-generated, re-run codegen. If hand-maintained, replace `"abandoned"` → `"cancelled"`. - -- [ ] **Step 4: Update StatusBadge and TaskGraphStatusIcon** - -Replace styling/icon for `"abandoned"` → `"cancelled"`. - -- [ ] **Step 5: Remove GraphDelegationEdge** - -Delete the file or remove the component. Remove from `edgeTypes` in `DAGCanvas.tsx`. - -- [ ] **Step 6: Handle node.cancelled in graphMutationReducer** - -Add handling for `"cancelled"` status in the reducer (same treatment as `"failed"` — mark terminal). - -- [ ] **Step 7: Run FE checks** - -```bash -pnpm run check:fe -``` - -- [ ] **Step 8: Commit** - -```bash -git add -A -git commit -m "feat(dashboard): align FE with CANCELLED status and removed delegation edges - -abandoned→cancelled in all types, badges, icons. Removed -GraphDelegationEdge (delegation edges no longer exist in DB). -graphMutationReducer handles node.cancelled." -``` - ---- - -## Phase 6: Final Verification - -### Task 6.1: Full check suite - -- [ ] **Step 1: Run full backend checks** - -```bash -pnpm run check:be -``` - -Expected: All pass (lint, format, type, slopcop) - -- [ ] **Step 2: Run full backend tests** - -```bash -pnpm run test:be:fast -``` - -Expected: All pass - -- [ ] **Step 3: Run FE checks** - -```bash -pnpm run check:fe -``` - -Expected: All pass - -- [ ] **Step 4: Verify no stale references** - -```bash -rg "ABANDONED\b|\"abandoned\"|'abandoned'|EDGE_ACTIVE|\"active\"" ergon_core/ ergon_builtins/ tests/ ergon-dashboard/src/ -``` - -Expected: Zero hits (or only in migration/changelog files) - -```bash -rg "TaskManagementToolkit|task_management_toolkit" ergon_core/ ergon_builtins/ tests/ -``` - -Expected: Zero hits - -- [ ] **Step 5: Git log review** - -```bash -git log --oneline main..HEAD -``` - -Verify commit chain is clean and each commit message accurately describes its change. - ---- - -## Summary of phases and subagent dispatch - -| Phase | Tasks | Can run as one subagent | Dependencies | -|-------|-------|------------------------|-------------| -| 0 — Schema | 0.1, 0.2, 0.3, 0.4 | Yes | None | -| 1 — Conditional writes | 1.1, 1.2, 1.3 | Yes | Phase 0 | -| 2 — Services | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6 | Yes | Phase 1 | -| 3 — Inngest + Propagation | 3.1, 3.2, 3.3, 3.4, 3.5 | Yes | Phase 2 | -| 4 — Tools + Workers | 4.1, 4.2, 4.3, 4.4 | Yes | Phase 3 | -| 5 — FE Alignment | 5.1 | Yes | Phase 3 (BE must be done) | -| 6 — Verification | 6.1 | Yes | All previous | - -Each phase should be dispatched as a single subagent. Review between phases. diff --git a/docs/superpowers/plans/2026-04-17-swebench-verified-benchmark.md b/docs/superpowers/plans/2026-04-17-swebench-verified-benchmark.md deleted file mode 100644 index e15c26cba..000000000 --- a/docs/superpowers/plans/2026-04-17-swebench-verified-benchmark.md +++ /dev/null @@ -1,1810 +0,0 @@ -# SWE-Bench Verified Benchmark Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add SWE-Bench Verified (500 curated real-world GitHub issue-fix tasks) as an Ergon benchmark, end-to-end: dataset loader, custom E2B sandbox template, generic bash + file-edit worker, and an evaluator that scores by running the official SWE-Bench test harness. - -**Architecture:** -- Dataset: `princeton-nlp/SWE-bench_Verified` loaded via `datasets.load_dataset`. Each row becomes one `BenchmarkTask` whose payload carries everything the evaluator needs (`instance_id`, `repo`, `base_commit`, `version`, `problem_statement`, `FAIL_TO_PASS`, `PASS_TO_PASS`, `test_patch`) but the worker's `description` contains ONLY the problem statement — gold `patch` is dropped entirely, `test_patch` is reserved for the evaluator. -- Sandbox: one E2B template (`ergon-swebench-v1`) with git, build toolchain, system libs for all 12 repos, `uv` as Python-version manager, `swebench` pip package, and a `UV_CACHE_DIR` on a writable path. Per-task setup is driven by `swebench.harness.test_spec.make_test_spec(instance).setup_env_script` and `install_repo_script` — we do not reimplement the install matrix. -- Worker: `SWEBenchReActWorker` extends `ReActWorker` and provides two generic tools: bash and str-replace file editor. On completion it runs `git diff HEAD` inside the sandbox and stores the unified diff as the output artifact. -- Evaluator: `SWEBenchRubric` holds a single `SWEBenchTestCriterion` that spawns a fresh sandbox of the same template, checks out `base_commit`, applies `test_patch`, applies the agent's patch, runs `eval_script`, and parses results with `swebench.harness.grading.get_eval_report`. - -**Tech Stack:** Python 3.13, pydantic, pydantic_ai (tool use), E2B `AsyncSandbox`, `datasets`, `huggingface_hub`, `swebench>=3.0`, uv (inside container). - ---- - -## File Structure - -**New files (created by this plan):** - -``` -ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/ -├── __init__.py -├── benchmark.py # SweBenchVerifiedBenchmark -├── task_schemas.py # SWEBenchInstance, SWEBenchTaskPayload -├── toolkit.py # SWEBenchToolkit (bash + file edit) -├── sandbox_manager.py # SWEBenchSandboxManager -├── criterion.py # SWEBenchTestCriterion -└── sandbox/ - ├── Dockerfile - └── e2b.toml.template - -ergon/ergon_builtins/ergon_builtins/workers/baselines/ -└── swebench_worker.py # SWEBenchReActWorker - -ergon/ergon_builtins/ergon_builtins/evaluators/rubrics/ -└── swebench_rubric.py # SWEBenchRubric (+ factory) - -ergon/tests/swebench_verified/ -├── __init__.py -├── test_benchmark.py # dataset loader (network-mocked) -├── test_task_schemas.py # pydantic round-trips, patch stripping -├── test_sandbox_manager.py # template threading -├── test_toolkit.py # bash + edit tool semantics, mocked sandbox -├── test_worker.py # toolkit wiring + patch extraction -├── test_criterion.py # eval flow on canned outputs (no E2B) -└── test_rubric.py # aggregation -``` - -**Modified files:** - -- `ergon/ergon_builtins/ergon_builtins/registry_core.py` — add SWE-Bench to `SANDBOX_MANAGERS` and `SANDBOX_TEMPLATES` -- `ergon/ergon_builtins/ergon_builtins/registry_data.py` — add `"swebench-verified"` to `BENCHMARKS`, `"swebench-react"` to `WORKERS`, `"swebench-rubric"` to `EVALUATORS` -- `ergon/ergon_builtins/pyproject.toml` — add `datasets` and `swebench` to `[data]` extra -- `ergon/ergon_cli/ergon_cli/commands/benchmark.py` — register template build path (if not auto-discovered via `SANDBOX_TEMPLATES`) - ---- - -## Task 1: Scaffolding and dependencies - -**Files:** -- Modify: `ergon/ergon_builtins/pyproject.toml` -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/__init__.py` -- Create: `ergon/tests/swebench_verified/__init__.py` - -- [ ] **Step 1: Add dependencies to `[data]` extra** - -Open `ergon/ergon_builtins/pyproject.toml`. Find the `[project.optional-dependencies]` / `data` section and add: - -```toml -[project.optional-dependencies] -data = [ - # ...existing entries kept... - "datasets>=2.20,<4", - "swebench>=3.0,<4", -] -``` - -- [ ] **Step 2: Install the extra and verify import** - -Run: `uv sync --all-packages --group dev --extra data` -Expected: resolver succeeds, `swebench` and `datasets` are installed. -Verify: `uv run python -c "import swebench.harness.test_spec as t; import datasets; print('ok')"` → prints `ok`. - -- [ ] **Step 3: Create empty package and test package** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/__init__.py` with contents: -```python -"""SWE-Bench Verified benchmark package.""" -``` - -Create `ergon/tests/swebench_verified/__init__.py` as an empty file (touch). - -- [ ] **Step 4: Commit** - -```bash -cd ergon -git add ergon_builtins/pyproject.toml uv.lock \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/__init__.py \ - tests/swebench_verified/__init__.py -git commit -m "chore(swebench): scaffold package and add datasets+swebench deps" -``` - ---- - -## Task 2: Task schemas - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/task_schemas.py` -- Test: `ergon/tests/swebench_verified/test_task_schemas.py` - -Note: SWE-Bench Verified rows carry `FAIL_TO_PASS` and `PASS_TO_PASS` as JSON-encoded strings that need parsing. We normalize them to `list[str]` in the payload. - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_task_schemas.py`: - -```python -"""Tests for SWE-Bench task schemas.""" - -from ergon_builtins.benchmarks.swebench_verified.task_schemas import ( - SWEBenchInstance, - SWEBenchTaskPayload, -) - - -RAW_ROW = { - "instance_id": "django__django-11999", - "repo": "django/django", - "base_commit": "deadbeef", - "patch": "--- gold patch, worker must not see ---", - "test_patch": "--- test patch, evaluator only ---", - "problem_statement": "Fix the thing.", - "hints_text": "maybe look at foo.py", - "version": "3.0", - "FAIL_TO_PASS": '["tests.test_foo.TestFoo.test_fix"]', - "PASS_TO_PASS": '["tests.test_foo.TestFoo.test_existing"]', - "environment_setup_commit": "cafebabe", -} - - -def test_instance_parses_json_encoded_test_lists() -> None: - instance = SWEBenchInstance.from_raw(RAW_ROW) - assert instance.fail_to_pass == ["tests.test_foo.TestFoo.test_fix"] - assert instance.pass_to_pass == ["tests.test_foo.TestFoo.test_existing"] - - -def test_payload_from_instance_strips_gold_patch() -> None: - instance = SWEBenchInstance.from_raw(RAW_ROW) - payload = SWEBenchTaskPayload.from_instance(instance) - dumped = payload.model_dump() - assert "patch" not in dumped - assert dumped["test_patch"] == RAW_ROW["test_patch"] - assert dumped["problem_statement"] == RAW_ROW["problem_statement"] - - -def test_worker_description_excludes_test_patch() -> None: - instance = SWEBenchInstance.from_raw(RAW_ROW) - payload = SWEBenchTaskPayload.from_instance(instance) - description = payload.build_worker_description() - assert RAW_ROW["problem_statement"] in description - assert "test patch" not in description - assert "gold patch" not in description -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_task_schemas.py -v` -Expected: `ImportError` or `ModuleNotFoundError` for `task_schemas`. - -- [ ] **Step 3: Implement `task_schemas.py`** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/task_schemas.py`: - -```python -"""Pydantic schemas for SWE-Bench Verified tasks. - -The raw dataset row carries both the gold ``patch`` and the ``test_patch`` -that defines the test cases. We deliberately drop ``patch`` before it ever -reaches the worker, and we keep ``test_patch`` in the payload for the -evaluator only — ``build_worker_description`` never includes it. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from typing import Any - -from pydantic import BaseModel, Field - - -class SWEBenchInstance(BaseModel): - """Parsed representation of one SWE-Bench Verified row.""" - - instance_id: str - repo: str - base_commit: str - problem_statement: str - hints_text: str = "" - version: str - fail_to_pass: list[str] - pass_to_pass: list[str] - environment_setup_commit: str - test_patch: str - - @classmethod - def from_raw(cls, row: Mapping[str, Any]) -> "SWEBenchInstance": - return cls( - instance_id=row["instance_id"], - repo=row["repo"], - base_commit=row["base_commit"], - problem_statement=row["problem_statement"], - hints_text=row.get("hints_text") or "", - version=str(row["version"]), - fail_to_pass=_parse_test_list(row["FAIL_TO_PASS"]), - pass_to_pass=_parse_test_list(row["PASS_TO_PASS"]), - environment_setup_commit=row.get("environment_setup_commit") or row["base_commit"], - test_patch=row["test_patch"], - ) - - -class SWEBenchTaskPayload(BaseModel): - """Payload attached to each ``BenchmarkTask``. - - Includes ``test_patch`` because the evaluator needs it, but - ``build_worker_description`` omits it so the worker cannot see the - tests it is supposed to make pass. - """ - - instance_id: str - repo: str - base_commit: str - version: str - problem_statement: str - hints_text: str = "" - fail_to_pass: list[str] - pass_to_pass: list[str] - environment_setup_commit: str - test_patch: str = Field(..., description="Gold test patch; evaluator-only.") - - @classmethod - def from_instance(cls, instance: SWEBenchInstance) -> "SWEBenchTaskPayload": - return cls(**instance.model_dump()) - - def build_worker_description(self) -> str: - parts = [ - f"Repository: {self.repo} (commit {self.base_commit[:12]})", - "", - "## Problem statement", - self.problem_statement.strip(), - ] - if self.hints_text.strip(): - parts.extend(["", "## Hints", self.hints_text.strip()]) - parts.extend([ - "", - "## Task", - "Modify the repository so that the described issue is fixed.", - "When done, your changes will be extracted as a `git diff HEAD` and", - "scored against a hidden test suite.", - ]) - return "\n".join(parts) - - -def _parse_test_list(value: Any) -> list[str]: - if isinstance(value, list): - return [str(v) for v in value] - if isinstance(value, str): - return [str(v) for v in json.loads(value)] - raise TypeError(f"Unsupported FAIL/PASS list type: {type(value)!r}") -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_task_schemas.py -v` -Expected: 3 passed. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/task_schemas.py \ - tests/swebench_verified/test_task_schemas.py -git commit -m "feat(swebench): task payload schemas with gold-patch stripping" -``` - ---- - -## Task 3: Benchmark class (dataset loader) - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py` -- Test: `ergon/tests/swebench_verified/test_benchmark.py` - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_benchmark.py`: - -```python -"""Tests for the SWE-Bench Verified benchmark loader.""" - -from __future__ import annotations - -from unittest.mock import patch - -from ergon_builtins.benchmarks.swebench_verified.benchmark import ( - SweBenchVerifiedBenchmark, -) - - -FAKE_ROWS = [ - { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "aaa", - "patch": "GOLD", - "test_patch": "TP1", - "problem_statement": "p1", - "hints_text": "", - "version": "3.0", - "FAIL_TO_PASS": '["t1"]', - "PASS_TO_PASS": '["t0"]', - "environment_setup_commit": "aaa", - }, - { - "instance_id": "sympy__sympy-2", - "repo": "sympy/sympy", - "base_commit": "bbb", - "patch": "GOLD", - "test_patch": "TP2", - "problem_statement": "p2", - "hints_text": "", - "version": "1.10", - "FAIL_TO_PASS": '["t2"]', - "PASS_TO_PASS": '["t0"]', - "environment_setup_commit": "bbb", - }, -] - - -def test_build_instances_yields_one_task_per_row() -> None: - with patch( - "ergon_builtins.benchmarks.swebench_verified.benchmark._load_rows", - return_value=FAKE_ROWS, - ): - benchmark = SweBenchVerifiedBenchmark() - instances = benchmark.build_instances() - - assert set(instances.keys()) == {"default"} - tasks = instances["default"] - assert len(tasks) == 2 - assert tasks[0].task_key == "django__django-1" - assert tasks[1].task_key == "sympy__sympy-2" - - -def test_limit_truncates() -> None: - with patch( - "ergon_builtins.benchmarks.swebench_verified.benchmark._load_rows", - return_value=FAKE_ROWS, - ): - benchmark = SweBenchVerifiedBenchmark(limit=1) - tasks = benchmark.build_instances()["default"] - - assert len(tasks) == 1 - assert tasks[0].task_key == "django__django-1" - - -def test_task_description_excludes_test_patch_and_gold_patch() -> None: - with patch( - "ergon_builtins.benchmarks.swebench_verified.benchmark._load_rows", - return_value=FAKE_ROWS, - ): - benchmark = SweBenchVerifiedBenchmark() - task = benchmark.build_instances()["default"][0] - - assert "TP1" not in task.description - assert "GOLD" not in task.description - assert "p1" in task.description - - -def test_task_payload_retains_test_patch_for_evaluator() -> None: - with patch( - "ergon_builtins.benchmarks.swebench_verified.benchmark._load_rows", - return_value=FAKE_ROWS, - ): - benchmark = SweBenchVerifiedBenchmark() - task = benchmark.build_instances()["default"][0] - - assert task.task_payload["test_patch"] == "TP1" - assert "patch" not in task.task_payload # gold patch was dropped -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_benchmark.py -v` -Expected: `ModuleNotFoundError` for `benchmark`. - -- [ ] **Step 3: Implement the benchmark** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py`: - -```python -"""SWE-Bench Verified benchmark loader. - -Pulls the ``princeton-nlp/SWE-bench_Verified`` HuggingFace dataset and yields -one ``BenchmarkTask`` per instance. The worker only sees the problem -statement; the evaluator receives ``test_patch`` via the task payload. -""" - -from __future__ import annotations - -import logging -from collections.abc import Mapping, Sequence -from typing import Any, ClassVar - -from ergon_core.api.benchmark import Benchmark -from ergon_core.api.task_types import BenchmarkTask - -from ergon_builtins.benchmarks.swebench_verified.task_schemas import ( - SWEBenchInstance, - SWEBenchTaskPayload, -) - -logger = logging.getLogger(__name__) - -HF_DATASET_ID = "princeton-nlp/SWE-bench_Verified" -HF_SPLIT = "test" - - -class SweBenchVerifiedBenchmark(Benchmark): - """Benchmark backed by SWE-Bench Verified (500 curated instances).""" - - type_slug: ClassVar[str] = "swebench-verified" - - def __init__( - self, - *, - limit: int | None = None, - name: str | None = None, - description: str | None = None, - metadata: Mapping[str, Any] | None = None, # slopcop: ignore[no-typing-any] - ) -> None: - super().__init__( - name=name or "swebench-verified", - description=description or "SWE-Bench Verified GitHub issue-fix benchmark", - metadata=metadata, - ) - self.limit = limit - - def build_instances(self) -> Mapping[str, Sequence[BenchmarkTask]]: - rows = _load_rows(limit=self.limit) - tasks: list[BenchmarkTask] = [] - for row in rows: - instance = SWEBenchInstance.from_raw(row) - payload = SWEBenchTaskPayload.from_instance(instance) - tasks.append( - BenchmarkTask( - task_key=instance.instance_id, - instance_key="default", - description=payload.build_worker_description(), - evaluator_binding_keys=("default",), - task_payload=payload.model_dump(), - ) - ) - logger.info("Loaded %d SWE-Bench Verified instances", len(tasks)) - return {"default": tasks} - - def evaluator_requirements(self) -> Sequence[str]: - return ("default",) - - -def _load_rows(*, limit: int | None = None) -> list[dict[str, Any]]: - """Load rows from the HF dataset. Isolated for testability.""" - # reason: optional dependency from ergon-builtins[data] - from datasets import load_dataset - - ds = load_dataset(HF_DATASET_ID, split=HF_SPLIT) - if limit is not None: - ds = ds.select(range(min(limit, len(ds)))) - return [dict(row) for row in ds] -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_benchmark.py -v` -Expected: 4 passed. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py \ - tests/swebench_verified/test_benchmark.py -git commit -m "feat(swebench): dataset loader yielding BenchmarkTasks" -``` - ---- - -## Task 4: Dockerfile and E2B template config - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox/Dockerfile` -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox/e2b.toml.template` - -This task produces files that are consumed by `ergon benchmark setup swebench-verified` in a later task. There is no TDD cycle for a Dockerfile — verification is building the template and running a smoke command. - -- [ ] **Step 1: Write the Dockerfile** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox/Dockerfile`: - -```dockerfile -# SWE-Bench Verified sandbox: one image that covers the 12 repos in Verified. -# Per-instance Python selection and dependency install is driven at runtime -# by swebench.harness.test_spec — this image only provides the substrate. -FROM ubuntu:22.04 - -ENV DEBIAN_FRONTEND=noninteractive -ENV PATH=/root/.local/bin:/usr/local/bin:$PATH -ENV UV_CACHE_DIR=/workspace/.uv-cache - -# --- system deps ------------------------------------------------------------- -# Kept in a single layer. Grouped by repo that needs them for readability. -RUN apt-get update && apt-get install -y --no-install-recommends \ - # core - git ca-certificates curl wget unzip xz-utils \ - build-essential pkg-config \ - # python build deps - libssl-dev libffi-dev libsqlite3-dev libbz2-dev \ - zlib1g-dev libreadline-dev liblzma-dev \ - # matplotlib / pillow - libfreetype6-dev libpng-dev libjpeg-dev libtiff-dev \ - # scikit-learn / scipy - libopenblas-dev liblapack-dev gfortran \ - # sphinx / lxml / xarray - libxml2-dev libxslt1-dev \ - # django (psycopg2) - libpq-dev \ - && rm -rf /var/lib/apt/lists/* - -# --- uv as Python-version manager ------------------------------------------- -RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ - ln -sf /root/.local/bin/uv /usr/local/bin/uv - -# --- swebench harness (for test_spec + grading) ----------------------------- -# Install into a dedicated venv so it does not collide with task-specific -# repo environments. -RUN uv venv /opt/swebench-harness --python 3.11 && \ - /opt/swebench-harness/bin/pip install --no-cache-dir "swebench>=3.0,<4" -ENV SWEBENCH_PY=/opt/swebench-harness/bin/python - -# --- workspace layout ------------------------------------------------------- -RUN mkdir -p /workspace/repo /workspace/.uv-cache /workspace/artifacts /inputs - -WORKDIR /workspace -CMD ["/bin/bash"] -``` - -- [ ] **Step 2: Write the E2B template config** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox/e2b.toml.template`: - -```toml -dockerfile = "Dockerfile" -template_name = "ergon-swebench-v1" -start_cmd = "/bin/bash" -cpu_count = 4 -memory_mb = 8192 -``` - -- [ ] **Step 3: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox/ -git commit -m "feat(swebench): E2B sandbox template (Dockerfile + e2b.toml)" -``` - -(Template build itself happens in Task 7 after registry wiring.) - ---- - -## Task 5: Sandbox manager - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py` -- Test: `ergon/tests/swebench_verified/test_sandbox_manager.py` - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_sandbox_manager.py`: - -```python -"""Tests for SWEBenchSandboxManager.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -import pytest - -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) - - -@pytest.fixture(autouse=True) -def _reset_singleton(): - SWEBenchSandboxManager._instances.clear() - yield - SWEBenchSandboxManager._instances.clear() - - -def test_resolves_template_from_registry() -> None: - with patch( - "ergon_builtins.benchmarks.swebench_verified.sandbox_manager.resolve_template", - return_value="tmpl-abc123", - ): - manager = SWEBenchSandboxManager() - assert manager.template == "tmpl-abc123" - - -@pytest.mark.asyncio -async def test_verify_setup_runs_git_version() -> None: - manager = SWEBenchSandboxManager.__new__(SWEBenchSandboxManager) - manager.template = "tmpl" - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock(return_value=AsyncMock(exit_code=0, stdout="git version 2.40")) - - await manager._verify_setup(sandbox, task_id="t1") - - sandbox.commands.run.assert_awaited() - args = sandbox.commands.run.call_args.args[0] - assert "git --version" in args -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_sandbox_manager.py -v` -Expected: `ModuleNotFoundError`. - -- [ ] **Step 3: Implement the sandbox manager** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py`: - -```python -"""Sandbox manager for the SWE-Bench Verified benchmark. - -Per-task setup (cloning the repo at ``base_commit``, creating the venv at -the right Python version, installing deps) is driven by -``swebench.harness.test_spec`` and is performed by the worker at task -start, not here. This manager only provisions the E2B sandbox from the -pre-built template. -""" - -from __future__ import annotations - -import logging - -from ergon_core.core.providers.sandbox.manager import BaseSandboxManager -from ergon_core.core.providers.sandbox.template_registry import resolve_template - -logger = logging.getLogger(__name__) - -TEMPLATE_SLUG = "swebench-verified" - - -class SWEBenchSandboxManager(BaseSandboxManager): - """Singleton manager that hands out E2B sandboxes built from ergon-swebench-v1.""" - - def __init__(self) -> None: - super().__init__() - self.template = resolve_template(TEMPLATE_SLUG) - - async def _install_dependencies(self, sandbox, task_id) -> None: # noqa: ARG002 - # Template is pre-built; per-task setup is driven by the worker - # using swebench.harness.test_spec scripts. - return None - - async def _verify_setup(self, sandbox, task_id) -> None: # noqa: ARG002 - result = await sandbox.commands.run("git --version && uv --version") - if result.exit_code != 0: - raise RuntimeError(f"SWE-Bench sandbox smoke check failed: {result.stdout}") -``` - -Note on `resolve_template`: this helper lives in `ergon_core.core.providers.sandbox.template_registry` and reads `~/.ergon/sandbox_templates.json`. If the import path differs in the current codebase, match the signature used by `MiniF2FSandboxManager.resolve_template` — grep for it before implementing if unsure. - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_sandbox_manager.py -v` -Expected: 2 passed. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py \ - tests/swebench_verified/test_sandbox_manager.py -git commit -m "feat(swebench): sandbox manager with template resolution" -``` - ---- - -## Task 6: Registry registration - -**Files:** -- Modify: `ergon/ergon_builtins/ergon_builtins/registry_core.py` -- Modify: `ergon/ergon_builtins/ergon_builtins/registry_data.py` - -- [ ] **Step 1: Register the sandbox manager and template in `registry_core.py`** - -In `ergon/ergon_builtins/ergon_builtins/registry_core.py`, add the import near the other benchmark-sandbox imports: - -```python -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) -``` - -And extend the existing dicts: - -```python -SANDBOX_MANAGERS: dict[str, type[BaseSandboxManager]] = { - "gdpeval": GDPEvalSandboxManager, - "minif2f": MiniF2FSandboxManager, - "swebench-verified": SWEBenchSandboxManager, -} - -SANDBOX_TEMPLATES: dict[str, Path] = { - "minif2f": Path(__file__).parent / "benchmarks/minif2f/sandbox", - "swebench-verified": Path(__file__).parent / "benchmarks/swebench_verified/sandbox", -} -``` - -- [ ] **Step 2: Register benchmark (+ placeholder worker/rubric entries) in `registry_data.py`** - -In `ergon/ergon_builtins/ergon_builtins/registry_data.py`, add: - -```python -from ergon_builtins.benchmarks.swebench_verified.benchmark import ( - SweBenchVerifiedBenchmark, -) -``` - -And extend the benchmarks dict: - -```python -BENCHMARKS["swebench-verified"] = SweBenchVerifiedBenchmark -``` - -Worker and evaluator entries are added in Tasks 9 and 11 respectively. - -- [ ] **Step 3: Sanity-check the registry loads** - -Run: `uv run python -c "from ergon_builtins.registry import BENCHMARKS, SANDBOX_MANAGERS; assert 'swebench-verified' in BENCHMARKS; assert 'swebench-verified' in SANDBOX_MANAGERS; print('ok')"` -Expected: `ok`. - -- [ ] **Step 4: Commit** - -```bash -git add ergon_builtins/ergon_builtins/registry_core.py \ - ergon_builtins/ergon_builtins/registry_data.py -git commit -m "feat(swebench): register benchmark and sandbox manager" -``` - ---- - -## Task 7: Build the E2B template - -**Files:** none created here — CLI action and artifact persistence to `~/.ergon/sandbox_templates.json`. - -- [ ] **Step 1: Build the template via CLI** - -Run: `uv run ergon benchmark setup swebench-verified` - -Expected behaviour (mirroring miniF2F): -- Reads `e2b.toml.template` + `Dockerfile` from the registered path. -- Calls E2B `Template.build(...)` with `template_name="ergon-swebench-v1"`. -- Writes the returned template_id to `~/.ergon/sandbox_templates.json`. - -If this command does not automatically pick up the new sandbox template, check `ergon_cli/commands/benchmark.py` — the loop over `SANDBOX_TEMPLATES` should be keyed off the registry, not a hard-coded list. If it's hard-coded, add the new slug there. - -- [ ] **Step 2: Smoke the built template** - -Run: `uv run python -c " -import asyncio -from e2b import AsyncSandbox -from ergon_core.core.providers.sandbox.template_registry import resolve_template - -async def go(): - tid = resolve_template('swebench-verified') - sbx = await AsyncSandbox.create(template=tid) - r = await sbx.commands.run('git --version && uv --version && \$SWEBENCH_PY -c \"import swebench; print(swebench.__version__)\"') - print(r.stdout) - await sbx.kill() - -asyncio.run(go()) -"` - -Expected: `git version ...`, `uv ...`, and a swebench version string. - -- [ ] **Step 3: Commit (template registry entry)** - -```bash -git add -u # picks up ~/.ergon changes IF they live in the repo; typically they don't -git commit --allow-empty -m "chore(swebench): build and register ergon-swebench-v1 template" -``` - -(If `~/.ergon/sandbox_templates.json` is outside the repo, just document the template id in the commit message.) - ---- - -## Task 8: Toolkit (bash + str-replace editor) - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py` -- Test: `ergon/tests/swebench_verified/test_toolkit.py` - -Keep the toolkit deliberately generic: `bash` and `str_replace_editor`. Both operate against the sandbox; no SWE-Bench-specific logic here. - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_toolkit.py`: - -```python -"""Tests for the SWE-Bench toolkit (bash + str-replace editor).""" - -from __future__ import annotations - -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from ergon_builtins.benchmarks.swebench_verified.toolkit import SWEBenchToolkit - - -@pytest.mark.asyncio -async def test_bash_tool_runs_command_in_workdir() -> None: - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock( - return_value=SimpleNamespace(exit_code=0, stdout="hello\n", stderr="") - ) - tk = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - - tool = next(t for t in tk.get_tools() if t.name == "bash") - response = await tool.function(command="echo hello") - - assert response.exit_code == 0 - assert "hello" in response.stdout - # Command should be wrapped to execute inside the workdir - invoked = sandbox.commands.run.call_args.args[0] - assert "/workspace/repo" in invoked - - -@pytest.mark.asyncio -async def test_str_replace_editor_view_reads_file() -> None: - sandbox = AsyncMock() - sandbox.files.read = AsyncMock(return_value="def foo():\n return 1\n") - tk = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - - tool = next(t for t in tk.get_tools() if t.name == "str_replace_editor") - response = await tool.function(command="view", path="src/foo.py") - - assert "def foo" in response.output - sandbox.files.read.assert_awaited_with("/workspace/repo/src/foo.py") - - -@pytest.mark.asyncio -async def test_str_replace_editor_replace_updates_file() -> None: - sandbox = AsyncMock() - sandbox.files.read = AsyncMock(return_value="def foo():\n return 1\n") - sandbox.files.write = AsyncMock() - tk = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - - tool = next(t for t in tk.get_tools() if t.name == "str_replace_editor") - response = await tool.function( - command="str_replace", - path="src/foo.py", - old_str=" return 1", - new_str=" return 2", - ) - - assert response.ok is True - sandbox.files.write.assert_awaited() - written_path, written_bytes = sandbox.files.write.call_args.args - assert written_path == "/workspace/repo/src/foo.py" - assert b"return 2" in written_bytes - - -@pytest.mark.asyncio -async def test_str_replace_editor_replace_fails_when_old_str_not_unique() -> None: - sandbox = AsyncMock() - sandbox.files.read = AsyncMock(return_value="x = 1\nx = 1\n") - tk = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - - tool = next(t for t in tk.get_tools() if t.name == "str_replace_editor") - response = await tool.function( - command="str_replace", path="x.py", old_str="x = 1", new_str="x = 2" - ) - - assert response.ok is False - assert "not unique" in response.error.lower() -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_toolkit.py -v` -Expected: `ModuleNotFoundError`. - -- [ ] **Step 3: Implement the toolkit** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py`: - -```python -"""Tools exposed to a SWE-Bench worker. - -Deliberately generic: one bash tool and one str-replace editor. Enough to -solve the benchmark end-to-end and portable to other code-editing tasks. -""" - -from __future__ import annotations - -import shlex -from collections.abc import Sequence -from typing import Literal - -from pydantic import BaseModel -from pydantic_ai.tools import Tool - - -class BashResponse(BaseModel): - exit_code: int - stdout: str - stderr: str - - -class EditorResponse(BaseModel): - ok: bool - output: str = "" - error: str = "" - - -class SWEBenchToolkit: - def __init__(self, *, sandbox, workdir: str = "/workspace/repo") -> None: - self._sandbox = sandbox - self._workdir = workdir - - def get_tools(self) -> Sequence[Tool]: - return [self._bash_tool(), self._editor_tool()] - - # --- bash --------------------------------------------------------------- - - def _bash_tool(self) -> Tool: - async def bash(command: str, timeout_sec: int = 300) -> BashResponse: - """Run a shell command inside the repo workdir.""" - wrapped = f"cd {shlex.quote(self._workdir)} && {command}" - result = await self._sandbox.commands.run(wrapped, timeout=timeout_sec) - return BashResponse( - exit_code=result.exit_code, - stdout=result.stdout or "", - stderr=getattr(result, "stderr", "") or "", - ) - - return Tool(function=bash, takes_ctx=False, name="bash") - - # --- str-replace editor ------------------------------------------------- - - def _editor_tool(self) -> Tool: - async def str_replace_editor( - command: Literal["view", "create", "str_replace"], - path: str, - file_text: str | None = None, - old_str: str | None = None, - new_str: str | None = None, - ) -> EditorResponse: - """View, create, or edit a file by exact string replacement.""" - abs_path = f"{self._workdir.rstrip('/')}/{path.lstrip('/')}" - try: - if command == "view": - content = await self._sandbox.files.read(abs_path) - return EditorResponse(ok=True, output=content) - - if command == "create": - if file_text is None: - return EditorResponse(ok=False, error="file_text required for create") - await self._sandbox.files.write(abs_path, file_text.encode()) - return EditorResponse(ok=True, output=f"created {abs_path}") - - if command == "str_replace": - if old_str is None or new_str is None: - return EditorResponse(ok=False, error="old_str and new_str required") - content = await self._sandbox.files.read(abs_path) - occurrences = content.count(old_str) - if occurrences == 0: - return EditorResponse(ok=False, error="old_str not found") - if occurrences > 1: - return EditorResponse( - ok=False, - error=f"old_str not unique ({occurrences} matches); add more context", - ) - new_content = content.replace(old_str, new_str, 1) - await self._sandbox.files.write(abs_path, new_content.encode()) - return EditorResponse(ok=True, output=f"edited {abs_path}") - - return EditorResponse(ok=False, error=f"unknown command {command!r}") - except Exception as exc: # noqa: BLE001 - surface as tool error - return EditorResponse(ok=False, error=str(exc)) - - return Tool(function=str_replace_editor, takes_ctx=False, name="str_replace_editor") -``` - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_toolkit.py -v` -Expected: 4 passed. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py \ - tests/swebench_verified/test_toolkit.py -git commit -m "feat(swebench): generic bash + str-replace editor toolkit" -``` - ---- - -## Task 9: Worker (with per-task setup + patch extraction) - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/workers/baselines/swebench_worker.py` -- Test: `ergon/tests/swebench_verified/test_worker.py` -- Modify: `ergon/ergon_builtins/ergon_builtins/registry_data.py` - -The worker does three things around the ReAct loop: -1. **Before** the loop: run `swebench.harness.test_spec.make_test_spec(instance).setup_env_script` and `install_repo_script` inside the sandbox so the repo is at `base_commit` with deps installed in a venv. -2. **During** the loop: ReAct with the bash + editor toolkit. -3. **After** the loop: run `git diff HEAD` and store the patch as the worker output. - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_worker.py`: - -```python -"""Tests for SWEBenchReActWorker.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from ergon_core.api.task_types import BenchmarkTask -from ergon_core.api.worker import WorkerContext -from ergon_builtins.workers.baselines.swebench_worker import SWEBenchReActWorker - - -def _fake_task() -> BenchmarkTask: - payload = { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "aaa", - "version": "3.0", - "problem_statement": "p", - "hints_text": "", - "fail_to_pass": ["t1"], - "pass_to_pass": ["t0"], - "environment_setup_commit": "aaa", - "test_patch": "TP", - } - return BenchmarkTask( - task_key="django__django-1", - instance_key="default", - description="Fix the thing", - evaluator_binding_keys=("default",), - task_payload=payload, - ) - - -@pytest.mark.asyncio -async def test_worker_runs_setup_scripts_before_react_loop() -> None: - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock(return_value=MagicMock(exit_code=0, stdout="", stderr="")) - manager = MagicMock() - manager.get_or_create = AsyncMock(return_value=sandbox) - - with patch( - "ergon_builtins.workers.baselines.swebench_worker.SWEBenchSandboxManager", - return_value=manager, - ), patch( - "ergon_builtins.workers.baselines.swebench_worker.make_test_spec", - ) as mk_spec: - mk_spec.return_value = MagicMock( - setup_env_script="echo SETUP", - install_repo_script="echo INSTALL", - eval_script="echo EVAL", # not used by worker - ) - worker = SWEBenchReActWorker(model="stub") - # Short-circuit the ReAct loop for this unit test - with patch.object(SWEBenchReActWorker.__bases__[0], "execute") as parent_execute: - async def _empty(*a, **kw): - if False: - yield - parent_execute.return_value = _empty() - - ctx = WorkerContext(run_id="r", task_id="t", execution_id="e", sandbox_id="s") - async for _ in worker.execute(_fake_task(), context=ctx): - pass - - # Setup and install scripts should both have been run - invoked = [call.args[0] for call in sandbox.commands.run.call_args_list] - assert any("SETUP" in c for c in invoked) - assert any("INSTALL" in c for c in invoked) - - -@pytest.mark.asyncio -async def test_worker_extracts_patch_via_git_diff_on_output() -> None: - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock( - return_value=MagicMock(exit_code=0, stdout="--- diff ---\n+foo", stderr="") - ) - worker = SWEBenchReActWorker(model="stub") - worker._sandbox = sandbox - worker._workdir = "/workspace/repo" - - ctx = WorkerContext(run_id="r", task_id="t", execution_id="e", sandbox_id="s") - output = await worker._extract_patch(ctx) - - assert "--- diff ---" in output - invoked = sandbox.commands.run.call_args.args[0] - assert "git diff HEAD" in invoked - assert "/workspace/repo" in invoked -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_worker.py -v` -Expected: `ModuleNotFoundError`. - -- [ ] **Step 3: Implement the worker** - -Create `ergon/ergon_builtins/ergon_builtins/workers/baselines/swebench_worker.py`: - -```python -"""SWE-Bench Verified worker. - -Uses swebench.harness.test_spec to perform per-task setup inside the E2B -sandbox, runs a ReAct loop over a generic bash + editor toolkit, and -returns the resulting ``git diff HEAD`` as the output artifact. -""" - -from __future__ import annotations - -import logging -from collections.abc import AsyncGenerator -from typing import ClassVar - -from ergon_core.api.task_types import BenchmarkTask -from ergon_core.api.worker import GenerationTurn, WorkerContext, WorkerOutput -from ergon_core.api.react_worker import ReActWorker # matches miniF2F base import - -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) -from ergon_builtins.benchmarks.swebench_verified.toolkit import SWEBenchToolkit - -logger = logging.getLogger(__name__) - -WORKDIR = "/workspace/repo" -SETUP_TIMEOUT_SEC = 1800 # 30 min — slowest installs (sklearn, matplotlib) - - -def make_test_spec(instance_row): # re-exported for test monkeypatching - from swebench.harness.test_spec import make_test_spec as _mk - return _mk(instance_row) - - -class SWEBenchReActWorker(ReActWorker): - type_slug: ClassVar[str] = "swebench-react" - - def __init__(self, *, model: str, **kwargs) -> None: - super().__init__(model=model, **kwargs) - self._sandbox = None - self._workdir = WORKDIR - self._patch: str = "" - - async def execute( - self, - task: BenchmarkTask, - *, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - manager = SWEBenchSandboxManager() - sandbox = await manager.get_or_create(context.task_id) - self._sandbox = sandbox - - await self._run_setup(task) - - toolkit = SWEBenchToolkit(sandbox=sandbox, workdir=self._workdir) - self.tools = list(toolkit.get_tools()) - - async for turn in super().execute(task, context=context): - yield turn - - self._patch = await self._extract_patch(context) - - async def _run_setup(self, task: BenchmarkTask) -> None: - # The raw instance row we'll feed to make_test_spec is the payload - # plus the derived fields needed by swebench's schema. - row = _payload_to_swebench_row(task.task_payload) - spec = make_test_spec(row) - - for label, script in ( - ("setup_env", spec.setup_env_script), - ("install_repo", spec.install_repo_script), - ): - logger.info("Running swebench %s script for %s", label, task.task_key) - result = await self._sandbox.commands.run( - f"bash -c {_shell_quote(script)}", - timeout=SETUP_TIMEOUT_SEC, - ) - if result.exit_code != 0: - raise RuntimeError( - f"swebench {label} failed for {task.task_key}: " - f"{(result.stdout or '')[-1000:]}" - ) - - async def _extract_patch(self, context: WorkerContext) -> str: # noqa: ARG002 - result = await self._sandbox.commands.run( - f"cd {self._workdir} && git add -A && git diff HEAD", - timeout=60, - ) - if result.exit_code != 0: - logger.warning("git diff failed: %s", result.stdout) - return "" - return result.stdout or "" - - def get_output(self, context: WorkerContext) -> WorkerOutput: # noqa: ARG002 - return WorkerOutput( - output=self._patch, - success=bool(self._patch.strip()), - artifacts={"patch": self._patch}, - ) - - -def _payload_to_swebench_row(payload: dict) -> dict: - """Re-expand a SWEBenchTaskPayload dict into the row shape swebench expects.""" - return { - "instance_id": payload["instance_id"], - "repo": payload["repo"], - "base_commit": payload["base_commit"], - "version": payload["version"], - "problem_statement": payload["problem_statement"], - "hints_text": payload.get("hints_text", ""), - "FAIL_TO_PASS": payload["fail_to_pass"], - "PASS_TO_PASS": payload["pass_to_pass"], - "environment_setup_commit": payload["environment_setup_commit"], - "test_patch": payload["test_patch"], - "patch": "", # intentionally blank - gold patch never reaches worker - } - - -def _shell_quote(script: str) -> str: - import shlex - return shlex.quote(script) -``` - -Note: the base-class import is written as `ergon_core.api.react_worker.ReActWorker` — match whichever module `MiniF2FReActWorker` imports from. Grep `minif2f_react_worker.py` for the exact line and mirror it. - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_worker.py -v` -Expected: 2 passed. - -- [ ] **Step 5: Register in `registry_data.py`** - -Open `ergon/ergon_builtins/ergon_builtins/registry_data.py`, add: - -```python -from ergon_builtins.workers.baselines.swebench_worker import SWEBenchReActWorker -# ... -WORKERS["swebench-react"] = SWEBenchReActWorker -``` - -- [ ] **Step 6: Commit** - -```bash -git add ergon_builtins/ergon_builtins/workers/baselines/swebench_worker.py \ - ergon_builtins/ergon_builtins/registry_data.py \ - tests/swebench_verified/test_worker.py -git commit -m "feat(swebench): worker with setup, ReAct loop, patch extraction" -``` - ---- - -## Task 10: Criterion (evaluation via swebench harness) - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` -- Test: `ergon/tests/swebench_verified/test_criterion.py` - -The criterion's job: given the agent's patch and the task payload, run the official SWE-Bench eval and return a 0/1 score. To avoid reusing the worker sandbox (which is messy post-agent), the criterion spawns a fresh sandbox of the same template, checks out `base_commit`, applies `test_patch`, applies the agent patch, runs `eval_script`, and parses the log with `swebench.harness.grading`. - -- [ ] **Step 1: Write failing tests** - -Create `ergon/tests/swebench_verified/test_criterion.py`: - -```python -"""Tests for SWEBenchTestCriterion. - -We mock the sandbox and swebench.harness.grading to isolate the flow: -patch application order, eval_script invocation, result parsing. -""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from ergon_core.api.evaluator import CriterionResult, EvaluationContext -from ergon_core.api.task_types import BenchmarkTask -from ergon_builtins.benchmarks.swebench_verified.criterion import ( - SWEBenchTestCriterion, -) - - -def _task() -> BenchmarkTask: - return BenchmarkTask( - task_key="django__django-1", - instance_key="default", - description="", - evaluator_binding_keys=("default",), - task_payload={ - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "aaa", - "version": "3.0", - "problem_statement": "p", - "hints_text": "", - "fail_to_pass": ["t1"], - "pass_to_pass": ["t0"], - "environment_setup_commit": "aaa", - "test_patch": "TP", - }, - ) - - -@pytest.mark.asyncio -async def test_criterion_scores_1_when_all_fail_to_pass_resolved() -> None: - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock(return_value=MagicMock(exit_code=0, stdout="log", stderr="")) - sandbox.kill = AsyncMock() - - with patch( - "ergon_builtins.benchmarks.swebench_verified.criterion._spawn_eval_sandbox", - AsyncMock(return_value=sandbox), - ), patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.make_test_spec", - return_value=MagicMock(eval_script="echo EVAL"), - ), patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.grade_log", - return_value={"resolved": True, "fail_to_pass": {"success": ["t1"], "failure": []}, - "pass_to_pass": {"success": ["t0"], "failure": []}}, - ): - crit = SWEBenchTestCriterion(name="test-resolution", weight=1.0) - ctx = EvaluationContext( - worker_output=MagicMock(output="PATCH", artifacts={"patch": "PATCH"}), - ) - result: CriterionResult = await crit.evaluate(_task(), ctx) - - assert result.score == 1.0 - assert result.passed is True - # patch application order: test_patch first, then agent patch - seq = [call.args[0] for call in sandbox.commands.run.call_args_list] - assert any("git apply" in s and "test_patch" in s.lower() for s in seq) or \ - any("TP" in s for s in seq) - - -@pytest.mark.asyncio -async def test_criterion_scores_0_when_empty_patch() -> None: - crit = SWEBenchTestCriterion(name="test-resolution", weight=1.0) - ctx = EvaluationContext(worker_output=MagicMock(output="", artifacts={"patch": ""})) - - result = await crit.evaluate(_task(), ctx) - - assert result.score == 0.0 - assert result.passed is False - assert "empty patch" in result.feedback.lower() - - -@pytest.mark.asyncio -async def test_criterion_scores_0_when_fail_to_pass_not_resolved() -> None: - sandbox = AsyncMock() - sandbox.commands.run = AsyncMock(return_value=MagicMock(exit_code=0, stdout="log", stderr="")) - sandbox.kill = AsyncMock() - - with patch( - "ergon_builtins.benchmarks.swebench_verified.criterion._spawn_eval_sandbox", - AsyncMock(return_value=sandbox), - ), patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.make_test_spec", - return_value=MagicMock(eval_script="echo EVAL"), - ), patch( - "ergon_builtins.benchmarks.swebench_verified.criterion.grade_log", - return_value={"resolved": False, "fail_to_pass": {"success": [], "failure": ["t1"]}, - "pass_to_pass": {"success": ["t0"], "failure": []}}, - ): - crit = SWEBenchTestCriterion(name="test-resolution", weight=1.0) - ctx = EvaluationContext(worker_output=MagicMock(output="PATCH", artifacts={"patch": "PATCH"})) - result = await crit.evaluate(_task(), ctx) - - assert result.score == 0.0 - assert result.passed is False -``` - -- [ ] **Step 2: Run tests to confirm they fail** - -Run: `uv run pytest tests/swebench_verified/test_criterion.py -v` -Expected: `ModuleNotFoundError`. - -- [ ] **Step 3: Implement the criterion** - -Create `ergon/ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py`: - -```python -"""Evaluator criterion that runs the SWE-Bench test harness. - -Spawns a fresh E2B sandbox from the ergon-swebench-v1 template, checks -out ``base_commit``, applies ``test_patch``, applies the agent's patch, -runs the swebench-generated ``eval_script``, and parses the log to -decide resolved/unresolved. -""" - -from __future__ import annotations - -import logging -import shlex - -from ergon_core.api.criterion import Criterion, CriterionResult -from ergon_core.api.evaluator import EvaluationContext -from ergon_core.api.task_types import BenchmarkTask - -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) -from ergon_builtins.workers.baselines.swebench_worker import _payload_to_swebench_row - -logger = logging.getLogger(__name__) - -WORKDIR = "/workspace/repo" -EVAL_TIMEOUT_SEC = 1800 - - -def make_test_spec(row): # re-exported for tests - from swebench.harness.test_spec import make_test_spec as _mk - return _mk(row) - - -def grade_log(instance_id: str, log: str, fail_to_pass: list[str], pass_to_pass: list[str]) -> dict: - """Adapt swebench.harness.grading to our inputs.""" - from swebench.harness.grading import get_logs_eval, get_eval_report - # swebench's API shape has drifted across versions. The cleanest stable - # surface: parse the eval log to a per-test status map, then compute - # resolved using get_eval_report with our FAIL/PASS lists. - test_status = get_logs_eval(log) # dict: test_name -> "PASSED"/"FAILED"/... - report = get_eval_report( - test_spec=None, - prediction={"instance_id": instance_id}, - test_log_path=None, - include_tests_status=True, - _status_map_override=test_status, - _fail_to_pass_override=fail_to_pass, - _pass_to_pass_override=pass_to_pass, - ) - return report[instance_id] - - -async def _spawn_eval_sandbox(): - manager = SWEBenchSandboxManager() - # A fresh sandbox keyed by a random UUID; never shared with a worker. - import uuid - sbx = await manager.get_or_create(uuid.uuid4()) - return sbx - - -class SWEBenchTestCriterion(Criterion): - """Scores 1 iff the agent patch resolves all FAIL_TO_PASS and does not break PASS_TO_PASS.""" - - def __init__(self, *, name: str, weight: float = 1.0) -> None: - super().__init__(name=name, weight=weight) - - async def evaluate( - self, task: BenchmarkTask, context: EvaluationContext - ) -> CriterionResult: - patch_text = (context.worker_output.artifacts or {}).get("patch") or context.worker_output.output or "" - if not patch_text.strip(): - return CriterionResult( - name=self.name, score=0.0, passed=False, - feedback="Empty patch — agent did not produce any edits.", - metadata={}, - ) - - payload = task.task_payload - row = _payload_to_swebench_row(payload) - spec = make_test_spec(row) - - sandbox = await _spawn_eval_sandbox() - try: - # 1. fresh checkout at base_commit (swebench install script does this; - # we rely on the sandbox being a fresh provision here). - inst_script = spec.install_repo_script # clones + checks out base_commit - r = await sandbox.commands.run( - f"bash -c {shlex.quote(inst_script)}", timeout=EVAL_TIMEOUT_SEC - ) - if r.exit_code != 0: - return _error_result(self.name, "install_repo failed", r.stdout) - - # 2. apply test_patch - await _write_and_apply(sandbox, "test.patch", payload["test_patch"]) - - # 3. apply agent patch - await _write_and_apply(sandbox, "agent.patch", patch_text) - - # 4. run eval script, capture full log - r = await sandbox.commands.run( - f"bash -c {shlex.quote(spec.eval_script)} 2>&1", - timeout=EVAL_TIMEOUT_SEC, - ) - log = r.stdout or "" - - # 5. grade - report = grade_log( - instance_id=payload["instance_id"], - log=log, - fail_to_pass=payload["fail_to_pass"], - pass_to_pass=payload["pass_to_pass"], - ) - resolved = bool(report.get("resolved")) - return CriterionResult( - name=self.name, - score=1.0 if resolved else 0.0, - passed=resolved, - feedback=_format_feedback(report), - metadata=report, - ) - finally: - try: - await sandbox.kill() - except Exception: # noqa: BLE001 - logger.warning("Failed to kill eval sandbox", exc_info=True) - - -async def _write_and_apply(sandbox, filename: str, content: str) -> None: - path = f"/tmp/{filename}" - await sandbox.files.write(path, content.encode()) - r = await sandbox.commands.run( - f"cd {WORKDIR} && git apply --allow-empty --verbose {path}", timeout=120 - ) - if r.exit_code != 0: - # Try 3-way; some patches need it - r = await sandbox.commands.run( - f"cd {WORKDIR} && git apply --3way --verbose {path}", timeout=120 - ) - if r.exit_code != 0: - raise RuntimeError(f"git apply {filename} failed: {(r.stdout or '')[-800:]}") - - -def _error_result(name: str, kind: str, detail: str) -> CriterionResult: - return CriterionResult( - name=name, score=0.0, passed=False, - feedback=f"{kind}: {(detail or '')[-400:]}", metadata={"error": kind}, - ) - - -def _format_feedback(report: dict) -> str: - f2p = report.get("fail_to_pass", {}) - p2p = report.get("pass_to_pass", {}) - return ( - f"FAIL_TO_PASS success={len(f2p.get('success', []))} " - f"failure={len(f2p.get('failure', []))}; " - f"PASS_TO_PASS success={len(p2p.get('success', []))} " - f"failure={len(p2p.get('failure', []))}" - ) -``` - -Note on `grade_log`: the `swebench` package's grading API surface has shifted between 2.x and 3.x. If the `_status_map_override` / `_fail_to_pass_override` kwargs don't exist in the installed version, adapt by writing the log to a temp file and feeding it to the public `get_eval_report(test_spec=spec, prediction={...}, test_log_path=log_path)`. The behavioural contract is the same. - -- [ ] **Step 4: Run tests to confirm they pass** - -Run: `uv run pytest tests/swebench_verified/test_criterion.py -v` -Expected: 3 passed. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py \ - tests/swebench_verified/test_criterion.py -git commit -m "feat(swebench): test-resolution criterion using swebench harness" -``` - ---- - -## Task 11: Rubric (evaluator wrapping the criterion) - -**Files:** -- Create: `ergon/ergon_builtins/ergon_builtins/evaluators/rubrics/swebench_rubric.py` -- Test: `ergon/tests/swebench_verified/test_rubric.py` -- Modify: `ergon/ergon_builtins/ergon_builtins/registry_data.py` - -- [ ] **Step 1: Write failing test** - -Create `ergon/tests/swebench_verified/test_rubric.py`: - -```python -"""Tests for SWEBenchRubric.""" - -from ergon_builtins.evaluators.rubrics.swebench_rubric import SWEBenchRubric - - -def test_rubric_contains_single_test_resolution_criterion() -> None: - rubric = SWEBenchRubric(name="swebench-rubric") - names = [c.name for c in rubric.criteria] - assert names == ["test-resolution"] - assert rubric.criteria[0].weight == 1.0 -``` - -- [ ] **Step 2: Run test to confirm it fails** - -Run: `uv run pytest tests/swebench_verified/test_rubric.py -v` -Expected: `ModuleNotFoundError`. - -- [ ] **Step 3: Implement the rubric** - -Create `ergon/ergon_builtins/ergon_builtins/evaluators/rubrics/swebench_rubric.py`: - -```python -"""Evaluator rubric for SWE-Bench Verified.""" - -from __future__ import annotations - -from typing import ClassVar - -from ergon_core.api.evaluator import Rubric - -from ergon_builtins.benchmarks.swebench_verified.criterion import ( - SWEBenchTestCriterion, -) - - -class SWEBenchRubric(Rubric): - type_slug: ClassVar[str] = "swebench-rubric" - - def __init__(self, *, name: str = "swebench-rubric") -> None: - super().__init__( - name=name, - criteria=[SWEBenchTestCriterion(name="test-resolution", weight=1.0)], - ) -``` - -- [ ] **Step 4: Run test to confirm it passes** - -Run: `uv run pytest tests/swebench_verified/test_rubric.py -v` -Expected: 1 passed. - -- [ ] **Step 5: Register in `registry_data.py`** - -Add to `ergon/ergon_builtins/ergon_builtins/registry_data.py`: - -```python -from ergon_builtins.evaluators.rubrics.swebench_rubric import SWEBenchRubric -# ... -EVALUATORS["swebench-rubric"] = SWEBenchRubric -``` - -- [ ] **Step 6: Commit** - -```bash -git add ergon_builtins/ergon_builtins/evaluators/rubrics/swebench_rubric.py \ - ergon_builtins/ergon_builtins/registry_data.py \ - tests/swebench_verified/test_rubric.py -git commit -m "feat(swebench): rubric wrapping test-resolution criterion" -``` - ---- - -## Task 12: CLI composition wiring - -**Files:** Verify only — composition should already work via registries. - -- [ ] **Step 1: Confirm the slugs compose into an experiment** - -Run: -```bash -uv run ergon benchmark list | grep swebench-verified -``` - -Expected: `swebench-verified` appears. - -Run: -```bash -uv run python -c " -from ergon_cli.composition import build_experiment -exp = build_experiment( - benchmark_slug='swebench-verified', - model='claude-opus-4-6', - worker_slug='swebench-react', - evaluator_slug='swebench-rubric', - limit=1, -) -print(type(exp).__name__, 'ok') -" -``` - -Expected: `Experiment ok`. - -- [ ] **Step 2: Commit (no-op if nothing changed)** - -If CLI wiring required changes, commit; otherwise skip. - ---- - -## Task 13: End-to-end smoke test - -**Files:** -- Create: `ergon/tests/swebench_verified/test_smoke_e2e.py` - -The smoke test runs **one** instance end-to-end with a stubbed worker that produces an empty patch, confirming the full pipeline (dataset load → task build → worker invocation → criterion scoring) wires without errors. We don't run a real agent here — that belongs in a manual/opt-in E2E run. - -- [ ] **Step 1: Write the smoke test** - -Create `ergon/tests/swebench_verified/test_smoke_e2e.py`: - -```python -"""End-to-end smoke for SWE-Bench Verified wiring (no LLM, no E2B).""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from ergon_cli.composition import build_experiment - - -@pytest.mark.asyncio -async def test_compose_and_build_instances_with_limit_1() -> None: - fake_row = { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "aaa", - "patch": "GOLD", - "test_patch": "TP", - "problem_statement": "p", - "hints_text": "", - "version": "3.0", - "FAIL_TO_PASS": '["t1"]', - "PASS_TO_PASS": '["t0"]', - "environment_setup_commit": "aaa", - } - with patch( - "ergon_builtins.benchmarks.swebench_verified.benchmark._load_rows", - return_value=[fake_row], - ): - exp = build_experiment( - benchmark_slug="swebench-verified", - model="stub", - worker_slug="swebench-react", - evaluator_slug="swebench-rubric", - limit=1, - ) - tasks = exp.benchmark.build_instances()["default"] - - assert len(tasks) == 1 - assert tasks[0].task_key == "django__django-1" - assert "GOLD" not in tasks[0].description -``` - -- [ ] **Step 2: Run it** - -Run: `uv run pytest tests/swebench_verified/test_smoke_e2e.py -v` -Expected: 1 passed. - -- [ ] **Step 3: Run the full suite for the new package** - -Run: `uv run pytest tests/swebench_verified/ -v` -Expected: all tests in the package pass (roughly 15 tests total across all test files). - -- [ ] **Step 4: Run repo-wide checks** - -Run: -```bash -pnpm run check:be -``` - -Expected: ruff, ty, slopcop all clean. - -- [ ] **Step 5: Commit** - -```bash -git add tests/swebench_verified/test_smoke_e2e.py -git commit -m "test(swebench): end-to-end composition smoke" -``` - ---- - -## Out of scope for this plan - -Deliberately deferred — call them out explicitly so future work knows: - -- **Large-scale parallel eval runs.** Concurrency tuning against 500 instances, Inngest throttling, E2B cost budgeting. First cut is `--limit 5` on a known-easy subset (e.g., flask / requests / pytest instances with small test suites). -- **Wheel cache pre-warming.** `UV_CACHE_DIR=/workspace/.uv-cache` is already wired but a persistent E2B volume mount is needed to amortize the cost across runs. That's a sandbox-infra change, not a benchmark change. -- **Retry logic for flaky tests.** SWE-Bench is known to have some non-determinism. The canonical harness retries; we currently don't. Add only if measured flakiness matters. -- **Leaderboard submission format.** The `predictions.jsonl` artifact format for the SWE-Bench leaderboard isn't produced here — easy to add a CLI `ergon benchmark export swebench-verified --format predictions`. -- **SWE-Bench full / Lite variants.** Same code path, different `HF_DATASET_ID`. Trivially pluggable via a subclass once this lands. diff --git a/docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md b/docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md deleted file mode 100644 index 4ef8ec6fa..000000000 --- a/docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md +++ /dev/null @@ -1,1292 +0,0 @@ -# Real-LLM debug harness — Implementation plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Ship the real-LLM debug harness described in -`docs/rfcs/active/2026-04-21-real-llm-debug-harness.md` — a pytest tier at -`tests/real_llm/` that runs real Sonnet-4.6-via-OpenRouter experiments, -asserts Postgres + Playwright, and is used as a bug-hunting instrument to -validate the three benchmark sandboxes end-to-end. - -**Architecture:** Generic ReAct worker (`react-generic` slug) + per-benchmark -DI'd toolkit via `benchmark_toolkit_composer`. Tests drive the shipped -`ergon benchmark run` CLI via subprocess against a full local stack -(`docker-compose.real-llm.yml`) with a `--assume-stack-up` escape for dev -iteration. OpenRouter cost-gated; Playwright dashboard assertions must-have. - -**Tech Stack:** pytest, `httpx`, Playwright (async), Docker Compose, Ergon CLI, -`ergon_core.core.providers.generation.openrouter_budget` (new), `ergon_builtins.tools.benchmark_toolkit_composer` (new). - -**Rollout:** This plan covers **PR 1 only** (Tasks 0–10). PR 2 (three-example -artifact) and the bug-hunt phase between them are driven by the RFC but run -as follow-ups on top of a merged PR 1. - ---- - -## Preconditions - -- `smoke-shared-infra` PR #25 **merged to `main`** — the `/api/test/*` - harness endpoints this plan polls are from that PR. If #25 is not yet - merged, **stop** and wait; do not proceed on a branched-off-PR-25 strategy - (merge conflicts multiply). This plan's branch is - `feature/real-llm-harness-infra` off a `main` that already contains #25. - -## Task 0 — Preflight - -**Files:** none yet. - -- [ ] **Step 0.1: Verify smoke-shared-infra PR is merged** - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon -git fetch origin main -git log origin/main --oneline | head -20 | grep -E "smoke shared infra|api/test" || { - echo "PR #25 not merged yet — stop here." - exit 1 -} -``` - -Expected: at least one line matches, confirming the harness endpoints are on -main. If not, stop and wait. - -- [ ] **Step 0.2: Branch off main** - -```bash -git checkout main -git pull origin main -git checkout -b feature/real-llm-harness-infra -``` - -- [ ] **Step 0.3: Sanity-check the harness endpoints are present on main** - -```bash -ls ergon_core/ergon_core/core/api/test_harness.py -grep -c "write/run/seed\|read/run\|write/reset" ergon_core/ergon_core/core/api/test_harness.py -``` - -Expected: file exists, ≥3 matches for the endpoint route strings. - ---- - -## Task 1 — OpenRouter budget gate module - -**Files:** -- Create: `ergon_core/ergon_core/core/providers/generation/openrouter_budget.py` -- Test: `tests/unit/test_openrouter_budget.py` - -Budget module is fully testable without any real API key — just mock -`httpx.AsyncClient.get`. TDD first. - -- [ ] **Step 1.1: Write the failing test** - -```python -# tests/unit/test_openrouter_budget.py -"""OpenRouterBudget: snapshot baseline, compute delta, gate spend.""" - -from unittest.mock import AsyncMock, patch - -import pytest - -from ergon_core.core.providers.generation.openrouter_budget import OpenRouterBudget - - -@pytest.mark.asyncio -async def test_remaining_usd_returns_limit_minus_delta() -> None: - budget = OpenRouterBudget(limit_usd=5.0, api_key="test-key") - - async def _mock_get(*_args: object, **_kwargs: object) -> object: - class _Resp: - status_code = 200 - def raise_for_status(self) -> None: - return None - def json(self) -> dict[str, object]: - return {"data": {"usage": 2.50, "limit": 100.0, "limit_remaining": 97.50}} - return _Resp() - - with patch("httpx.AsyncClient.get", new=AsyncMock(side_effect=_mock_get)): - await budget.snapshot_baseline() - - # usage is same as baseline on first call => spent 0, remaining = limit - assert (await budget.remaining_usd()) == pytest.approx(5.0) - - -@pytest.mark.asyncio -async def test_remaining_usd_after_spend() -> None: - budget = OpenRouterBudget(limit_usd=5.0, api_key="test-key") - - # First call sets baseline at usage=2.50; second call reports usage=3.70; - # delta is 1.20; remaining = 5.0 - 1.20 = 3.80. - usages = iter([2.50, 3.70]) - - async def _mock_get(*_args: object, **_kwargs: object) -> object: - next_usage = next(usages) - - class _Resp: - status_code = 200 - def raise_for_status(self) -> None: - return None - def json(self) -> dict[str, object]: - return {"data": {"usage": next_usage, "limit": 100.0, "limit_remaining": 97.5}} - return _Resp() - - with patch("httpx.AsyncClient.get", new=AsyncMock(side_effect=_mock_get)): - await budget.snapshot_baseline() - assert (await budget.remaining_usd()) == pytest.approx(3.80) - - -@pytest.mark.asyncio -async def test_remaining_usd_raises_without_snapshot() -> None: - budget = OpenRouterBudget(limit_usd=5.0, api_key="test-key") - - with pytest.raises(RuntimeError, match="baseline"): - await budget.remaining_usd() -``` - -- [ ] **Step 1.2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/test_openrouter_budget.py -v -``` - -Expected: FAIL with `ModuleNotFoundError: No module named -'ergon_core.core.providers.generation.openrouter_budget'`. - -- [ ] **Step 1.3: Write the module** - -```python -# ergon_core/ergon_core/core/providers/generation/openrouter_budget.py -"""Track cumulative OpenRouter spend against a per-session budget. - -Usage: - budget = OpenRouterBudget(limit_usd=5.0, api_key=os.environ["OPENROUTER_API_KEY"]) - await budget.snapshot_baseline() # at pytest session start - ... - if await budget.remaining_usd() <= 0: - pytest.skip("OpenRouter budget exhausted") -""" - -import httpx - - -_KEY_ENDPOINT = "https://openrouter.ai/api/v1/auth/key" - - -class OpenRouterBudget: - """Snapshot cumulative OpenRouter spend and compare against a limit.""" - - def __init__(self, *, limit_usd: float, api_key: str) -> None: - self._limit = limit_usd - self._api_key = api_key - self._baseline: float | None = None - - async def snapshot_baseline(self) -> None: - self._baseline = await self._current_usage() - - async def remaining_usd(self) -> float: - if self._baseline is None: - raise RuntimeError("snapshot_baseline must be called before remaining_usd") - current = await self._current_usage() - return self._limit - (current - self._baseline) - - async def _current_usage(self) -> float: - async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.get( - _KEY_ENDPOINT, - headers={"Authorization": f"Bearer {self._api_key}"}, - ) - resp.raise_for_status() - return float(resp.json()["data"]["usage"]) -``` - -- [ ] **Step 1.4: Run test to verify it passes** - -```bash -uv run pytest tests/unit/test_openrouter_budget.py -v -``` - -Expected: 3 passed. - -- [ ] **Step 1.5: Commit** - -```bash -git add ergon_core/ergon_core/core/providers/generation/openrouter_budget.py \ - tests/unit/test_openrouter_budget.py -git commit -m "feat(budget): OpenRouterBudget snapshot + delta gate (module + unit tests)" -``` - ---- - -## Task 2 — `benchmark_toolkit_composer` module - -**Files:** -- Create: `ergon_builtins/ergon_builtins/tools/benchmark_toolkit_composer.py` -- Test: `tests/unit/test_benchmark_toolkit_composer.py` - -The composer takes `benchmark_slug`, `ctx: WorkerContext`, and `sandbox` -and returns the union of tools a generic ReAct worker needs for that -benchmark. **Does not** construct tools that require live sandbox skills — -returns callable protocols that will be satisfied at execute time. - -- [ ] **Step 2.1: Write the failing test** - -```python -# tests/unit/test_benchmark_toolkit_composer.py -"""benchmark_toolkit_composer: per-benchmark DI factory for generic ReAct.""" - -from types import SimpleNamespace -from unittest.mock import MagicMock -from uuid import uuid4 - -import pytest - -from ergon_builtins.tools.benchmark_toolkit_composer import compose_benchmark_toolkit - - -def _make_ctx() -> SimpleNamespace: - return SimpleNamespace( - run_id=uuid4(), - node_id=uuid4(), - execution_id=uuid4(), - sandbox_id="sb-test", - task_id=uuid4(), - definition_id=None, - metadata={}, - ) - - -def test_compose_researchrubrics_unions_lifecycle_rr_and_graph() -> None: - tools = compose_benchmark_toolkit( - benchmark_slug="researchrubrics", - ctx=_make_ctx(), - sandbox=MagicMock(), - run_skill=MagicMock(), - publisher_sync=MagicMock(), - ) - # Minimum union size: 8 (lifecycle) + 6 (rr) + 6 (graph) = 20 - assert len(tools) >= 20 - - -def test_compose_minif2f_unions_lifecycle_and_minif2f() -> None: - tools = compose_benchmark_toolkit( - benchmark_slug="minif2f", - ctx=_make_ctx(), - sandbox=MagicMock(), - ) - # Minimum: 8 (lifecycle) + Lean toolkit (≥5) = 13 - assert len(tools) >= 13 - - -def test_compose_swebench_unions_lifecycle_and_swebench() -> None: - tools = compose_benchmark_toolkit( - benchmark_slug="swebench-verified", - ctx=_make_ctx(), - sandbox=MagicMock(), - ) - # Minimum: 8 (lifecycle) + bash + str-replace = 10 - assert len(tools) >= 10 - - -def test_compose_unknown_slug_raises() -> None: - with pytest.raises(ValueError, match="no toolkit composer for"): - compose_benchmark_toolkit( - benchmark_slug="unknown", - ctx=_make_ctx(), - sandbox=MagicMock(), - ) -``` - -- [ ] **Step 2.2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/test_benchmark_toolkit_composer.py -v -``` - -Expected: FAIL with `ModuleNotFoundError`. - -- [ ] **Step 2.3: Write the module** - -```python -# ergon_builtins/ergon_builtins/tools/benchmark_toolkit_composer.py -"""Per-benchmark DI factory — unions `SubtaskLifecycleToolkit` with the -env-specific toolkit so a single generic ReAct worker can exercise any -of the three target benchmarks.""" - -from collections.abc import Awaitable, Callable -from typing import Any, Protocol - - -class _HasContextFields(Protocol): - run_id: Any # slopcop: ignore[no-typing-any] - node_id: Any # slopcop: ignore[no-typing-any] - execution_id: Any # slopcop: ignore[no-typing-any] - sandbox_id: str - - -def compose_benchmark_toolkit( - *, - benchmark_slug: str, - ctx: _HasContextFields, - sandbox: Any, # slopcop: ignore[no-typing-any] - run_skill: Callable[..., Awaitable[Any]] | None = None, # slopcop: ignore[no-typing-any] - publisher_sync: Callable[[], Awaitable[list[Any]]] | None = None, # slopcop: ignore[no-typing-any] -) -> list[Any]: # slopcop: ignore[no-typing-any] - """Return the union of Tools a generic ReAct worker needs for benchmark_slug.""" - from ergon_builtins.tools.subtask_lifecycle_toolkit import SubtaskLifecycleToolkit - - lifecycle = SubtaskLifecycleToolkit( - run_id=ctx.run_id, - parent_node_id=ctx.node_id, - sandbox_id=ctx.sandbox_id, - ).get_tools() - - match benchmark_slug: - case "researchrubrics": - from ergon_builtins.tools.graph_toolkit import ResearchGraphToolkit - from ergon_builtins.tools.research_rubrics_toolkit import ( - ResearchRubricsToolkit, - ) - - if run_skill is None or publisher_sync is None: - raise ValueError( - "researchrubrics composer requires run_skill + publisher_sync" - ) - rr = ResearchRubricsToolkit( - run_skill=run_skill, - publisher_sync=publisher_sync, - ).build_tools() - graph = ResearchGraphToolkit( - run_id=ctx.run_id, - task_execution_id=ctx.execution_id, - ).build_tools() - return [*lifecycle, *rr, *graph] - case "minif2f": - from ergon_builtins.benchmarks.minif2f.toolkit import MiniF2FToolkit - - return [*lifecycle, *MiniF2FToolkit(sandbox=sandbox).get_tools()] - case "swebench-verified": - from ergon_builtins.benchmarks.swebench_verified.toolkit import SWEBenchToolkit - - return [*lifecycle, *SWEBenchToolkit(sandbox=sandbox).get_tools()] - case _: - raise ValueError(f"no toolkit composer for {benchmark_slug!r}") -``` - -- [ ] **Step 2.4: Run test to verify it passes** - -```bash -uv run pytest tests/unit/test_benchmark_toolkit_composer.py -v -``` - -Expected: 4 passed. If the existing toolkits raise at `get_tools()` / -`build_tools()` because of missing `pydantic_ai.tools.Tool`, the test -environment already has pydantic-ai; confirm by `uv pip list | grep pydantic-ai`. - -- [ ] **Step 2.5: Commit** - -```bash -git add ergon_builtins/ergon_builtins/tools/benchmark_toolkit_composer.py \ - tests/unit/test_benchmark_toolkit_composer.py -git commit -m "feat(tools): benchmark_toolkit_composer DI factory (3 envs + unit tests)" -``` - ---- - -## Task 3 — `ReActGenericWorker` + registration - -**Files:** -- Create: `ergon_builtins/ergon_builtins/workers/baselines/react_generic_worker.py` -- Modify: `ergon_builtins/ergon_builtins/registry_core.py:47` (add `"react-generic"` to WORKERS) -- Test: `tests/unit/test_react_generic_worker.py` - -Thin subclass of `ReActWorker` that reads its benchmark slug from -`ctx.metadata["toolkit_benchmark"]`, composes the toolkit at execute-time -against the live sandbox, and delegates to `super().execute()`. Pattern -mirrors `minif2f_react_worker.py`. - -- [ ] **Step 3.1: Write the failing test** - -```python -# tests/unit/test_react_generic_worker.py -"""ReActGenericWorker: composes toolkit from ctx.metadata['toolkit_benchmark'].""" - -from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import pytest - -from ergon_builtins.workers.baselines.react_generic_worker import ReActGenericWorker - - -def _ctx(benchmark_slug: str) -> SimpleNamespace: - return SimpleNamespace( - run_id=uuid4(), - node_id=uuid4(), - execution_id=uuid4(), - sandbox_id="sb-test", - task_id=uuid4(), - definition_id=None, - metadata={"toolkit_benchmark": benchmark_slug}, - ) - - -@pytest.mark.asyncio -async def test_execute_composes_toolkit_from_metadata_for_swebench() -> None: - worker = ReActGenericWorker(name="w", model="x") - ctx = _ctx("swebench-verified") - fake_sandbox = MagicMock() - - called: dict[str, object] = {} - - def _spy(**kwargs: object) -> list[object]: - called.update(kwargs) - return ["tool-a", "tool-b"] - - # Patch both sandbox-connect and the composer. The worker must yield at - # least once; we short-circuit super().execute by monkeypatching ReActWorker. - with ( - patch( - "ergon_builtins.workers.baselines.react_generic_worker.AsyncSandbox.connect", - AsyncMock(return_value=fake_sandbox), - ), - patch( - "ergon_builtins.workers.baselines.react_generic_worker.compose_benchmark_toolkit", - side_effect=_spy, - ), - patch.object( - ReActGenericWorker.__mro__[1], - "execute", - return_value=_async_iter([]), - ), - ): - _turns = [t async for t in worker.execute(task=None, context=ctx)] - - assert called["benchmark_slug"] == "swebench-verified" - assert worker.tools == ["tool-a", "tool-b"] - - -def _async_iter(items: list[object]) -> object: - async def _gen(): - for i in items: - yield i - return _gen() - - -def test_raises_if_metadata_missing_toolkit_benchmark() -> None: - worker = ReActGenericWorker(name="w", model="x") - ctx = SimpleNamespace( - run_id=uuid4(), - node_id=uuid4(), - execution_id=uuid4(), - sandbox_id="sb-test", - task_id=uuid4(), - definition_id=None, - metadata={}, - ) - with pytest.raises(ValueError, match="toolkit_benchmark"): - worker._benchmark_slug(ctx) -``` - -- [ ] **Step 3.2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/test_react_generic_worker.py -v -``` - -Expected: FAIL (module not found). - -- [ ] **Step 3.3: Write the module** - -```python -# ergon_builtins/ergon_builtins/workers/baselines/react_generic_worker.py -"""Generic ReAct worker that composes its toolkit per benchmark from metadata. - -Reads `ctx.metadata["toolkit_benchmark"]` at execute() time, opens the -sandbox, calls `compose_benchmark_toolkit(...)`, stashes the result on -`self.tools`, and delegates to `super().execute()`. - -Intended for the real-LLM debug harness; validates that `ReActWorker` + -the three composed toolkits behave correctly end-to-end against a real -model, without us having to maintain per-benchmark specialised workers. -""" - -from collections.abc import AsyncGenerator -from typing import Any - -from e2b_code_interpreter import AsyncSandbox - -from ergon_core.api import BenchmarkTask, WorkerContext, WorkerOutput -from ergon_core.api.generation import GenerationTurn - -from ergon_builtins.tools.benchmark_toolkit_composer import compose_benchmark_toolkit -from ergon_builtins.workers.baselines.react_worker import ReActWorker - - -class ReActGenericWorker(ReActWorker): - type_slug = "react-generic" - - def _benchmark_slug(self, ctx: WorkerContext) -> str: - slug = ctx.metadata.get("toolkit_benchmark") if ctx.metadata else None - if not isinstance(slug, str) or not slug: - raise ValueError( - "ReActGenericWorker requires ctx.metadata['toolkit_benchmark']" - ) - return slug - - async def execute( - self, - task: BenchmarkTask | None, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - slug = self._benchmark_slug(context) - sandbox: Any = await AsyncSandbox.connect(context.sandbox_id) # slopcop: ignore[no-typing-any] - - # For researchrubrics we'd also need run_skill + publisher_sync; leave - # None-default and let the composer raise if that branch is hit without - # them being wired at a higher layer. PR 2 adds the wiring. - self.tools = compose_benchmark_toolkit( - benchmark_slug=slug, - ctx=context, - sandbox=sandbox, - ) - - async for turn in super().execute(task, context=context): - yield turn -``` - -- [ ] **Step 3.4: Register** - -Modify `ergon_builtins/ergon_builtins/registry_core.py` at line 47-56 -(`WORKERS` dict). Add import: - -```python -from ergon_builtins.workers.baselines.react_generic_worker import ReActGenericWorker -``` - -And add to the dict: - -```python - "react-generic": ReActGenericWorker, -``` - -- [ ] **Step 3.5: Run test to verify it passes** - -```bash -uv run pytest tests/unit/test_react_generic_worker.py -v -uv run pytest tests/unit/ -k "registry or react_generic" -v -``` - -Expected: all green. - -- [ ] **Step 3.6: Commit** - -```bash -git add ergon_builtins/ergon_builtins/workers/baselines/react_generic_worker.py \ - ergon_builtins/ergon_builtins/registry_core.py \ - tests/unit/test_react_generic_worker.py -git commit -m "feat(worker): ReActGenericWorker reads toolkit_benchmark from ctx.metadata" -``` - ---- - -## Task 4 — CLI wiring: `--toolkit-benchmark` flag - -**Files:** -- Modify: `ergon_cli/ergon_cli/commands/benchmark.py` (add CLI arg; pass as metadata) -- Modify: `ergon_cli/ergon_cli/composition/__init__.py` (honour metadata→worker) -- Test: `tests/unit/test_cli_react_generic_composition.py` - -`build_experiment` already resolves `worker_slug` from the registry. We need -to pipe a `toolkit_benchmark: str` value into `Experiment.metadata` so that, -when the runtime calls `worker.execute(task, context)`, `context.metadata` -contains `toolkit_benchmark`. The simplest wiring: add a kwarg to -`build_experiment`, route it onto each `BenchmarkTask.metadata`. - -- [ ] **Step 4.1: Write the failing test** - -```python -# tests/unit/test_cli_react_generic_composition.py -"""Smoke: build_experiment(worker=react-generic, toolkit_benchmark=...) puts -the slug into BenchmarkTask metadata so ReActGenericWorker can read it.""" - -from ergon_cli.composition import build_experiment - - -def test_react_generic_toolkit_benchmark_propagates_into_task_metadata() -> None: - exp = build_experiment( - benchmark_slug="smoke-test", - model="stub:constant", - worker_slug="react-generic", - evaluator_slug="stub-rubric", - toolkit_benchmark="swebench-verified", - limit=1, - ) - instances = exp.benchmark.build_instances() - tasks = [t for tasks_for_cohort in instances.values() for t in tasks_for_cohort] - assert tasks, "benchmark produced no tasks" - assert all( - t.metadata.get("toolkit_benchmark") == "swebench-verified" for t in tasks - ) -``` - -- [ ] **Step 4.2: Run to verify failure** - -```bash -uv run pytest tests/unit/test_cli_react_generic_composition.py -v -``` - -Expected: FAIL — `build_experiment()` has no `toolkit_benchmark` kwarg. - -- [ ] **Step 4.3: Add the kwarg in composition** - -In `ergon_cli/ergon_cli/composition/__init__.py`: - -- Add `toolkit_benchmark: str | None = None` to `build_experiment` signature. -- After `benchmark = _construct_benchmark(...)`, if `toolkit_benchmark` is - not None, walk `benchmark.build_instances()` and mutate each - `BenchmarkTask.metadata["toolkit_benchmark"] = toolkit_benchmark`. -- For `worker_slug == "react-generic"`, fall into the default - `Experiment.from_single_worker(...)` branch (the existing catch-all - already handles it once the worker is in WORKERS). - -Smallest diff shape (inside the `_` catch-all case before -`Experiment.from_single_worker`): - -```python -if toolkit_benchmark is not None: - for tasks in benchmark.build_instances().values(): - for task in tasks: - # BenchmarkTask is immutable where it's a frozen model; fall back - # to the mutable metadata dict. - task.metadata["toolkit_benchmark"] = toolkit_benchmark -``` - -(If `BenchmarkTask.metadata` is frozen in your build, construct new tasks; -inspect `ergon_core/api/task_types.py` and adapt.) - -- [ ] **Step 4.4: Expose on the CLI** - -In `ergon_cli/ergon_cli/commands/benchmark.py`, find the -`benchmark run` arg parser section and add: - -```python -parser.add_argument( - "--toolkit-benchmark", - default=None, - help="When --worker=react-generic, which benchmark's toolkit to compose.", -) -``` - -And pass `toolkit_benchmark=args.toolkit_benchmark` into the -`build_experiment(...)` call. - -- [ ] **Step 4.5: Run tests** - -```bash -uv run pytest tests/unit/test_cli_react_generic_composition.py tests/cli -v -``` - -Expected: new test passes; existing CLI tests unaffected. - -- [ ] **Step 4.6: Commit** - -```bash -git add ergon_cli/ergon_cli/commands/benchmark.py \ - ergon_cli/ergon_cli/composition/__init__.py \ - tests/unit/test_cli_react_generic_composition.py -git commit -m "feat(cli): --toolkit-benchmark flag propagates into task metadata" -``` - ---- - -## Task 5 — `tests/real_llm/` scaffolding + conftest - -**Files:** -- Create: `tests/real_llm/__init__.py` (empty) -- Create: `tests/real_llm/conftest.py` -- Modify: `pyproject.toml` to register `real_llm` pytest marker - -- [ ] **Step 5.1: Create empty package marker** - -```bash -mkdir -p tests/real_llm -touch tests/real_llm/__init__.py -``` - -- [ ] **Step 5.2: Add marker to `pyproject.toml`** - -Find the `[tool.pytest.ini_options]` block (currently contains `markers = -[...]` with `integration`, etc.). Add: - -```toml - "real_llm: real-LLM end-to-end tests (requires ERGON_REAL_LLM=1 + OPENROUTER_API_KEY)", -``` - -- [ ] **Step 5.3: Write the conftest** - -```python -# tests/real_llm/conftest.py -"""Session-level fixtures for the real-LLM tier. - -Gates: - - ERGON_REAL_LLM=1 must be set (else the entire tier skips). - - OPENROUTER_API_KEY must be set (else real-LLM tests skip; stub canary - continues to run if it opts in explicitly). - - --assume-stack-up flag skips the docker-compose fixture and trusts the - developer to have the stack running (pnpm dev:test + postgres + inngest - + fastapi). - -Session fixtures (docker stack, OpenRouter budget) live here; per-benchmark -fixtures live inside each test module. -""" - -import os - -import pytest - - -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - "--assume-stack-up", - action="store_true", - default=False, - help="Skip docker-compose fixture; trust the developer to have the " - "full stack (dashboard + backend + postgres + inngest) running.", - ) - - -@pytest.fixture(scope="session") -def real_llm_enabled() -> bool: - return os.environ.get("ERGON_REAL_LLM") == "1" - - -@pytest.fixture(autouse=True) -def _skip_if_not_enabled(real_llm_enabled: bool, request: pytest.FixtureRequest) -> None: - if request.node.get_closest_marker("real_llm") and not real_llm_enabled: - pytest.skip("ERGON_REAL_LLM=1 not set; real-LLM tier is opt-in") -``` - -- [ ] **Step 5.4: Confirm discovery** - -```bash -uv run pytest tests/real_llm --collect-only -``` - -Expected: 0 tests collected (no test files yet), no errors. - -- [ ] **Step 5.5: Commit** - -```bash -git add tests/real_llm/__init__.py tests/real_llm/conftest.py pyproject.toml -git commit -m "feat(real-llm): tests/real_llm scaffolding + pytest marker + conftest" -``` - ---- - -## Task 6 — `docker-compose.real-llm.yml` + stack fixture - -**Files:** -- Create: `docker-compose.real-llm.yml` -- Create: `tests/real_llm/fixtures/__init__.py` -- Create: `tests/real_llm/fixtures/stack.py` - -Overlay includes Postgres, Inngest, FastAPI (ENABLE_TEST_HARNESS=1), and a -headed `pnpm dev:test` dashboard — everything the harness polls / probes / -Playwright-asserts. - -- [ ] **Step 6.1: Write the compose file** - -```yaml -# docker-compose.real-llm.yml -services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: ergon - POSTGRES_PASSWORD: ergon - POSTGRES_DB: ergon - ports: ["5433:5432"] - inngest: - image: inngest/inngest:latest - command: inngest dev --port 8288 - ports: ["8288:8288"] - api: - build: { context: ., dockerfile: infra/Dockerfile.api } - environment: - DATABASE_URL: postgres://ergon:ergon@postgres:5432/ergon - ENABLE_TEST_HARNESS: "1" - TEST_HARNESS_SECRET: "real-llm-secret" - INNGEST_DEV: "http://inngest:8288" - ERGON_API_BASE_URL: "http://api:9000" - depends_on: [postgres, inngest] - ports: ["9000:9000"] - dashboard: - build: { context: ergon-dashboard, dockerfile: Dockerfile.test } - environment: - ERGON_API_BASE_URL: "http://api:9000" - ENABLE_TEST_HARNESS: "1" - depends_on: [api] - ports: ["3101:3101"] -``` - -(Note: if `infra/Dockerfile.api` and `ergon-dashboard/Dockerfile.test` do -not yet exist, create minimal stubs based on the repo's existing compose -files; run `ls infra/` + `ls ergon-dashboard/` first to discover.) - -- [ ] **Step 6.2: Write the stack fixture** - -```python -# tests/real_llm/fixtures/stack.py -"""docker-compose up/down session fixture with --assume-stack-up flag.""" - -import subprocess -import time -from collections.abc import Generator - -import httpx -import pytest - -_COMPOSE_FILE = "docker-compose.real-llm.yml" -_API_URL = "http://127.0.0.1:9000" -_DASHBOARD_URL = "http://127.0.0.1:3101" -_UP_TIMEOUT_S = 120 - - -def _wait_for(url: str, timeout: float) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - try: - with httpx.Client(timeout=2.0) as client: - client.get(url) - return - except (httpx.ConnectError, httpx.ReadTimeout): - time.sleep(2.0) - raise RuntimeError(f"timed out waiting for {url}") - - -@pytest.fixture(scope="session") -def real_llm_stack(request: pytest.FixtureRequest) -> Generator[None, None, None]: - if request.config.getoption("--assume-stack-up"): - _wait_for(f"{_API_URL}/health", 10) - _wait_for(f"{_DASHBOARD_URL}", 10) - yield - return - - subprocess.run( - ["docker", "compose", "-f", _COMPOSE_FILE, "up", "-d", "--wait"], - check=True, - ) - try: - _wait_for(f"{_API_URL}/health", _UP_TIMEOUT_S) - _wait_for(f"{_DASHBOARD_URL}", _UP_TIMEOUT_S) - yield - finally: - subprocess.run( - ["docker", "compose", "-f", _COMPOSE_FILE, "down", "-v"], - check=False, - ) -``` - -- [ ] **Step 6.3: Smoke-test `--assume-stack-up` path** - -With a local `pnpm dev:test` + backend already running (if you have one), -collect the fixture without running any real test: - -```bash -uv run pytest tests/real_llm --collect-only --assume-stack-up -``` - -Expected: no errors; if no stack is up, the fixture isn't triggered by -collect-only anyway. - -- [ ] **Step 6.4: Commit** - -```bash -git add docker-compose.real-llm.yml tests/real_llm/fixtures/__init__.py \ - tests/real_llm/fixtures/stack.py -git commit -m "feat(real-llm): docker-compose overlay + stack session fixture" -``` - ---- - -## Task 7 — Budget + Playwright + harness client fixtures - -**Files:** -- Create: `tests/real_llm/fixtures/openrouter_budget.py` -- Create: `tests/real_llm/fixtures/playwright_client.py` -- Create: `tests/real_llm/fixtures/harness_client.py` (Python twin of the TS `BackendHarnessClient`) - -- [ ] **Step 7.1: Budget fixture** - -```python -# tests/real_llm/fixtures/openrouter_budget.py -import os -from collections.abc import AsyncGenerator - -import pytest - -from ergon_core.core.providers.generation.openrouter_budget import OpenRouterBudget - - -@pytest.fixture(scope="session") -async def openrouter_budget() -> AsyncGenerator[OpenRouterBudget, None]: - key = os.environ.get("OPENROUTER_API_KEY") - if not key: - pytest.skip("OPENROUTER_API_KEY not set — skipping real-LLM tests") - limit = float(os.environ.get("ERGON_REAL_LLM_BUDGET_USD", "5.0")) - budget = OpenRouterBudget(limit_usd=limit, api_key=key) - await budget.snapshot_baseline() - yield budget - - -@pytest.fixture(autouse=True) -async def _budget_gate(openrouter_budget: OpenRouterBudget) -> None: - remaining = await openrouter_budget.remaining_usd() - if remaining <= 0: - pytest.skip(f"OpenRouter budget exhausted (remaining=${remaining:.2f})") -``` - -- [ ] **Step 7.2: Harness client (Python, for `/api/test/read/run/*/state`)** - -```python -# tests/real_llm/fixtures/harness_client.py -import os -from typing import Any - -import httpx - - -class BackendHarnessClient: - """Python twin of ergon-dashboard/tests/helpers/testHarnessClient.ts.""" - - def __init__(self, base_url: str) -> None: - self._base = base_url - - def get_run_state(self, run_id: str) -> dict[str, Any]: # slopcop: ignore[no-typing-any] - with httpx.Client(timeout=10.0) as client: - r = client.get(f"{self._base}/api/test/read/run/{run_id}/state") - r.raise_for_status() - return r.json() - - def wait_for_terminal( - self, run_id: str, *, timeout_s: float = 600.0, poll_s: float = 3.0 - ) -> dict[str, Any]: # slopcop: ignore[no-typing-any] - import time - - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - state = self.get_run_state(run_id) - if state["status"] in {"completed", "failed", "cancelled"}: - return state - time.sleep(poll_s) - raise TimeoutError(f"run {run_id} did not reach terminal status in {timeout_s}s") - - -import pytest - -@pytest.fixture -def harness_client() -> BackendHarnessClient: - return BackendHarnessClient( - os.environ.get("ERGON_API_BASE_URL", "http://127.0.0.1:9000") - ) -``` - -- [ ] **Step 7.3: Playwright client** - -```python -# tests/real_llm/fixtures/playwright_client.py -import os -from collections.abc import AsyncGenerator - -import pytest -from playwright.async_api import Browser, BrowserContext, async_playwright - - -@pytest.fixture(scope="session") -async def playwright_browser() -> AsyncGenerator[Browser, None]: - async with async_playwright() as pw: - browser = await pw.chromium.launch() - yield browser - await browser.close() - - -@pytest.fixture -async def playwright_context( - playwright_browser: Browser, -) -> AsyncGenerator[BrowserContext, None]: - ctx = await playwright_browser.new_context( - base_url=os.environ.get("ERGON_DASHBOARD_URL", "http://127.0.0.1:3101"), - ) - yield ctx - await ctx.close() -``` - -- [ ] **Step 7.4: Wire into conftest** - -Edit `tests/real_llm/conftest.py` — append fixture imports so pytest finds -them: - -```python -from tests.real_llm.fixtures.openrouter_budget import openrouter_budget, _budget_gate # noqa: F401 -from tests.real_llm.fixtures.harness_client import harness_client # noqa: F401 -from tests.real_llm.fixtures.playwright_client import playwright_browser, playwright_context # noqa: F401 -from tests.real_llm.fixtures.stack import real_llm_stack # noqa: F401 -``` - -- [ ] **Step 7.5: Collect-check** - -```bash -uv run pytest tests/real_llm --collect-only -``` - -Expected: no import errors. - -- [ ] **Step 7.6: Commit** - -```bash -git add tests/real_llm/fixtures/ tests/real_llm/conftest.py -git commit -m "feat(real-llm): budget/harness/playwright fixtures wired into conftest" -``` - ---- - -## Task 8 — Canary: `test_smoke_stub.py` - -**Files:** -- Create: `tests/real_llm/benchmarks/__init__.py` (empty) -- Create: `tests/real_llm/benchmarks/test_smoke_stub.py` - -The canary uses the existing `smoke-test` benchmark with **stub workers** — -zero OpenRouter cost — to prove every layer of the harness wired correctly -(stack up, subprocess CLI, harness client, DB query, Playwright). - -- [ ] **Step 8.1: Write the canary** - -```python -# tests/real_llm/benchmarks/test_smoke_stub.py -"""Real-LLM harness canary — exercises the whole harness pipeline without -actually spending tokens. Uses the smoke-test benchmark + stub-worker path. - -Validates: - - docker stack up (or --assume-stack-up), stack fixture did not skip - - `ergon benchmark run` CLI path works - - /api/test/read/run/{id}/state returns a terminal state - - Postgres row exists with the right relationships - - Playwright can find the cohort in the dashboard -""" - -import os -import subprocess -from uuid import UUID - -import pytest - -pytestmark = [pytest.mark.real_llm, pytest.mark.asyncio] - - -async def test_harness_canary_smoke_stub( - real_llm_stack: None, - harness_client, # noqa: ANN001 - playwright_context, # noqa: ANN001 -) -> None: - # Run the CLI as a user would. - result = subprocess.run( - [ - "uv", "run", "ergon", "benchmark", "run", - "smoke-test", - "--model", "stub:constant", - "--worker", "stub-worker", - "--evaluator", "stub-rubric", - "--limit", "1", - "--json", - ], - capture_output=True, - text=True, - timeout=180, - ) - assert result.returncode == 0, f"CLI failed: {result.stderr}" - - # CLI prints a JSON blob with run_id when --json is passed; parse it. - import json - payload = json.loads(result.stdout.strip().splitlines()[-1]) - run_id = payload["run_id"] - UUID(run_id) # validate shape - - # Poll the harness until terminal. - state = harness_client.wait_for_terminal(run_id, timeout_s=120) - assert state["status"] == "completed", f"run did not complete: {state}" - assert len(state["graph_nodes"]) >= 1 - - # Playwright: cohort index renders, find the run row. - page = await playwright_context.new_page() - await page.goto("/") - await page.wait_for_load_state("networkidle") - # Loose assertion: the page rendered and didn't crash. - assert "cohort" in (await page.content()).lower() -``` - -- [ ] **Step 8.2: Confirm `--json` flag exists on `ergon benchmark run`** - -```bash -uv run ergon benchmark run --help 2>&1 | grep -- "--json" || { - echo "NOTE: --json flag may not exist; adjust Step 8.1 to parse run_id from table output" -} -``` - -If it doesn't exist, fall back to querying the most recent run via -`/api/test/read` (requires a seed-by-cohort helper) **or** add a -`--json` flag to the CLI (small PR) — defer to Step 8.3. - -- [ ] **Step 8.3: If no `--json` flag, extract run_id another way** - -Acceptable alternatives, in order of preference: - -1. Grep the CLI stdout for a UUID matching `RunRecord` created - within the last 60s. -2. Query `get_session()` for the newest `RunRecord` by `created_at desc` - and use that (in-process DB access is already used elsewhere in - `tests/integration/`). - -- [ ] **Step 8.4: Run the canary** - -With a stack up (`docker compose -f docker-compose.real-llm.yml up -d` or -equivalent dev stack), run: - -```bash -ERGON_REAL_LLM=1 uv run pytest tests/real_llm/benchmarks/test_smoke_stub.py -v -``` - -Expected: 1 passed. If any layer fails, iterate: -- subprocess timeout → check CLI path / stub registry -- harness 404 → confirm `ENABLE_TEST_HARNESS=1` set on API container -- Playwright can't reach dashboard → confirm port 3101 is exposed - -- [ ] **Step 8.5: Commit** - -```bash -git add tests/real_llm/benchmarks/__init__.py \ - tests/real_llm/benchmarks/test_smoke_stub.py -git commit -m "test(real-llm): canary smoke using stub workers (cost=0)" -``` - ---- - -## Task 9 — Architecture doc updates - -**Files:** -- Modify: `docs/architecture/07_testing.md` -- Modify: `docs/architecture/06_builtins.md` - -- [ ] **Step 9.1: Update testing doc** - -Append a new row to the testing-tier matrix section in -`docs/architecture/07_testing.md`: - -```markdown -| Tier | Path | Runs in CI? | Activates on | Assertions | -|---|---|---|---|---| -| real-LLM | `tests/real_llm/` | **No** (manual dispatch only) | `ERGON_REAL_LLM=1` + `OPENROUTER_API_KEY` | Postgres + Playwright + `/api/test/*` harness; OpenRouter budget gate skips when exhausted | -``` - -(Insert after the existing e2e row; if the doc uses a different format, -adapt to it.) - -- [ ] **Step 9.2: Update builtins doc** - -Append a paragraph to the tools section of -`docs/architecture/06_builtins.md`: - -```markdown -`benchmark_toolkit_composer` is a DI factory that, given a benchmark slug -and a `WorkerContext`, returns the union of tools the generic ReAct worker -(`react-generic` slug) needs to exercise that benchmark. It's the -mechanism by which the real-LLM debug harness -(`tests/real_llm/`) runs one worker against all three benchmark sandboxes -without per-benchmark specialised workers. -``` - -- [ ] **Step 9.3: Commit** - -```bash -git add docs/architecture/07_testing.md docs/architecture/06_builtins.md -git commit -m "docs(arch): add real-LLM tier row + benchmark_toolkit_composer note" -``` - ---- - -## Task 10 — Open PR 1 - -**Files:** none directly. - -- [ ] **Step 10.1: Full check suite** - -```bash -pnpm run check:fast -uv run pytest tests/unit tests/state tests/cli -v -``` - -Expected: all green. Fix anything that broke; iterate. - -- [ ] **Step 10.2: Push + PR** - -```bash -git push -u origin feature/real-llm-harness-infra -gh pr create \ - --title "feat(real-llm): debug harness infra — tests/real_llm/ + canary (PR 1 of 2)" \ - --body "$(cat <<'EOF' -## Summary -- New `tests/real_llm/` pytest tier, marker-gated (`ERGON_REAL_LLM=1` + `OPENROUTER_API_KEY`). -- `benchmark_toolkit_composer` DI factory + `react-generic` worker slug + CLI `--toolkit-benchmark` flag. -- `OpenRouterBudget` module polls `/api/v1/auth/key` for spend gating (default $5 cap). -- `docker-compose.real-llm.yml` stack overlay + `--assume-stack-up` dev flag. -- Canary test (`test_smoke_stub.py`) exercises stack + CLI + harness-read + Playwright with stub workers (cost = $0). -- No real-LLM runs in this PR. PR 2 adds the 3-random-instances-per-benchmark artifact; between PR 1 and PR 2 the overnight loop runs the harness, files bugs in `docs/bugs/open/`, and ships fix PRs. - -## Test plan -- [x] `pnpm run check:fast` green -- [x] `uv run pytest tests/unit -v` green (new openrouter_budget, benchmark_toolkit_composer, react_generic_worker, cli react-generic composition tests) -- [x] Canary test runs against a local stack and exits 0 (stub workers, zero OpenRouter cost) - -Follows up on `docs/rfcs/active/2026-04-21-real-llm-debug-harness.md`. - -🤖 Generated with [Claude Code](https://claude.com/claude-code) -EOF -)" -``` - -- [ ] **Step 10.3: Watch CI, iterate** - -If CI fails, fix on the branch with fresh commits — never `--amend` a -pushed commit, never `--no-verify`. - ---- - -# After PR 1 merges: Bug-hunt phase - -Run `ERGON_REAL_LLM=1 OPENROUTER_API_KEY=... uv run pytest tests/real_llm/ --m real_llm --assume-stack-up` in a loop. For each non-LLM bug: - -1. Reproduce manually to confirm it's not a transient. -2. File `docs/bugs/open/YYYY-MM-DD-<slug>.md` from `docs/bugs/TEMPLATE.md`. -3. If trivial: open a fix PR that moves the bug file to `fixed/` on merge. -4. If non-trivial: promote to RFC at `docs/rfcs/active/...`, link via - `related_rfc`. - -The loop terminates when each of the three benchmarks (researchrubrics, -minif2f, swebench-verified) passes the hard-gate assertions on a fresh run. -That condition is the PR 2 freeze signal. - ---- - -# PR 2 — Three-example artifact (follow-up plan) - -A separate plan at `docs/superpowers/plans/2026-04-21-real-llm-three-examples.md` -(written after PR 1 merges and the bug-hunt phase stabilises). Shape -previewed here: - -- `test_researchrubrics.py`, `test_minif2f.py`, `test_swebench.py` - parametrised over 3 random benchmark instances (seeded). -- Hard gates (as in the RFC). -- Soft gate: "at least 1/3 non-zero score per benchmark." -- Results reporter writes `.results/YYYY-MM-DD-HHMM-<benchmark>.md` + emits - a combined PR-body artifact. diff --git a/docs/superpowers/plans/2026-04-22-worker-interface-and-artifact-routing.md b/docs/superpowers/plans/2026-04-22-worker-interface-and-artifact-routing.md deleted file mode 100644 index 22dd4dcc9..000000000 --- a/docs/superpowers/plans/2026-04-22-worker-interface-and-artifact-routing.md +++ /dev/null @@ -1,2696 +0,0 @@ -# Worker interface + artifact routing cleanup — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Execute the four cleanups in `docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md` on `feature/real-llm-harness-infra` (PR #27) — tighten the Worker construction contract, move SWE-Bench per-task setup into the sandbox manager's `_install_dependencies` hook driven by a new data-layer payload lookup, rewire artifact routing to flow via sandbox files + `CriterionRuntime`, and rename the ambiguous `output_text` field to `final_assistant_message` end-to-end. - -**Architecture:** Four parallel changes that converge on PR #27 without a DB schema addition. `ReActWorker` becomes a plain `(tools, prompt, iterations)` worker with all kwargs required; benchmark-specific wiring moves from a `BenchmarkAdapter` ABC into registry-level factory closures. SWE-Bench setup scripts run inside `BaseSandboxManager._install_dependencies` after the new method `queries.task_executions.get_task_payload` joins `run_task_executions → experiment_definition_tasks`. Criteria stop reading `WorkerOutput.artifacts` entirely — MiniF2F reads its proof via `context.runtime.read_resource("final_solution.lean")`, SWE-Bench computes the patch itself via `context.runtime.run_command("git diff HEAD")`. One Alembic migration renames the `run_task_executions.output_text` column. - -**Tech Stack:** Python 3.13, PEP 695 type aliases (`type Tool = Any`), SQLModel, Alembic, pydantic-ai `Agent`, Inngest steps, `ty` type checker, `ruff`, `slopcop`, `uv run pytest`. - -**Branch:** `feature/real-llm-harness-infra` (already exists — no worktree needed; PR #27 open). - ---- - -## Pre-flight - -- [ ] **Confirm branch + clean tree** - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon -git status -git rev-parse --abbrev-ref HEAD -``` - -Expected: on `feature/real-llm-harness-infra`, working tree clean (or uncommitted RFC edits only). - -- [ ] **Run baseline fast tests to confirm green start** - -```bash -pnpm run test:be:fast -``` - -Expected: all pass. If anything is already red, surface to human before touching code. - -- [ ] **Commit any outstanding RFC doc edits before starting** - -```bash -git add docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md -git status -git diff --cached --stat -git commit -m "rfc(worker-interface): finalize before implementation" -``` - -Skip if nothing staged. - ---- - -## Task 1: Add `Tool` type alias module + re-export - -**Files:** -- Create: `ergon_core/ergon_core/api/types.py` -- Modify: `ergon_core/ergon_core/api/__init__.py` -- Test: `tests/unit/api/test_types_reexport.py` - -**Why:** The RFC hoists `Tool = Any` to a single definition in `ergon_core.api.types` so call sites read as "list of tools" rather than "list of Any" and the `slopcop: ignore[no-typing-any]` lives at one site only. - -- [ ] **Step 1: Write the failing test** - -```python -# tests/unit/api/test_types_reexport.py -"""Ensure the public ``Tool`` alias is exported from ``ergon_core.api``.""" - -from typing import get_type_hints - - -def test_tool_is_reexported_from_api_root() -> None: - from ergon_core.api import Tool # noqa: F401 — import is the assertion - - # Defining a function with list[Tool] must type-check (Tool is callable as a type hint). - def _takes_tools(tools: list[Tool]) -> int: - return len(tools) - - assert _takes_tools([]) == 0 - assert _takes_tools([object(), object()]) == 2 - - -def test_tool_module_is_importable() -> None: - from ergon_core.api import types as api_types - - assert hasattr(api_types, "Tool") -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/api/test_types_reexport.py -v -``` - -Expected: `ImportError: cannot import name 'Tool' from 'ergon_core.api'`. - -- [ ] **Step 3: Create `ergon_core/ergon_core/api/types.py`** - -```python -# ergon_core/ergon_core/api/types.py -"""Shared type aliases for the public API surface.""" - -from typing import Any - -type Tool = Any # slopcop: ignore[no-typing-any] -"""Framework-agnostic tool carrier. - -Intentionally unconstrained so workers can integrate with any agent -framework. ``ReActWorker`` passes these through to pydantic-ai's -``Agent(tools=...)``; nothing in our code enforces a structural protocol. -If we ever pin to pydantic-ai, tighten this to -``pydantic_ai.tools.Tool | Callable[..., Any]``. -""" - -__all__ = ["Tool"] -``` - -- [ ] **Step 4: Re-export `Tool` from `ergon_core/ergon_core/api/__init__.py`** - -Add the import alphabetically (after `task_types`, before `worker`) and add `"Tool"` to `__all__`: - -```python -from ergon_core.api.types import Tool -``` - -And add `"Tool",` to `__all__` between `"TaskEvaluationResult"` and `"Worker"`. - -- [ ] **Step 5: Run test to verify it passes** - -```bash -uv run pytest tests/unit/api/test_types_reexport.py -v -``` - -Expected: both tests pass. - -- [ ] **Step 6: Run full check** - -```bash -pnpm run check:be -``` - -Expected: clean (lint/format/ty/slopcop all pass). Fix any issues. - -- [ ] **Step 7: Commit** - -```bash -git add ergon_core/ergon_core/api/types.py ergon_core/ergon_core/api/__init__.py tests/unit/api/test_types_reexport.py -git commit -m "$(cat <<'EOF' -feat(api): introduce public Tool type alias - -Hoist `Tool = Any` from inline usage in workers into `ergon_core.api.types` -so call sites read as `list[Tool]` and the `slopcop: ignore[no-typing-any]` -lives at exactly one definition site. Re-exported from `ergon_core.api`. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §1 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 2: Tighten base `Worker.__init__` — drop `model` default - -**Files:** -- Modify: `ergon_core/ergon_core/api/worker.py` -- Test: `tests/unit/api/test_worker_base_contract.py` - -**Why:** The RFC commits to making `model: str | None` required on the base class. The union stays (caller may pass `None` to opt into the platform resolver), but the `= None` default is dropped. Concrete subclasses and their test fixtures will update in the next task. - -- [ ] **Step 1: Write the failing test** - -```python -# tests/unit/api/test_worker_base_contract.py -"""Contract tests for the base `Worker.__init__` signature.""" - -import inspect - -from ergon_core.api import Worker - - -def test_model_kwarg_has_no_default() -> None: - """`model` must be keyword-only AND have no default value. - - Defaults on worker `__init__` are an anti-pattern (RFC 2026-04-22): - they hide sizing decisions. Factories must pass `model=` explicitly. - """ - sig = inspect.signature(Worker.__init__) - model_param = sig.parameters["model"] - assert model_param.kind == inspect.Parameter.KEYWORD_ONLY, ( - f"`model` must be keyword-only, got {model_param.kind}" - ) - assert model_param.default is inspect.Parameter.empty, ( - f"`model` must have no default; got {model_param.default!r}" - ) - - -def test_name_kwarg_has_no_default() -> None: - sig = inspect.signature(Worker.__init__) - name_param = sig.parameters["name"] - assert name_param.kind == inspect.Parameter.KEYWORD_ONLY - assert name_param.default is inspect.Parameter.empty -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/api/test_worker_base_contract.py -v -``` - -Expected: `test_model_kwarg_has_no_default` FAILS with "`model` must have no default; got None". - -- [ ] **Step 3: Edit `ergon_core/ergon_core/api/worker.py`** - -Replace the `__init__` signature starting at line 30: - -```python - def __init__( - self, - *, - name: str, - model: str | None, - metadata: Mapping[str, Any] | None = None, # slopcop: ignore[no-typing-any] - ) -> None: - self.name = name - self.model = model - self.metadata: dict[str, Any] = dict(metadata or {}) # slopcop: ignore[no-typing-any] - self._turn_repo = GenerationTurnRepository() -``` - -(The only change: delete `= None` after `model: str | None` on line 34.) - -- [ ] **Step 4: Run test to verify it passes** - -```bash -uv run pytest tests/unit/api/test_worker_base_contract.py -v -``` - -Expected: both tests pass. - -- [ ] **Step 5: Run `ty` to find subclasses that now fail** - -```bash -uv run ty check ergon_core ergon_builtins tests -``` - -Expected: failures at every site that constructed a `Worker` subclass without `model=`. Note the list — they're fixed in Task 3. - -- [ ] **Step 6: DO NOT commit yet** — Task 3 fixes the ripple. Keep the tree uncommitted and move on. - ---- - -## Task 3: Fix non-benchmark worker subclasses + their call-sites - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/stub_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/smoke_test_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/manager_researcher_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/stubs/canonical_smoke_worker.py` -- Modify: all test files that construct any of the above without `model=` -- Test: existing fast suite acts as the regression check - -**Why:** Resolves Open Question 1 in favor of **option (a)** — every subclass propagates the "no default" rule so the registry factory closures (built in Task 6) can wrap them uniformly. This is noisier than option (b) but keeps the contract consistent from the top of the hierarchy down: a reader of any worker class sees the same story about `model`. - -**Strategy:** Each subclass that forwards `model` to `super().__init__` simply drops its own `= None`. Test fixtures that constructed with `StubWorker(name="x")` now write `StubWorker(name="x", model=None)` — the explicit `None` is intentional and documents the "use platform default" choice. - -- [ ] **Step 1: Inventory all subclasses and their constructors** - -```bash -uv run ruff check --select=F401 --fix ergon_builtins ergon_core tests 2>/dev/null || true -``` - -Then: - -```bash -grep -n "def __init__" ergon_builtins/ergon_builtins/workers/baselines/stub_worker.py \ - ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py \ - ergon_builtins/ergon_builtins/workers/baselines/smoke_test_worker.py \ - ergon_builtins/ergon_builtins/workers/baselines/manager_researcher_worker.py \ - ergon_builtins/ergon_builtins/workers/research_rubrics/stub_worker.py \ - ergon_builtins/ergon_builtins/workers/stubs/canonical_smoke_worker.py -``` - -Note which ones define their own `__init__` vs. inherit from the base (the latter need no change). - -- [ ] **Step 2: For each subclass with its own `__init__`, drop the `model` default** - -If the subclass signature reads: - -```python -def __init__(self, *, name: str, model: str | None = None, ...) -> None: -``` - -change to: - -```python -def __init__(self, *, name: str, model: str | None, ...) -> None: -``` - -If the subclass only passes `name=` to `super()` and doesn't mention `model` in its own signature, it inherits from base and no source change is needed — callers pass `model=` directly. - -- [ ] **Step 3: Update every construction call-site that omits `model=`** - -Run: - -```bash -uv run ty check ergon_core ergon_builtins tests 2>&1 | tee /tmp/ty-report.txt -``` - -For each error reporting "missing argument `model`", open the file and add `model=None` explicitly: - -```python -# Before -StubWorker(name="test") -# After -StubWorker(name="test", model=None) -``` - -Pattern to grep if `ty` output is noisy: - -```bash -grep -rn --include="*.py" "StubWorker(name=" tests/ ergon_builtins/ ergon_core/ -grep -rn --include="*.py" "TrainingStubWorker(name=" tests/ ergon_builtins/ ergon_core/ -grep -rn --include="*.py" "SmokeTestWorker(name=" tests/ ergon_builtins/ ergon_core/ -grep -rn --include="*.py" "ManagerResearcherWorker(name=" tests/ ergon_builtins/ ergon_core/ -grep -rn --include="*.py" "StubResearchRubricsWorker(name=" tests/ ergon_builtins/ ergon_core/ -grep -rn --include="*.py" "CanonicalSmokeWorker(name=" tests/ ergon_builtins/ ergon_core/ -``` - -Pass `model=None` wherever `model=` is absent. - -- [ ] **Step 4: Run `ty` again to confirm clean** - -```bash -uv run ty check ergon_core ergon_builtins tests -``` - -Expected: no missing-argument errors. Ignore unrelated warnings that pre-date this change. - -- [ ] **Step 5: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: all green. If anything fails, it's either (a) a site you missed; or (b) a subclass whose signature ripple missed a nested `super().__init__` call — fix inline. - -- [ ] **Step 6: Run full check** - -```bash -pnpm run check:be -``` - -Expected: clean. - -- [ ] **Step 7: Commit the base + subclass tightening together** - -```bash -git add ergon_core/ergon_core/api/worker.py \ - ergon_builtins/ergon_builtins/workers \ - tests/unit/api/test_worker_base_contract.py \ - tests/ -git commit -m "$(cat <<'EOF' -refactor(worker): make `model` kwarg required on base Worker and subclasses - -Drop `model: str | None = None` defaults across the Worker hierarchy -(base class, StubWorker, TrainingStubWorker, SmokeTestWorker, -ManagerResearcherWorker, StubResearchRubricsWorker, CanonicalSmokeWorker) -and update call-sites to pass `model=None` explicitly where the platform -resolver default is intended. The union `str | None` stays; the default -goes. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §1 -(anti-pattern: nullable-with-default on worker __init__) - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 4: Tighten `ReActWorker.__init__` — drop adapter + all-kwargs-required - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Test: `tests/unit/workers/test_react_worker_contract.py` - -**Why:** `ReActWorker` becomes a plain `(tools, system_prompt, max_iterations)` worker. The adapter ABC and its hooks disappear from the worker signature (actual file deletion happens in Task 5). All kwargs are required — no defaults. - -- [ ] **Step 1: Write the failing contract tests** - -```python -# tests/unit/workers/test_react_worker_contract.py -"""Contract tests for the post-RFC `ReActWorker.__init__` signature.""" - -import inspect - -import pytest - -from ergon_builtins.workers.baselines.react_worker import ReActWorker - - -def test_no_adapter_kwarg() -> None: - sig = inspect.signature(ReActWorker.__init__) - assert "adapter" not in sig.parameters, ( - "BenchmarkAdapter ABC is being deleted — ReActWorker must not accept an adapter kwarg." - ) - - -@pytest.mark.parametrize( - "kwarg", - ["name", "model", "tools", "system_prompt", "max_iterations"], -) -def test_all_kwargs_required_and_keyword_only(kwarg: str) -> None: - sig = inspect.signature(ReActWorker.__init__) - param = sig.parameters[kwarg] - assert param.kind == inspect.Parameter.KEYWORD_ONLY, ( - f"`{kwarg}` must be keyword-only; got {param.kind}" - ) - assert param.default is inspect.Parameter.empty, ( - f"`{kwarg}` must have no default (RFC 2026-04-22 forbids nullable-with-default); " - f"got {param.default!r}" - ) - - -def test_construct_with_minimal_explicit_kwargs() -> None: - """A ReActWorker can be built with explicit [] tools and None prompt.""" - worker = ReActWorker( - name="unit", - model=None, - tools=[], - system_prompt=None, - max_iterations=1, - ) - assert worker.name == "unit" - assert worker.tools == [] - assert worker.system_prompt is None - assert worker.max_iterations == 1 -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py -v -``` - -Expected: `test_no_adapter_kwarg` and all parametrized `test_all_kwargs_required_and_keyword_only` cases fail. - -- [ ] **Step 3: Rewrite `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py`** - -Open the file. Make the following edits (line numbers from the current read): - -- Remove line 28: `from ergon_builtins.workers.baselines.adapters.base import BenchmarkAdapter` (not needed after adapter dir is deleted in Task 5 — removing the import early keeps intermediate state runnable once Task 5 lands; for THIS task, keep it until Task 5 deletes it). -- Add import: `from ergon_core.api import Tool` near the other `ergon_core.api` imports (line ~11). -- Replace the entire `__init__` method (lines 82–109) with: - -```python - def __init__( - self, - *, - name: str, - model: str | None, - tools: list[Tool], - system_prompt: str | None, - max_iterations: int, - ) -> None: - super().__init__(name=name, model=model) - self.tools: list[Tool] = tools - self.system_prompt: str | None = system_prompt - self.max_iterations: int = max_iterations - self._seed_messages: list[ModelMessage] | None = None -``` - -- Replace the `execute` method body (lines 111–126) — remove all `self._adapter` references: - -```python - async def execute( - self, - task: BenchmarkTask, - *, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - async for turn in self._run_agent(task): - yield turn -``` - -- Delete `self._adapter = adapter` + the three adapter-resolution branches in the old `__init__`. -- Replace `get_output` (lines 173–178) with: - -```python - def get_output(self, context: WorkerContext) -> WorkerOutput: - """Extract the agent's text output from the last context event.""" - return self._base_output(context) -``` - -- Remove the `BenchmarkAdapter` type annotation on the class docstring (lines 74–77) — the class no longer knows about adapters. - -- Keep everything else (`_run_agent`, `_base_output`, `from_buffer`, helpers) unchanged. - -- [ ] **Step 4: Run the contract tests to verify they pass** - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py -v -``` - -Expected: all four tests pass. - -- [ ] **Step 5: Expected secondary failures** - -```bash -uv run pytest tests/minif2f/test_react_worker.py -v 2>&1 | head -30 -``` - -Expected: existing MiniF2F adapter-style tests will now fail (they construct `ReActWorker(adapter=...)`). These tests are deleted/rewritten in Task 5 + later tasks — DO NOT fix them here. - -- [ ] **Step 6: DO NOT commit yet** — Tasks 5–7 must land atomically with this so registry factories and adapter-test deletion reach a consistent state in one PR. Keep the tree in flight. - ---- - -## Task 5: Delete adapter module + adapter tests - -**Files:** -- Delete: `ergon_builtins/ergon_builtins/workers/baselines/adapters/__init__.py` -- Delete: `ergon_builtins/ergon_builtins/workers/baselines/adapters/base.py` -- Delete: `ergon_builtins/ergon_builtins/workers/baselines/adapters/minif2f.py` -- Delete: `ergon_builtins/ergon_builtins/workers/baselines/adapters/swebench.py` -- Delete: any `tests/**/test_*adapter*.py` -- Delete: `tests/minif2f/test_react_worker.py` (will be rewritten in Task 13) -- Modify: `ergon_builtins/ergon_builtins/registry_core.py` (drop adapter imports) -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` (drop lingering adapter import if still present from Task 4) - -**Why:** The ABC is the wrong abstraction (RFC §1). Nothing imports from `adapters/` outside the files being deleted or modified here. - -- [ ] **Step 1: Confirm nothing outside scope imports from `adapters/`** - -```bash -grep -rn --include="*.py" "from ergon_builtins.workers.baselines.adapters" \ - ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -``` - -Expected only hits: `react_worker.py` (will drop import now) and `registry_core.py` (will drop in Task 6). If there are others, stop and flag to human. - -- [ ] **Step 2: Delete the adapter directory** - -```bash -rm -r ergon_builtins/ergon_builtins/workers/baselines/adapters/ -``` - -- [ ] **Step 3: Find + delete adapter-targeted tests** - -```bash -find tests -type f -name "test_*adapter*.py" -``` - -Delete each matching file. Also delete `tests/minif2f/test_react_worker.py` (the current version constructs `ReActWorker(adapter=...)`; a new `test_react_worker.py` will be written in Task 13 covering the new surface). - -```bash -rm tests/minif2f/test_react_worker.py -# For each finding from the find command above: -rm <path> -``` - -- [ ] **Step 4: Remove any lingering adapter import in `react_worker.py`** - -Search the file: - -```bash -grep -n "adapter" ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -``` - -If any remain, delete those lines. - -- [ ] **Step 5: Remove adapter imports from `registry_core.py` (partial — factories still wrong; fixed in Task 6)** - -Delete the line: - -```python -from ergon_builtins.workers.baselines.adapters import MiniF2FAdapter, SWEBenchAdapter -``` - -The `_minif2f_react` and `_swebench_react` functions will fail to resolve — that's fine because Task 6 rewrites them next. - -- [ ] **Step 6: Run lint to confirm no broken imports in files that remain** - -```bash -uv run ruff check ergon_core ergon_builtins tests -``` - -Ignore errors that come from `registry_core.py`'s missing `MiniF2FAdapter` / `SWEBenchAdapter` symbols — Task 6 fixes them. - -- [ ] **Step 7: Commit Tasks 4 + 5 together** - -```bash -git add ergon_builtins/ergon_builtins/workers/baselines/react_worker.py \ - tests/unit/workers/test_react_worker_contract.py -git add -u # stage deletions -git status # verify: react_worker.py modified, adapters/ deleted, adapter tests deleted, minif2f react test deleted -git commit -m "$(cat <<'EOF' -refactor(react-worker): drop BenchmarkAdapter ABC; require all kwargs - -`ReActWorker.__init__` now takes `(name, model, tools, system_prompt, -max_iterations)` as required keyword-only kwargs — no defaults, no -adapter. The ABC, the four hooks (`build_tools`, `on_run_start`, -`on_run_end`, `transform_output`), and both concrete adapters -(`MiniF2FAdapter`, `SWEBenchAdapter`) are deleted. Benchmark-specific -wiring moves into registry factory closures (next commit). - -Tests targeting the deleted adapter ABC are removed. A replacement -`tests/unit/workers/test_react_worker_contract.py` locks in the -new signature. A new behavioral test for MiniF2F will land with -the criterion rewrite. - -Registry factories (`_minif2f_react`, `_swebench_react`) are TEMPORARILY -broken by this commit — the very next commit rewires them. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §1 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - -Note: this commit intentionally leaves `registry_core.py` in a broken state for exactly one commit. Task 6 immediately repairs it. **Do NOT push the branch between Tasks 5 and 6.** - ---- - -## Task 6: Rewrite registry factories with inline toolkits + new kwargs - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/registry_core.py` -- Create: `ergon_builtins/ergon_builtins/workers/baselines/react_prompts.py` -- Test: `tests/unit/registry/test_react_factories.py` - -**Why:** With `BenchmarkAdapter` gone, per-benchmark wiring (toolkit, system prompt, `max_iterations`) lives in the registry factory. The factory call signature grows `task_id: UUID` and `sandbox_id: str` kwargs so benchmark factories can build toolkits bound to the live sandbox. Non-benchmark factories accept-and-ignore the extras (option (a) from Open Question 1). - -- [ ] **Step 1: Write the failing factory test** - -```python -# tests/unit/registry/test_react_factories.py -"""Smoke-test the new registry factory signatures.""" - -from unittest.mock import MagicMock -from uuid import uuid4 - -import pytest - -from ergon_builtins.registry_core import WORKERS -from ergon_core.api import Worker - - -def test_no_bare_react_v1_entry() -> None: - """RFC §1: `react-v1` bare entry removed — every factory binds a concrete toolkit.""" - assert "react-v1" not in WORKERS, ( - "Bare `react-v1` entry must not exist post-RFC. Use `minif2f-react` or " - "`swebench-react` instead." - ) - - -def test_stub_factory_accepts_new_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: - """Non-benchmark factories must accept `task_id` / `sandbox_id` kwargs (option a).""" - factory = WORKERS["stub-worker"] - worker = factory( - name="stub-under-test", - model=None, - task_id=uuid4(), - sandbox_id="sbx-abc", - ) - assert isinstance(worker, Worker) - assert worker.name == "stub-under-test" - - -def test_minif2f_factory_builds_toolkit(monkeypatch: pytest.MonkeyPatch) -> None: - """The minif2f factory must construct a live toolkit bound to the sandbox.""" - from ergon_builtins.benchmarks.minif2f import sandbox_manager as sm_mod - - fake_sandbox = MagicMock(name="fake-sandbox") - fake_manager = MagicMock(spec=sm_mod.MiniF2FSandboxManager) - fake_manager.get_sandbox.return_value = fake_sandbox - monkeypatch.setattr(sm_mod, "MiniF2FSandboxManager", lambda: fake_manager) - - factory = WORKERS["minif2f-react"] - task_id = uuid4() - worker = factory( - name="minif2f-test", - model=None, - task_id=task_id, - sandbox_id="sbx-minif2f", - ) - assert isinstance(worker, Worker) - # Factory should have asked the manager for the sandbox - fake_manager.get_sandbox.assert_called_once_with(task_id) - # Tools list must be non-empty (the MiniF2F toolkit publishes ≥1 tool) - assert worker.tools != [] - # `max_iterations` must be explicit — 30 is the MiniF2F budget from the old adapter - assert worker.max_iterations == 30 -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/registry/test_react_factories.py -v -``` - -Expected: all three tests fail (registry still references deleted adapters). - -- [ ] **Step 3: Create `ergon_builtins/ergon_builtins/workers/baselines/react_prompts.py`** - -Move the two prompts here (they previously lived on the adapter subclasses): - -```python -# ergon_builtins/ergon_builtins/workers/baselines/react_prompts.py -"""System prompts for ReAct worker factories. - -Each benchmark's registry factory binds one of these at worker-construction -time; the prompts are otherwise framework-agnostic and contain no runtime -state. -""" - -MINIF2F_SYSTEM_PROMPT = ( - "You are an expert Lean 4 theorem prover. Your task is to produce a " - "complete, verified proof of the given theorem using Mathlib4.\n\n" - "Workflow:\n" - "1. Call write_lean_file to save a candidate proof to " - "/workspace/scratchpad/draft.lean. Use 'sorry' as a placeholder while " - "exploring.\n" - "2. Call check_lean_file to see compilation errors and remaining goals.\n" - "3. Iterate until the proof has no 'sorry' and no errors.\n" - "4. Write the final proof to /workspace/final_output/final_solution.lean " - "and call verify_lean_proof to confirm the Lean kernel accepts it.\n\n" - "Always import Mathlib at the top. Keep proofs short and use high-level " - "tactics (ring, linarith, nlinarith, simp, omega) when possible." -) - -SWEBENCH_SYSTEM_PROMPT = ( - "You are a senior software engineer fixing an issue in a Python repo.\n\n" - "You have two tools:\n" - "- bash: run shell commands in the repo workdir.\n" - "- str_replace_editor: view/create/str_replace files.\n\n" - "Workflow:\n" - "1. Read the problem statement and explore the repo layout.\n" - "2. Locate the relevant files; run failing tests to reproduce.\n" - "3. Edit code via str_replace_editor; re-run tests until they pass.\n" - "4. Keep the patch minimal — do not modify test files.\n" - "The final answer is whatever `git diff HEAD` shows when you stop." -) -``` - -- [ ] **Step 4: Rewrite `ergon_builtins/ergon_builtins/registry_core.py`** - -Replace the old factory section. Updated imports (drop the adapter import, add toolkit imports + UUID): - -```python -from uuid import UUID - -from ergon_builtins.benchmarks.minif2f.sandbox_manager import MiniF2FSandboxManager -from ergon_builtins.benchmarks.minif2f.toolkit import MiniF2FToolkit -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import SWEBenchSandboxManager -from ergon_builtins.benchmarks.swebench_verified.toolkit import SWEBenchToolkit -from ergon_builtins.workers.baselines.react_prompts import ( - MINIF2F_SYSTEM_PROMPT, - SWEBENCH_SYSTEM_PROMPT, -) -from ergon_builtins.workers.baselines.react_worker import ReActWorker -# ... rest of imports unchanged (drop the adapters import line) -``` - -Replace the `_minif2f_react` factory closure: - -```python -def _minif2f_react( - *, - name: str, - model: str | None, - task_id: UUID, - sandbox_id: str, # unused; sandbox is resolved via the singleton manager -) -> ReActWorker: - """Registry factory: ReActWorker wired with a live MiniF2F toolkit.""" - _ = sandbox_id # factory signature requires it; manager lookup uses task_id - sandbox = MiniF2FSandboxManager().get_sandbox(task_id) - if sandbox is None: - raise RuntimeError( - f"MiniF2F factory requires a live sandbox for task_id={task_id}; " - "SandboxSetupRequest must have completed before worker-execute runs." - ) - toolkit = MiniF2FToolkit( - sandbox=sandbox, - sandbox_run_skill=_minif2f_run_skill(sandbox), - run_id=task_id, # MiniF2FToolkit uses this only for tracing - ) - return ReActWorker( - name=name, - model=model, - tools=list(toolkit.get_tools()), - system_prompt=MINIF2F_SYSTEM_PROMPT, - max_iterations=30, - ) - - -def _minif2f_run_skill(sandbox): # slopcop: ignore[no-return-annotation] - """Return the write_lean_file run_skill callback bound to ``sandbox``. - - Extracted from the old MiniF2FAdapter verbatim. The MiniF2F toolkit - only routes ``write_lean_file`` through this callback; the other - tools drive ``sandbox.commands.run`` directly. - """ - from typing import Any - from uuid import UUID - - async def run_skill( - _run_id: UUID, - skill_name: str, - response_model: type, - **kwargs: Any, # slopcop: ignore[no-typing-any] - ) -> Any: # slopcop: ignore[no-typing-any] - if skill_name != "write_lean_file": - raise ValueError(f"MiniF2F factory does not support skill {skill_name!r}") - file_path = kwargs["file_path"] - content = kwargs["content"] - payload = content.encode("utf-8") if isinstance(content, str) else content - await sandbox.files.write(file_path, payload) - return response_model( - success=True, - filename=file_path, - bytes_written=len(payload), - ) - - return run_skill -``` - -Replace `_swebench_react`: - -```python -def _swebench_react( - *, - name: str, - model: str | None, - task_id: UUID, - sandbox_id: str, -) -> ReActWorker: - """Registry factory: ReActWorker wired with a live SWE-Bench toolkit.""" - _ = sandbox_id # see note in _minif2f_react - sandbox = SWEBenchSandboxManager().get_sandbox(task_id) - if sandbox is None: - raise RuntimeError( - f"SWE-Bench factory requires a live sandbox for task_id={task_id}; " - "SandboxSetupRequest must have completed (including " - "_install_dependencies) before worker-execute runs." - ) - toolkit = SWEBenchToolkit(sandbox=sandbox, workdir="/workspace/repo") - return ReActWorker( - name=name, - model=model, - tools=list(toolkit.get_tools()), - system_prompt=SWEBENCH_SYSTEM_PROMPT, - max_iterations=50, - ) -``` - -Add a helper that wraps plain `Worker` subclasses so the registry signature is uniform — **resolves Open Question 1 in favor of option (a)** but keeps the wrapping minimal: - -```python -def _plain(cls: type[Worker]): # slopcop: ignore[no-return-annotation] - """Wrap a plain ``Worker`` subclass so it ignores registry-injected kwargs. - - Non-benchmark workers (``StubWorker``, ``SmokeTestWorker``, …) don't need - ``task_id`` or ``sandbox_id``. The registry call site always passes them - (see ``worker_execute.py``); this shim drops the extras before forwarding. - """ - - def factory( - *, - name: str, - model: str | None, - task_id: UUID, # noqa: ARG001 - sandbox_id: str, # noqa: ARG001 - ) -> Worker: - return cls(name=name, model=model) - - factory.__name__ = f"_{cls.__name__}_factory" - factory.__qualname__ = factory.__name__ - return factory -``` - -Then the `WORKERS` dict loses the bare `"react-v1"` entry and wraps plain classes through `_plain`: - -```python -WORKERS: dict[str, Callable[..., Worker]] = { - "stub-worker": _plain(StubWorker), - "training-stub": _plain(TrainingStubWorker), - "smoke-test-worker": _plain(SmokeTestWorker), - # NOTE: `react-v1` bare entry removed (RFC 2026-04-22). - # Every real use binds a concrete toolkit via a factory closure below. - "minif2f-react": _minif2f_react, - "swebench-react": _swebench_react, - "manager-researcher": _plain(ManagerResearcherWorker), - "researcher": _plain(StubWorker), - "researchrubrics-stub": _plain(StubResearchRubricsWorker), - "canonical-smoke": _plain(CanonicalSmokeWorker), -} -``` - -- [ ] **Step 5: Run the factory tests to verify they pass** - -```bash -uv run pytest tests/unit/registry/test_react_factories.py -v -``` - -Expected: all three pass. - -- [ ] **Step 6: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: all green (MiniF2F `test_react_worker.py` was deleted in Task 5; no other suite exercises the old adapter paths). - -- [ ] **Step 7: Run full check** - -```bash -pnpm run check:be -``` - -Expected: clean. - -- [ ] **Step 8: Commit** - -```bash -git add ergon_builtins/ergon_builtins/registry_core.py \ - ergon_builtins/ergon_builtins/workers/baselines/react_prompts.py \ - tests/unit/registry/test_react_factories.py -git commit -m "$(cat <<'EOF' -refactor(registry): move benchmark wiring into ReAct factory closures - -Registry factories now build toolkits inline and pass concrete -`tools=`, `system_prompt=`, `max_iterations=` into `ReActWorker(...)`. -Signature grows `task_id: UUID` and `sandbox_id: str` kwargs so -benchmark factories can resolve the live sandbox from their singleton -manager. Plain workers (Stub, SmokeTest, …) are wrapped in a -`_plain(cls)` shim that accepts-and-ignores the extras. - -The bare `"react-v1": ReActWorker` entry is removed — every real use -now binds a concrete toolkit + prompt + iteration budget, so a generic -ReAct registration is meaningless. - -System prompts move from deleted adapters to -`workers/baselines/react_prompts.py` as plain module constants. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §1 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 7: Update `worker_execute.py` call-site to pass `task_id` / `sandbox_id` - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` -- Test: `tests/unit/runtime/test_worker_execute_factory_call.py` (or add to existing unit test if one covers this function) - -**Why:** The `worker_cls(...)` call site is the one place the registry-wrapped factory is actually invoked. Now that every factory in the registry expects `task_id=` and `sandbox_id=`, the call must supply them. Both fields are already in `WorkerExecuteRequest` (line 33–34). - -- [ ] **Step 1: Write the failing call-site test** - -```python -# tests/unit/runtime/test_worker_execute_factory_call.py -"""Verify worker_execute passes task_id / sandbox_id into the factory.""" - -from unittest.mock import MagicMock -from uuid import uuid4 - -from ergon_builtins.registry_core import WORKERS - - -def test_factory_receives_task_and_sandbox(monkeypatch) -> None: - """The factory registered in WORKERS must receive task_id + sandbox_id kwargs.""" - captured: dict[str, object] = {} - - def capturing_factory(**kwargs: object) -> MagicMock: - captured.update(kwargs) - w = MagicMock() - w.name = "captured" - return w - - monkeypatch.setitem(WORKERS, "capturing", capturing_factory) - - task_id = uuid4() - sandbox_id = "sbx-xyz" - - # Direct call imitating worker_execute.py:60 - worker_cls = WORKERS["capturing"] - worker_cls( - name="captured", - model="anthropic:claude-sonnet-4", - task_id=task_id, - sandbox_id=sandbox_id, - ) - - assert captured == { - "name": "captured", - "model": "anthropic:claude-sonnet-4", - "task_id": task_id, - "sandbox_id": sandbox_id, - } -``` - -- [ ] **Step 2: Run test** - -```bash -uv run pytest tests/unit/runtime/test_worker_execute_factory_call.py -v -``` - -Expected: this test should actually **pass** already given Task 6's factory shape. But the real fix is in `worker_execute.py`:60 — it must pass those kwargs when invoking the factory. Confirm by grep: - -```bash -grep -n "worker_cls(" ergon_core/ergon_core/core/runtime/inngest/worker_execute.py -``` - -If the call still reads `worker_cls(name=payload.assigned_worker_slug, model=payload.model_target)` without `task_id=` / `sandbox_id=`, the real runtime path is broken even though the unit test isolates the factory. - -- [ ] **Step 3: Edit `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` line ~60** - -Change: - -```python -worker = worker_cls( - name=payload.assigned_worker_slug, - model=payload.model_target, -) -``` - -to: - -```python -worker = worker_cls( - name=payload.assigned_worker_slug, - model=payload.model_target, - task_id=payload.task_id, - sandbox_id=payload.sandbox_id, -) -``` - -- [ ] **Step 4: Add a behavioral assertion to the existing `worker_execute` test if present** - -```bash -find tests -name "test_worker_execute*.py" -``` - -If such a file exists, add a case that asserts the factory signature match. If not, the unit test from Step 1 is sufficient. - -- [ ] **Step 5: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: all green. - -- [ ] **Step 6: Commit** - -```bash -git add ergon_core/ergon_core/core/runtime/inngest/worker_execute.py \ - tests/unit/runtime/test_worker_execute_factory_call.py -git commit -m "$(cat <<'EOF' -feat(runtime): pass task_id + sandbox_id into worker factory - -The registry-wrapped factory call at worker_execute.py:60 now supplies -`task_id=payload.task_id` and `sandbox_id=payload.sandbox_id` in addition -to the existing `name=` / `model=` kwargs. Both are already present on -`WorkerExecuteRequest`; this is purely a call-site propagation. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §1 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 8: Add `TaskExecutionsQueries.get_task_payload` - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/queries.py` -- Test: `tests/state/test_task_executions_queries.py` (add or extend) - -**Why:** The data layer owns the `run_task_executions` → `experiment_definition_tasks` JOIN. Sandbox managers read task_payload through this helper; they do NOT re-implement the JOIN themselves. - -- [ ] **Step 1: Write the failing test** - -```python -# tests/state/test_task_executions_queries.py (add to existing file or create) -"""Tests for TaskExecutionsQueries.get_task_payload.""" - -from uuid import uuid4 - -from ergon_core.core.persistence.definitions.models import ( - ExperimentDefinition, - ExperimentDefinitionTask, -) -from ergon_core.core.persistence.queries import queries -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.telemetry.models import RunRecord, RunTaskExecution - - -def _insert_task_execution_with_payload(payload: dict[str, object]) -> tuple: - """Insert minimal fixture rows and return (execution_id, definition_task_id).""" - with get_session() as session: - ed = ExperimentDefinition( - id=uuid4(), - experiment_id=uuid4(), - benchmark_type="swebench-verified", - name="test-def", - ) - edt = ExperimentDefinitionTask( - id=uuid4(), - experiment_definition_id=ed.id, - task_slug="test-task", - task_description="fixture", - task_payload=payload, - ) - run = RunRecord(id=uuid4(), experiment_definition_id=ed.id, name="test-run") - exe = RunTaskExecution( - id=uuid4(), - run_id=run.id, - definition_task_id=edt.id, - task_slug="test-task", - attempt_number=1, - status="pending", - ) - session.add_all([ed, edt, run, exe]) - session.commit() - return exe.id, edt.id - - -def test_get_task_payload_returns_joined_payload(db_session) -> None: # noqa: ARG001 - payload = {"instance_id": "django__django-12345", "repo": "django/django"} - exe_id, _ = _insert_task_execution_with_payload(payload) - - result = queries.task_executions.get_task_payload(exe_id) - assert result == payload - - -def test_get_task_payload_returns_none_for_missing_execution(db_session) -> None: # noqa: ARG001 - assert queries.task_executions.get_task_payload(uuid4()) is None -``` - -(Use whatever `db_session` fixture the existing state tests use; check an existing file like `tests/state/test_runs_queries.py` for the fixture name.) - -- [ ] **Step 2: Run test** - -```bash -uv run pytest tests/state/test_task_executions_queries.py -v -``` - -Expected: `AttributeError: 'TaskExecutionsQueries' object has no attribute 'get_task_payload'`. - -- [ ] **Step 3: Add the method to `TaskExecutionsQueries`** - -In `ergon_core/ergon_core/core/persistence/queries.py`, after `update_status` (line ~257): - -```python - def get_task_payload(self, task_execution_id: UUID) -> dict[str, Any] | None: - """Return the immutable task_payload for a task execution. - - Joins ``run_task_executions`` → ``experiment_definition_tasks``. - Returns ``None`` if the execution row does not exist or its - ``definition_task_id`` points at nothing (run-scoped tasks that - weren't tied to a definition — should not happen in normal - benchmark flow). - """ - with get_session() as session: - stmt = ( - select(ExperimentDefinitionTask.task_payload) - .join( - RunTaskExecution, - RunTaskExecution.definition_task_id == ExperimentDefinitionTask.id, - ) - .where(RunTaskExecution.id == task_execution_id) - ) - return session.exec(stmt).first() -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -uv run pytest tests/state/test_task_executions_queries.py::test_get_task_payload_returns_joined_payload -v -uv run pytest tests/state/test_task_executions_queries.py::test_get_task_payload_returns_none_for_missing_execution -v -``` - -Expected: both pass. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_core/ergon_core/core/persistence/queries.py \ - tests/state/test_task_executions_queries.py -git commit -m "$(cat <<'EOF' -feat(queries): add TaskExecutionsQueries.get_task_payload - -JOIN helper that reads `experiment_definition_tasks.task_payload` through -`run_task_executions.definition_task_id`. Callers (next commit: the -SWE-Bench sandbox manager's `_install_dependencies` hook) no longer need -to know about the JOIN — they ask the data layer for the payload by -execution id. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §2 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 9: Rewrite `SWEBenchSandboxManager._install_dependencies` - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py` -- Create: `ergon_core/ergon_core/core/providers/sandbox/errors.py` (if `SandboxSetupError` doesn't already exist — check first) -- Test: `tests/unit/benchmarks/test_swebench_sandbox_manager.py` - -**Why:** The adapter's `on_run_start` used to run `setup_env_script` + `install_repo_script`. The hooks are gone; the sandbox manager owns per-task setup now. The hook has `task_id` — fetch the payload via `queries.task_executions.get_task_payload` and drive the harness scripts. - -- [ ] **Step 1: Check whether `SandboxSetupError` exists** - -```bash -grep -rn "class SandboxSetupError" ergon_core/ -``` - -If not found, create it in `ergon_core/ergon_core/core/providers/sandbox/errors.py`: - -```python -# ergon_core/ergon_core/core/providers/sandbox/errors.py -"""Exceptions raised by sandbox lifecycle code paths.""" - - -class SandboxSetupError(RuntimeError): - """Raised when `BaseSandboxManager._install_dependencies` cannot complete. - - Carries the original stderr/stdout tail in its message so Inngest - retries surface actionable errors without digging through logs. - """ -``` - -- [ ] **Step 2: Write the failing test** - -```python -# tests/unit/benchmarks/test_swebench_sandbox_manager.py -"""Install-dependencies behavior for SWE-Bench.""" - -from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 - -import pytest - -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager import ( - SWEBenchSandboxManager, -) - - -SAMPLE_PAYLOAD = { - "instance_id": "django__django-12345", - "repo": "django/django", - "base_commit": "abcdef1234567890", - "version": "4.2", - "problem_statement": "fix foo", - "fail_to_pass": ["tests.test_x"], - "pass_to_pass": ["tests.test_y"], - "environment_setup_commit": "setup123", - "test_patch": "", - "hints_text": "", -} - - -@pytest.mark.asyncio -async def test_install_runs_setup_and_install_scripts(monkeypatch: pytest.MonkeyPatch) -> None: - from ergon_core.core.persistence import queries as q_mod - - monkeypatch.setattr( - q_mod.queries.task_executions, - "get_task_payload", - lambda _tid: SAMPLE_PAYLOAD, - ) - - fake_spec = MagicMock( - setup_env_script="echo setup", - install_repo_script="echo install", - ) - from ergon_builtins.benchmarks.swebench_verified import sandbox_manager as sm - - monkeypatch.setattr(sm, "make_test_spec", lambda _row: fake_spec) - - sandbox = MagicMock() - sandbox.commands.run = AsyncMock(return_value=MagicMock(exit_code=0, stdout="ok")) - - manager = SWEBenchSandboxManager() - await manager._install_dependencies(sandbox, uuid4()) # type: ignore[attr-defined] - - assert sandbox.commands.run.call_count == 2 - cmds = [c.args[0] for c in sandbox.commands.run.call_args_list] - assert any("echo setup" in cmd for cmd in cmds) - assert any("echo install" in cmd for cmd in cmds) - - -@pytest.mark.asyncio -async def test_install_raises_when_payload_missing(monkeypatch: pytest.MonkeyPatch) -> None: - from ergon_core.core.persistence import queries as q_mod - from ergon_core.core.providers.sandbox.errors import SandboxSetupError - - monkeypatch.setattr( - q_mod.queries.task_executions, "get_task_payload", lambda _tid: None - ) - - manager = SWEBenchSandboxManager() - with pytest.raises(SandboxSetupError, match="No task_payload"): - await manager._install_dependencies(MagicMock(), uuid4()) # type: ignore[attr-defined] - - -@pytest.mark.asyncio -async def test_install_raises_on_nonzero_exit(monkeypatch: pytest.MonkeyPatch) -> None: - from ergon_core.core.persistence import queries as q_mod - from ergon_core.core.providers.sandbox.errors import SandboxSetupError - from ergon_builtins.benchmarks.swebench_verified import sandbox_manager as sm - - monkeypatch.setattr( - q_mod.queries.task_executions, "get_task_payload", lambda _tid: SAMPLE_PAYLOAD - ) - monkeypatch.setattr( - sm, - "make_test_spec", - lambda _row: MagicMock(setup_env_script="false", install_repo_script="true"), - ) - - sandbox = MagicMock() - sandbox.commands.run = AsyncMock( - return_value=MagicMock(exit_code=1, stdout="boom") - ) - - manager = SWEBenchSandboxManager() - with pytest.raises(SandboxSetupError, match="setup_env"): - await manager._install_dependencies(sandbox, uuid4()) # type: ignore[attr-defined] -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_sandbox_manager.py -v -``` - -Expected: 3 failures — current `_install_dependencies` is a `pass`. - -- [ ] **Step 4: Rewrite `_install_dependencies` in `sandbox_manager.py`** - -Replace the body (lines 58–61): - -```python - async def _install_dependencies(self, sandbox: AsyncSandbox, task_id: UUID) -> None: - """Clone the repo at base_commit and install deps for this SWE-Bench instance. - - Payload is fetched from the data layer (no event-payload leak); - `make_test_spec` produces the canonical setup + install shell - scripts. Called exactly once per sandbox by - `BaseSandboxManager.create()` — the early-return at `create()` - guards idempotence, so re-entry does not re-run these scripts. - """ - from ergon_core.core.persistence.queries import queries - from ergon_core.core.providers.sandbox.errors import SandboxSetupError - - from ergon_builtins.benchmarks.swebench_verified.criterion import ( - make_test_spec, - ) - from ergon_builtins.benchmarks.swebench_verified.sandbox_manager_support import ( - payload_to_swebench_row, - ) - - payload = queries.task_executions.get_task_payload(task_id) - if payload is None: - raise SandboxSetupError( - f"No task_payload for task_id={task_id}; prepare step must commit " - "before sandbox-setup dispatches." - ) - row = payload_to_swebench_row(payload) - spec = make_test_spec(row) - - import shlex - - for label, script in ( - ("setup_env", spec.setup_env_script), - ("install_repo", spec.install_repo_script), - ): - logger.info("SWE-Bench _install_dependencies running %s for task_id=%s", label, task_id) - r = await sandbox.commands.run( - f"bash -c {shlex.quote(script)}", - timeout=1800, - ) - if r.exit_code != 0: - tail = (r.stdout or "")[-1000:] - raise SandboxSetupError( - f"swebench {label} failed for task_id={task_id}: exit={r.exit_code} " - f"tail={tail!r}" - ) -``` - -Note the two imports at the top of the function body — `make_test_spec` and `payload_to_swebench_row`. The latter is moved out of the deleted `adapters/swebench.py` into a new small module: - -- [ ] **Step 5: Create `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager_support.py`** - -```python -# ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager_support.py -"""Small helpers for SWE-Bench sandbox setup. - -Formerly `_payload_to_swebench_row` lived on the deleted -`BenchmarkAdapter` subclass; it belongs to the sandbox manager (and the -evaluation criterion) now. -""" - -from typing import Any - - -def payload_to_swebench_row( - payload: dict[str, Any], # slopcop: ignore[no-typing-any] -) -> dict[str, Any]: # slopcop: ignore[no-typing-any] - """Translate a ``SWEBenchTaskPayload`` dict into a harness row. - - The harness expects UPPER_CASE keys for ``FAIL_TO_PASS`` / ``PASS_TO_PASS`` - and a ``patch`` field (we always pass the empty string since the gold - patch must never reach the worker). - """ - return { - "instance_id": payload["instance_id"], - "repo": payload["repo"], - "base_commit": payload["base_commit"], - "version": payload["version"], - "problem_statement": payload["problem_statement"], - "hints_text": payload.get("hints_text", ""), - "FAIL_TO_PASS": payload["fail_to_pass"], - "PASS_TO_PASS": payload["pass_to_pass"], - "environment_setup_commit": payload["environment_setup_commit"], - "test_patch": payload["test_patch"], - "patch": "", - } -``` - -- [ ] **Step 6: Update `criterion.py` to use the moved helper** - -In `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` line 26: - -```python -# Before -from ergon_builtins.workers.baselines.adapters.swebench import _payload_to_swebench_row -# After -from ergon_builtins.benchmarks.swebench_verified.sandbox_manager_support import ( - payload_to_swebench_row as _payload_to_swebench_row, -) -``` - -(Keep the local alias with the leading underscore if the criterion's body still references the private name; changing the body is Task 11's scope.) - -- [ ] **Step 7: Run tests** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_sandbox_manager.py -v -``` - -Expected: all three pass. - -- [ ] **Step 8: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: green. - -- [ ] **Step 9: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager_support.py \ - ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py \ - ergon_core/ergon_core/core/providers/sandbox/errors.py \ - tests/unit/benchmarks/test_swebench_sandbox_manager.py -git commit -m "$(cat <<'EOF' -feat(swebench): move per-task setup into _install_dependencies - -`SWEBenchSandboxManager._install_dependencies` now fetches the -instance payload through the new `queries.task_executions.get_task_payload` -helper and runs `setup_env_script` + `install_repo_script` via the -harness. `_payload_to_swebench_row` moves out of the deleted -`adapters/swebench.py` into a new `sandbox_manager_support` module -shared by the manager and the criterion. - -`SandboxSetupError` is added to the sandbox-provider package. Raised on -missing payload or non-zero exit; propagates through Inngest. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §2 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 10: `ensure_sandbox()` idempotence regression test (resolves Open Question 2) - -**Files:** -- Test: `tests/unit/sandbox/test_ensure_sandbox_idempotence.py` - -**Why:** With per-task setup scripts now living in `_install_dependencies`, the cost of a redundant `ensure_sandbox()` call from a criterion must remain zero (scripts must NOT re-run). `BaseSandboxManager.create()` already early-returns when the sandbox_key is in `_sandboxes`; lock that guarantee down with a regression test so a future refactor doesn't silently re-run setup. - -- [ ] **Step 1: Write the regression test** - -```python -# tests/unit/sandbox/test_ensure_sandbox_idempotence.py -"""Regression: `_install_dependencies` runs exactly once across repeat -`ensure_sandbox()` / `create()` calls for the same key. - -RFC 2026-04-22 moves SWE-Bench per-task setup into `_install_dependencies`. -If `BaseSandboxManager.create()` ever stops early-returning on a cached -sandbox, setup scripts would re-run on every criterion-level -`ensure_sandbox()`. That would be silent but expensive — this test keeps -it caught.""" - -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import pytest - -from ergon_core.core.providers.sandbox.manager import BaseSandboxManager - - -class _ProbeManager(BaseSandboxManager): - """Tiny subclass that counts `_install_dependencies` invocations.""" - - install_calls: int = 0 - template = "test-template" - - async def _create_directory_structure(self, sandbox, sandbox_key) -> None: # noqa: ANN001 - return None - - async def _install_dependencies(self, sandbox, task_id) -> None: # noqa: ANN001 - type(self).install_calls += 1 - - async def _verify_setup(self, sandbox, task_id) -> None: # noqa: ANN001 - return None - - -@pytest.mark.asyncio -async def test_install_dependencies_runs_exactly_once_on_repeated_create() -> None: - _ProbeManager.install_calls = 0 - task_id = uuid4() - - fake_sandbox = MagicMock() - fake_sandbox.commands.run = AsyncMock( - return_value=MagicMock(exit_code=0, stdout="") - ) - - # Stub out the E2B sandbox creation path used by the base class. - with patch.object( - BaseSandboxManager, - "_open_async_sandbox", - AsyncMock(return_value=fake_sandbox), - ): - mgr = _ProbeManager() - await mgr.create(task_id, run_id=task_id, timeout_minutes=30) - await mgr.create(task_id, run_id=task_id, timeout_minutes=30) - await mgr.create(task_id, run_id=task_id, timeout_minutes=30) - - assert _ProbeManager.install_calls == 1, ( - "BaseSandboxManager.create must early-return on a cached sandbox " - "and NOT re-invoke _install_dependencies; otherwise criterion-level " - "ensure_sandbox() calls will silently re-run SWE-Bench setup scripts." - ) -``` - -Note: if `BaseSandboxManager` does not expose a `_open_async_sandbox` helper, find the actual low-level factory used in `create()` (check lines around 231–300 of `manager.py`) and patch that instead. The spirit of the test — three creates, one install — is the invariant. - -- [ ] **Step 2: Run the test** - -```bash -uv run pytest tests/unit/sandbox/test_ensure_sandbox_idempotence.py -v -``` - -Expected: passes (the early-return already exists). If it fails, that itself is a bug — surface to human before moving on. - -- [ ] **Step 3: Commit** - -```bash -git add tests/unit/sandbox/test_ensure_sandbox_idempotence.py -git commit -m "$(cat <<'EOF' -test(sandbox): regression — _install_dependencies runs exactly once - -With SWE-Bench per-task setup scripts now running in -`_install_dependencies`, `BaseSandboxManager.create()`'s early-return -on a cached sandbox_key is load-bearing: any refactor that drops it -would silently re-run `setup_env_script` + `install_repo_script` on -every criterion-level `ensure_sandbox()` call. This test locks the -invariant in. - -Resolves open question 2 of RFC 2026-04-22. - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 11: Add `CriterionRuntime.get_all_files_for_task` - -**Files:** -- Modify: `ergon_core/ergon_core/api/criterion_runtime.py` -- Modify: `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` -- Test: `tests/unit/runtime/test_criterion_runtime_get_all_files.py` - -**Why:** RFC §3 adds this single materializing helper so criteria can pick up every file a task published without iterating `list_resources()` + `read_resource()` themselves. - -- [ ] **Step 1: Write the failing test** - -```python -# tests/unit/runtime/test_criterion_runtime_get_all_files.py -"""Unit tests for DefaultCriterionRuntime.get_all_files_for_task.""" - -from datetime import UTC, datetime, timedelta -from pathlib import Path -from unittest.mock import MagicMock -from uuid import uuid4 - -import pytest - -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.telemetry.models import RunResource - - -@pytest.fixture -def runtime_for_task(db_session, tmp_path): # noqa: ARG001 — db_session side-effect - """Yield (runtime, run_id, task_id).""" - from ergon_core.core.runtime.evaluation.criterion_runtime import DefaultCriterionRuntime - from ergon_core.core.runtime.evaluation.evaluation_schemas import CriterionContext - - run_id = uuid4() - task_id = uuid4() - ctx = CriterionContext(run_id=run_id, task_execution_id=task_id) - runtime = DefaultCriterionRuntime( - context=ctx, - sandbox_manager=MagicMock(), - run_id=run_id, - task_id=task_id, - ) - return runtime, run_id, task_id, tmp_path - - -def _write_resource( - *, - run_id, - task_id, - name: str, - content: bytes, - tmp_path: Path, - created_at: datetime, -) -> None: - path = tmp_path / f"{name}-{created_at.timestamp()}.bin" - path.write_bytes(content) - with get_session() as session: - session.add( - RunResource( - id=uuid4(), - run_id=run_id, - task_execution_id=task_id, - kind="output", - name=name, - mime_type="application/octet-stream", - file_path=str(path), - size_bytes=len(content), - created_at=created_at, - ) - ) - session.commit() - - -@pytest.mark.asyncio -async def test_returns_materialized_bytes(runtime_for_task) -> None: - runtime, run_id, task_id, tmp = runtime_for_task - now = datetime.now(UTC) - _write_resource( - run_id=run_id, task_id=task_id, name="a.txt", - content=b"hello", tmp_path=tmp, created_at=now, - ) - _write_resource( - run_id=run_id, task_id=task_id, name="b.bin", - content=b"\x00\x01\x02", tmp_path=tmp, created_at=now, - ) - - result = await runtime.get_all_files_for_task() - assert result == {"a.txt": b"hello", "b.bin": b"\x00\x01\x02"} - - -@pytest.mark.asyncio -async def test_dedups_keeping_newest(runtime_for_task) -> None: - runtime, run_id, task_id, tmp = runtime_for_task - now = datetime.now(UTC) - _write_resource( - run_id=run_id, task_id=task_id, name="proof.lean", - content=b"old", tmp_path=tmp, created_at=now - timedelta(seconds=5), - ) - _write_resource( - run_id=run_id, task_id=task_id, name="proof.lean", - content=b"NEW", tmp_path=tmp, created_at=now, - ) - - result = await runtime.get_all_files_for_task() - assert result == {"proof.lean": b"NEW"} - - -@pytest.mark.asyncio -async def test_scoped_to_own_task(runtime_for_task) -> None: - runtime, run_id, task_id, tmp = runtime_for_task - other_task_id = uuid4() - now = datetime.now(UTC) - _write_resource( - run_id=run_id, task_id=task_id, name="mine.txt", - content=b"mine", tmp_path=tmp, created_at=now, - ) - _write_resource( - run_id=run_id, task_id=other_task_id, name="not-mine.txt", - content=b"other", tmp_path=tmp, created_at=now, - ) - - result = await runtime.get_all_files_for_task() - assert "not-mine.txt" not in result - assert result["mine.txt"] == b"mine" -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -uv run pytest tests/unit/runtime/test_criterion_runtime_get_all_files.py -v -``` - -Expected: `AttributeError: 'DefaultCriterionRuntime' object has no attribute 'get_all_files_for_task'`. - -- [ ] **Step 3: Extend the Protocol** - -In `ergon_core/ergon_core/api/criterion_runtime.py`, add to the `# ── resource I/O ──` section: - -```python - async def get_all_files_for_task(self) -> "dict[str, bytes]": - """Return `{name: bytes}` for every resource produced by this task. - - Scoped to the `(run_id, task_id)` the runtime was constructed with. - On duplicate `name`s (same file published multiple times), the - newest `created_at` wins. Not size-capped — callers expecting - large resources should use `list_resources()` + `read_resource()`. - """ - ... -``` - -- [ ] **Step 4: Implement on `DefaultCriterionRuntime`** - -In `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py`, after `list_resources` (line ~231): - -```python - async def get_all_files_for_task(self) -> dict[str, bytes]: - """See `CriterionRuntime.get_all_files_for_task`.""" - if self._task_id is None: - return {} - with get_session() as session: - stmt = ( - select(RunResource) - .where(RunResource.run_id == self._run_id) - .where(RunResource.task_execution_id == self._task_id) - .order_by(RunResource.created_at.desc()) # type: ignore[arg-type] - ) - rows = list(session.exec(stmt).all()) - - seen: set[str] = set() - out: dict[str, bytes] = {} - for row in rows: - if row.name in seen: - continue - seen.add(row.name) - out[row.name] = Path(row.file_path).read_bytes() - return out -``` - -- [ ] **Step 5: Run tests** - -```bash -uv run pytest tests/unit/runtime/test_criterion_runtime_get_all_files.py -v -``` - -Expected: all three pass. - -- [ ] **Step 6: Commit** - -```bash -git add ergon_core/ergon_core/api/criterion_runtime.py \ - ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py \ - tests/unit/runtime/test_criterion_runtime_get_all_files.py -git commit -m "$(cat <<'EOF' -feat(criterion-runtime): add get_all_files_for_task helper - -Materializing `(run_id, task_id)`-scoped helper that returns -`{name: bytes}` for every `run_resources` row produced by this task, -keeping the newest revision of each name. Criteria that want the whole -output bundle (MiniF2F proof + verifier log, etc.) call this once -instead of iterating `list_resources()` + `read_resource()`. - -Protocol (api) + concrete impl both updated. Brings the Protocol -surface to 12 methods. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §3 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 12: Rewrite MiniF2F criterion to read from RunResources - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/rules/proof_verification.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/toolkit.py` (verify writes go to `/workspace/final_output/`) -- Test: `tests/unit/benchmarks/test_minif2f_proof_verification.py` - -**Why:** RFC §3, path A. `lean_write_file` writes to `/workspace/final_output/final_solution.lean` (already confirmed from summary); `SandboxResourcePublisher.sync()` auto-publishes it; the criterion reads via `context.runtime.read_resource`. - -- [ ] **Step 1: Open `toolkit.py` and confirm the write path** - -```bash -grep -n "final_output\|/workspace" ergon_builtins/ergon_builtins/benchmarks/minif2f/toolkit.py -``` - -If the `lean_write_file` tool already writes to `/workspace/final_output/final_solution.lean`, no change is needed. If any path writes elsewhere (e.g. only to `/workspace/scratchpad/`), update the default `file_path` the docstring recommends to the final_output location. Log the finding. - -- [ ] **Step 2: Write the failing criterion test** - -```python -# tests/unit/benchmarks/test_minif2f_proof_verification.py -"""MiniF2F criterion reads proof via CriterionRuntime, not WorkerOutput.artifacts.""" - -from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 - -import pytest - -from ergon_builtins.benchmarks.minif2f.rules.proof_verification import ( - ProofVerificationCriterion, -) -from ergon_core.api import WorkerOutput -from ergon_core.api.criterion_runtime import CommandResult -from ergon_core.api.evaluation_context import EvaluationContext -from ergon_core.api.task_types import BenchmarkTask - - -@pytest.mark.asyncio -async def test_reads_proof_via_runtime_read_resource() -> None: - runtime = MagicMock() - runtime.read_resource = AsyncMock(return_value=b"theorem t : True := by trivial") - runtime.run_command = AsyncMock( - return_value=CommandResult(stdout="[ok]", stderr="", exit_code=0) - ) - - context = EvaluationContext( - run_id=uuid4(), - task=BenchmarkTask(task_slug="t1", description="d", task_payload={}), - worker_result=WorkerOutput(output="irrelevant", success=True), - runtime=runtime, - ) - - criterion = ProofVerificationCriterion() - result = await criterion.evaluate(context) - - runtime.read_resource.assert_awaited_once_with("final_solution.lean") - # Whatever the criterion's verification impl decides, it must NOT - # have touched `worker_result.artifacts`. - assert result.name # smoke: result is a well-formed CriterionResult - - -@pytest.mark.asyncio -async def test_scores_zero_when_proof_missing() -> None: - from ergon_core.core.runtime.evaluation.criterion_runtime import ResourceNotFoundError - - runtime = MagicMock() - runtime.read_resource = AsyncMock(side_effect=ResourceNotFoundError("missing")) - - context = EvaluationContext( - run_id=uuid4(), - task=BenchmarkTask(task_slug="t1", description="d", task_payload={}), - worker_result=WorkerOutput(output="irrelevant", success=True), - runtime=runtime, - ) - - criterion = ProofVerificationCriterion() - result = await criterion.evaluate(context) - assert result.score == 0.0 - assert not result.passed -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -uv run pytest tests/unit/benchmarks/test_minif2f_proof_verification.py -v -``` - -Expected: fails — current `_extract_proof` reads `context.worker_result.artifacts.get("final_solution.lean")`. - -- [ ] **Step 4: Rewrite `_extract_proof` in `proof_verification.py`** - -Open the file at line ~97. Replace whatever the current body does with: - -```python - async def _extract_proof(self, context: EvaluationContext) -> str | None: - """Return the Lean source the agent wrote, or None if missing. - - Reads from the task-scoped run_resource named - ``final_solution.lean`` — published by - ``SandboxResourcePublisher.sync()`` after the worker writes to - ``/workspace/final_output/final_solution.lean``. The - pre-RFC-2026-04-22 path through ``worker_result.artifacts`` is - not used: ``artifacts`` is dropped at the Inngest - ``worker_execute`` boundary. - """ - if context.runtime is None: - return None - from ergon_core.core.runtime.evaluation.criterion_runtime import ( - ResourceNotFoundError, - ) - - try: - raw = await context.runtime.read_resource("final_solution.lean") - except ResourceNotFoundError: - return None - return raw.decode("utf-8", errors="replace") -``` - -If `_extract_proof` is called from a synchronous entry point, the caller must become `async`; trace upward and add `await` as needed. - -- [ ] **Step 5: Update any other adjacent call that still reads `worker_result.artifacts`** - -```bash -grep -n "worker_result.artifacts\|worker.artifacts" ergon_builtins/ergon_builtins/benchmarks/minif2f/ -``` - -Remove any remaining reads. The criterion must read nothing from the artifacts dict. - -- [ ] **Step 6: Run tests** - -```bash -uv run pytest tests/unit/benchmarks/test_minif2f_proof_verification.py -v -``` - -Expected: both pass. - -- [ ] **Step 7: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: green. - -- [ ] **Step 8: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/minif2f \ - tests/unit/benchmarks/test_minif2f_proof_verification.py -git commit -m "$(cat <<'EOF' -refactor(minif2f): criterion reads proof via CriterionRuntime - -`ProofVerificationCriterion._extract_proof` now calls -`context.runtime.read_resource("final_solution.lean")` and returns -`None` on `ResourceNotFoundError`. The pre-RFC path through -`worker_result.artifacts` is removed — `artifacts` is dropped at the -Inngest `worker_execute` boundary so reading from it was always -dead code masked by the deleted `MiniF2FAdapter.transform_output`. - -The `lean_write_file` tool writes to -`/workspace/final_output/final_solution.lean`; auto-published by -`SandboxResourcePublisher.sync()` before sandbox teardown. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §3 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 13: Rewrite SWE-Bench criterion to compute patch via run_command - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` -- Test: `tests/unit/benchmarks/test_swebench_criterion_patch_source.py` - -**Why:** RFC §3, path B. The criterion owns the patch extraction now (`git add -A && git diff HEAD`); it no longer reads `worker.artifacts["patch"]` or falls back to `worker.output`. - -- [ ] **Step 1: Write the failing test** - -```python -# tests/unit/benchmarks/test_swebench_criterion_patch_source.py -"""SWE-Bench criterion computes its own patch via runtime.run_command.""" - -from unittest.mock import AsyncMock, MagicMock -from uuid import uuid4 - -import pytest - -from ergon_builtins.benchmarks.swebench_verified.criterion import SWEBenchTestCriterion -from ergon_core.api import WorkerOutput -from ergon_core.api.criterion_runtime import CommandResult -from ergon_core.api.evaluation_context import EvaluationContext -from ergon_core.api.task_types import BenchmarkTask - - -@pytest.mark.asyncio -async def test_criterion_computes_patch_via_run_command(monkeypatch) -> None: - """The criterion must NOT read `worker.artifacts` or `worker.output` - for the patch — it runs `git diff HEAD` itself.""" - runtime = MagicMock() - # run_command is called many times (install, apply patches, run tests, - # and — critically — to compute the patch). Return a benign patch for - # the `git diff` invocation. - def _fake_run(cmd: str, timeout: int = 30) -> CommandResult: - if "git diff HEAD" in cmd: - return CommandResult( - stdout="diff --git a/x b/x\n-old\n+new\n", - stderr="", - exit_code=0, - ) - return CommandResult(stdout="", stderr="", exit_code=0) - - runtime.run_command = AsyncMock(side_effect=_fake_run) - runtime.ensure_sandbox = AsyncMock() - - # The old criterion still uses the sandbox-manager-under-runtime path; - # we inject that. - sandbox = MagicMock() - sandbox.files.write = AsyncMock() - sandbox.commands.run = AsyncMock( - return_value=MagicMock(exit_code=0, stdout="PASSED") - ) - - runtime.sandbox_manager = MagicMock() - runtime.sandbox_manager.get_sandbox = MagicMock(return_value=sandbox) - - payload = { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "abc", - "version": "4.2", - "problem_statement": "x", - "fail_to_pass": ["tests.t"], - "pass_to_pass": [], - "environment_setup_commit": "setup", - "test_patch": "", - "hints_text": "", - } - - # Worker produces NO artifacts and empty output; criterion must still - # derive the patch from the sandbox. - context = EvaluationContext( - run_id=uuid4(), - task=BenchmarkTask( - task_slug="django-1", description="d", task_payload=payload - ), - worker_result=WorkerOutput(output="", success=True), - runtime=runtime, - ) - - criterion = SWEBenchTestCriterion() - - # Skip the heavy harness-grading path with a monkeypatch: - monkeypatch.setattr( - "ergon_builtins.benchmarks.swebench_verified.criterion.get_eval_report", - lambda **kwargs: { - payload["instance_id"]: {"resolved": True, "tests_status": {}} - }, - ) - monkeypatch.setattr( - "ergon_builtins.benchmarks.swebench_verified.criterion.make_test_spec", - lambda row: MagicMock(install_repo_script=":", eval_script=":"), - ) - - result = await criterion.evaluate(context) - - # At least one call to run_command must have been `git diff HEAD`. - git_diff_calls = [ - call for call in runtime.run_command.await_args_list - if "git diff HEAD" in call.args[0] - ] - assert git_diff_calls, ( - "criterion must compute its own patch via runtime.run_command('… git diff HEAD …')" - ) - assert result.passed is True # matches the monkeypatched harness report -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_criterion_patch_source.py -v -``` - -Expected: fails — current criterion reads `(worker.artifacts or {}).get("patch") or worker.output` at line 110. - -- [ ] **Step 3: Rewrite the `evaluate` patch-acquisition block** - -In `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py`: - -Replace line 110: - -```python -patch_text = (worker.artifacts or {}).get("patch") or worker.output or "" -``` - -with: - -```python -patch_text = await _extract_patch_via_runtime(context) -``` - -Add at module level, below the imports: - -```python -async def _extract_patch_via_runtime(context: EvaluationContext) -> str: - """Compute `git add -A && git diff HEAD` via the criterion runtime.""" - if context.runtime is None: - raise RuntimeError( - "SWEBenchTestCriterion requires a CriterionRuntime for patch " - "extraction; none was injected." - ) - await context.runtime.ensure_sandbox() - result = await context.runtime.run_command( - f"cd {WORKDIR} && git add -A && git diff HEAD", - timeout=120, - ) - if result.exit_code != 0: - return "" - return result.stdout or "" -``` - -The rest of `evaluate()` (the empty-patch check, harness invocation, grading) stays unchanged — it already operates on a local `patch_text` variable. - -Also remove the now-unused variable reference: `worker = context.worker_result` at line 109 may still be used later (e.g. for `worker.output` as a fallback). Confirm and drop if dead. - -- [ ] **Step 4: Run the test** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_criterion_patch_source.py -v -``` - -Expected: pass. - -- [ ] **Step 5: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: green. - -- [ ] **Step 6: Commit** - -```bash -git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py \ - tests/unit/benchmarks/test_swebench_criterion_patch_source.py -git commit -m "$(cat <<'EOF' -refactor(swebench): criterion computes patch via runtime.run_command - -`SWEBenchTestCriterion.evaluate()` no longer reads `worker.artifacts["patch"]` -or the worker's output as a fallback. It extracts the patch itself by -running `git add -A && git diff HEAD` via `context.runtime.run_command` -after `ensure_sandbox()`. - -This closes the original "artifacts are dropped at the Inngest boundary" -workaround: the criterion goes straight to the source of truth (the -sandbox working tree), so the data path no longer depends on the -worker routing the patch through `WorkerOutput.output` as a fallback. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §3 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 14: Deprecate `WorkerOutput.artifacts` docstring - -**Files:** -- Modify: `ergon_core/ergon_core/api/results.py` - -**Why:** RFC §3 leaves the field in place for a follow-up PR (removal would ripple into every Worker subclass + test) but marks it as deprecated in the docstring so future readers see the anti-pattern. - -- [ ] **Step 1: Edit `results.py`** - -Locate the `WorkerOutput.artifacts` field and replace its `Field(...)` with the deprecation docstring from the RFC: - -```python - artifacts: dict[str, Any] = Field( # slopcop: ignore[no-typing-any] - default_factory=dict, - description=( - "DEPRECATED. This field is NOT carried across the durable " - "worker→evaluator boundary (dropped at " - "inngest/worker_execute.py). Do not use for files or data " - "the criterion needs to read. Files → write to " - "/workspace/final_output/ (auto-published as RunResources by " - "SandboxResourcePublisher.sync). Computed artifacts → have " - "the criterion run commands in the sandbox via " - "CriterionRuntime.run_command. " - "Slated for removal once no in-tree worker writes to it." - ), - ) -``` - -(If the field already lacks `slopcop: ignore[no-typing-any]` but `dict[str, Any]` needs it, add the trailing `# slopcop: ignore[no-typing-any]` comment to the `artifacts:` type annotation line.) - -- [ ] **Step 2: Run full check** - -```bash -pnpm run check:be -``` - -Expected: clean. - -- [ ] **Step 3: Commit** - -```bash -git add ergon_core/ergon_core/api/results.py -git commit -m "$(cat <<'EOF' -docs(api): deprecate WorkerOutput.artifacts with explanatory docstring - -The field stays on the return contract (removal is a separate PR) but -its description now warns that it is dropped at the Inngest -worker_execute boundary and points readers at -/workspace/final_output/ + CriterionRuntime as the supported channels. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §3 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 15: Rename `output_text` → `final_assistant_message` (code side) - -**Files (27 tracked files per prior grep):** -- Modify: `ergon_core/ergon_core/core/runtime/services/inngest_function_results.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_execution_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_inspection_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/child_function_payloads.py` (`worker_output_text` → `worker_final_assistant_message`) -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluator_dispatch_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/persist_outputs.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` (`RunTaskExecution` column attribute) -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/repositories.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` (line 162) -- Modify: `ergon_core/ergon_core/core/api/schemas.py` (line 64) -- Modify: `ergon_core/ergon_core/core/dashboard/emitter.py` (if the grep hit it) -- Modify: every test file found in the grep that references `output_text` (or `worker_output_text`) in production-path assertions; tests under `.claude/worktrees/` are siblings and should NOT be touched -- Modify: `docs/event-wal/01_AUDIT.md`, `docs/event-wal/02_INCREMENTAL_PERSISTENCE.md`, `docs/rfcs/active/2026-04-17-cleanup-cancelled-task-release-sandbox.md` (doc updates) -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox_utils.py` (the unexpected hit — investigate; may be a legitimate reader) - -**Why:** RFC §4. Fifteen+ files carry the same field through three layers. Do the rename atomically in one commit (excluding the Alembic migration, which is Task 16). - -- [ ] **Step 1: Refresh the full inventory** - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon -git ls-files | xargs grep -l "output_text\|worker_output_text" 2>/dev/null | \ - grep -v "^\.claude/worktrees" | \ - grep -v "^docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md" | \ - sort -u -``` - -The second `grep -v` excludes this RFC and the current plan doc (which legitimately reference `output_text` in their explanations — those stay as historical record). Write the resulting list to `/tmp/rename-inventory.txt`. - -- [ ] **Step 2: Investigate `ergon_builtins/benchmarks/gdpeval/sandbox_utils.py`** - -```bash -grep -n "output_text" ergon_builtins/ergon_builtins/benchmarks/gdpeval/sandbox_utils.py -``` - -If it's a legitimate reader of `RunTaskExecution.output_text`, rename the attribute access. If it's a local variable named `output_text` (unrelated), leave it alone. Note the choice in the commit message. - -- [ ] **Step 3: Rename the dataclass / pydantic field in `inngest_function_results.py`** - -```python -# Before -output_text: str | None = None -# After -final_assistant_message: str | None = None -``` - -Same in `orchestration_dto.py` for `FinalizeTaskExecutionCommand`. - -- [ ] **Step 4: Rename `RunTaskExecution.output_text` in `telemetry/models.py`** - -Locate line 120. Change: - -```python -output_text: str | None = None -``` - -to: - -```python -final_assistant_message: str | None = None -``` - -- [ ] **Step 5: Rename `child_function_payloads.py` carrier field** - -`PersistOutputsRequest.worker_output_text` → `PersistOutputsRequest.worker_final_assistant_message`. Update the docstring comment accordingly. - -- [ ] **Step 6: Cascade every call-site and attribute access** - -For each file in `/tmp/rename-inventory.txt`, open and rename. The two patterns are: -- `.output_text` → `.final_assistant_message` -- `output_text=` → `final_assistant_message=` -- `worker_output_text` → `worker_final_assistant_message` - -**Do NOT blindly sed** — some test strings may contain "output_text" as assertion text about the old name; those should rename. But raw string literals in docs/event-wal that are quoting the *deprecated name for historical context* stay. - -Recommended: for each file, open, read, make each rename consciously. The fast cross-cut: - -```bash -for f in $(cat /tmp/rename-inventory.txt); do - echo "=== $f ===" - grep -n "output_text\|worker_output_text" "$f" -done -``` - -- [ ] **Step 7: Verify nothing outside docs/RFC references the old names** - -```bash -git ls-files | xargs grep -l "output_text\|worker_output_text" 2>/dev/null | \ - grep -v "^docs/" | \ - grep -v "^\.claude/worktrees" -``` - -Expected output: empty (or only this plan file if it's been committed). - -- [ ] **Step 8: Run `ty` to confirm all attribute accesses resolve** - -```bash -uv run ty check ergon_core ergon_builtins tests -``` - -Expected: clean. Any "unresolved attribute output_text" means a site was missed. - -- [ ] **Step 9: Run fast suite** - -```bash -pnpm run test:be:fast -``` - -Expected: green. The rename is a pure symbol substitution; behavior is unchanged. - -- [ ] **Step 10: Commit (without migration — next task)** - -```bash -git add -A -git status # verify scope -git commit -m "$(cat <<'EOF' -refactor(rename): output_text → final_assistant_message across layers - -Rename carried through three layers end-to-end: - -- `WorkerExecuteResult.output_text` → `final_assistant_message` -- `FinalizeTaskExecutionCommand.output_text` → `final_assistant_message` -- `PersistOutputsRequest.worker_output_text` → `worker_final_assistant_message` -- `RunTaskExecution.output_text` (ORM attribute) → `final_assistant_message` - (DB column rename lands in next commit's Alembic migration) -- Dashboard API + CLI readers of `output_text` renamed -- Tests updated to assert the new name - -The old name collided with `CommandResult.stdout` ("output") and with -files under `/workspace/final_output/`. The new name matches the -`assistant_text` context event that produces this value. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §4 - -NOTE: ORM and DB column are temporarily out of sync until the next -commit applies the Alembic migration. Do NOT run against a live -database between these two commits. - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 16: Alembic migration for column rename - -**Files:** -- Create: `ergon_core/migrations/versions/<timestamp>_rename_output_text_to_final_assistant_message.py` - -**Why:** Simple column rename on `run_task_executions`. The ORM attribute is already renamed (Task 15); this brings the DB schema into sync. - -- [ ] **Step 1: Identify the current head revision** - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon -ls ergon_core/migrations/versions/*.py | sort -``` - -From prior inspection the newest revision is `f9075c2ddbc9_run_resource_append_only_log.py` or newer. Confirm: - -```bash -grep -l "down_revision.*=.*None" ergon_core/migrations/versions/*.py 2>/dev/null # earliest -# Find the head by looking for no down_revision referencing it: -for f in ergon_core/migrations/versions/*.py; do - rev=$(grep -oE "revision: str = \"[^\"]+\"" "$f" | grep -oE "\"[^\"]+\"" | tr -d '"') - down=$(grep -oE "down_revision: Union\[str, None\] = \"[^\"]+\"" "$f" | grep -oE "\"[^\"]+\"" | tr -d '"') - echo "$rev → $down ($f)" -done -``` - -The head is the `revision` value that does not appear as any file's `down_revision`. Save it as `HEAD_REV`. - -- [ ] **Step 2: Generate the new revision file** - -Name: `<new_hash>_rename_output_text_to_final_assistant_message.py`. Use `uv run alembic revision` if available; otherwise hand-write: - -```bash -uv run alembic -c ergon_core/alembic.ini revision -m "rename output_text to final_assistant_message" 2>&1 -``` - -If alembic autogen isn't set up here, create the file manually as below. - -- [ ] **Step 3: Write the migration** - -```python -# ergon_core/migrations/versions/<new_hash>_rename_output_text_to_final_assistant_message.py -"""rename output_text to final_assistant_message - -Revision ID: <generated> -Revises: <HEAD_REV> -Create Date: 2026-04-22 00:00:00.000000 - -Column rename on run_task_executions. No data transform. The ORM -attribute was renamed in the preceding commit; this migration brings -the DB schema into sync. RFC: 2026-04-22-worker-interface-and-artifact-routing. -""" - -from typing import Sequence, Union - -from alembic import op - -revision: str = "<generated>" -down_revision: Union[str, None] = "<HEAD_REV>" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.alter_column( - "run_task_executions", - "output_text", - new_column_name="final_assistant_message", - ) - - -def downgrade() -> None: - op.alter_column( - "run_task_executions", - "final_assistant_message", - new_column_name="output_text", - ) -``` - -Replace `<generated>` with a fresh 12-char hex id (you can use `python -c "import secrets; print(secrets.token_hex(6))"`). Replace `<HEAD_REV>` with the value saved in Step 1. - -- [ ] **Step 4: Apply the migration locally and run tests** - -```bash -uv run alembic -c ergon_core/alembic.ini upgrade head -pnpm run test:be:fast -``` - -Expected: migration applies cleanly; tests green. - -- [ ] **Step 5: Commit** - -```bash -git add ergon_core/migrations/versions/<new_hash>_rename_output_text_to_final_assistant_message.py -git commit -m "$(cat <<'EOF' -migrate: rename run_task_executions.output_text column - -Simple column rename; no data transform. Paired with the prior commit -that renamed the ORM attribute from `output_text` to -`final_assistant_message`. Revision chains onto the previous head. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md §4 - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 17: Update architecture docs in the same PR - -**Files:** -- Modify: `docs/architecture/01_public_api.md` -- Modify: `docs/architecture/06_builtins.md` -- Modify: `docs/architecture/cross_cutting/artifacts.md` -- Modify: `docs/architecture/04_sandbox_lifecycle.md` - -**Why:** CLAUDE.md mandates in-PR architecture-doc updates for every feature PR that changes public API, invariants, or anti-patterns. This PR changes all three. - -- [ ] **Step 1: Update `docs/architecture/01_public_api.md`** - -Add / revise the Worker contract section. Use the RFC "Invariants affected → 01_public_api.md" block verbatim as the source. - -- Document `Tool` alias exported from `ergon_core.api`. -- Document the base `Worker.__init__` kwargs (no defaults). -- Document `ReActWorker.__init__` kwargs (all required). -- Update `CriterionRuntime` method list to include `get_all_files_for_task`. State surface-area count (12). -- Add the `final_assistant_message` naming invariant line. - -- [ ] **Step 2: Update `docs/architecture/06_builtins.md`** - -Remove the "ReAct adapter composition" section (if present, added in commit `a9819fe`). Replace with a "ReAct toolkit composition" section describing the factory-closure pattern. - -Add three anti-patterns to the anti-patterns list (copy verbatim from RFC "Invariants affected → 06_builtins.md"): - -- Worker subclasses for per-benchmark glue -- Per-task setup inside workers -- Nullable-with-default kwargs on concrete Worker `__init__` - -- [ ] **Step 3: Update `docs/architecture/cross_cutting/artifacts.md`** - -Add the "Invariants" bullet (copy verbatim from RFC "Invariants affected → cross_cutting/artifacts.md"): - -> - `WorkerOutput.artifacts` is a non-durable field. It is dropped at the Inngest `worker_execute` step boundary and is not a channel to criteria. File-shaped artifacts are published via `SandboxResourcePublisher.sync()` from `/workspace/final_output/`; criteria read them via `CriterionRuntime.read_resource(name)` or `get_all_files_for_task()`. Computed artifacts (e.g. `git diff`) are produced by the criterion itself via `CriterionRuntime.run_command(...)`. - -Remove any obsolete "use `WorkerOutput.artifacts` for evaluator-visible data" guidance. - -- [ ] **Step 4: Update `docs/architecture/04_sandbox_lifecycle.md`** - -Add the per-task-setup invariant (copy verbatim from RFC "Invariants affected → 04_sandbox_lifecycle.md"): - -> For benchmarks that require per-task environment setup (clone a specific commit, install version-pinned deps, apply a harness spec), that work runs inside `BaseSandboxManager._install_dependencies(sandbox, task_id)` — not inside the worker's `execute()`, not inside a separate `on_run_start` hook, and not inside the criterion. Managers that need per-task data (payload, instance-id metadata) read it from the data layer via `queries.task_executions.get_task_payload(task_id)`; `SandboxSetupRequest` carries only `task_id`, not the full payload. - -- [ ] **Step 5: Verify all arch docs still render (no broken cross-refs)** - -```bash -grep -rn "output_text\|BenchmarkAdapter\|WorkerOutput.artifacts" docs/architecture/ -``` - -Fix any stale references. - -- [ ] **Step 6: Commit** - -```bash -git add docs/architecture/ -git commit -m "$(cat <<'EOF' -docs(arch): update public-api + builtins + artifacts + sandbox docs - -In-PR architecture-doc updates paired with the code changes: - -- 01_public_api.md — tightened Worker contract, `Tool` alias, - CriterionRuntime surface (12 methods), `final_assistant_message` - naming invariant. -- 06_builtins.md — replace "ReAct adapter composition" with - "ReAct toolkit composition"; add three anti-pattern bullets. -- cross_cutting/artifacts.md — `WorkerOutput.artifacts` non-durability - invariant. -- 04_sandbox_lifecycle.md — per-task setup runs in - `_install_dependencies`, payload fetched via `queries.task_executions`. - -RFC: docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" -``` - ---- - -## Task 18: Full verification + push - -**Files:** none (meta-task) - -- [ ] **Step 1: Run the full `check:fast` pipeline** - -```bash -pnpm run check:fast -``` - -Expected: backend + frontend both clean. Fix any fallout from upstream renames in the dashboard (`ergon-dashboard/`) if the generated TypeScript bindings reference `output_text`. - -- [ ] **Step 2: Run both test suites** - -```bash -pnpm run test:be:fast -pnpm run test:be:state -``` - -Expected: all green. - -- [ ] **Step 3: Confirm PR #27 still tracks the branch** - -```bash -git log --oneline main..HEAD | head -30 -gh pr view 27 --json title,state,baseRefName,headRefName -``` - -Expected: PR exists, open, head is `feature/real-llm-harness-infra`. - -- [ ] **Step 4: Push** - -```bash -git push origin feature/real-llm-harness-infra -``` - -- [ ] **Step 5: Update the RFC open-questions list to reflect resolutions** - -Open `docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md`. In the "Open questions" section, mark each resolved: - -- Q1 (non-benchmark worker kwarg shape): **Resolved — option (a)**: every plain worker is wrapped in `_plain(cls)` in the registry; subclasses themselves don't learn about `task_id` / `sandbox_id`. -- Q2 (ensure_sandbox idempotence): **Resolved** — regression test at `tests/unit/sandbox/test_ensure_sandbox_idempotence.py`. -- Q3 (read_resource_text helper): **Resolved — no**. Criteria decode explicitly at the call site. - -Commit + push: - -```bash -git add docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md -git commit -m "$(cat <<'EOF' -rfc(worker-interface): resolve open questions 1/2/3 - -- Q1: plain workers wrapped in `_plain(cls)` registry shim; subclasses - don't learn about task_id / sandbox_id. -- Q2: regression test locks _install_dependencies=1 across repeat - ensure_sandbox(). -- Q3: no `read_resource_text` helper; criteria decode at call site. - -Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> -EOF -)" - -git push origin feature/real-llm-harness-infra -``` - -- [ ] **Step 6: On merge (post-review)** - -After PR #27 merges, a follow-up (not part of this plan): - -1. `git mv docs/rfcs/active/2026-04-22-worker-interface-and-artifact-routing.md docs/rfcs/accepted/` -2. Open a tracking issue for the `WorkerOutput.artifacts` field removal once no in-tree worker writes to it. - ---- - -## Self-review checklist - -**1. Spec coverage.** Walked the RFC section-by-section: - -- Proposal §1 (Worker interface) → Tasks 1 (Tool alias), 2 (base Worker), 3 (subclasses), 4 (ReActWorker), 5 (delete adapters), 6 (registry factories), 7 (worker_execute call-site). -- Proposal §2 (setup scripts + payload lookup) → Tasks 8 (`get_task_payload`), 9 (`_install_dependencies`), 10 (ensure_sandbox idempotence regression — Open Q2). -- Proposal §3 (artifact→evaluator routing) → Tasks 11 (`get_all_files_for_task`), 12 (MiniF2F criterion), 13 (SWE-Bench criterion), 14 (`WorkerOutput.artifacts` deprecation). -- Proposal §4 (output_text rename) → Tasks 15 (code), 16 (Alembic migration). -- Invariants affected (all four arch docs) → Task 17. -- Open Q1 resolved by Task 6's `_plain(cls)` shim. Open Q3 resolved in Task 18 RFC update. - -**2. Placeholder scan.** No "TBD" / "fill in" / "add appropriate error handling" / "similar to Task N" strings. Every step has concrete code or an exact command. - -**3. Type consistency.** - -- `Tool` is used consistently as `list[Tool]` in Tasks 1 (definition), 4 (ReActWorker signature), 6 (factory closures). -- `queries.task_executions.get_task_payload(task_execution_id)` signature matches in Task 8 (definition), Task 9 (first usage). -- `get_all_files_for_task()` return type `dict[str, bytes]` consistent between Task 11's Protocol extension, implementation, and test assertions. -- `final_assistant_message` is the final attribute name in every layer: `WorkerExecuteResult`, `FinalizeTaskExecutionCommand`, `RunTaskExecution`, with the carrier `PersistOutputsRequest.worker_final_assistant_message` prefixed as the existing `worker_output_text` was. - -**4. Commit-granularity invariant.** Tasks 4 + 5 are the only pair that commits together (Task 4 leaves `registry_core.py` temporarily broken and Task 5 repairs it in the very next commit). Task 15 (code rename) + Task 16 (Alembic migration) likewise form a two-commit pair with a DO-NOT-RUN-AGAINST-PROD note. Every other task is atomically committable. This matches CLAUDE.md's "frequent commits" guidance while preserving the atomic nature of the rename. - ---- - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-04-22-worker-interface-and-artifact-routing.md`. - -Per the controller context (this is the continuation of writing-plans → subagent-driven-development flow), the next step is to execute this plan via the `superpowers:subagent-driven-development` skill on branch `feature/real-llm-harness-infra`, dispatching one implementer subagent per task, followed by two-stage review (spec compliance, then code quality) before moving to the next task. diff --git a/docs/superpowers/plans/2026-04-23-code-smell-and-ci-cleanup.md b/docs/superpowers/plans/2026-04-23-code-smell-and-ci-cleanup.md deleted file mode 100644 index 97aeaf1ea..000000000 --- a/docs/superpowers/plans/2026-04-23-code-smell-and-ci-cleanup.md +++ /dev/null @@ -1,749 +0,0 @@ -# Code Smell and CI Cleanup Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove all identified code smells (lazy imports, sentinel value, slop suppression comments) and confirm every CI gate — ruff lint/format, ty, slopcop, xenon, frontend ESLint/TypeScript, and unit tests — is green. - -**Architecture:** Each change is a surgical edit to a single file. No new abstractions. Lazy imports move to module top-level; the UUID sentinel is replaced with `UUID | None`; noqa comments that are no longer needed are deleted. Part 2 runs the full CI suite locally and fixes any remaining failures. - -**Tech Stack:** Python 3.13, uv workspace (ergon_core / ergon_builtins / ergon_cli / ergon_infra), ruff, ty, slopcop, xenon, pnpm / ESLint / TypeScript. - ---- - -## Part 1 — Code Smell Removal - -### Lazy-import policy reminder (from CLAUDE.md) - -The only acceptable reason to defer an import to call-site scope is a genuine -circular import that cannot be resolved by restructuring modules. "Heavy deps", -"startup cost", "tests can monkeypatch" are **not** acceptable. The package -extras (`ergon-builtins[local-models]`, `ergon-builtins[data]`) are the isolation -mechanism — not lazy imports. - ---- - -### Task 1: `transformers_backend.py` — promote lazy imports to top-level - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/models/transformers_backend.py` - -Context: `import torch` and `import outlines` are *already* top-level (lines 14–16). -`from transformers import AutoModelForCausalLM, AutoTokenizer` is deferred inside -`_ensure_loaded` with reason "defer heavy deps" — slop. A second `import torch` -inside `_compute_logprobs` is deferred with reason "keeps torch out of module load -path for tests that mock the model" — also slop per CLAUDE.md. - -- [ ] **Step 1: Add `transformers` import to module top-level** - - In `ergon_builtins/ergon_builtins/models/transformers_backend.py`, after the - existing `import torch` line (currently line 16), add: - - ```python - from transformers import AutoModelForCausalLM, AutoTokenizer - ``` - -- [ ] **Step 2: Remove both lazy imports from `_ensure_loaded`** - - Delete the two deferred-import lines and their `# reason:` comments inside - `_ensure_loaded`: - - ```python - # DELETE these two blocks: - # reason: defer heavy deps until first use so importing this module does not load torch/transformers. - import torch - - # reason: (same as torch above) - from transformers import AutoModelForCausalLM, AutoTokenizer - ``` - -- [ ] **Step 3: Remove lazy `import torch` from `_compute_logprobs`** - - Delete this block inside `_compute_logprobs`: - - ```python - # DELETE: - # reason: local import keeps torch import out of module load path for tests that mock the model. - import torch - ``` - -- [ ] **Step 4: Run unit tests to verify nothing broke** - - ```bash - cd /path/to/ergon - uv run pytest tests/unit -q -k "transformers" --durations=5 - ``` - - Expected: all pass (or no tests collected — the model is exercised via integration). - -- [ ] **Step 5: Commit** - - ```bash - git add ergon_builtins/ergon_builtins/models/transformers_backend.py - git commit -m "fix: promote lazy transformers/torch imports to module top-level" - ``` - ---- - -### Task 2: `gdpeval/loader.py` — promote lazy imports to top-level - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/loader.py` - -Context: `pandas` and `huggingface_hub` symbols are imported lazily inside four -functions (`_load_parquet`, two helper functions at lines 78 and 105, and a fourth -at line 127). All are covered by `ergon-builtins[data]`. - -- [ ] **Step 1: Add top-level imports** - - Replace the existing top-level import block (currently `functools`, `json`, - `pathlib`, `typing`) with: - - ```python - import functools - import json - from pathlib import Path - from typing import Any - - import pandas as pd - from huggingface_hub import hf_hub_download, snapshot_download - ``` - -- [ ] **Step 2: Remove all lazy import lines in `_load_parquet`** - - Delete: - ```python - # Deferred: optional dependency - import pandas as pd - - # Deferred: optional dependency - from huggingface_hub import hf_hub_download - ``` - -- [ ] **Step 3: Remove all remaining lazy `hf_hub_download` / `snapshot_download` lines** - - Search for and delete every `from huggingface_hub import ...` inside function - bodies (lines 78, 105, 127). The symbol is now available from the top-level import. - -- [ ] **Step 4: Run unit tests** - - ```bash - uv run pytest tests/unit -q -k "gdpeval" --durations=5 - ``` - - Expected: pass (or no tests collected). - -- [ ] **Step 5: Commit** - - ```bash - git add ergon_builtins/ergon_builtins/benchmarks/gdpeval/loader.py - git commit -m "fix: promote lazy pandas/huggingface_hub imports in gdpeval loader" - ``` - ---- - -### Task 3: Benchmark files — promote lazy `datasets` / `huggingface_hub` imports - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py` - -All three are in `ergon-builtins[data]` territory. - -- [ ] **Step 1: `swebench_verified/benchmark.py` — add top-level import, remove lazy** - - Add to the top-level import block: - ```python - from datasets import load_dataset - ``` - - Delete from inside the method body at line 79: - ```python - from datasets import load_dataset - ``` - -- [ ] **Step 2: `researchrubrics/benchmark.py` — add top-level imports, remove lazy** - - Add to the top-level import block: - ```python - from datasets import load_dataset - from huggingface_hub import HfApi - ``` - - Delete from inside the method body at lines 97–100: - ```python - from datasets import load_dataset - from huggingface_hub import HfApi - ``` - -- [ ] **Step 3: `minif2f/benchmark.py` — add top-level import, remove lazy** - - Add to the top-level import block: - ```python - from huggingface_hub import hf_hub_download - ``` - - Delete from inside the method body at line 90: - ```python - from huggingface_hub import hf_hub_download - ``` - -- [ ] **Step 4: Run unit tests** - - ```bash - uv run pytest tests/unit -q -k "swebench or researchrubrics or minif2f" --durations=5 - ``` - - Expected: pass (or no tests collected). - -- [ ] **Step 5: Commit** - - ```bash - git add ergon_builtins/ergon_builtins/benchmarks/swebench_verified/benchmark.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py \ - ergon_builtins/ergon_builtins/benchmarks/minif2f/benchmark.py - git commit -m "fix: promote lazy datasets/huggingface_hub imports in benchmark files" - ``` - ---- - -### Task 4: `ergon_cli/main.py` — promote lazy CLI handler imports - -**Files:** -- Modify: `ergon_cli/ergon_cli/main.py` - -Context: Eight CLI command handlers are imported lazily inside the `main()` dispatch -block with reason "CLI startup cost" — not an acceptable reason per CLAUDE.md. - -- [ ] **Step 1: Add all handler imports to module top-level** - - After the existing top-level imports in `main.py`, add: - - ```python - from ergon_cli.commands.benchmark import handle_benchmark - from ergon_cli.commands.doctor import handle_doctor - from ergon_cli.commands.eval import handle_eval - from ergon_cli.commands.evaluator import handle_evaluator - from ergon_cli.commands.onboard import handle_onboard - from ergon_cli.commands.run import handle_run - from ergon_cli.commands.train import handle_train - from ergon_cli.commands.worker import handle_worker - ``` - -- [ ] **Step 2: Replace each lazy-import dispatch block with a direct call** - - Each block like: - ```python - elif args.command == "benchmark": - # Deferred: CLI startup cost - from ergon_cli.commands.benchmark import handle_benchmark - return handle_benchmark(args) - ``` - - Becomes: - ```python - elif args.command == "benchmark": - return handle_benchmark(args) - ``` - - Apply to all eight command branches. - -- [ ] **Step 3: Verify the CLI still works** - - ```bash - uv run ergon --help - uv run ergon benchmark --help - ``` - - Expected: help text printed, exit 0. - -- [ ] **Step 4: Run unit tests** - - ```bash - uv run pytest tests/unit/cli -q --durations=5 - ``` - - Expected: all pass. - -- [ ] **Step 5: Commit** - - ```bash - git add ergon_cli/ergon_cli/main.py - git commit -m "fix: promote lazy CLI handler imports to module top-level" - ``` - ---- - -### Task 5: `app.py:70` — fix env-var-conditional import - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/app.py` - -Context: `app.py:70` defers `from ergon_core.core.api.test_harness import router` -inside an `if os.environ.get("ENABLE_TEST_HARNESS") == "1":` guard with -`# noqa: PLC0415`. This is not a circular dependency — it is conditional -behaviour. The import itself is safe at module level; only the *mounting* should -be conditional. - -Note: `app.py:42` (`from ergon_builtins.registry import SANDBOX_MANAGERS`) is a -genuine circular import (ergon_builtins → ergon_core, not the reverse) and must -**stay** deferred with its existing `# reason:` comment and `# noqa: PLC0415`. -Do not touch line 42. - -- [ ] **Step 1: Move the test harness import to module top-level** - - Add to the top-level import block in `app.py`: - - ```python - from ergon_core.core.api.test_harness import router as _test_harness_router - ``` - -- [ ] **Step 2: Remove the lazy import from the conditional block, keep the mount** - - The block currently reads: - ```python - if os.environ.get("ENABLE_TEST_HARNESS") == "1": - from ergon_core.core.api.test_harness import router as _test_harness_router # noqa: PLC0415 - app.include_router(_test_harness_router) - ``` - - Change it to: - ```python - if os.environ.get("ENABLE_TEST_HARNESS") == "1": - app.include_router(_test_harness_router) - ``` - -- [ ] **Step 3: Run the test that guards this behaviour** - - ```bash - uv run pytest tests/unit/test_app_mounts_harness_conditionally.py -v - ``` - - This test imports `app` after setting/unsetting `ENABLE_TEST_HARNESS` — confirm - it still passes. If it fails because it relied on the late import (post-env-set), - update the test to patch `_test_harness_router` at the top-level name instead. - -- [ ] **Step 4: Commit** - - ```bash - git add ergon_core/ergon_core/core/api/app.py \ - tests/unit/test_app_mounts_harness_conditionally.py - git commit -m "fix: promote test_harness conditional import to module top-level" - ``` - ---- - -### Task 6: Replace `DYNAMIC_TASK_SENTINEL_ID` sentinel with `UUID | None` - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/events/task_events.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_management_service.py` -- Modify (if needed): any downstream consumer that reads `event.task_id` and checks - for the zero UUID - -Context: `DYNAMIC_TASK_SENTINEL_ID = UUID("00000000-0000-0000-0000-000000000000")` is -used in two call sites: -1. `task_propagation_service.py:84` — `task_id=rn.definition_task_id or DYNAMIC_TASK_SENTINEL_ID` -2. `task_management_service.py:702` — `task_id=DYNAMIC_TASK_SENTINEL_ID` (hardcoded for dynamic dispatch) - -The four `TaskReadyEvent`-family classes all have `task_id: UUID`. Changing to -`UUID | None` requires updating those schemas and any downstream consumers that -compare `task_id` against the zero UUID. - -- [ ] **Step 1: Audit downstream consumers before changing anything** - - ```bash - grep -rn "task_id" ergon_core/ergon_core/core/runtime/ --include="*.py" | \ - grep -v "definition_task_id\|completed_task_id\|#" - ``` - - Read each hit. Note any code that does `if event.task_id == DYNAMIC_TASK_SENTINEL_ID` - or equivalent — these need updating to `if event.task_id is None`. - -- [ ] **Step 2: Update event schemas in `task_events.py`** - - Change `task_id: UUID` → `task_id: UUID | None` on every event class that uses it - (`TaskReadyEvent`, `TaskStartedEvent`, `TaskCompletedEvent`, `TaskFailedEvent` — check - each one). Remove `DYNAMIC_TASK_SENTINEL_ID` and its import of `UUID` if UUID is - no longer used standalone (it still will be for other fields, so just remove the - sentinel constant). - - Example diff for `TaskReadyEvent`: - ```python - # Before - task_id: UUID - - # After - task_id: UUID | None - ``` - -- [ ] **Step 3: Update `task_propagation_service.py`** - - ```python - # Before - task_id=rn.definition_task_id or DYNAMIC_TASK_SENTINEL_ID, - - # After - task_id=rn.definition_task_id, - ``` - - Remove the `from ergon_core.core.runtime.events.task_events import DYNAMIC_TASK_SENTINEL_ID` - import. - -- [ ] **Step 4: Update `task_management_service.py`** - - ```python - # Before - task_id=DYNAMIC_TASK_SENTINEL_ID, - - # After - task_id=None, - ``` - - Remove the `DYNAMIC_TASK_SENTINEL_ID` import. - -- [ ] **Step 5: Update any sentinel comparisons found in Step 1** - - For each site that reads `task_id` and compares it to the zero UUID: - ```python - # Before - if event.task_id == DYNAMIC_TASK_SENTINEL_ID: - - # After - if event.task_id is None: - ``` - -- [ ] **Step 6: Run affected unit and state tests** - - ```bash - uv run pytest tests/unit/state/ tests/unit -q -k "task" --durations=10 - ``` - - Expected: all pass. - -- [ ] **Step 7: Commit** - - ```bash - git add ergon_core/ergon_core/core/runtime/events/task_events.py \ - ergon_core/ergon_core/core/runtime/services/task_propagation_service.py \ - ergon_core/ergon_core/core/runtime/services/task_management_service.py - # Add any other files touched in Step 5 - git commit -m "fix: replace DYNAMIC_TASK_SENTINEL_ID with UUID | None" - ``` - ---- - -## Part 2 — CI Gate Audit and Fixes - -Run every check that CI runs (`pnpm run check:fast`) plus unit tests. Triage each -failure, fix it, and re-run until all gates are green. The fixes in Part 1 may -introduce new ty errors (e.g. if `UUID | None` is now passed where `UUID` was -expected) — those surface here. - ---- - -### Task 7: Ruff lint (`check:be:lint`) - -- [ ] **Step 1: Run ruff check** - - ```bash - cd /path/to/ergon - uv run ruff check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - - Capture the output. Common failures after Part 1: - - `RUF100` (unused `# noqa` directive) — any `# noqa: PLC0415` left on imports - that are now top-level - - Anything new introduced by the import moves - -- [ ] **Step 2: Auto-fix what ruff can fix** - - ```bash - uv run ruff check --fix ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - -- [ ] **Step 3: Manually fix remaining errors** - - For each remaining ruff error, make the minimal edit. The most likely survivors: - - Stale `# noqa: PLC0415` comments — delete the comment (not the import). - - Any import ordering issue from the new top-level additions — ruff's `I` rules - are not selected, so this is unlikely; if they surface check the select config. - -- [ ] **Step 4: Re-run to confirm zero errors** - - ```bash - uv run ruff check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - - Expected: no output (exit 0). - -- [ ] **Step 5: Commit any fixes** - - ```bash - git add -p # stage only lint-fix changes - git commit -m "fix: ruff lint cleanup after lazy-import promotion" - ``` - ---- - -### Task 8: Ruff format (`check:be:fmt`) - -- [ ] **Step 1: Run format check** - - ```bash - uv run ruff format --check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - -- [ ] **Step 2: Auto-format if needed** - - ```bash - uv run ruff format ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - -- [ ] **Step 3: Re-run check to confirm zero diffs** - - ```bash - uv run ruff format --check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts - ``` - - Expected: exit 0. - -- [ ] **Step 4: Commit if any formatting changed** - - ```bash - git add -p - git commit -m "fix: ruff format after lazy-import promotion" - ``` - ---- - -### Task 9: Type check — ty (`check:be:type`) - -- [ ] **Step 1: Run ty** - - ```bash - uv run ty check ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - Expected new failures after Part 1: - - `task_id: UUID | None` passed to a function expecting `UUID` — fix by adding - a guard (`if task_id is None: ...`) or asserting non-None at the call site. - - Any cascade from `from transformers import AutoModelForCausalLM, AutoTokenizer` - at module level — these are already in `allowed-unresolved-imports` in - `pyproject.toml` so ty ignores them; this should be fine. - -- [ ] **Step 2: Fix each error** - - For each `ty` error: - - **`UUID | None` passed where `UUID` expected** — add `assert task_id is not None` - or an early return at the relevant call site. Do not widen the receiving type - unless the None case is genuinely handled there. - - **Unresolved attribute after optional-dep import** — check the `ty.overrides` - section in `pyproject.toml`; the optional-dep modules already have - `invalid-argument-type = "warn"` and similar rules set to `"warn"`, so these - should not be errors. If a new error appears, add it to the appropriate - existing `[[tool.ty.overrides]]` block. - -- [ ] **Step 3: Re-run to confirm zero errors** - - ```bash - uv run ty check ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - Expected: exit 0. - -- [ ] **Step 4: Commit any type fixes** - - ```bash - git add -p - git commit -m "fix: ty errors after UUID | None sentinel replacement" - ``` - ---- - -### Task 10: slopcop (`check:be:slopcop`) - -- [ ] **Step 1: Run slopcop** - - ```bash - uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - Expected failures after Part 1: - - Any `# Deferred:` comments left in files after the lazy imports were removed — - these become dangling comments, not slopcop violations per se, but ruff may - flag them. Delete them. - - Any noqa comments referencing now-removed lazy imports. - - Unexpected failures (pre-existing, may have been masked): - - `no-broad-except` on the `BLE001` sites in `subtask_lifecycle_toolkit.py` - (7 instances) and `bash_sandbox_tool.py` — these currently have - `# slopcop: ignore[no-broad-except]` or `# noqa: BLE001`; verify the - annotation is the form slopcop accepts. - - `global-modified` on `rollouts.py:35` and `openrlhf_http.py:31` — check - `PLW0603` annotation form. - -- [ ] **Step 2: Fix any violations** - - For a slopcop violation with no legitimate justification, fix the underlying code. - For a violation with a legitimate justification that is missing its annotation, - add `# slopcop: ignore[<rule>] -- <reason>` on the offending line. - -- [ ] **Step 3: Re-run to confirm zero violations** - - ```bash - uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - Expected: exit 0. - -- [ ] **Step 4: Commit if anything changed** - - ```bash - git add -p - git commit -m "fix: slopcop violations after lazy-import cleanup" - ``` - ---- - -### Task 11: Cyclomatic complexity — xenon (`check:be:complexity`) - -- [ ] **Step 1: Run xenon** - - ```bash - uv run xenon --max-absolute F --max-modules E --max-average C \ - ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - The import promotions in Part 1 do not change function complexity, so this gate - should already pass. If it fails, it is a pre-existing issue unrelated to this - branch. - -- [ ] **Step 2: If failing, identify the offending function** - - ```bash - uv run radon cc -s ergon_core ergon_builtins ergon_cli ergon_infra | grep -E "^[A-Z] " - ``` - - Any function graded F is a blocker. Refactor it to bring the score within - threshold. (Note: `pyproject.toml` already has `[tool.ruff.lint.per-file-ignores]` - entries for known complexity debt — xenon thresholds are independent of those.) - -- [ ] **Step 3: Re-run to confirm passing** - - ```bash - uv run xenon --max-absolute F --max-modules E --max-average C \ - ergon_core ergon_builtins ergon_cli ergon_infra - ``` - - Expected: exit 0. - ---- - -### Task 12: Frontend checks (`check:fe`) - -- [ ] **Step 1: Install frontend deps** - - ```bash - pnpm -C ergon-dashboard install --frozen-lockfile - ``` - -- [ ] **Step 2: Run ESLint** - - ```bash - pnpm -C ergon-dashboard run lint - ``` - - Capture output. These are independent of the Python changes in Part 1. - -- [ ] **Step 3: Run TypeScript check** - - ```bash - pnpm -C ergon-dashboard run typecheck - ``` - -- [ ] **Step 4: Fix any failures** - - ESLint failures: follow the rule message; most are auto-fixable with - `pnpm -C ergon-dashboard run lint --fix`. - - TypeScript errors: address each one; do not add `// @ts-ignore`. - -- [ ] **Step 5: Re-run both checks to confirm zero errors** - - ```bash - pnpm -C ergon-dashboard run lint && pnpm -C ergon-dashboard run typecheck - ``` - - Expected: both exit 0. - -- [ ] **Step 6: Commit any frontend fixes** - - ```bash - git add ergon-dashboard/ - git commit -m "fix: frontend lint/type errors" - ``` - ---- - -### Task 13: Unit tests (`test:be:fast`) - -- [ ] **Step 1: Run the full fast unit suite** - - ```bash - uv run pytest tests/unit -q -n auto --durations=20 - ``` - - Expected: all pass. Pay attention to any test that relied on the lazy-import - pattern (e.g. tests that set env vars and then imported a module expecting - deferred behaviour — `test_app_mounts_harness_conditionally.py` is the known - candidate; it was already addressed in Task 5). - -- [ ] **Step 2: Fix any failures** - - For each failing test, read the failure message. Common causes: - - Test imported a module at the top and expected a lazy import to not have fired — - update the test to mock at the top-level name instead of deferring import. - - `task_id: UUID | None` assertion in an event snapshot test — update the snapshot. - -- [ ] **Step 3: Re-run to confirm all pass** - - ```bash - uv run pytest tests/unit -q -n auto --durations=20 - ``` - - Expected: all pass, no failures, no errors. - -- [ ] **Step 4: Commit any test fixes** - - ```bash - git add tests/ - git commit -m "fix: update unit tests after import promotion and sentinel removal" - ``` - ---- - -### Task 14: Full CI gate verification - -- [ ] **Step 1: Run the complete fast check** - - ```bash - pnpm run check:fast - ``` - - This runs: ruff lint → ruff format → ty → slopcop → xenon → ESLint → TypeScript. - All must exit 0. - -- [ ] **Step 2: Run fast unit tests** - - ```bash - pnpm run test:be:fast - ``` - - Expected: all pass. - -- [ ] **Step 3: If anything is still red, fix it before opening the PR** - - Repeat the relevant task from above. Do not open the PR until `pnpm run check:fast` - and `pnpm run test:be:fast` are both fully green. diff --git a/docs/superpowers/plans/2026-04-23-test-quality-improvements.md b/docs/superpowers/plans/2026-04-23-test-quality-improvements.md deleted file mode 100644 index 3f9d43ff1..000000000 --- a/docs/superpowers/plans/2026-04-23-test-quality-improvements.md +++ /dev/null @@ -1,2089 +0,0 @@ -# Test Quality Improvements - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Address 45 identified issues across unit and integration tests — covering reuse/DRY, code quality, coverage gaps, assertion strength, performance, isolation (Part 1), and weak-test patterns identified against the pydantic-ai gold standard (Part 2). - ---- - -## Part 2: Weak Tests (pydantic-ai gold standard comparison) - -### pydantic-ai testing philosophy applied - -- **Precise `pytest.raises`** — always specifies the exact exception type + `match=` regex, never bare `Exception` -- **Parametrize aggressively** — any multi-case boolean/string logic becomes a single parametrized test -- **Error paths as first-class citizens** — every happy path has a corresponding raises/error-state counterpart with message content verified -- **Assert all fields, not just presence** — `result.passed`, `result.score`, `result.feedback` all asserted, not just `result.name` -- **Test behaviour, not structure** — never `inspect.getsource`, never positional index into a list, never `cls.__name__` -- **Exact status codes** — `assert resp.status_code == 404`, never `in (404, 500)` - ---- - -### Task 26: Fix bare `except Exception` in frozen-model tests - -**Files:** -- Modify: `tests/unit/state/test_benchmark_contract.py` -- Modify: `tests/unit/state/test_llm_judge_runtime_injection.py` - -- [ ] **Step 1: Find all bare `pytest.raises(Exception)` in unit tests** - -```bash -rg -n "pytest\.raises\(Exception\)" tests/unit/ -``` - -- [ ] **Step 2: Replace with precise exception type + match=** - -Each should become: - -```python -with pytest.raises(ValidationError, match="frozen instance"): - record.some_field = new_value -``` - -Import `from pydantic import ValidationError` if not already present. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_benchmark_contract.py tests/unit/state/test_llm_judge_runtime_injection.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_benchmark_contract.py tests/unit/state/test_llm_judge_runtime_injection.py -git commit -m "test(state): replace bare pytest.raises(Exception) with precise ValidationError + match" -``` - ---- - -### Task 27: Replace `cls.__name__` assertions with `type_slug` / `is` checks - -**Files:** -- Modify: `tests/unit/state/test_research_rubrics_benchmark.py` - -- [ ] **Step 1: Read lines 15 and 23** - -Find both `assert cls.__name__ == "..."` assertions. - -- [ ] **Step 2: Replace with behavioural contract** - -```python -# Before -assert cls.__name__ == "ResearchRubricsBenchmark" - -# After -assert BENCHMARKS["researchrubrics-ablated"] is ResearchRubricsBenchmark -assert issubclass(ResearchRubricsBenchmark, Benchmark) -``` - -The `is` identity check is the real contract — the slug maps to this exact class. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_research_rubrics_benchmark.py -git commit -m "test(registry): assert registry slug maps to correct class identity, not __name__ string" -``` - ---- - -### Task 28: Parametrize `test_type_invariants.py` - -**Files:** -- Modify: `tests/unit/state/test_type_invariants.py` - -- [ ] **Step 1: Read the full file** - -```bash -cat -n tests/unit/state/test_type_invariants.py -``` - -Identify every test class and list the `(ModelClass, field_name, input_value, expected)` tuples. - -- [ ] **Step 2: Rewrite as parametrized tests** - -```python -import pytest -from pydantic import ValidationError - -@pytest.mark.parametrize("cls,kwargs,field,expected", [ - (RunRecord, {...}, "status", RunStatus.PENDING), - (RunTaskExecution, {...}, "status", ExecutionStatus.PENDING), - (RunResource, {...}, "node_id", some_node_id), - # ... all variants -]) -def test_field_stores_value(cls, kwargs, field, expected): - obj = cls(**kwargs) - assert getattr(obj, field) == expected - - -@pytest.mark.parametrize("cls,frozen_field,new_value", [ - (RunRecord, "status", RunStatus.COMPLETED), - # ... all frozen model mutation cases -]) -def test_frozen_model_rejects_mutation(cls, frozen_field, new_value): - obj = cls(...) - with pytest.raises(ValidationError, match="frozen instance"): - setattr(obj, frozen_field, new_value) -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_type_invariants.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_type_invariants.py -git commit -m "test(state): parametrize type invariant tests to eliminate per-model boilerplate" -``` - ---- - -### Task 29: Fix positional-index tool assertions in `test_subtask_lifecycle_toolkit.py` - -**Files:** -- Modify: `tests/unit/smoke_base/test_subtask_lifecycle_toolkit.py` - -- [ ] **Step 1: Read the full file** - -```bash -cat -n tests/unit/smoke_base/test_subtask_lifecycle_toolkit.py -``` - -Identify all uses of `tools[0]`, `tools[2]`, etc. - -- [ ] **Step 2: Replace count + name checks with a single set assertion** - -```python -def test_get_tools_returns_expected_set() -> None: - tools = _make_toolkit().get_tools() - names = {t.__name__ for t in tools} - assert names == { - "add_subtask", "plan_subtasks", "cancel_task", - "refine_task", "restart_task", "list_subtasks", - "get_subtask", "bash", - } - - -def test_add_subtask_is_async() -> None: - tools = _make_toolkit().get_tools() - add = next(t for t in tools if t.__name__ == "add_subtask") - assert asyncio.iscoroutinefunction(add) -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/smoke_base/test_subtask_lifecycle_toolkit.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/smoke_base/test_subtask_lifecycle_toolkit.py -git commit -m "test(subtask-lifecycle): replace positional tool[i] indexing with name-based lookup" -``` - ---- - -### Task 30: Pin error message content in lifecycle toolkit error-path tests - -**Files:** -- Modify: `tests/unit/smoke_base/test_subtask_lifecycle_toolkit.py` (or `tests/unit/state/test_subtask_lifecycle_toolkit.py` — whichever has `assert isinstance(result["error"], str)`) - -- [ ] **Step 1: Find the loose error assertions** - -```bash -rg -n 'isinstance.*"error".*str\|len.*"error".*> 0' tests/unit/ -``` - -- [ ] **Step 2: Add content assertions** - -```python -# Before -assert isinstance(result["error"], str) -assert len(result["error"]) > 0 - -# After -assert isinstance(result["error"], str) -assert "uuid" in result["error"].lower() # or "invalid" — match actual message -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest -k "lifecycle_toolkit" -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/ -git commit -m "test(subtask-lifecycle): assert error message content, not just non-empty string" -``` - ---- - -### Task 31: Replace `inspect.getsource` ordering test with runtime behaviour test - -**Files:** -- Modify: `tests/unit/smoke_base/test_leaf_sends_completion_message.py` - -- [ ] **Step 1: Read lines 104-144** - -Find the test that does `inspect.getsource(lb.BaseSmokeLeafWorker.execute)` and asserts on text positions. - -- [ ] **Step 2: Rewrite as a runtime behaviour test** - -Instead of asserting source-code order, execute the code path and observe the outcome: - -```python -async def test_completion_message_not_sent_when_subworker_raises(monkeypatch): - """_send_completion_message must not be called if subworker.work() raises.""" - send_calls = [] - - async def _fake_send(*args, **kwargs): - send_calls.append((args, kwargs)) - - monkeypatch.setattr(BaseSmokeLeafWorker, "_send_completion_message", _fake_send) - - leaf = _FailingLeaf(...) # subworker raises during work() - with pytest.raises(SomeExpectedException): - async for _ in leaf.execute(task, context=context): - pass - - assert send_calls == [], "_send_completion_message must not be called when subworker raises" -``` - -Adapt to the actual class/exception structure from reading the file. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/smoke_base/test_leaf_sends_completion_message.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/smoke_base/test_leaf_sends_completion_message.py -git commit -m "test(smoke_base): replace inspect.getsource ordering check with runtime behaviour assertion" -``` - ---- - -### Task 32: Rename and strengthen context assembly test - -**Files:** -- Modify: `tests/unit/state/test_context_assembly.py` - -- [ ] **Step 1: Read lines 203-214** - -Find `test_only_request_events_no_response`. - -- [ ] **Step 2: Rename + strengthen assertion** - -```python -def test_pending_request_parts_not_flushed_without_response_event(self): - """Request events alone must not produce assembled messages — flush requires a response.""" - events = [ - _make_event("system_prompt", SystemPromptPayload(text="sys"), 0), - _make_event("user_message", UserMessagePayload(text="hi"), 1), - ] - messages = assemble_pydantic_ai_messages(events) - assert messages == [] # exact equality, not just len == 0 -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_context_assembly.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_context_assembly.py -git commit -m "test(state): rename and strengthen context assembly flush-semantics test" -``` - ---- - -### Task 33: Strengthen MiniF2F criterion result assertions - -**Files:** -- Modify: `tests/unit/benchmarks/test_minif2f_proof_verification.py` - -- [ ] **Step 1: Read lines 63-86** - -Find both tests — the happy path (proof present) and the zero-score path. - -- [ ] **Step 2: Assert all result fields** - -```python -# Happy path -assert result.passed is True -assert result.score == 1.0 -assert result.name # existing - -# Error path (proof missing) -assert result.passed is False -assert result.score == 0.0 -assert result.feedback is not None -assert "not found" in result.feedback.lower() # or whatever the actual message is -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/benchmarks/test_minif2f_proof_verification.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/benchmarks/test_minif2f_proof_verification.py -git commit -m "test(minif2f): assert passed/score/feedback on criterion results, not just name presence" -``` - ---- - -### Task 34: Fix tool count assertion in `test_react_factories.py` - -**Files:** -- Modify: `tests/unit/registry/test_react_factories.py` - -- [ ] **Step 1: Read line 62 and surrounding context** - -Find `assert worker.tools != []`. Determine the expected tool count for the MiniF2F factory (check what toolkit it uses). - -- [ ] **Step 2: Replace with exact count** - -```python -# Before -assert worker.tools != [] - -# After -assert len(worker.tools) == <N> # e.g. 6 for MiniF2FToolkit — verify from toolkit source -``` - -Look at `MiniF2FToolkit.get_tools()` to confirm N. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/registry/test_react_factories.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/registry/test_react_factories.py -git commit -m "test(registry): assert exact tool count for react factory, not just non-empty" -``` - ---- - -### Task 35: Parametrize `test_event_schema_phase0.py` - -**Files:** -- Modify: `tests/unit/state/test_event_schema_phase0.py` - -- [ ] **Step 1: Read the full file** - -```bash -cat -n tests/unit/state/test_event_schema_phase0.py -``` - -List every model class being tested and the constructor kwargs for each. - -- [ ] **Step 2: Collapse 8 test class pairs into parametrized functions** - -```python -_NODE_ID_MODELS = [ - (TaskReadyEvent, {"run_id": _RUN_ID, "definition_id": _DEF_ID, "task_id": _TASK_ID}), - (TaskCompletedEvent, {"run_id": _RUN_ID, "node_id": None, ...}), - # ... all 8 classes -] - -@pytest.mark.parametrize("cls,base_kwargs", _NODE_ID_MODELS) -def test_node_id_accepts_value(cls, base_kwargs): - nid = NodeId(uuid4()) - obj = cls(**base_kwargs, node_id=nid) - assert obj.node_id == nid - # Round-trip: - assert cls.model_validate(obj.model_dump()).node_id == nid - - -@pytest.mark.parametrize("cls,base_kwargs", _NODE_ID_MODELS) -def test_node_id_defaults_to_none(cls, base_kwargs): - obj = cls(**base_kwargs) - assert obj.node_id is None -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_event_schema_phase0.py -v -``` - -Expected: same number of passing tests as before, now parametrized. - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_event_schema_phase0.py -git commit -m "test(state): parametrize event schema node_id tests across all 8 model classes" -``` - ---- - -### Task 36: Parametrize LLM judge pass/fail verdict test - -**Files:** -- Modify: `tests/unit/state/test_llm_judge_runtime_injection.py` - -- [ ] **Step 1: Read lines 57-84** - -Find the two near-identical `test_evaluate_calls_runtime` / `test_evaluate_failing_verdict` functions. - -- [ ] **Step 2: Collapse to one parametrized test** - -```python -@pytest.mark.parametrize("passed,expected_score,reasoning", [ - (True, 1.0, "Good coverage"), - (False, 0.0, "Report lacks sources"), -]) -async def test_evaluate_verdict(passed, expected_score, reasoning, mock_runtime): - criterion = LLMJudgeCriterion(...) - mock_runtime.return_value = _JudgeVerdict(passed=passed, reasoning=reasoning) - result = await criterion.evaluate(ctx) - assert result.passed is passed - assert result.score == expected_score - assert reasoning in result.feedback -``` - -Adapt to the actual API from reading the file. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_llm_judge_runtime_injection.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_llm_judge_runtime_injection.py -git commit -m "test(criterion): parametrize LLM judge pass/fail verdict into single test" -``` - ---- - -### Task 37: Add second-script exit-code test to `test_swebench_sandbox_manager.py` - -**Files:** -- Modify: `tests/unit/benchmarks/test_swebench_sandbox_manager.py` - -- [ ] **Step 1: Read lines 27-52** - -Understand `test_install_raises_on_nonzero_exit` and the `_fake_run` structure. - -- [ ] **Step 2: Parametrize across both scripts** - -```python -@pytest.mark.parametrize("failing_script", ["setup_env_script", "install_repo_script"]) -def test_install_raises_when_script_fails(failing_script, monkeypatch): - def _fake_run(cmd, **kwargs): - if failing_script in cmd: - return FakeResult(exit_code=1, stderr=f"{failing_script} failed") - return FakeResult(exit_code=0) - - monkeypatch.setattr(sandbox, "run_command", _fake_run) - with pytest.raises(SandboxSetupError, match=failing_script): - sandbox.install(task_id=some_uuid) -``` - -Adapt to actual class/method names from reading the file. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_sandbox_manager.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/benchmarks/test_swebench_sandbox_manager.py -git commit -m "test(swebench): parametrize sandbox install failure across both setup scripts" -``` - ---- - -### Task 38: Fix non-deterministic status code in harness mount test - -**Files:** -- Modify: `tests/unit/test_app_mounts_harness_conditionally.py` - -- [ ] **Step 1: Read lines 29-39** - -Find the `assert resp.status_code in (404, 500)` assertion and understand the app loading pattern. - -- [ ] **Step 2: Override DB dependency to make response deterministic** - -Follow the pattern from `test_test_harness.py` — inject a stub session so DB absence returns 404, not 500: - -```python -def test_harness_mounted_when_env_set(monkeypatch): - app = _reload_app_with(monkeypatch, "1") - app.dependency_overrides[get_session] = lambda: _null_session() - client = TestClient(app) - resp = client.get(f"/api/test/read/run/{uuid4()}/state") - assert resp.status_code == 404 # route exists, run not found — deterministic -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/test_app_mounts_harness_conditionally.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/test_app_mounts_harness_conditionally.py -git commit -m "test(harness): inject null DB session to get deterministic 404 instead of 500" -``` - ---- - -### Task 39: Fix over-mocked `ReActWorker.execute` in `test_research_rubrics_workers.py` - -**Files:** -- Modify: `tests/unit/state/test_research_rubrics_workers.py` - -- [ ] **Step 1: Read lines 63-118** - -Understand what `patch("...ReActWorker.execute")` is replacing and what `worker.tools` is expected to contain. - -- [ ] **Step 2: Remove the wholesale execute mock; mock only I/O** - -The goal is to verify that `worker.tools` is set up by the `execute` preamble. Instead of mocking `execute` entirely, extract the tool-registration into a dedicated test that doesn't call `execute` at all: - -```python -async def test_tools_registered_after_execute_setup(monkeypatch): - """Tool registration happens in execute() preamble before super().execute().""" - # Patch super().execute() to stop after preamble, not the whole method - stopped = [] - async def _stop(*args, **kwargs): - stopped.append(True) - return - yield # make it an async generator - - monkeypatch.setattr(ReActWorker, "execute", _stop) - worker = ResearchRubricsManagerWorker(...) - async for _ in worker.execute(task, context=ctx): - pass - assert len(worker.tools) == 12 - assert stopped # confirm we did enter execute -``` - -Adapt based on actual class hierarchy. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_research_rubrics_workers.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_research_rubrics_workers.py -git commit -m "test(workers): mock only super().execute to keep tool-registration preamble under test" -``` - ---- - -### Task 40: Fix `test_no_retired_slugs_present` to check registry, not module attributes - -**Files:** -- Modify: `tests/unit/smoke_base/test_registry_smoke_entries.py` - -- [ ] **Step 1: Read lines 40-44** - -Find `assert not hasattr(fixtures, "CanonicalSmokeWorker")`. - -- [ ] **Step 2: Replace with registry key check** - -```python -def test_no_retired_slugs_present() -> None: - from ergon_builtins.registry_core import WORKERS, EVALUATORS - retired_worker_slugs = {"canonical-smoke"} - assert not (retired_worker_slugs & set(WORKERS.keys())), ( - f"Retired worker slugs still in registry: {retired_worker_slugs & set(WORKERS.keys())}" - ) -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/smoke_base/test_registry_smoke_entries.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/smoke_base/test_registry_smoke_entries.py -git commit -m "test(registry): check registry keys for retired slugs, not module attribute presence" -``` - ---- - -### Task 41: Assert full expected worker slug set in `test_research_rubrics_benchmark.py` - -**Files:** -- Modify: `tests/unit/state/test_research_rubrics_benchmark.py` - -- [ ] **Step 1: Read lines 25-29** - -Find the single `assert "researchrubrics-researcher" in WORKERS` check. - -- [ ] **Step 2: Assert the full expected set** - -```python -def test_worker_slugs_registered(self): - from ergon_builtins.registry_data import WORKERS - expected = {"researchrubrics-researcher"} # full authoritative list - missing = expected - set(WORKERS.keys()) - assert not missing, f"Expected worker slugs missing from registry: {missing}" -``` - -(Add any additional slugs that should be present beyond the researcher.) - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_research_rubrics_benchmark.py -git commit -m "test(registry): assert full expected worker slug set, not just one slug" -``` - ---- - -### Task 42: Assert `feedback` content on MiniF2F proof-missing path - -**Files:** -- Modify: `tests/unit/benchmarks/test_minif2f_proof_verification.py` - -- [ ] **Step 1: Read lines 67-86** - -Find `test_scores_zero_when_proof_missing`. - -- [ ] **Step 2: Add feedback content assertion** - -```python -assert result.score == 0.0 -assert not result.passed -assert result.feedback is not None -assert "not found" in result.feedback.lower() # verify actual message from production code -``` - -Read the production criterion code to confirm the exact error string used. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/benchmarks/test_minif2f_proof_verification.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/benchmarks/test_minif2f_proof_verification.py -git commit -m "test(minif2f): assert feedback message content on proof-missing error path" -``` - ---- - -### Task 43: Consolidate CLI error tests into parametrized rc + stderr checks - -**Files:** -- Modify: `tests/unit/cli/test_benchmark_setup.py` - -- [ ] **Step 1: Read lines 67-83** - -Find `test_fails_when_api_key_unset`, `test_error_message_mentions_api_key`, `test_fails_for_unknown_slug`. - -- [ ] **Step 2: Collapse to single parametrized test** - -```python -@pytest.mark.parametrize("setup_fn,expected_stderr_fragment", [ - (_unset_api_key, "api_key"), - (_set_unknown_slug, "unknown"), -]) -def test_setup_fails_with_informative_error(setup_fn, expected_stderr_fragment, monkeypatch, capsys): - setup_fn(monkeypatch) - rc = setup_benchmark(_make_args()) - assert rc != 0 - captured = capsys.readouterr() - assert expected_stderr_fragment in (captured.err + captured.out).lower() -``` - -Each case simultaneously checks both the exit code and the message content. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/cli/test_benchmark_setup.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/cli/test_benchmark_setup.py -git commit -m "test(cli): parametrize error cases to assert both exit code and error message content" -``` - ---- - -## Part 3: Zero-Value Test Deletions - -Tests that pass unconditionally regardless of whether the production code works. Delete outright — no replacement needed. - ---- - -### Task 44: Delete `test_add_subtask_tool_is_async_callable` and `test_get_tools_returns_eight_callables` - -**Files:** -- Modify: `tests/unit/state/test_subtask_lifecycle_toolkit.py` - -Both functions are redundant noise: -- `test_get_tools_returns_eight_callables` — `len(tools) == 8` is already proved by `test_tools_have_correct_function_names` which lists all 8 names. `all(callable(t) for t in tools)` tests Python's `def` keyword. -- `test_add_subtask_tool_is_async_callable` — `asyncio.iscoroutinefunction(tools[0])` tests Python's `async` keyword via a positional index. Type-checking catches accidental sync-ification; the positional index also makes it fragile to reordering. - -- [ ] **Step 1: Delete both functions** - -Remove lines 17-27 (both `test_get_tools_returns_eight_callables` and `test_add_subtask_tool_is_async_callable`) from `tests/unit/state/test_subtask_lifecycle_toolkit.py`. Also remove the now-unused `import asyncio` at the top if it's only used by those two tests. - -- [ ] **Step 2: Run tests to confirm nothing else was relying on them** - -```bash -uv run pytest tests/unit/state/test_subtask_lifecycle_toolkit.py -v -``` - -Expected: remaining tests still pass. - -- [ ] **Step 3: Commit** - -```bash -git add tests/unit/state/test_subtask_lifecycle_toolkit.py -git commit -m "test(subtask-lifecycle): delete zero-value async/callable assertions covered by name test" -``` - ---- - -### Task 45: Delete `tests/unit/api/test_types_reexport.py` - -**Files:** -- Delete: `tests/unit/api/test_types_reexport.py` - -Both functions in this file test Python language mechanics, not application logic: - -- `test_tool_is_reexported_from_api_root` — the two assertions (`_takes_tools([]) == 0`, `_takes_tools([object(), object()]) == 2`) test `len()` on literal lists. The comment in the file itself says `# noqa: F401 — import is the assertion`. The import is covered by `ty` type checking. -- `test_tool_module_is_importable` — `hasattr(api_types, "Tool")` is a pure import test. Covered by `ty`. - -- [ ] **Step 1: Delete the file** - -```bash -git rm tests/unit/api/test_types_reexport.py -``` - -- [ ] **Step 2: Check nothing imports from it** - -```bash -rg "test_types_reexport" tests/ -``` - -Expected: zero matches. - -- [ ] **Step 3: Run the api test suite** - -```bash -uv run pytest tests/unit/api/ -v -``` - -- [ ] **Step 4: Commit** - -```bash -git commit -m "test(api): delete test_types_reexport — pure import tests covered by ty type checker" -``` - ---- - -## Part 4: Gaps to reach 8+/10 - -Three categories not covered above that are required to reach 8/10: the docstring lie in `test_type_invariants.py`, a small DRY consolidation left unfinished, and zero test coverage for three production abstractions added in the recent refactor. - ---- - -### Task 46: Add invalid-value rejection tests to `test_type_invariants.py` - -**Files:** -- Modify: `tests/unit/state/test_type_invariants.py` - -The file's docstring says *"Verifies that enum/Literal fields reject invalid values at model construction time"* but contains **zero** tests that pass an invalid value. Every test only constructs with a valid value and asserts it is stored. Task 28 (Part 2) adds parametrize + frozen-mutation tests; this task adds the construction-time rejection tests that the docstring actually promises. - -- [ ] **Step 1: Read the file and list every constrained field** - -```bash -cat -n tests/unit/state/test_type_invariants.py -``` - -For each model/field being tested, identify what "invalid" looks like (a string literal outside the enum, `None` for a required field, a negative integer for a `PositiveInt`, etc.). - -- [ ] **Step 2: Write rejection tests — one per constrained field** - -```python -from pydantic import ValidationError -import pytest - -@pytest.mark.parametrize("cls,kwargs_override,invalid_field,invalid_value", [ - ( - RunRecord, - {"experiment_definition_id": uuid4()}, - "status", - "not-a-status", - ), - ( - RunTaskExecution, - {"run_id": uuid4(), "definition_task_id": uuid4()}, - "status", - "garbage", - ), - ( - RunGenerationTurn, - {"run_id": uuid4(), "task_execution_id": uuid4(), - "worker_binding_key": "w", "turn_index": 0, "raw_response": {}}, - "execution_outcome", - "unknown-outcome", - ), - # ... add a case for every model with a constrained field -]) -def test_invalid_field_value_rejected(cls, kwargs_override, invalid_field, invalid_value): - """Enum/Literal fields must raise ValidationError for invalid values at construction.""" - with pytest.raises(ValidationError): - cls(**{**kwargs_override, invalid_field: invalid_value}) -``` - -Run it to confirm it fails first (production code should raise — if it doesn't, that's a real bug to fix): - -```bash -uv run pytest tests/unit/state/test_type_invariants.py::test_invalid_field_value_rejected -v -``` - -Expected: PASS (Pydantic should reject these). If any case unexpectedly passes (no error raised), the production model is missing a validator — file a bug and tighten the model. - -- [ ] **Step 3: Run the full file** - -```bash -uv run pytest tests/unit/state/test_type_invariants.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_type_invariants.py -git commit -m "test(state): add construction-time rejection tests — fulfill docstring contract in test_type_invariants" -``` - ---- - -### Task 47: Consolidate cancel/refine/restart error-path tests into one parametrized test - -**Files:** -- Modify: `tests/unit/state/test_subtask_lifecycle_toolkit.py` - -Task 30 (Part 2) pins the error message content in these three functions. This task consolidates the three near-identical functions into a single parametrized test to remove the copy-paste. Do this after Task 30 so the content assertions are already in place. - -- [ ] **Step 1: Read the three functions after Task 30 is applied** - -The three functions (`test_cancel_task_handles_invalid_uuid_gracefully`, `test_refine_task_handles_invalid_uuid_gracefully`, `test_restart_task_handles_invalid_uuid_gracefully`) will differ only by which tool they call and what args they pass. Note each: `(tool_name, args_tuple)`. - -- [ ] **Step 2: Collapse into one parametrized test** - -```python -@pytest.mark.parametrize("tool_name,args", [ - ("cancel_task", ("not-a-uuid",)), - ("refine_task", ("not-a-uuid", "new description")), - ("restart_task", ("not-a-uuid",)), -]) -async def test_invalid_uuid_returns_error(tool_name: str, args: tuple) -> None: - tools = _make_toolkit().get_tools() - tool = next(t for t in tools if t.__name__ == tool_name) - result = await tool(*args) - assert result["success"] is False - assert "uuid" in result["error"].lower() -``` - -- [ ] **Step 3: Delete the three original functions** - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/unit/state/test_subtask_lifecycle_toolkit.py -v -``` - -Expected: 3 parametrize variants passing, replacing 3 functions. - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/state/test_subtask_lifecycle_toolkit.py -git commit -m "test(subtask-lifecycle): consolidate cancel/refine/restart error paths into parametrized test" -``` - ---- - -### Task 48: Add unit tests for `is_stub_sandbox_id()`, `VLLMDiscoveryError`, `RunRecordMissingError` - -Three production abstractions added in the recent refactor have **zero test coverage**. Confirmed with: - -```bash -rg "is_stub_sandbox_id|VLLMDiscoveryError|RunRecordMissingError" tests/unit/ tests/integration/ -# → zero matches -``` - -**Files:** -- Create: `tests/unit/sandbox/test_stub_sandbox_id.py` -- Create: `tests/unit/runtime/test_vllm_discovery_error.py` -- Create: `tests/unit/runtime/test_run_record_missing_error.py` - -(Or consolidate into nearby existing test files if they fit naturally — see steps below.) - -- [ ] **Step 1: Locate the production implementations** - -```bash -rg -n "def is_stub_sandbox_id\|class VLLMDiscoveryError\|class RunRecordMissingError" ergon_core/ ergon_builtins/ -``` - -Read each to understand the contract. - -- [ ] **Step 2: Write tests for `is_stub_sandbox_id()`** - -```python -# tests/unit/sandbox/test_stub_sandbox_id.py -import pytest -from ergon_core.core.runtime.inngest.execute_task import is_stub_sandbox_id # adjust import - -@pytest.mark.parametrize("sandbox_id,expected", [ - ("stub-sbx-abc123", True), # adjust prefix to match actual implementation - ("sbx-real-123", False), - ("", False), - (None, False), # if None is a valid input -]) -def test_is_stub_sandbox_id(sandbox_id, expected): - assert is_stub_sandbox_id(sandbox_id) is expected -``` - -Adjust the parametrize cases to match the actual stub prefix/pattern from reading the source. - -- [ ] **Step 3: Write tests for `VLLMDiscoveryError`** - -```python -# tests/unit/runtime/test_vllm_discovery_error.py -import pytest -from ergon_core.core.providers.generation.vllm_model import VLLMDiscoveryError, _discover_vllm_model_name - -def test_discovery_raises_when_endpoint_unreachable(monkeypatch): - """_discover_vllm_model_name must raise VLLMDiscoveryError, not return 'default'.""" - import urllib.error - monkeypatch.setattr( - "ergon_core.core.providers.generation.vllm_model.urllib.request.urlopen", - lambda *a, **kw: (_ for _ in ()).throw(urllib.error.URLError("connection refused")), - ) - with pytest.raises(VLLMDiscoveryError, match="connection refused"): - _discover_vllm_model_name("http://localhost:8000") - - -def test_discovery_raises_on_malformed_json(monkeypatch): - """Malformed JSON response must raise VLLMDiscoveryError, not crash.""" - # monkeypatch urlopen to return b"not json" - ... - with pytest.raises(VLLMDiscoveryError): - _discover_vllm_model_name("http://localhost:8000") -``` - -Adapt to the actual function signature and import path from step 1. - -- [ ] **Step 4: Write tests for `RunRecordMissingError`** - -```python -# tests/unit/runtime/test_run_record_missing_error.py -import pytest -from ergon_core.core.runtime.errors.delegation_errors import RunRecordMissingError -from ergon_core.core.runtime.services.task_management_service import TaskManagementService - -def test_raises_when_run_record_missing(mock_session): - """TaskManagementService must raise RunRecordMissingError, not AttributeError or None.""" - # mock_session returns None for RunRecord query - svc = TaskManagementService() - with pytest.raises(RunRecordMissingError, match="RunRecord"): - svc.some_method(mock_session, run_id=uuid4()) -``` - -Read the production code to identify which method triggers this and adapt accordingly. - -- [ ] **Step 5: Run all three new test files** - -```bash -uv run pytest tests/unit/sandbox/test_stub_sandbox_id.py \ - tests/unit/runtime/test_vllm_discovery_error.py \ - tests/unit/runtime/test_run_record_missing_error.py -v -``` - -- [ ] **Step 6: Commit** - -```bash -git add tests/unit/sandbox/test_stub_sandbox_id.py \ - tests/unit/runtime/test_vllm_discovery_error.py \ - tests/unit/runtime/test_run_record_missing_error.py -git commit -m "test(runtime,sandbox): add unit tests for is_stub_sandbox_id, VLLMDiscoveryError, RunRecordMissingError" -``` - ---- - -**Goal:** Address 48 identified issues across unit and integration tests — covering reuse/DRY, code quality, coverage gaps, assertion strength, performance, isolation (Part 1), weak-test patterns against the pydantic-ai gold standard (Part 2), zero-value test deletions (Part 3), and the remaining gaps to reach 8+/10 cleanliness (Part 4). - -**Architecture:** No production code changes. All fixes are confined to `tests/unit/` and `tests/integration/` (plus new conftest files). Changes are independent and can be done in any order. - -**Tech Stack:** pytest, pytest-asyncio (`asyncio_mode=auto`), pytest-xdist (`-n auto`), SQLModel, unittest.mock - ---- - -## Category 1: Reuse / DRY - -### Task 1: Deduplicate `_patch_session_with_rows()` across runtime tests - -**Files:** -- Create: `tests/unit/runtime/conftest.py` -- Modify: `tests/unit/runtime/test_criterion_runtime_get_all_files.py` -- Modify: any other runtime test files that define the same helper (verify with `rg "_patch_session_with_rows" tests/unit/runtime/`) - -- [ ] **Step 1: Locate all duplicates** - -```bash -rg "_patch_session_with_rows" tests/unit/runtime/ -``` - -Note every file that defines this helper. - -- [ ] **Step 2: Write the shared fixture** - -Create `tests/unit/runtime/conftest.py`: - -```python -import contextlib -from collections.abc import Generator -from unittest.mock import MagicMock, patch - - -@contextlib.contextmanager -def patch_session_with_rows(rows: list) -> Generator[MagicMock, None, None]: - """Patch get_session() to yield a mock session with the given query rows.""" - mock_session = MagicMock() - mock_session.exec.return_value.all.return_value = rows - with patch("ergon_core.core.persistence.shared.db.get_session") as mock_get: - mock_get.return_value.__enter__ = MagicMock(return_value=mock_session) - mock_get.return_value.__exit__ = MagicMock(return_value=False) - yield mock_session -``` - -(Adjust the patch target to match the actual import path found in step 1.) - -- [ ] **Step 3: Replace the inline definitions** - -In each file identified in step 1, remove the local `_patch_session_with_rows` definition and import the shared one: - -```python -from tests.unit.runtime.conftest import patch_session_with_rows -``` - -- [ ] **Step 4: Run tests to verify nothing broke** - -```bash -uv run pytest tests/unit/runtime/ -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/runtime/ -git commit -m "test(runtime): extract _patch_session_with_rows to shared conftest" -``` - ---- - -### Task 2: Extract shared smoke_base fixtures to conftest - -**Files:** -- Create: `tests/unit/smoke_base/conftest.py` -- Modify: `tests/unit/smoke_base/test_smoke_criterion_shape.py` -- Modify: `tests/unit/smoke_base/test_smoke_criterion_completed.py` -- Modify: `tests/unit/smoke_base/test_smoke_criterion_probe.py` - -- [ ] **Step 1: Read the three files and identify the duplicated code** - -```bash -rg "_FakeNode|class _Crit" tests/unit/smoke_base/ -l -``` - -Read each file to extract the exact `_FakeNode` dataclass and `_Crit` subclass definitions. - -- [ ] **Step 2: Write the shared conftest** - -Create `tests/unit/smoke_base/conftest.py` with the shared `_FakeNode` dataclass and `_Crit` base subclass (copy the canonical version from one of the three files). - -- [ ] **Step 3: Remove local definitions and import from conftest** - -In each of the three test files, delete the local `_FakeNode` / `_Crit` definitions and replace with imports. Pytest automatically discovers `conftest.py` in the same directory, so no explicit import of fixtures is needed — but for plain helper classes, use a direct import: - -```python -from tests.unit.smoke_base.conftest import _FakeNode, _Crit -``` - -- [ ] **Step 4: Run and verify** - -```bash -uv run pytest tests/unit/smoke_base/ -v -``` - -Expected: all tests pass. - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/smoke_base/ -git commit -m "test(smoke_base): move shared _FakeNode and _Crit to conftest" -``` - ---- - -### Task 3: Deduplicate `_mock_runtime()` / `_ctx()` between swebench tests - -**Files:** -- Create: `tests/integration/swebench_verified/conftest.py` -- Modify: `tests/integration/swebench_verified/test_criterion.py` -- Modify: `tests/unit/test_swebench_criterion_no_sandbox.py` (or wherever the unit-side copy lives — verify with `rg "_mock_runtime|def _ctx" tests/`) - -- [ ] **Step 1: Confirm both definitions are identical** - -```bash -rg -A 20 "def _mock_runtime\|def _ctx" tests/integration/swebench_verified/test_criterion.py -rg -A 20 "def _mock_runtime\|def _ctx" tests/unit/ -``` - -If they differ, note the differences and produce a single merged version that satisfies both. - -- [ ] **Step 2: Extract to integration conftest** - -Create `tests/integration/swebench_verified/conftest.py`: - -```python -# Shared helpers for swebench criterion tests. -# Also used by tests/unit/test_swebench_criterion_no_sandbox.py via direct import. -``` - -Move the canonical `_mock_runtime()` factory and `_ctx()` helper into this file. - -- [ ] **Step 3: Update importers** - -In both test files, remove the local definitions and import: - -```python -from tests.integration.swebench_verified.conftest import _mock_runtime, _ctx -``` - -- [ ] **Step 4: Run both test files** - -```bash -uv run pytest tests/integration/swebench_verified/test_criterion.py tests/unit/test_swebench_criterion_no_sandbox.py -v -``` - -- [ ] **Step 5: Commit** - -```bash -git add tests/ -git commit -m "test(swebench): extract _mock_runtime/_ctx to shared conftest" -``` - ---- - -### Task 4: Move CLI test helpers to `tests/unit/cli/conftest.py` - -**Files:** -- Create: `tests/unit/cli/conftest.py` -- Modify: `tests/unit/cli/test_benchmark_setup.py` - -- [ ] **Step 1: Read the current helpers** - -Read `tests/unit/cli/test_benchmark_setup.py` lines 12-59 to extract `_FakeBuildInfo` and `_patch_sdk()`. - -- [ ] **Step 2: Create conftest with the helpers** - -```python -# tests/unit/cli/conftest.py -import pytest -from unittest.mock import patch -# ... move _FakeBuildInfo and _patch_sdk() here verbatim -``` - -- [ ] **Step 3: Update the test file** - -Remove the local definitions and import from conftest. - -- [ ] **Step 4: Run tests** - -```bash -uv run pytest tests/unit/cli/ -v -``` - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/cli/ -git commit -m "test(cli): move _FakeBuildInfo and _patch_sdk to conftest" -``` - ---- - -## Category 2: Code Quality - -### Task 5: Deduplicate `_Resp` mock class in `test_openrouter_budget.py` - -**Files:** -- Modify: `tests/unit/test_openrouter_budget.py` - -- [ ] **Step 1: Read the file and identify both `_Resp` definitions** - -```bash -rg -n "_Resp\|_make_mock_response" tests/unit/test_openrouter_budget.py -``` - -- [ ] **Step 2: Replace with a module-level factory** - -Remove the two inline class definitions and add at the top of the file: - -```python -def _make_mock_response(exit_code: int, usage: dict | None = None): - class _Resp: - def __init__(self): - self.exit_code = exit_code - self.usage = usage or {} - return _Resp() -``` - -Update each test function to call `_make_mock_response(exit_code=0)` etc. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/test_openrouter_budget.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/test_openrouter_budget.py -git commit -m "test(openrouter): extract _Resp to module-level factory" -``` - ---- - -### Task 6: Narrow bare `except Exception` in `tests/integration/conftest.py` - -**Files:** -- Modify: `tests/integration/conftest.py` - -- [ ] **Step 1: Read the current except clause (~line 62)** - -Read `tests/integration/conftest.py` to find the broad except in `_reset_inngest_http_client`. - -- [ ] **Step 2: Replace with specific exceptions** - -Identify what can actually be raised when tearing down the HTTP client (e.g. `RuntimeError`, `asyncio.CancelledError`). Replace: - -```python -except Exception: - pass -``` - -with something like: - -```python -except (RuntimeError, asyncio.CancelledError): - pass # client already closed or event loop torn down -``` - -Add an inline comment explaining why these are tolerated. - -- [ ] **Step 3: Run integration tests (with infra available) or just run check:be** - -```bash -pnpm run check:be -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/integration/conftest.py -git commit -m "test(integration): narrow broad except in inngest client teardown" -``` - ---- - -### Task 7: Add missing `@pytest.mark.asyncio` in `test_worker_execute_factory_call.py` - -**Files:** -- Modify: `tests/unit/runtime/test_worker_execute_factory_call.py` - -- [ ] **Step 1: Read the file** - -```bash -cat -n tests/unit/runtime/test_worker_execute_factory_call.py -``` - -- [ ] **Step 2: Add marker** - -Add `@pytest.mark.asyncio` above any async test function missing it, or confirm the test is actually synchronous and clean up any async-looking patterns. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/runtime/test_worker_execute_factory_call.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/runtime/test_worker_execute_factory_call.py -git commit -m "test(runtime): add missing asyncio marker on worker execute test" -``` - ---- - -### Task 8: Fix fragile string matching in swebench integration criterion test - -**Files:** -- Modify: `tests/integration/swebench_verified/test_criterion.py` - -- [ ] **Step 1: Read the `_dispatch` inner function (~lines 60-68)** - -Find where it does `"git diff HEAD" in cmd` or similar string matching. - -- [ ] **Step 2: Replace with shlex-based argument parsing** - -```python -import shlex - -def _dispatch(cmd: str, **kwargs): - parts = shlex.split(cmd) - if parts[:3] == ["git", "diff", "HEAD"]: - return _fake_diff_output() - ... -``` - -This is robust to extra whitespace and argument order variations. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/integration/swebench_verified/test_criterion.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/integration/swebench_verified/test_criterion.py -git commit -m "test(swebench): use shlex for robust command matching in _dispatch" -``` - ---- - -### Task 9: Fix mutable sequence state in `test_event_schema_phase0.py` - -**Files:** -- Modify: `tests/unit/state/test_event_schema_phase0.py` - -- [ ] **Step 1: Read the `_make_event()` function (~line 43)** - -Identify the class-level `sequence` counter or mutable default that tests rely on. - -- [ ] **Step 2: Make sequence a required positional argument** - -```python -def _make_event(sequence: int, **kwargs): - ... -``` - -Update all call sites to pass an explicit sequence integer. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_event_schema_phase0.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_event_schema_phase0.py -git commit -m "test(state): make sequence a required arg in _make_event to remove mutable default" -``` - ---- - -### Task 10: Clarify misleading docstring in `test_context_assembly.py` - -**Files:** -- Modify: `tests/unit/state/test_context_assembly.py` - -- [ ] **Step 1: Read lines 204-214** - -Find the test that asserts `len(messages) == 0` with a comment about "not flushed because no response event". - -- [ ] **Step 2: Rename and re-document the test** - -Rename the test to something like `test_request_only_produces_no_assembled_messages` and update the docstring/comment to say the empty result is intentional behaviour (a request without a paired response yields no complete message). - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_context_assembly.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_context_assembly.py -git commit -m "test(state): clarify intent of zero-message assertion in context assembly" -``` - ---- - -## Category 3: Coverage Gaps - -### Task 11: Add test for harness-enabled + missing secret header → 401 - -**Files:** -- Modify: `tests/unit/test_test_harness.py` - -- [ ] **Step 1: Read the existing harness tests** - -```bash -cat -n tests/unit/test_test_harness.py -``` - -Understand the existing 401 test to model the new one. - -- [ ] **Step 2: Write the failing test first** - -```python -def test_seed_requires_secret_when_harness_enabled(client_with_harness): - """Requests without the secret header should be rejected 401 when harness is on.""" - resp = client_with_harness.post("/harness/seed", json={}) - assert resp.status_code == 401 -``` - -Run it: - -```bash -uv run pytest tests/unit/test_test_harness.py::test_seed_requires_secret_when_harness_enabled -v -``` - -Expected: FAIL (test not yet wired). - -- [ ] **Step 3: Verify production code handles this case** - -Check the harness route handler. If the 401 is already raised, the test will pass immediately — which means it was just missing coverage. If not, fix the route handler. - -- [ ] **Step 4: Run the full file** - -```bash -uv run pytest tests/unit/test_test_harness.py -v -``` - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/test_test_harness.py -git commit -m "test(harness): add coverage for missing secret header when harness is enabled" -``` - ---- - -### Task 12: Add happy-path tests to `test_subtask_lifecycle_toolkit.py` - -**Files:** -- Modify: `tests/unit/state/test_subtask_lifecycle_toolkit.py` - -- [ ] **Step 1: Read the existing test file** - -```bash -cat -n tests/unit/state/test_subtask_lifecycle_toolkit.py -``` - -Note which tools are tested (only error paths currently) and what the mock structure looks like. - -- [ ] **Step 2: Write failing happy-path tests** - -```python -async def test_add_subtask_succeeds_with_valid_args(mock_session): - tool = build_subtask_lifecycle_tools(run_id=..., parent_node_id=..., sandbox_id=...)[0] - result = await tool( - task_slug="subq-1", - description="Do the thing", - assigned_worker_slug="researchrubrics-researcher", - ) - assert result["success"] is True - assert "node_id" in result - - -async def test_cancel_subtask_succeeds(mock_session): - ... - - -async def test_refine_subtask_succeeds(mock_session): - ... -``` - -(Adapt to the actual tool surface; read the toolkit to understand what `mock_session` needs to return.) - -- [ ] **Step 3: Run tests to confirm they fail first, then pass after fixing mocks** - -```bash -uv run pytest tests/unit/state/test_subtask_lifecycle_toolkit.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_subtask_lifecycle_toolkit.py -git commit -m "test(subtask-lifecycle): add happy-path coverage for add/cancel/refine" -``` - ---- - -### Task 13: Add test for cached sandbox early-return path - -**Files:** -- Modify: `tests/unit/sandbox/test_ensure_sandbox_idempotence.py` - -- [ ] **Step 1: Read the existing tests (~lines 59-104)** - -Understand the mock structure for `get_sandbox()` and `create()`. - -- [ ] **Step 2: Write failing test** - -```python -def test_create_returns_cached_sandbox_without_install(monkeypatch): - """If get_sandbox() already returns a sandbox, create() must not call install.""" - manager = SomeSandboxManager() - fake_sandbox = object() - monkeypatch.setattr(manager, "get_sandbox", lambda task_id: fake_sandbox) - install_calls = [] - monkeypatch.setattr(manager, "_install", lambda *a, **kw: install_calls.append(1)) - - result = manager.create(task_id=some_uuid) - - assert result is fake_sandbox - assert install_calls == [], "install must not be called when sandbox already exists" -``` - -Run it, expect FAIL, then verify against the production code path. - -- [ ] **Step 3: Run the full sandbox test suite** - -```bash -uv run pytest tests/unit/sandbox/ -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/sandbox/test_ensure_sandbox_idempotence.py -git commit -m "test(sandbox): add coverage for cached sandbox early-return in create()" -``` - ---- - -### Task 14: Add evaluation-row assertion to `test_full_lifecycle.py` - -**Files:** -- Modify: `tests/integration/test_full_lifecycle.py` - -- [ ] **Step 1: Read the test (~lines 41-121)** - -Find where it asserts run/task status and understand the DB session fixture. - -- [ ] **Step 2: Add evaluation assertion** - -After the existing completion assertions, add: - -```python -from ergon_core.core.persistence.models import RunTaskEvaluation - -evals = session.exec( - select(RunTaskEvaluation).where(RunTaskEvaluation.run_id == run.id) -).all() -assert len(evals) > 0, "expected at least one evaluation row after full lifecycle" -``` - -- [ ] **Step 3: Run the integration test (requires running stack)** - -```bash -uv run pytest tests/integration/test_full_lifecycle.py -v -``` - -If the stack is not available, add the assertion anyway and note it for CI. - -- [ ] **Step 4: Commit** - -```bash -git add tests/integration/test_full_lifecycle.py -git commit -m "test(integration): assert evaluation rows exist after full lifecycle" -``` - ---- - -### Task 15: Add exit-code-127 variant to swebench criterion patch test - -**Files:** -- Modify: `tests/unit/benchmarks/test_swebench_criterion_patch_source.py` - -- [ ] **Step 1: Read the file and the `_fake_run()` helper (~lines 21-29)** - -Understand what the current fake run does and how exit codes map to criterion outcomes. - -- [ ] **Step 2: Write the failing test** - -```python -def test_install_failure_exit_127_fails_criterion(): - """Install script exiting 127 (command not found) should fail the criterion.""" - def _fake_run_127(cmd: str, **kwargs): - if "install" in cmd: - return FakeRunResult(exit_code=127, stdout="", stderr="bash: command not found") - return FakeRunResult(exit_code=0, stdout="", stderr="") - - criterion = SWEBenchCriterion(...) - result = criterion.evaluate(..., run_command=_fake_run_127) - assert result.passed is False - assert "127" in result.message or "not found" in result.message.lower() -``` - -Adapt to the actual API from reading the file. - -- [ ] **Step 3: Run test** - -```bash -uv run pytest tests/unit/benchmarks/test_swebench_criterion_patch_source.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/benchmarks/test_swebench_criterion_patch_source.py -git commit -m "test(swebench-criterion): add coverage for install script exit 127" -``` - ---- - -## Category 4: Assertion Strength - -### Task 16: Strengthen `test_type_invariants.py` status assertion - -**Files:** -- Modify: `tests/unit/state/test_type_invariants.py` - -- [ ] **Step 1: Read lines 29-40** - -Find the test asserting `record.status == RunStatus.PENDING`. - -- [ ] **Step 2: Split into two assertions** - -```python -# Assert the field was set by the constructor, not a default -record = RunRecord(status=RunStatus.PENDING, ...) -assert record.status == RunStatus.PENDING # explicitly set - -# Assert the field survives a round-trip (not just an in-memory default) -roundtripped = RunRecord.model_validate(record.model_dump()) -assert roundtripped.status == RunStatus.PENDING -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_type_invariants.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_type_invariants.py -git commit -m "test(state): strengthen RunRecord status assertion with explicit construction + round-trip check" -``` - ---- - -### Task 17: Add round-trip assertion in `test_event_schema_phase0.py` - -**Files:** -- Modify: `tests/unit/state/test_event_schema_phase0.py` - -- [ ] **Step 1: Read lines 20-35** - -Find the tests that set and check `node_id`. - -- [ ] **Step 2: Add `model_validate()` round-trip verification** - -```python -raw = evt.model_dump() -roundtripped = EventSchema.model_validate(raw) -assert roundtripped.node_id == nid, "node_id must survive model_dump/model_validate round-trip" -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_event_schema_phase0.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_event_schema_phase0.py -git commit -m "test(state): add round-trip assertion for node_id in event schema test" -``` - ---- - -### Task 18: Assert on `type_slug` in `test_research_rubrics_benchmark.py` - -**Files:** -- Modify: `tests/unit/state/test_research_rubrics_benchmark.py` - -- [ ] **Step 1: Read lines 9-23** - -Find the test checking `cls.__name__`. - -- [ ] **Step 2: Replace `__name__` check with `type_slug` check** - -```python -# Before: -assert cls.__name__ == "ResearchRubricsBenchmark" - -# After: -assert "researchrubrics-ablated" in BENCHMARKS -assert BENCHMARKS["researchrubrics-ablated"] is ResearchRubricsBenchmark -``` - -This is more intent-revealing: we care that the slug resolves to the right class, not what Python named the class. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_research_rubrics_benchmark.py -git commit -m "test(registry): assert registry maps to correct class, not class __name__" -``` - ---- - -### Task 19: Assert build not called in `test_idempotent_skip()` - -**Files:** -- Modify: `tests/unit/cli/test_benchmark_setup.py` - -- [ ] **Step 1: Read lines 102-113** - -Find `test_idempotent_skip()`. - -- [ ] **Step 2: Add negative assertion** - -After the `assert rc == 0`: - -```python -fake.build.assert_not_called() -# OR if the mock structure is different: -assert mock_build.call_count == 0, "build must not be triggered on idempotent skip" -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/cli/test_benchmark_setup.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/cli/test_benchmark_setup.py -git commit -m "test(cli): assert build is not triggered on idempotent skip" -``` - ---- - -## Category 5: Performance - -### Task 20: Guard smoke harness reset fixture against skipped tests - -**Files:** -- Modify: `tests/integration/smokes/test_smoke_harness.py` - -- [ ] **Step 1: Read the `_reset_before_each()` fixture (~lines 74-92)** - -Understand what the HTTP reset call does and when it's skipped. - -- [ ] **Step 2: Add a guard** - -```python -@pytest.fixture(autouse=True) -def _reset_before_each(request): - if request.node.get_closest_marker("skip"): - return - # ... existing HTTP reset logic -``` - -Alternatively, change `autouse=True` to explicit usage in tests that actually need the reset. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/integration/smokes/test_smoke_harness.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/integration/smokes/test_smoke_harness.py -git commit -m "test(smoke-harness): skip DB reset fixture on skipped tests" -``` - ---- - -### Task 21: Promote message factories to module-scoped fixtures in `test_generation_turn_build.py` - -**Files:** -- Modify: `tests/unit/state/test_generation_turn_build.py` - -- [ ] **Step 1: Read lines 34-90** - -Identify `_make_messages_text_only()` and `_make_messages_with_tool_call()` and whether any test mutates the returned objects. - -- [ ] **Step 2: Convert to module-scoped fixtures if safe** - -```python -@pytest.fixture(scope="module") -def messages_text_only(): - return _make_messages_text_only() - - -@pytest.fixture(scope="module") -def messages_with_tool_call(): - return _make_messages_with_tool_call() -``` - -Update test signatures to accept the fixtures. If any test mutates the list/objects, copy them first: `msgs = list(messages_text_only)`. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_generation_turn_build.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_generation_turn_build.py -git commit -m "test(state): promote message factories to module-scoped fixtures" -``` - ---- - -### Task 22: Parametrize slug-variant tests in `test_smoke_criterion_shape.py` - -**Files:** -- Modify: `tests/unit/smoke_base/test_smoke_criterion_shape.py` - -- [ ] **Step 1: Read lines 35-60** - -Find the four test functions for missing/extra/renamed slug variants (e.g. `test_missing_slug_fails`, `test_extra_slug_fails`, `test_renamed_slug_fails`, `test_correct_slugs_pass`). - -- [ ] **Step 2: Rewrite as parametrized test** - -```python -@pytest.mark.parametrize("mutation,expected_pass", [ - (lambda slugs: slugs[:-1], False), # missing one slug - (lambda slugs: slugs + ["extra"], False), # extra slug - (lambda slugs: slugs[:1] + ["renamed"] + slugs[2:], False), # renamed slug - (lambda slugs: slugs, True), # correct — all slugs present -]) -def test_smoke_criterion_slug_shape(mutation, expected_pass, fake_criterion): - slugs = mutation(list(EXPECTED_SUBTASK_SLUGS)) - result = fake_criterion.evaluate(build_graph_with_slugs(slugs)) - assert result.passed is expected_pass -``` - -Adapt to the actual API from reading the file. - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/smoke_base/test_smoke_criterion_shape.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/smoke_base/test_smoke_criterion_shape.py -git commit -m "test(smoke_base): parametrize slug shape tests to eliminate four-function repetition" -``` - ---- - -## Category 6: Isolation - -### Task 23: Fix singleton state mutation race in `test_ensure_sandbox_idempotence.py` - -**Files:** -- Modify: `tests/unit/sandbox/test_ensure_sandbox_idempotence.py` - -- [ ] **Step 1: Read lines 35-56** - -Find `_reset_sandbox_singleton()` with `autouse=True` and see which class attributes it mutates on `BaseSandboxManager` / `_ProbeManager`. - -- [ ] **Step 2: Replace with `monkeypatch` per-test** - -Remove the `autouse` fixture entirely. In each test that needs clean state, use `monkeypatch` to reset the class attribute: - -```python -def test_create_is_idempotent(monkeypatch): - monkeypatch.setattr(BaseSandboxManager, "_cache", {}) - monkeypatch.setattr(_ProbeManager, "_instance", None) - # ... rest of test -``` - -`monkeypatch` is function-scoped by default and xdist-safe. - -- [ ] **Step 3: Run with parallelism to verify no flakiness** - -```bash -uv run pytest tests/unit/sandbox/ -n 4 -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/sandbox/test_ensure_sandbox_idempotence.py -git commit -m "test(sandbox): replace autouse singleton reset with per-test monkeypatch to fix xdist race" -``` - ---- - -### Task 24: Add missing `@pytest.mark.asyncio` in `test_subtask_lifecycle_toolkit.py` - -**Files:** -- Modify: `tests/unit/state/test_subtask_lifecycle_toolkit.py` - -- [ ] **Step 1: Read lines 30-60** - -Identify all `async def test_*` functions missing the `@pytest.mark.asyncio` marker. (With `asyncio_mode = "auto"` in pyproject.toml this may be a no-op — verify the config first.) - -```bash -grep "asyncio_mode" pyproject.toml -``` - -If `asyncio_mode = "auto"` is set, the marker is implicit and this task is already resolved — skip to commit with a note. - -- [ ] **Step 2: Add explicit markers if not using auto mode** - -```python -@pytest.mark.asyncio -async def test_cancel_with_invalid_uuid_returns_error(): - ... -``` - -- [ ] **Step 3: Run tests** - -```bash -uv run pytest tests/unit/state/test_subtask_lifecycle_toolkit.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/unit/state/test_subtask_lifecycle_toolkit.py -git commit -m "test(subtask-lifecycle): add explicit asyncio markers (belt-and-suspenders with asyncio_mode=auto)" -``` - ---- - -### Task 25: Add run-level namespacing to `test_full_lifecycle_with_eval.py` DB polling - -**Files:** -- Modify: `tests/integration/test_full_lifecycle_with_eval.py` - -- [ ] **Step 1: Read lines 84-105** - -Find the polling loop and the DB query that could race with `test_full_lifecycle.py`. - -- [ ] **Step 2: Add a unique cohort or run_id prefix** - -If the existing integration test fixtures don't already namespace by run, generate a unique `cohort` slug per test invocation: - -```python -import uuid - -@pytest.fixture -def unique_cohort(): - return f"test-{uuid.uuid4().hex[:8]}" -``` - -Use this cohort in all queries and run submissions within the test so results from parallel runs don't bleed across. - -- [ ] **Step 3: Run both lifecycle tests together to verify no races** - -```bash -uv run pytest tests/integration/test_full_lifecycle.py tests/integration/test_full_lifecycle_with_eval.py -v -``` - -- [ ] **Step 4: Commit** - -```bash -git add tests/integration/test_full_lifecycle_with_eval.py -git commit -m "test(integration): namespace lifecycle-with-eval DB queries by unique cohort to prevent parallel-run races" -``` diff --git a/docs/superpowers/plans/2026-04-25-e2e-contract-playwright-hardening.md b/docs/superpowers/plans/2026-04-25-e2e-contract-playwright-hardening.md deleted file mode 100644 index 3a8b6c37d..000000000 --- a/docs/superpowers/plans/2026-04-25-e2e-contract-playwright-hardening.md +++ /dev/null @@ -1,410 +0,0 @@ -# E2E Contract And Playwright Hardening Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make E2E tests prove the real product path from PostgreSQL persistence through repository/read-service/API DTOs into the dashboard UI. - -**Architecture:** Move smoke assertions upward from ad hoc SQL toward repository/read-service contracts, while keeping narrow storage-level checks only where the behavior being tested is explicitly storage-level. Expand live Playwright smoke coverage so real PG-backed runs are clicked, inspected, and asserted in the frontend rather than only screenshotted after `graph-canvas` appears. - -**Tech Stack:** Python `pytest`, SQLModel repositories/read services, FastAPI test harness endpoints, Next.js dashboard, Playwright, TypeScript contract parsers. - ---- - -## Current Findings - -The Python smoke tests are deep but mostly direct-SQL. They assert graph shape, resource rows, context-event counts, sandbox WAL/lifecycle rows, thread ordering, blob roundtrips, temporal ordering, cohort membership, and evaluation rows. That is useful for persistence regression, but it can miss bugs where data exists in PostgreSQL and is then dropped or reshaped incorrectly by repository/read-service/API layers. - -The live Playwright smoke tests are too shallow for the current dashboard. `ergon-dashboard/tests/e2e/_shared/smoke.ts` validates a reduced backend harness DTO, navigates to the run page, checks `graph-canvas`, and captures screenshots. The dashboard now exposes richer `data-testid` hooks for graph nodes, workspace panels, event stream, timeline, status counts, and cohort run rows, so the E2Es should assert those real UI surfaces. - -The seeded dashboard tests cover richer UI interactions, but against synthetic dashboard harness fixtures rather than real smoke runs from PG. They are still valuable, but they do not prove real persisted data hydrates into the frontend. - -There is one semantic decision before implementation: task-level evaluations for dynamic subtasks currently do not appear in `RunSnapshotDto.evaluations_by_task`, because `RunTaskEvaluation` stores `definition_task_id`, while dynamic subtask nodes are runtime `RunGraphNode` rows that may not map to an `ExperimentDefinitionTask`. - -## Semantic Decision: Dynamic Leaf Evaluations - -Today, `RunTaskEvaluation` has: - -- `run_id` -- `definition_task_id` -- `definition_evaluator_id` -- score/pass/failure summary fields - -`RunReadService.build_run_snapshot()` builds a `defn_to_node` map from static definition task IDs to run graph node IDs, then `_task_keyed_evaluations()` attaches evaluations to frontend task IDs through that map. If an evaluation's `definition_task_id` does not map to a node, the code skips it. The comment says dynamic-task evaluation would need a `node_id` foreign key. - -This means: - -- The run-level final score can still appear, because it is averaged from all evaluation rows. -- Static definition task evaluations can appear in `evaluations_by_task`. -- Dynamic subtask evaluations cannot reliably appear in `evaluations_by_task` unless the evaluation row also records the runtime node ID or execution ID. - -My recommendation: - -- **Run-level benchmark/evaluator results should appear at the run level.** They answer “did this run pass the benchmark?” -- **Task-level evaluations should appear in the task workspace only when they are genuinely about that task node.** -- **Dynamic leaf evaluations should appear in the frontend if we evaluate dynamic leaves individually.** To support that correctly, add `node_id` and preferably `task_execution_id` to `RunTaskEvaluation`, then key `evaluations_by_task` by runtime node ID. -- **Do not fake dynamic task evaluations by guessing through `definition_task_id`.** That would hide the actual model mismatch and make E2Es less trustworthy. - -Implementation can proceed in two phases: first harden E2Es around the current behavior, then add dynamic-task evaluation mapping if we decide task workspaces should show those evaluations. - -## File Structure - -### Backend E2E Assertion Layer - -- `tests/e2e/_asserts.py`: Replace most direct SQL assertions with assertions over `RunReadService.build_run_snapshot()`, `RunReadService.list_mutations()`, and existing query/repository APIs. -- `tests/e2e/_read_contracts.py`: New focused helper module for fetching and asserting read-service DTOs in tests. -- `tests/e2e/test_researchrubrics_smoke.py`: Keep environment-specific artifact content checks and route shared assertions through the new helper layer. -- `tests/e2e/test_minif2f_smoke.py`: Add out-of-band MiniF2F artifact checks. -- `tests/e2e/test_swebench_smoke.py`: Add out-of-band SWE-Bench artifact checks. - -### Backend API/Test Harness - -- `ergon_core/ergon_core/core/api/test_harness.py`: Expand the live Playwright harness DTO with node IDs and the omitted `executions`, `mutations`, and per-task counts that Playwright will assert. -- `ergon_core/ergon_core/core/api/runs.py`: No broad refactor. Only change if dynamic evaluation semantics are explicitly approved. -- `ergon_core/ergon_core/core/runtime/services/run_read_service.py`: Use existing snapshot behavior for read-service assertions. Only change for dynamic evaluation mapping if approved. -- Potential migration: add `node_id` / `task_execution_id` to `RunTaskEvaluation` only if we choose dynamic task evaluation UI support. - -### Frontend E2E - -- `ergon-dashboard/tests/helpers/backendHarnessClient.ts`: Update `BackendRunState` to match the backend harness DTO exactly. -- `ergon-dashboard/tests/e2e/_shared/smoke.ts`: Assert the real live UI path: cohort route, run header, graph node selection, workspace panels, status counts, event stream, timeline, screenshots. -- `ergon-dashboard/tests/e2e/_shared/expected.ts`: Keep as the TS topology mirror for now, but add a drift test. -- `ergon-dashboard/tests/e2e/run.snapshot.spec.ts` and `run.delta.spec.ts`: Keep seeded UI tests; do not replace them with live tests. - -## Task 1: Add Read-Service Contract Helpers - -**Files:** - -- Create: `tests/e2e/_read_contracts.py` -- Modify: `tests/e2e/_asserts.py` - -- [x] **Step 1: Create a helper for required run snapshots** - -Create `tests/e2e/_read_contracts.py`: - -```python -from __future__ import annotations - -from uuid import UUID - -from ergon_core.core.api.schemas import RunSnapshotDto -from ergon_core.core.runtime.services.run_read_service import RunReadService - - -def require_run_snapshot(run_id: UUID) -> RunSnapshotDto: - snapshot = RunReadService().build_run_snapshot(run_id) - assert snapshot is not None, f"RunReadService returned no snapshot for run {run_id}" - return snapshot -``` - -- [x] **Step 2: Add snapshot-backed graph assertions** - -In `tests/e2e/_asserts.py`, rewrite `_assert_run_graph()` to assert over `snapshot.tasks`, `snapshot.root_task_id`, `snapshot.total_tasks`, `snapshot.total_leaf_tasks`, and dependency IDs. Keep a tiny direct graph repository check only if the read-service DTO cannot express a needed edge invariant. - -- [x] **Step 3: Add snapshot-backed resources/executions/evaluations/thread assertions** - -Rewrite these functions to use the snapshot first: - -- `_assert_run_resources` -- `_assert_run_turn_counts` -- `_assert_thread_messages_ordered` -- `_assert_run_evaluation` - -Keep `_assert_blob_roundtrip()` as storage-level because it intentionally proves blob bytes exist on disk and are stable across reads. - -- [x] **Step 4: Keep WAL/lifecycle checks explicit until repository methods exist** - -For `SandboxCommandWalEntry` and `SandboxEvent`, either use existing repository methods if available or leave the direct SQL in place with a comment explaining that these are storage-level observability checks pending a read-service API. - -- [x] **Step 5: Verify focused Python E2E helpers** - -Run: - -```bash -uv run ruff check tests/e2e/_asserts.py tests/e2e/_read_contracts.py -uv run pytest tests/e2e/test_researchrubrics_smoke.py -v --timeout=330 -``` - -Expected: the helper-level assertions pass or reveal a real read-service/API gap where direct SQL previously passed. - -## Task 2: Resolve Sad-Path Semantics - -**Files:** - -- Modify: `tests/e2e/_asserts.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/workers/researchrubrics_smoke_sadpath.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/leaf_base.py` - -- [x] **Step 1: Confirm actual sad-path behavior from current runtime code** - -Run: - -```bash -uv run pytest tests/e2e/test_researchrubrics_smoke.py -v --timeout=330 -``` - -Inspect the sad run graph statuses and thread count through the test failure/output or a short one-off query using the existing test harness endpoint. - -- [x] **Step 2: Pick one intended behavior** - -Chosen behavior: - -2. **Score-zero completion semantics:** all leaves complete, `l_2` produces score-zero output, `l_3` still runs, and the smoke-completion thread has 8 messages because only `l_2` suppresses completion. - -Do not keep comments and tests split across both meanings. - -- [x] **Step 3: Update tests and comments to match the chosen behavior** - -Updated stale comments in the sad-path worker, leaf base, and temporal-ordering -assertion. Existing E2E assertions already encoded score-zero semantics: -all leaves completed, 8 completion-thread messages, one partial artifact, and -run evaluation score 0. - -## Task 3: Expand The Backend Harness DTO - -**Files:** - -- Modify: `ergon_core/ergon_core/core/api/test_harness.py` -- Modify: `ergon-dashboard/tests/helpers/backendHarnessClient.ts` - -- [x] **Step 1: Add node IDs and counts to the backend DTO** - -Extend `TestGraphNodeDto` with: - -```python -id: UUID -parent_node_id: UUID | None -``` - -Extend `TestRunStateDto` with fields Playwright will assert: - -```python -execution_count: int -mutation_count: int -resource_count: int -thread_count: int -context_event_count: int -``` - -If per-task counts are straightforward to compute, include them keyed by `task_slug` or node ID. - -- [x] **Step 2: Update the TS harness type** - -In `ergon-dashboard/tests/helpers/backendHarnessClient.ts`, update `BackendRunState` to include every backend DTO field, especially `executions`, which is currently omitted. - -- [x] **Step 3: Add a narrow backend unit/API test if one exists nearby** - -If there is an existing test harness API test, add assertions that `read_run_state` includes the new fields. If not, rely on live E2E coverage in Task 4 and keep this change small. - -## Task 4: Add Real Live Playwright Assertions - -**Files:** - -- Modify: `ergon-dashboard/tests/e2e/_shared/smoke.ts` -- Possibly create: `ergon-dashboard/tests/e2e/_shared/liveRunAssertions.ts` - -- [x] **Step 1: Assert the expanded backend DTO** - -For happy runs, assert: - -```ts -expect(state.status).toBe("completed"); -expect(state.graph_nodes.length).toBe(10); -expect(state.resource_count).toBeGreaterThanOrEqual(18); -expect(state.mutations.length).toBeGreaterThan(0); -expect(state.executions.length).toBeGreaterThan(0); -expect(state.evaluations.some((e) => e.score === 1.0)).toBe(true); -``` - -For sad runs, assert the chosen sad-path semantics from Task 2. - -- [x] **Step 2: Navigate through the cohort-aware route** - -Use: - -```ts -const cohortId = await client.getCohortId(cohortKey); -await page.goto(`/cohorts/${cohortId}/runs/${run_id}`); -``` - -This validates breadcrumb context and the same route users reach from the cohort detail page. - -- [x] **Step 3: Assert run header and status UI** - -Assert: - -- `run-header` is visible -- status text is completed -- score is visible when expected -- `run-status-bar` and relevant `run-status-count-*` chips exist - -- [x] **Step 4: Click a real graph node** - -Pick a stable leaf node from the backend DTO, preferably `d_root` or another completed leaf: - -```ts -const leaf = state.graph_nodes.find((n) => n.task_slug === "d_root"); -expect(leaf).toBeTruthy(); -await page.getByTestId(`graph-node-${leaf!.id}`).click(); -``` - -- [x] **Step 5: Assert workspace sections against live data** - -After clicking the node, assert: - -- `workspace-header` -- `workspace-actions` -- `workspace-outputs` -- `workspace-executions` -- `workspace-sandbox` -- `workspace-communication` -- `workspace-transitions` - -Only assert `workspace-evaluation` content if the selected node actually has a task-level evaluation in the snapshot. Do not force dynamic-leaf evaluation visibility until the semantic decision is implemented. - -- [x] **Step 6: Assert event stream and timeline** - -Click `event-stream-toggle`, assert `event-stream-region` and one or more `event-row-*` entries. Switch to timeline mode and assert `timeline-region` when mutations exist. - -- [x] **Step 7: Capture screenshots after assertions** - -Keep the screenshot filenames based on real run IDs: - -```ts -path.join(screenshotDir, cfg.env, `${run_id}-${kind}.png`) -``` - -The screenshots should now represent a validated UI state, not just a loaded canvas. - -## Task 5: Add Benchmark-Specific Out-Of-Band Artifact Checks - -**Files:** - -- Modify: `tests/e2e/test_minif2f_smoke.py` -- Modify: `tests/e2e/test_swebench_smoke.py` - -- [x] **Step 1: Add MiniF2F artifact verification** - -Add an out-of-band check like ResearchRubrics: - -- 9 `proof_*.lean` resources -- file content includes `theorem smoke_trivial` -- file content includes `:=` - -- [x] **Step 2: Add SWE-Bench artifact verification** - -Add: - -- 9 `patch_*.py` resources -- file parses as Python AST -- a function named `add` exists - -- [x] **Step 3: Keep criteria checks as in-workflow checks** - -Do not remove criterion checks. The E2E out-of-band checks intentionally catch silent criterion regressions or missing evaluation dispatch. - -## Task 6: Prevent Python/TypeScript Topology Drift - -**Files:** - -- Modify: `ergon-dashboard/tests/e2e/_shared/expected.ts` -- Add or modify: `tests/unit/...` drift test location to be selected during implementation - -- [x] **Step 1: Add a drift test** - -Create a small unit test that compares: - -- Python `EXPECTED_SUBTASK_SLUGS` -- TS `EXPECTED_SUBTASK_SLUGS` - -Use a simple parser or emit a JSON file during test setup. Avoid adding a generation pipeline unless drift becomes frequent. - -- [x] **Step 2: Keep the TS mirror for Playwright** - -Do not add cross-language runtime imports to Playwright. The mirror is acceptable if drift is tested. - -## Task 7: Dynamic Evaluation Mapping - -Approved: dynamic task-level evaluations should render by runtime graph node ID, -not by static definition task ID. - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Add migration under `ergon_core/migrations/versions/` -- Modify: evaluation persistence call sites -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/run_read_service.py` -- Modify: frontend types/contracts if task-level dynamic evaluations become visible - -- [x] **Step 1: Add runtime node references to evaluations** - -Add required runtime identity fields: - -```python -node_id: UUID -task_execution_id: UUID -``` - -Keep `definition_task_id` nullable as static provenance only; dynamic nodes may not -have a static definition task row. The migration backfills existing rows from task -executions and drops rows that cannot be truthfully mapped to a runtime node. - -- [x] **Step 2: Persist node/execution IDs when evaluation is task-specific** - -Update evaluation dispatch/persistence so task-specific evaluator results record the runtime node and execution IDs. - -- [x] **Step 3: Key `_task_keyed_evaluations()` by `node_id`** - -In `ergon_core/core/api/runs.py`, key evaluations by `node_id`. Do not guess from -`definition_task_id`; evaluation rows without runtime identity are not truthfully -renderable in a task workspace. - -- [x] **Step 4: Add frontend assertions** - -Once dynamic evaluations are correctly keyed, live Playwright can assert `workspace-evaluation` content after clicking evaluated dynamic leaves. - -## Verification Plan - -Run focused Python checks: - -```bash -uv run ruff check tests/e2e ergon_core/ergon_core/core/api/test_harness.py -uv run pytest tests/e2e/test_researchrubrics_smoke.py -v --timeout=330 -uv run pytest tests/e2e/test_minif2f_smoke.py -v --timeout=330 -uv run pytest tests/e2e/test_swebench_smoke.py -v --timeout=330 -``` - -Run focused frontend checks: - -```bash -pnpm --dir ergon-dashboard exec playwright test tests/e2e/*.smoke.spec.ts --project=chromium -pnpm --dir ergon-dashboard test -pnpm --dir ergon-dashboard lint -``` - -Run CI-level validation: - -```bash -gh pr checks -gh run list --branch "$(git branch --show-current)" --limit 10 -``` - -After CI E2E completes, verify the screenshots ref contains PNGs: - -```bash -hash=$(git ls-remote https://github.com/DeepFlow-research/ergon.git refs/heads/screenshots/pr-<PR_NUMBER> | awk '{print $1}') -git fetch origin "$hash" -git ls-tree -r --name-only "$hash" -``` - -Expected: each smoke env has `*.png`, not only `EMPTY.txt`. - -## Recommended Execution Order - -1. Task 2: resolve sad-path semantics before encoding stronger assertions. -2. Task 1: move Python assertions to read-service contracts. -3. Task 3: expand backend harness DTO. -4. Task 4: add real live Playwright assertions. -5. Task 5: add benchmark-specific out-of-band checks. -6. Task 6: add topology drift guard. -7. Task 7 only if we decide dynamic task evaluations should appear in the FE. diff --git a/docs/superpowers/plans/2026-04-25-slopcop-0-1-1-cleanup.md b/docs/superpowers/plans/2026-04-25-slopcop-0-1-1-cleanup.md deleted file mode 100644 index 5a500f447..000000000 --- a/docs/superpowers/plans/2026-04-25-slopcop-0-1-1-cleanup.md +++ /dev/null @@ -1,584 +0,0 @@ -# Slopcop 0.1.1 Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra` pass on `slopcop 0.1.1` by fixing the flagged code rather than growing suppressions. - -**Architecture:** Treat every `0.1.1` finding as a real review item. Fix `no-async-from-sync` by moving async flow upward to the CLI entrypoint, fix `no-or-empty-coalesce` by distinguishing `None` from valid falsy values, and fix `no-typing-any`/bare `object` by introducing narrow JSON aliases, protocols, or domain models only where they make the code more accurate. Suppress only stable boundaries such as the console-script event-loop bridge or untyped third-party SDK surfaces, and every suppression must include an inline reason. - -**Tech Stack:** Python 3.13, uv workspace, Slopcop 0.1.1, Ruff, ty, pytest, Pydantic, SQLModel. - ---- - -## Current State - -- `pyproject.toml` now requires `slopcop>=0.1.1`. -- `uv.lock` resolves `slopcop==0.1.1`. -- `uv run slopcop --format json ergon_core ergon_builtins ergon_cli ergon_infra` reports: - - `no-async-from-sync`: 1 error at `ergon_cli/ergon_cli/main.py`. - - `no-or-empty-coalesce`: 42 warnings across CLI, runtime, builtins, and smoke fixtures. - - `no-typing-any`: 121 warnings, including both `Any` and bare `object`. -- The previous smoke-fixture move is complete; live paths are under `ergon_core/ergon_core/test_support/smoke_fixtures`. -- The suppression budget currently fails because the console-script bridge added one `slopcop: ignore`; this is acceptable only if the ignore is correctly placed on the line above and documented as the single event-loop boundary. - -## Working Rules - -- Do not convert `Any` to `object`; Slopcop 0.1.1 correctly flags both. -- Do not replace `x or ""`, `x or []`, `x or {}`, or `x or 0` mechanically with a different fallback. Decide whether the field can be `None`. -- Preserve meaningful falsy values: - - Empty string `""` is a valid command output, cohort name, model target, etc. - - Empty list `[]` and empty dict `{}` are valid persisted JSON values. - - Numeric `0` and `0.0` are valid return codes and scores. -- Do not manufacture empty strings for missing domain values. If a value is - optional, keep it optional and let the boundary decide how to represent - missingness. For trace attributes specifically, `normalize_attributes()` skips - `None`, so passing `None` is usually better than serializing `""`. -- Prefer direct, local `is None` handling at the use site. A generic helper like - `_text_or_empty()` hides the question Slopcop is forcing us to answer: is an - empty string a valid value here, or are we normalizing a missing value from an - external boundary? -- Add a helper only when it has a domain name that preserves intent, such as - `_sandbox_stream_text(stream: str | None) -> str` in one sandbox adapter file. - Do not create generic "or empty" helpers. - -- For JSON-shaped data, prefer a recursive alias in a shared local module or the file that owns the boundary: - -```python -JsonScalar = str | int | float | bool | None -JsonValue = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -JsonObject = dict[str, JsonValue] -``` - ---- - -## Task 1: Finish the CLI Async Boundary Fix - -**Files:** -- Modify: `ergon_cli/ergon_cli/main.py` -- Modify: `ergon_cli/ergon_cli/commands/benchmark.py` -- Modify: `ergon_cli/ergon_cli/commands/eval.py` -- Modify: `ergon_core/ergon_core/core/rl/eval_runner.py` -- Test: existing CLI import/type checks - -- [ ] **Step 1: Ensure the only sync/async bridge is `main()`** - -`ergon_cli/ergon_cli/main.py` should keep `main()` sync for the console script. Slopcop 0.1.1 recognizes this suppression only on the `asyncio.run(...)` call line, so keep the call short enough that Ruff does not move the suppression to a continuation line: - -```python -async def _main(argv: list[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv) - - if args.command == "benchmark": - return await handle_benchmark(args) - elif args.command == "eval": - return await handle_eval(args) - # existing sync handlers remain direct returns - - -def main(argv: list[str] | None = None) -> int: - coroutine = _main(argv) - return asyncio.run(coroutine) # slopcop: ignore[no-async-from-sync] -- CLI entrypoint -``` - -- [ ] **Step 2: Keep async handlers async all the way down** - -`ergon_cli/ergon_cli/commands/benchmark.py`: - -```python -async def handle_benchmark(args: Namespace) -> int: - if args.bench_action == "list": - benchmarks = list_benchmarks() - render_table(["Slug", "Name", "Description"], benchmarks) - return 0 - elif args.bench_action == "run": - return await run_benchmark(args) - elif args.bench_action == "setup": - return setup_benchmark(args) - else: - print("Usage: ergon benchmark {list|run|setup}") - return 1 - - -async def run_benchmark(args: Namespace) -> int: - # existing setup/persist code unchanged - run_handle = await _create_and_dispatch( - persisted, - timeout=args.timeout, - cohort_id=cohort.id, - ) -``` - -`ergon_cli/ergon_cli/commands/eval.py`: - -```python -async def handle_eval(args: Namespace) -> int: - if args.eval_action == "watch": - return await _watch(args) - elif args.eval_action == "checkpoint": - return await _checkpoint(args) - else: - print("Usage: ergon eval {watch|checkpoint}") - return 1 -``` - -`ergon_core/ergon_core/core/rl/eval_runner.py`: - -```python -async def evaluate_checkpoint( - checkpoint_path: str, - benchmark_type: str, - *, - evaluator_type: str = "stub-rubric", - model_base: str | None = None, - eval_limit: int | None = None, -) -> int: - ckpt = CheckpointInfo(path=checkpoint_path, step=0, has_config=True, has_model=True) - return await _run_local_eval( - ckpt, - benchmark_type=benchmark_type, - evaluator_type=evaluator_type, - model_base=model_base, - eval_limit=eval_limit, - ) -``` - -- [ ] **Step 3: Verify the async rule** - -Run: - -```bash -uv run ruff format ergon_cli/ergon_cli/main.py ergon_cli/ergon_cli/commands/benchmark.py ergon_cli/ergon_cli/commands/eval.py ergon_core/ergon_core/core/rl/eval_runner.py -uv run ruff check ergon_cli/ergon_cli/main.py ergon_cli/ergon_cli/commands/benchmark.py ergon_cli/ergon_cli/commands/eval.py ergon_core/ergon_core/core/rl/eval_runner.py -uv run slopcop ergon_cli ergon_core/ergon_core/core/rl/eval_runner.py -uv run python scripts/check_suppression_budget.py -``` - -Expected: -- Ruff passes. -- `no-async-from-sync` is absent except the accepted console-script boundary. -- Suppression budget either passes with `slopcop_ignore=240` after an explicit budget update, or the bridge is handled without adding a counted ignore if Slopcop recognizes a better layout. - ---- - -## Task 2: Fix Command Output Coalescing - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/rules/proof_verification.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/sandbox_manager.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/workers/minif2f_smoke.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/workers/researchrubrics_smoke.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/workers/swebench_smoke.py` -- Test: existing benchmark/toolkit and smoke fixture unit tests - -- [ ] **Step 1: Make every stdout/stderr fallback explicit** - -Do not introduce a generic `_text_or_empty()` helper. For command output, keep -the `None` normalization visible at the sandbox or subprocess boundary: - -If the value comes from an untyped exception via `getattr`, split the operations: - -```python -stdout = getattr(exc, "stdout", None) # slopcop: ignore[no-hasattr-getattr] -- sandbox SDK exception may carry stdout -stderr = getattr(exc, "stderr", None) # slopcop: ignore[no-hasattr-getattr] -- sandbox SDK exception may carry stderr -stdout_text = "" if stdout is None else stdout -stderr_text = "" if stderr is None else stderr -output = stdout_text + stderr_text -``` - -- [ ] **Step 2: Replace stdout/stderr coalescing** - -Examples to apply: - -```python -# Before -output = (result.stdout or "") + (result.stderr or "") - -# After -stdout = "" if result.stdout is None else result.stdout -stderr = "" if result.stderr is None else result.stderr -output = stdout + stderr -``` - -```python -# Before -probe_stdout = (probe.stdout or "").strip()[:4096] - -# After -probe_stdout = ("" if probe.stdout is None else probe.stdout).strip()[:4096] -``` - -```python -# Before -tail = (r.stdout or "")[-1000:] - -# After -stdout = "" if r.stdout is None else r.stdout -tail = stdout[-1000:] -``` - -- [ ] **Step 3: Fix SWE-Bench grading text without losing empty logs** - -In `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py`: - -```python -log = "" if r.stdout is None else r.stdout -``` - -For error detail, preserve first non-`None` text while allowing empty strings: - -```python -detail = r.stdout if r.stdout is not None else r.stderr -return _error_result(self.name, self.weight, "install_repo failed", "" if detail is None else detail) -``` - -For `_write_and_apply`: - -```python -stdout = "" if r.stdout is None else r.stdout -raise RuntimeError(f"git apply {path} failed: {stdout[-800:]}") -``` - -- [ ] **Step 4: Run focused tests** - -Run: - -```bash -uv run ruff check ergon_builtins/ergon_builtins/benchmarks/minif2f ergon_builtins/ergon_builtins/benchmarks/swebench_verified ergon_core/ergon_core/test_support/smoke_fixtures -uv run pytest tests/unit/smoke_base tests/unit/benchmarks tests/integration/minif2f -q -uv run slopcop ergon_builtins/ergon_builtins/benchmarks/minif2f ergon_builtins/ergon_builtins/benchmarks/swebench_verified ergon_core/ergon_core/test_support/smoke_fixtures -``` - -Expected: -- No `no-or-empty-coalesce` findings in the listed files. -- Existing smoke and benchmark tests pass or fail only for known external-service requirements. - ---- - -## Task 3: Fix Runtime and API Coalescing - -**Files:** -- Modify: `ergon_cli/ergon_cli/commands/benchmark.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/api/test_harness.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/repositories.py` -- Modify: `ergon_core/ergon_core/core/rl/eval_runner.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/sandbox_setup.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluator_dispatch_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/experiment_persistence_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/run_service.py` -- Test: focused unit tests for runtime DTOs, Inngest services, and API run views - -- [ ] **Step 1: Fix CLI cohort fallback** - -In `ergon_cli/ergon_cli/commands/benchmark.py`: - -```python -cohort_name = args.slug if args.cohort is None else args.cohort -``` - -This preserves `--cohort ""` if a caller intentionally passes an empty string. - -- [ ] **Step 2: Fix API DTO fallbacks** - -In `ergon_core/ergon_core/core/api/runs.py`, replace these patterns: - -```python -worker = worker_by_binding.get(node.assigned_worker_slug or "") -error_msg = ex.error_json.get("message") or str(ex.error_json) -total_score=ev.score or 0.0 -``` - -With: - -```python -worker = ( - worker_by_binding.get(node.assigned_worker_slug) - if node.assigned_worker_slug is not None - else None -) -message = ex.error_json.get("message") -error_msg = message if isinstance(message, str) else str(ex.error_json) -total_score=0.0 if ev.score is None else ev.score -``` - -- [ ] **Step 3: Fix JSON defaults in test harness and telemetry** - -Use explicit `None` checks: - -```python -meta = {} if r.summary_json is None else r.summary_json -``` - -For list JSON accessors: - -```python -tool_calls = [] if self.tool_calls_json is None else self.tool_calls_json -return [ToolCall.model_validate(tc) for tc in tool_calls] -``` - -For summary merging: - -```python -existing_summary = dict({} if run.summary_json is None else run.summary_json) -``` - -- [ ] **Step 4: Fix runtime trace attributes without inventing empty strings** - -In runtime/Inngest files, do not replace `x or ""` with -`"" if x is None else x`. That is the same behavior with different syntax. -Instead, decide whether each attribute is required or optional: - -- Required fields should be enforced before the span is emitted. If task - execution cannot continue without the value, raise the existing - `ConfigurationError`/`ContractViolationError` before tracing. -- Optional trace-only fields should stay as `None` in `CompletedSpan.attributes`. - `ergon_core.core.runtime.tracing.normalize_attributes()` drops `None`, so the - exported span omits absent attributes instead of pretending the value is an - empty string. - -Then replace: - -```python -"worker_type": prepared.worker_type or "", -"assigned_worker_slug": prepared.assigned_worker_slug or "", -"model_target": prepared.model_target or "", -"sandbox_id": result.sandbox_id or "", -``` - -With invariant checks or nullable attributes: - -```python -if prepared.worker_type is None: - raise ConfigurationError( - "Task has no worker_type configured", - run_id=payload.run_id, - task_id=payload.task_id, - ) - -attributes={ - "run_id": str(payload.run_id), - "definition_id": str(payload.definition_id), - "task_id": str(payload.task_id), - "execution_id": str(prepared.execution_id), - "task_slug": prepared.task_slug, - "benchmark_type": prepared.benchmark_type, - "worker_type": prepared.worker_type, - "assigned_worker_slug": prepared.assigned_worker_slug, - "model_target": prepared.model_target, - "skipped": False, - "status": "completed", -} -``` - -For sandbox setup, `sandbox_id` should be treated as part of the returned -contract. If it can truly be absent, omit the trace attribute by passing `None`; -if downstream requires it, raise a contract error before emitting: - -```python -if result.sandbox_id is None: - raise ContractViolationError( - "sandbox-setup returned no sandbox_id", - run_id=run_id, - task_id=task_id, - ) - -attributes={ - "run_id": str(run_id), - "task_id": str(task_id), - "benchmark_type": benchmark_type, - "sandbox_id": result.sandbox_id, - "input_resource_count": len(payload.input_resource_ids), -} -``` - -- [ ] **Step 5: Fix subprocess return code handling** - -In `ergon_core/ergon_core/core/rl/eval_runner.py`: - -```python -exit_code = 0 if proc.returncode is None else proc.returncode -``` - -- [ ] **Step 6: Run focused verification** - -Run: - -```bash -uv run ruff check ergon_cli/ergon_cli/commands/benchmark.py ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/core/api/test_harness.py ergon_core/ergon_core/core/persistence/telemetry ergon_core/ergon_core/core/runtime ergon_core/ergon_core/core/rl/eval_runner.py -uv run pytest tests/unit/state tests/unit/runtime tests/unit/test_app_mounts_harness_conditionally.py -q -uv run slopcop ergon_cli/ergon_cli/commands/benchmark.py ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/core/api/test_harness.py ergon_core/ergon_core/core/persistence/telemetry ergon_core/ergon_core/core/runtime ergon_core/ergon_core/core/rl/eval_runner.py -``` - -Expected: -- No `no-or-empty-coalesce` findings in runtime/API files. -- No new ty diagnostics in touched files. - ---- - -## Task 4: Audit `Any` and Bare `object` by Boundary - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/definitions/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/saved_specs/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/graph/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/*_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/evaluation/evaluation_schemas.py` -- Modify: `ergon_core/ergon_core/api/*.py` -- Modify: `ergon_builtins/ergon_builtins/tools/*.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/*/*.py` -- Modify: `ergon_infra/ergon_infra/adapters/*.py` -- Test: corresponding unit tests and `ty` - -- [ ] **Step 1: Introduce a shared JSON type alias** - -If there is no existing equivalent, create `ergon_core/ergon_core/api/json_types.py`: - -```python -"""JSON-compatible public type aliases.""" - -JsonScalar = str | int | float | bool | None -JsonValue = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -JsonObject = dict[str, JsonValue] -``` - -Use this only for real JSON boundaries: persisted model JSON columns, API metadata bags, trace attributes, tool payloads, and third-party HTTP payloads. - -- [ ] **Step 2: Replace bare `object` when code expects JSON** - -Examples: - -```python -from ergon_core.api.json_types import JsonObject, JsonValue - -metadata: JsonObject -task_payload: JsonObject -attributes: dict[str, JsonValue] -``` - -Do not use `JsonValue` when the code expects a model, SDK object, callable, or protocol. - -- [ ] **Step 3: Replace tool call shapes with typed models or aliases** - -For toolkits in `ergon_builtins/ergon_builtins/tools`, inspect how each value is accessed. If the function returns a fixed response shape, create a Pydantic response model instead of `dict[str, object]`: - -```python -class ToolResult(BaseModel): - success: bool - message: str - data: JsonObject | None = None -``` - -Use existing project response models when they already exist. - -- [ ] **Step 4: Keep true untyped third-party boundaries suppressed with reasons** - -Acceptable examples: - -```python -def get_eval_report( - *, - test_spec: Any, # slopcop: ignore[no-typing-any] -- SWE-Bench returns an untyped TestSpec object - prediction: dict[str, str], - test_log_path: str, - include_tests_status: bool = True, -) -> dict[str, JsonValue]: -``` - -Unacceptable examples: - -```python -payload: dict[str, Any] -metadata: dict[str, object] -``` - -These should become `JsonObject`, a Pydantic model, or a protocol. - -- [ ] **Step 5: Run type and Slopcop verification by domain** - -Run after each domain batch: - -```bash -uv run ty check ergon_core/ergon_core ergon_builtins ergon_infra -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -uv run pytest tests/unit -q -``` - -Expected: -- `no-typing-any` count falls after each batch. -- Remaining `Any` suppressions have reasons tied to real external or framework boundaries. - ---- - -## Task 5: Tighten the CI Contract - -**Files:** -- Modify: `package.json` -- Modify: `.github/workflows/ci-fast.yml` -- Modify: `scripts/check_suppression_budget.py` -- Test: `tests/unit/test_suppression_budget.py` - -- [ ] **Step 1: Keep Slopcop strict** - -Do not change CI to `--warn-only`. `slopcop 0.1.1` is repo-specific enough that warnings should be treated as work items. - -Keep: - -```json -"check:be:slopcop": "uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra" -``` - -And: - -```yaml -- name: slopcop check - run: uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -``` - -- [ ] **Step 2: Update the suppression budget only for reviewed suppressions** - -If the console-script bridge remains the only new suppression: - -```python -BUDGET = SuppressionCounts( - slopcop_ignore=240, - noqa=0, - type_ignore=83, -) -``` - -Do not raise the budget for avoidable `Any`, coalescing, or broad exception suppressions. - -- [ ] **Step 3: Verify the final gates** - -Run: - -```bash -uv run ruff check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -uv run ruff format --check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -uv run ty check ergon_core/ergon_core ergon_builtins ergon_cli ergon_infra -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -uv run python scripts/check_suppression_budget.py -uv run pytest tests/unit -q -``` - -Expected: -- Slopcop exits 0 with no errors or warnings. -- Suppression budget passes. -- Unit tests pass. - ---- - -## Self-Review - -- Spec coverage: The plan covers the new `slopcop 0.1.1` rules currently failing the gate: async bridge, empty coalescing, and `Any`/bare `object`. -- Placeholder scan: No task says "TBD" or "fix appropriately"; each task includes concrete file paths, example code, and commands. -- Type consistency: `JsonValue` and `JsonObject` are consistently named across tasks; no generic empty-string helper remains. -- Remaining risk: The `Any` cleanup is large and should be executed in domain batches. Do not attempt a single mechanical all-repo replacement. diff --git a/docs/superpowers/plans/2026-04-25-slopcop-ignore-cleanup.md b/docs/superpowers/plans/2026-04-25-slopcop-ignore-cleanup.md deleted file mode 100644 index ab020f9a6..000000000 --- a/docs/superpowers/plans/2026-04-25-slopcop-ignore-cleanup.md +++ /dev/null @@ -1,336 +0,0 @@ -# Slopcop Ignore Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace avoidable `slopcop`, `noqa`, and `type: ignore` suppressions with real fixes, starting with the currently failing slopcop gate. - -**Architecture:** Treat suppressions as lint debt, not as the fix. First make `uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra` pass without adding new ignores, then remove broad clusters of existing suppressions by introducing narrow project types, moving imports, and replacing dataclasses with project-standard value models. Keep any remaining ignores only where the code is intentionally adapting to untyped third-party APIs or tool limitations, and require an inline reason. - -**Tech Stack:** Python 3.13, uv workspace, slopcop, ruff, ty, pytest. - ---- - -## Current Inventory - -- No literal `SlotCop`, `slotcop`, or `slot_cop` symbols were found in `ergon`; the tool is `slopcop`. -- `slopcop` is a dev dependency in `pyproject.toml` and CI runs `uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra`. -- The suppression budget currently counts these inline comment suppressions outside docs: - - `slopcop: ignore`: 239 - - `# noqa`: 0 - - `# type: ignore`: 89 -- The earlier broad `rg` inventory found 241 `# slopcop: ignore[...]` suppressions: - - `ergon_core`: 125 - - `ergon_builtins`: 92 - - `ergon_cli`: 8 - - `ergon_infra`: 2 - - `tests`: 14 - - `scripts`: 0 -- At initial inspection, `uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra` failed with 16 warnings across 4 files. On the current dirty branch, it now passes; do not reintroduce suppressions for these items. - - `ergon_core/ergon_core/api/benchmark.py`: `no-typing-any` - - `ergon_core/ergon_core/core/api/runs.py`: seven `guarded-function-import` warnings - - `ergon_core/ergon_core/test_support/smoke_fixtures/sandbox.py`: `no-future-annotations`, `no-dataclass`, `no-str-empty-default` - - `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/subworker.py`: `no-dataclass` - -## Fix Policy - -- Do not add new `slopcop: ignore[...]` unless the violation is a stable public API boundary, an untyped third-party API boundary, or a known slopcop false positive. -- Every remaining `slopcop: ignore[...]` must include a short reason after the rule name. -- Prefer moving imports to module scope over adding `# reason:` comments for `guarded-function-import`. -- Prefer narrow aliases or Pydantic models over `Any` only when they make the code more accurate. Do not replace `Any` with `object` just to satisfy the linter; if callers intentionally pass arbitrary JSON or framework payloads, keep `Any` and document the boundary. -- Prefer Pydantic `BaseModel` with `model_config = {"frozen": True}` over dataclasses. -- For `noqa` and `type: ignore`, remove first and run the owning checker before deciding whether to keep it. - ---- - -### Task 1: Make The Current Slopcop Gate Pass - -**Files:** -- Modify: `ergon_core/ergon_core/api/benchmark.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/sandbox.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/subworker.py` -- Test: `tests/unit/state/test_benchmark_contract.py` -- Test: `tests/unit/smoke_base/test_leaf_sends_completion_message.py` -- Test: `tests/unit/test_app_mounts_harness_conditionally.py` - -- [ ] **Step 1: Establish the failing baseline** - -Run: - -```bash -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -``` - -Expected: 16 warnings across the 4 files listed above. - -- [ ] **Step 2: Keep `Benchmark.parse_task_payload` honest** - -In `ergon_core/ergon_core/api/benchmark.py`, do not replace `Any` with `object`. This method accepts arbitrary persisted JSON or a Pydantic model and then validates through `task_payload_model`; `Any` is the accurate public boundary type here. - -Add a justified slopcop suppression to the signature: - -```python -def parse_task_payload( # slopcop: ignore[no-typing-any] -- arbitrary persisted JSON validated below - cls, - payload: BaseModel | Mapping[str, Any] | None, -) -> BaseModel: -``` - -This is an intentional boundary annotation, not a cleanup-by-renaming. Keep the existing `isinstance(payload, BaseModel)` branch before `model_validate`. - -- [ ] **Step 3: Move `RunReadService` import to module scope** - -In `ergon_core/ergon_core/core/api/runs.py`, add this top-level import with the other project imports: - -```python -from ergon_core.core.runtime.services.run_read_service import RunReadService -``` - -Then delete the seven function-scope imports of `RunReadService` in `build_run_snapshot`, `get_mutations`, `get_generations`, `get_resource_content`, `get_training_curves`, `get_training_sessions`, and `get_training_metrics`. - -- [ ] **Step 4: Re-check smoke sandbox dataclasses before changing them** - -On the current branch, slopcop passes with these dataclasses and `SubworkerResult` has an explicit comment explaining why positional dataclass construction is more ergonomic than Pydantic for test fixtures. Do not replace these dataclasses for the sake of the linter. Only revisit this step if slopcop starts failing here again or if a concrete behavior/type-safety improvement is identified. - -If slopcop fails again, inspect `ergon_core/ergon_core/test_support/smoke_fixtures/sandbox.py`. Avoid this mechanical replacement unless it genuinely improves the code: - -In `ergon_core/ergon_core/test_support/smoke_fixtures/sandbox.py`, remove: - -```python -from __future__ import annotations -from dataclasses import dataclass -``` - -Add: - -```python -from pydantic import BaseModel -``` - -Replace `_CommandResult` and `_EntryInfo` with frozen Pydantic models: - -```python -class _CommandResult(BaseModel): - model_config = {"frozen": True} - - stdout: str = Field(default="") - stderr: str = Field(default="") - exit_code: int = 0 - - -class _EntryInfo(BaseModel): - model_config = {"frozen": True} - - name: str -``` - -If slopcop still flags the empty string defaults, make `stdout` and `stderr` required and update every `_CommandResult()` call in the file to pass `stdout=""` and `stderr=""` explicitly. - -- [ ] **Step 5: Replace `SubworkerResult` dataclass** - -In `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/subworker.py`, replace the dataclass import and decorator with Pydantic: - -```python -from pydantic import BaseModel -``` - -```python -class SubworkerResult(BaseModel): - model_config = {"frozen": True} - - file_path: str - probe_stdout: str - probe_exit_code: int -``` - -Check callers. If any tests construct it positionally, update those test fixtures to keyword construction. - -- [ ] **Step 6: Verify Task 1** - -Run: - -```bash -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -uv run ruff check ergon_core/ergon_core/api/benchmark.py ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/test_support/smoke_fixtures/sandbox.py ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/subworker.py -uv run ty check ergon_core/ergon_core -uv run pytest tests/unit/state/test_benchmark_contract.py tests/unit/smoke_base/test_leaf_sends_completion_message.py tests/unit/test_app_mounts_harness_conditionally.py -q -``` - -Expected: slopcop passes; ruff passes; ty has no new errors in touched files; tests pass. - ---- - -### Task 2: Audit `no-typing-any` Suppressions For Real Type Improvements - -**Files:** -- Modify: `ergon_core/ergon_core/api/benchmark.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/criterion_base.py` -- Modify: `ergon_core/ergon_core/test_support/smoke_fixtures/criteria/smoke_rubrics.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/criterion.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Test: focused unit/integration tests that cover each changed benchmark or worker. - -- [ ] **Step 1: Classify each `Any` before editing** - -For every `slopcop: ignore[no-typing-any]`, classify it into one of three buckets: - -- Keep: arbitrary framework payloads, persisted JSON before validation, plugin/tool interfaces, or third-party SDK surfaces. -- Replace: places where the code already assumes a concrete structure, such as dictionaries with fixed keys or homogeneous lists. -- Centralize: repeated unavoidable framework types that can live behind one project alias. - -Do not perform substitutions like `Any` -> `object` unless all downstream operations work with `object` without casts and the new type communicates the domain better. - -- [ ] **Step 2: Type smoke probe payloads** - -In `criterion_base.py`, introduce a local Pydantic model: - -```python -class ProbeResult(BaseModel): - exit_code: int | None = None - stdout: str = "" -``` - -Then replace `dict[UUID, dict[str, Any]]` with `dict[UUID, ProbeResult]` throughout that file and update call sites to use attributes instead of dictionary indexing. - -- [ ] **Step 3: Type HuggingFace research-rubrics rows** - -In `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py`, add: - -```python -class ResearchRubricsRow(TypedDict, total=False): - sample_id: str - domain: str - ablated_prompt: str - rubrics: list[RubricRow] - removed_elements: list[str] | None - ablation_type: str | None - - -class RubricRow(TypedDict): - criterion: str - axis: str - weight: float -``` - -Change `_payload_from_row(row: Mapping[str, Any])` to `_payload_from_row(row: ResearchRubricsRow)`. - -- [ ] **Step 4: Centralize unavoidable framework `Any`** - -For PydanticAI message parts in `react_worker.py`, introduce project-local aliases such as: - -```python -type ModelMessagePart = object -type JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] -``` - -Replace `list[Any]`, `type[Any]`, and JSON-safe return annotations where the implementation only inspects attributes or recursively serializes JSON-like values. - -- [ ] **Step 5: Verify Task 2** - -Run: - -```bash -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -uv run ty check ergon_core/ergon_core ergon_builtins -uv run pytest tests/unit/state/test_research_rubrics_benchmark.py tests/integration/swebench_verified/test_criterion.py tests/unit/state/test_research_rubrics_workers.py -q -``` - -Expected: fewer `slopcop: ignore[no-typing-any]` comments, no new ty failures, tests pass. - ---- - -### Task 3: Audit `noqa` Suppressions - -**Files:** -- Modify as needed across `ergon_core`, `ergon_builtins`, `ergon_cli`, `ergon_infra`, `tests`, and `scripts`. - -- [ ] **Step 1: List active `noqa` usage** - -Run: - -```bash -rg -n '#\s*noqa' ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -``` - -- [ ] **Step 2: Remove stale import-order/import-location suppressions first** - -For each `# noqa: E402` or `# noqa: PLC0415`, try moving the import to module scope. Keep a function-scope import only if it prevents a real circular import, avoids an optional dependency import on cold paths, or is required for fixture registration order. In that case add a `# reason:` comment immediately above the import. - -- [ ] **Step 3: Re-run ruff** - -Run: - -```bash -uv run ruff check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -``` - -Expected: no stale `noqa` comments; any surviving suppressions are justified by runtime behavior. - ---- - -### Task 4: Audit `type: ignore` Suppressions - -**Files:** -- Modify as needed across `ergon_core`, `ergon_builtins`, `ergon_cli`, `ergon_infra`, `tests`, and `scripts`. - -- [ ] **Step 1: List active type ignores** - -Run: - -```bash -rg -n '#\s*type:\s*ignore' ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -``` - -- [ ] **Step 2: Prefer typed fakes over ignored test calls** - -For smoke tests that call private methods with `_FakeNode` and `# type: ignore[arg-type]`, replace `_FakeNode` with a tiny builder that returns a real `RunGraphNode` or define a `Protocol` accepted by the private method. Do not keep the ignore if the method only needs `id`, `task_slug`, and `status`. - -- [ ] **Step 3: Prefer local protocols for sandbox mocks** - -For sandbox manager tests using `# type: ignore[attr-defined]`, expose a narrow protocol or public test helper for `_install_dependencies` calls instead of ignoring private attribute access. - -- [ ] **Step 4: Re-run ty** - -Run: - -```bash -uv run ty check ergon_core/ergon_core ergon_builtins ergon_cli ergon_infra -``` - -Expected: no new type errors; a smaller, justified set of `type: ignore` comments remains. - ---- - -### Task 5: Add A Suppression Budget Check - -**Files:** -- Create: `scripts/check_suppression_budget.py` -- Modify: `.github/workflows/ci-fast.yml` - -- [ ] **Step 1: Create the budget checker** - -Add a script that counts `slopcop: ignore`, `# noqa`, and `# type: ignore` in code paths, excludes `docs/**`, and fails if counts increase beyond a checked-in baseline. - -- [ ] **Step 2: Wire CI** - -Add this after slopcop in `.github/workflows/ci-fast.yml`: - -```yaml -- name: Suppression budget - run: uv run python scripts/check_suppression_budget.py -``` - -- [ ] **Step 3: Verify the full backend gate** - -Run: - -```bash -uv run ruff check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -uv run ruff format --check ergon_core ergon_builtins ergon_cli ergon_infra tests scripts -uv run ty check ergon_core/ergon_core ergon_builtins ergon_cli ergon_infra -uv run slopcop ergon_core ergon_builtins ergon_cli ergon_infra -uv run python scripts/check_suppression_budget.py -``` - -Expected: all commands pass, and future agents cannot quietly add suppressions without changing the baseline. diff --git a/docs/superpowers/plans/2026-04-26-communication-thread-workspace.md b/docs/superpowers/plans/2026-04-26-communication-thread-workspace.md deleted file mode 100644 index b5a67c55c..000000000 --- a/docs/superpowers/plans/2026-04-26-communication-thread-workspace.md +++ /dev/null @@ -1,558 +0,0 @@ -# Communication Thread Workspace Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make inter-agent communication a first-class, time-aware workspace view with agent-authored thread summaries, task anchoring, and a clickable WhatsApp-style thread trace. - -**Architecture:** Extend the communication schema from agent tool request through persistence, dashboard DTOs, live events, run snapshots, and frontend rendering. Preserve the current `(run_id, topic)` thread identity and add nullable `summary` metadata so agents can set a human-readable thread summary when creating the first message. Frontend should work with summary absent, but prefer it when present. - -**Tech Stack:** Python, SQLModel/Alembic, Pydantic DTOs, dashboard event contracts, React/TypeScript, Playwright. - ---- - -## File Structure - -- Modify `ergon_core/ergon_core/core/runtime/services/communication_schemas.py`: add nullable `thread_summary` to `CreateMessageRequest`, `ThreadSummary`, and `ThreadWithMessages`. -- Modify `ergon_core/ergon_core/core/persistence/telemetry/models.py`: add nullable `summary` column to `Thread`. -- Create migration under `ergon_core/migrations/versions/`: add nullable `summary` column to `threads`. -- Modify `ergon_core/ergon_core/core/runtime/services/communication_service.py`: persist `thread_summary` only when creating a thread or when an existing thread has no summary. -- Modify `ergon_core/ergon_core/core/api/schemas.py`: add nullable `summary` to `RunCommunicationThreadDto`. -- Modify `ergon_core/ergon_core/core/api/runs.py`: populate `thread.summary`, `thread.task_id`, and `message.task_id` in `_build_communication_threads`. -- Modify `ergon_core/ergon_core/core/dashboard/event_contracts.py` only if generated event contracts need explicit schema references refreshed. -- Modify `ergon-dashboard/src/generated/rest/contracts.ts` after schema generation or manually in lockstep if generation is not available in this branch. -- Modify `ergon-dashboard/src/lib/contracts/rest.ts`: ensure normalized `RunCommunicationThread` includes `summary: string | null`. -- Modify `ergon-dashboard/src/components/panels/CommunicationPanel.tsx`: replace always-expanded cards with thread list + selected chat trace. -- Modify `ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.ts`: keep existing time filtering, and ensure summaries/counts are based on visible messages at selected time. -- Test `tests/unit/smoke_base/test_leaf_sends_completion_message.py`: existing callers remain valid without summaries. -- Test `tests/unit/dashboard/test_event_contract_types.py`: DTO exposes `summary`, `task_id`, and `task_execution_id`. -- Add backend unit tests for communication summary persistence and task anchoring. -- Update Playwright tests in `ergon-dashboard/tests/e2e/run.snapshot.spec.ts` or `run.delta.spec.ts` for clickable thread list and chat bubbles. - ---- - -## Task 1: Extend Communication Schema and Persistence - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/communication_schemas.py` -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Create: `ergon_core/migrations/versions/<revision>_add_thread_summary.py` -- Test: `tests/unit/smoke_base/test_leaf_sends_completion_message.py` - -- [ ] **Step 1: Write compatibility assertion for summary-optional requests** - -Add to `tests/unit/smoke_base/test_leaf_sends_completion_message.py` inside `test_send_completion_message_posts_request`: - -```python -assert req.thread_summary is None -``` - -- [ ] **Step 2: Run test to verify current schema fails** - -Run: - -```bash -pytest tests/unit/smoke_base/test_leaf_sends_completion_message.py::test_send_completion_message_posts_request -q -``` - -Expected: FAIL because `CreateMessageRequest` has no `thread_summary` attribute. - -- [ ] **Step 3: Add nullable summary field to request/response schemas** - -In `communication_schemas.py`, update `CreateMessageRequest`: - -```python -class CreateMessageRequest(BaseModel): - run_id: UUID - from_agent_id: str = Field( - description="ID of the sending agent, e.g. '{run_id}:worker'", - ) - to_agent_id: str = Field( - description="ID of the receiving agent, e.g. '{run_id}:stakeholder'", - ) - thread_topic: str - thread_summary: str | None = Field( - default=None, - description="Optional human-readable summary set when the thread is first created.", - ) - content: str - task_execution_id: UUID | None = None -``` - -Also add `summary: str | None = None` to `ThreadSummary` and `ThreadWithMessages`. - -- [ ] **Step 4: Add persistence field** - -In `models.py`, add to `Thread`: - -```python -summary: str | None = None -``` - -- [ ] **Step 5: Add migration** - -Create an Alembic migration adding: - -```python -op.add_column("threads", sa.Column("summary", sqlmodel.sql.sqltypes.AutoString(), nullable=True)) -``` - -Downgrade removes the column. - -- [ ] **Step 6: Run schema compatibility test** - -Run: - -```bash -pytest tests/unit/smoke_base/test_leaf_sends_completion_message.py::test_send_completion_message_posts_request -q -``` - -Expected: PASS. - ---- - -## Task 2: Persist Agent-Authored Thread Summary - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/communication_service.py` -- Test: add or extend communication service unit test near existing service tests - -- [ ] **Step 1: Write failing service test** - -Create a test that calls `communication_service.save_message` with: - -```python -CreateMessageRequest( - run_id=run_id, - from_agent_id="leaf-l_1", - to_agent_id="parent", - thread_topic="smoke-completion", - thread_summary="Leaf workers report completion artifacts and probe exit status.", - content="l_1: done exit=0", - task_execution_id=execution_id, -) -``` - -Assert the persisted `Thread.summary` equals the provided summary. - -- [ ] **Step 2: Run test to verify it fails** - -Run: - -```bash -pytest tests/unit -q -k "communication and summary" -``` - -Expected: FAIL because `CommunicationService` does not persist summary. - -- [ ] **Step 3: Update thread creation/update semantics** - -In `CommunicationService.save_message`, pass `thread_summary=request.thread_summary` into `_get_or_create_thread`. - -Update `_get_or_create_thread` signature: - -```python -def _get_or_create_thread( - session, - *, - run_id: UUID, - agent_a_id: str, - agent_b_id: str, - topic: str, - thread_summary: str | None = None, -) -> Thread: -``` - -When an existing thread is found: - -```python -if existing is not None: - if existing.summary is None and thread_summary: - existing.summary = thread_summary - session.add(existing) - return existing -``` - -When creating: - -```python -thread = Thread(run_id=run_id, topic=topic, agent_a_id=a, agent_b_id=b, summary=thread_summary) -``` - -- [ ] **Step 4: Include summary in service response DTOs** - -When building `RunCommunicationThreadDto`, `ThreadSummary`, and `ThreadWithMessages`, populate `summary=thread.summary`. - -- [ ] **Step 5: Run service tests** - -Run: - -```bash -pytest tests/unit -q -k "communication" -``` - -Expected: PASS. - ---- - -## Task 3: Populate Dashboard Thread DTOs and Task Anchors - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Test: `tests/unit/dashboard/test_event_contract_types.py` -- Test: add focused test for `_build_communication_threads` - -- [ ] **Step 1: Extend DTO contract test** - -Add to `test_event_contract_types.py`: - -```python -def test_thread_dto_exposes_summary_and_task_identity() -> None: - assert "summary" in RunCommunicationThreadDto.model_fields - assert "task_id" in RunCommunicationThreadDto.model_fields - assert "task_id" in RunCommunicationMessageDto.model_fields -``` - -- [ ] **Step 2: Run contract test to verify failure** - -Run: - -```bash -pytest tests/unit/dashboard/test_event_contract_types.py -q -``` - -Expected: FAIL until `summary` is added. - -- [ ] **Step 3: Add summary field to API schema** - -In `RunCommunicationThreadDto`: - -```python -summary: str | None = None -``` - -- [ ] **Step 4: Populate message task IDs in snapshots** - -Update `_build_communication_threads` to accept `execution_task_map: dict[UUID, UUID]`. - -For each message: - -```python -task_id = execution_task_map.get(m.task_execution_id) if m.task_execution_id else None -``` - -Set: - -```python -task_id=str(task_id) if task_id else None -``` - -on `RunCommunicationMessageDto`. - -- [ ] **Step 5: Populate thread task ID** - -For each thread, collect message task IDs. If exactly one unique non-null task ID exists, set `RunCommunicationThreadDto.task_id` to that ID. Otherwise set it to `None`. - -- [ ] **Step 6: Pass execution map from run read service** - -In `run_read_service.py`, change: - -```python -threads=run_api_helpers._build_communication_threads(threads, thread_messages), -``` - -to: - -```python -threads=run_api_helpers._build_communication_threads( - threads, - thread_messages, - execution_task_map, -), -``` - -- [ ] **Step 7: Run backend contract tests** - -Run: - -```bash -pytest tests/unit/dashboard/test_event_contract_types.py -q -``` - -Expected: PASS. - ---- - -## Task 4: Carry Summary and Anchors Through Live Events - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/communication_service.py` -- Modify: dashboard event contract tests if needed -- Test: add or extend dashboard event contract/unit test - -- [ ] **Step 1: Write failing live event assertion** - -In a communication service test, patch `dashboard_emitter.thread_message_created` and assert the emitted `thread.summary` equals the request `thread_summary`, and emitted `message.task_execution_id` equals request `task_execution_id`. - -- [ ] **Step 2: Run test to verify failure** - -Run: - -```bash -pytest tests/unit -q -k "thread_message_created" -``` - -Expected: FAIL because live DTO currently omits summary and task execution identity in the emitted message. - -- [ ] **Step 3: Populate live DTO fields** - -In `CommunicationService.save_message`, set: - -```python -thread_dto = RunCommunicationThreadDto( - id=str(thread.id), - run_id=str(thread.run_id), - topic=thread.topic, - summary=thread.summary, - agent_a_id=thread.agent_a_id, - agent_b_id=thread.agent_b_id, - created_at=thread.created_at, - updated_at=thread.updated_at, - messages=[], -) -``` - -Set on `message_dto`: - -```python -task_execution_id=str(message.task_execution_id) if message.task_execution_id else None, -``` - -If task ID derivation is available in this service path, also set `task_id`. If not, leave `task_id=None` and rely on snapshot enrichment until the service has an execution lookup helper. - -- [ ] **Step 4: Run live event test** - -Run: - -```bash -pytest tests/unit -q -k "thread_message_created" -``` - -Expected: PASS. - ---- - -## Task 5: Update Frontend Contracts - -**Files:** -- Modify: `ergon-dashboard/src/generated/rest/contracts.ts` -- Modify: `ergon-dashboard/src/lib/contracts/rest.ts` -- Test: `ergon-dashboard/tests/contracts/contracts.test.ts` - -- [ ] **Step 1: Write frontend contract assertion** - -In `tests/contracts/contracts.test.ts`, assert a thread fixture can contain: - -```ts -summary: "Leaf workers report completion artifacts and probe exit status." -``` - -and parsing preserves `thread.summary`. - -- [ ] **Step 2: Run test to verify failure** - -Run: - -```bash -pnpm exec vitest run tests/contracts/contracts.test.ts -``` - -Expected: FAIL if generated contract does not include `summary`. - -- [ ] **Step 3: Update generated/rest contract** - -Add `summary: z.string().nullable().optional()` to `RunCommunicationThreadDto` schema if codegen is not run in this task. - -- [ ] **Step 4: Normalize summary** - -Ensure `RunCommunicationThread` exposes: - -```ts -summary: string | null; -``` - -and normalization defaults missing `summary` to `null`. - -- [ ] **Step 5: Run frontend contract test** - -Run: - -```bash -pnpm exec vitest run tests/contracts/contracts.test.ts -``` - -Expected: PASS. - ---- - -## Task 6: Restyle Communication Panel as Thread List + Chat Trace - -**Files:** -- Modify: `ergon-dashboard/src/components/panels/CommunicationPanel.tsx` -- Test: `ergon-dashboard/tests/e2e/run.snapshot.spec.ts` - -- [ ] **Step 1: Add E2E assertions for clickable thread list** - -In `run.snapshot.spec.ts`, after opening communication tab, assert: - -```ts -await expect(page.getByTestId("communication-thread-list")).toBeVisible(); -await expect(page.getByTestId("communication-thread-card").first()).toContainText("smoke-completion"); -await page.getByTestId("communication-thread-card").first().click(); -await expect(page.getByTestId("communication-chat-trace")).toBeVisible(); -await expect(page.getByTestId("communication-chat-message").first()).toBeVisible(); -``` - -- [ ] **Step 2: Run E2E test to verify failure** - -Run: - -```bash -pnpm exec playwright test tests/e2e/run.snapshot.spec.ts --project=chromium -g "graph selection opens workspace evidence sections" -``` - -Expected: FAIL because the current panel has no thread-list/chat-trace test IDs. - -- [ ] **Step 3: Implement selected thread state** - -In `CommunicationPanel.tsx`, add: - -```ts -const [selectedThreadId, setSelectedThreadId] = useState<string | null>(threads[0]?.id ?? null); -const selectedThread = threads.find((thread) => thread.id === selectedThreadId) ?? threads[0] ?? null; -``` - -Use `useEffect` to reset selection when `threads[0]?.id` changes. - -- [ ] **Step 4: Render thread list** - -For each thread, render a button card with: - -- topic -- `thread.summary ?? summarizeThread(thread)` -- message count -- created/updated time -- participant chips derived from `messages[].fromAgentId` plus `agentAId`/`agentBId` - -- [ ] **Step 5: Render WhatsApp-style chat trace** - -For selected thread, render messages sorted by `sequenceNum` with: - -- sender label from `fromAgentId` -- timestamp from `createdAt` -- content as wrapped text -- bubble alignment keyed by sender -- small metadata row: `#${sequenceNum}` and `taskId` when available - -- [ ] **Step 6: Keep empty state clear** - -If no threads are visible at time `t`, show: - -```tsx -No communication threads yet at this point in the run. -``` - -- [ ] **Step 7: Run E2E test** - -Run: - -```bash -pnpm exec playwright test tests/e2e/run.snapshot.spec.ts --project=chromium -g "graph selection opens workspace evidence sections" -``` - -Expected: PASS. - ---- - -## Task 7: Ensure Time-Step Filtering Reads Correctly - -**Files:** -- Modify: `ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.ts` if needed -- Test: `ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.test.ts` - -- [ ] **Step 1: Add test for visible-at-time thread summaries** - -Create a test where: - -- thread exists at 10:00 -- messages exist at 10:01 and 10:02 -- selected time is 10:01:30 - -Assert returned thread contains only the first message. - -- [ ] **Step 2: Run test** - -Run: - -```bash -pnpm exec tsx --test src/components/workspace/filterTaskEvidenceForTime.test.ts -``` - -Expected: PASS if current filtering already handles this. If it fails, patch only filtering logic. - ---- - -## Task 8: Final Verification - -**Files:** -- All touched files - -- [ ] **Step 1: Run backend unit tests** - -Run: - -```bash -pytest tests/unit/smoke_base/test_leaf_sends_completion_message.py tests/unit/dashboard/test_event_contract_types.py -q -``` - -Expected: PASS. - -- [ ] **Step 2: Run frontend typecheck** - -Run from `ergon-dashboard`: - -```bash -pnpm exec tsc --noEmit -``` - -Expected: exit 0. - -- [ ] **Step 3: Run frontend E2E tests sequentially** - -Run from `ergon-dashboard`: - -```bash -pnpm exec playwright test tests/e2e/run.snapshot.spec.ts --project=chromium -pnpm exec playwright test tests/e2e/run.delta.spec.ts --project=chromium -``` - -Expected: both pass. Run sequentially to avoid shared dev-server port collisions. - -- [ ] **Step 4: Lint recently edited files** - -Use the IDE linter diagnostics for: - -- `CommunicationPanel.tsx` -- `filterTaskEvidenceForTime.ts` -- generated/normalized contract files -- backend communication service/schema files - -Expected: no new linter errors. - ---- - -## Self-Review - -- Spec coverage: covers agent-authored nullable thread summary, first-message creation path, backend summary/task anchoring, live event DTOs, frontend clickable thread list, chat trace, and time-step filtering. -- Placeholder scan: no `TBD`/`TODO` placeholders; migration revision filename remains intentionally parameterized because Alembic generates revision IDs. -- Type consistency: backend uses `thread_summary` for request input and `summary` for persisted/output thread metadata; frontend uses `summary`. diff --git a/docs/superpowers/plans/2026-04-26-core-test-logic-audit.md b/docs/superpowers/plans/2026-04-26-core-test-logic-audit.md deleted file mode 100644 index efe7a06d9..000000000 --- a/docs/superpowers/plans/2026-04-26-core-test-logic-audit.md +++ /dev/null @@ -1,1113 +0,0 @@ -# Core Test Logic Audit Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove testing-specific logic that has crept into core runtime code, starting with sandbox sentinel handling, while preserving the non-null `sandbox_id` contract. - -**Architecture:** Core orchestration should depend on provider-owned lifecycle APIs, not on test/stub identities or sentinel parsing. Production sandbox setup requires E2B configuration and must fail loudly if E2B is unavailable; there is no core "no remote sandbox was provisioned" fallback. Test doubles such as `StubSandboxManager`, smoke workers, smoke fixtures, local sandbox implementations, and any placeholder sentinel IDs stay under `ergon_core.test_support` and are only wired through explicitly gated harness/bootstrap paths. - -**Tech Stack:** Python, FastAPI, Inngest, SQLModel, pytest, Playwright, Docker Compose. - ---- - -## Current Findings - -The immediate issue is not that placeholder sandbox IDs exist in tests. The issue is that runtime/core code knows too much about why a placeholder exists. Core should require real sandbox provisioning; test support can still provide sentinel-backed managers for unit/integration tests. - -Current leaks: - -- `ergon_core/ergon_core/core/providers/sandbox/manager.py` defines `StubSandboxManager` and `is_stub_sandbox_id`; the manager can continue to exist, but it belongs under `ergon_core.test_support`, not `core`. -- `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` imports `StubSandboxManager` to mint skipped-task sandbox IDs. -- `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` imports `is_stub_sandbox_id` before terminating sandboxes. -- `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` imports `is_stub_sandbox_id` before terminating run-level sandboxes. -- `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` currently imports `is_stub_sandbox_id` for failed-task cleanup. -- `ergon_core/ergon_core/test_support/smoke_fixtures/*` is acceptable test-owned code, but core runtime must not import it. -- `ergon_core/ergon_core/core/api/test_harness.py` is acceptable only because it is explicitly mounted behind `ENABLE_TEST_HARNESS`; this plan adds guardrails so that pattern does not spread. - -## Audit Results - -Audit command run on 2026-04-26: - -```bash -rg "test_support|tests\.|smoke|fake|mock|stub|fixture|ENABLE_TEST|ENABLE_SMOKE|test_harness|StubSandboxManager|is_stub_sandbox_id|stub-sandbox" ergon_core/ergon_core/core -``` - -### Must Fix - -- `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` - - Imports `BaseSandboxManager` and `is_stub_sandbox_id`. - - Branches on `is_stub_sandbox_id` during failed-task sandbox cleanup. - - Classification: **test/provider sentinel knowledge leaked into runtime orchestration.** - - Fix: route through provider-owned `terminate_sandbox_by_id`. - -- `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` - - Imports `StubSandboxManager`. - - Creates `stub_sandbox_id` for skipped tasks. - - Comments instruct downstream teardown to inspect `is_stub_sandbox_id`. - - Classification: **test double implementation leaked into task execution.** - - Fix: remove the skipped-task stub path from core. If skipped tasks must emit completion events, they should still have a real sandbox ID from normal setup or the event contract should be redesigned deliberately; do not mint provider placeholders in core. - -- `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` - - Imports and branches on `is_stub_sandbox_id`. - - Classification: **runtime teardown knows provider sentinel details.** - - Fix: call provider-owned lifecycle termination API. - -- `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` - - Imports and branches on `is_stub_sandbox_id`. - - Classification: **run cleanup knows provider sentinel details.** - - Fix: call provider-owned lifecycle termination API. - -- `ergon_core/ergon_core/core/runtime/events/task_events.py` - - Comments describe `StubSandboxManager` and `is_stub_sandbox_id`. - - Classification: **contract docs encode the wrong abstraction.** - - Fix: document that production task execution uses real sandbox IDs. Test-support managers may emit sentinel IDs, but core consumers must not branch on them. - -- `ergon_core/ergon_core/core/providers/sandbox/manager.py` - - Defines `_STUB_SANDBOX_PREFIX`, `is_stub_sandbox_id`, and `StubSandboxManager`. - - `DefaultSandboxManager.create` delegates to `StubSandboxManager` when `E2B_API_KEY` is missing. - - Classification: **a test double is mixed into core, and core incorrectly treats missing E2B configuration as a recoverable execution mode.** - - Fix: move `StubSandboxManager` to `ergon_core.test_support`; remove the `DefaultSandboxManager` no-E2B fallback and let `BaseSandboxManager.create` fail loudly when `E2B_API_KEY` is absent. - -- `ergon_core/ergon_core/core/api/app.py` - - Imports `ergon_core.test_support.smoke_fixtures.register_smoke_fixtures` under `settings.smoke_fixtures_enabled`. - - Classification: **core app bootstrap imports test-support code.** The flag helps, but the dependency direction is still wrong. - - Fix: replace this with a generic startup-plugin mechanism, then configure the smoke fixture registration callable from local/CI environment. - -### Probably Fix / Rename - -- `ergon_core/ergon_core/core/runtime/inngest/benchmark_run_start.py` - - Defaults `worker_slug` to `"stub-worker"` and `evaluator_slug` to `"stub-rubric"`. - - Classification: **not necessarily test logic, but the default names read as test doubles inside a production request contract.** - - Fix: make both fields required. The benchmark-run request contract should not invent worker/evaluator defaults. - -- `ergon_core/ergon_core/core/rl/eval_runner.py` - - Defaults `evaluator_type` to `"stub-rubric"` and uses `"stub-worker"` when no `model_base` is provided. - - Classification: **RL/dev utility behavior may be legitimate, but the naming implies test doubles.** - - Fix: make evaluator/model inputs explicit. Do not default to stub worker or stub evaluator slugs. - -- `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` - - Comment says `release-sandbox — stub`. - - Classification: **stale implementation comment.** - - Fix: update to lifecycle-service language when cancelled-task cleanup is wired. - -### Allowed / No Code Change - -- `ergon_core/ergon_core/core/api/test_harness.py` - - Test-only router behind `ENABLE_TEST_HARNESS`. - - Classification: **allowed explicitly gated integration surface.** - - Constraint: may depend on test concepts, but should stay isolated to this file/package. - -- `ergon_core/ergon_core/core/settings.py` - - Defines `ENABLE_TEST_HARNESS`, `ENABLE_SMOKE_FIXTURES`, and `smoke_fixtures_enabled`. - - Classification: **allowed configuration surface for gated dev/test behavior.** - - Follow-up: `ENABLE_SMOKE_FIXTURES` should be replaced or backed by a generic startup-plugin setting when `core/api/app.py` is fixed. - -- `ergon_core/ergon_core/core/runtime/services/task_management_service.py` - - Comment says tests must seed `RunRecord` via factories/fixtures. - - Classification: **allowed test guidance in invariant documentation.** - - No runtime behavior depends on test code. - -- `ergon_core/ergon_core/core/runtime/errors/delegation_errors.py` - - Comment says missing fixtures in tests should fail loudly. - - Classification: **allowed explanatory comment.** - - No runtime behavior depends on test code. - -- `ergon_core/ergon_core/core/providers/sandbox/manager.py` - - ImportError fallback exception classes for missing E2B SDK. - - Classification: **allowed optional dependency shim, but rename/comment carefully to avoid "test stub" language.** - -- `ergon_core/ergon_core/core/persistence/graph/models.py` - - Comment includes `"canonical-smoke"` as an example worker slug. - - Classification: **allowed example but should be refreshed if smoke naming changes.** - -Desired boundary: - -- Core runtime may say: "terminate the sandbox for this ID." -- Core runtime may not say: "skip because this ID is a stub." -- Core production sandbox creation must fail loudly if E2B is unavailable. -- Test-support managers may use sentinel IDs, but only test-support code should create or name them. -- The `sandbox_id` field should remain non-null for normal lifecycle events that require it. - -## File Structure - -Create: - -- `ergon_core/ergon_core/core/providers/sandbox/lifecycle.py` - - Owns sandbox lifecycle decisions by ID. - - Defines a termination result and termination service. - - Does not define or parse test sentinel IDs. - -- `ergon_core/ergon_core/test_support/sandbox/stub_manager.py` - - Contains `StubSandboxManager` as a test double for unit tests and harness-specific tests. - - Owns its sentinel prefix and any fake sandbox lifecycle bookkeeping. - - Must not be imported by `ergon_core.core`. - -- `tests/unit/sandbox/test_sandbox_lifecycle_service.py` - - Tests real-ID termination dispatch. - - Tests malformed or missing IDs are handled explicitly. - -- `tests/unit/architecture/test_no_test_logic_in_core.py` - - Regression guard that scans core runtime/provider modules for forbidden imports and terms. - - Allows explicitly approved files such as `core/api/test_harness.py` and `core/settings.py`. - -Modify: - -- `ergon_core/ergon_core/core/providers/sandbox/manager.py` - - Remove `is_stub_sandbox_id` from this file. - - Move `StubSandboxManager` to `ergon_core/ergon_core/test_support/sandbox/stub_manager.py`. - - Remove `DefaultSandboxManager.create`'s no-E2B fallback so production setup inherits the loud failure in `BaseSandboxManager.create`. - -- `ergon_core/ergon_core/core/providers/sandbox/__init__.py` - - Export lifecycle primitives that runtime code is allowed to use. - -- `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` - - Stop importing `StubSandboxManager`. - - Remove skipped-task placeholder minting from core. - -- `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` - - Replace `is_stub_sandbox_id` branching with the lifecycle service termination call. - -- `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` - - Replace `is_stub_sandbox_id` branching with the lifecycle service termination call. - -- `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` - - Replace failed-task cleanup branching with the lifecycle service termination call. - -- `ergon_core/ergon_core/core/runtime/events/task_events.py` - - Update comments that reference stub mode or `is_stub_sandbox_id`. - -- `ergon_core/ergon_core/core/api/app.py` - - Remove direct imports from `ergon_core.test_support`. - - Load optional startup hooks through a generic plugin setting. - -- `ergon_core/ergon_core/core/settings.py` - - Add a generic startup plugin setting. - - Keep test harness routing gated, but stop hardcoding smoke fixture registration in core app startup. - -- `tests/unit/runtime/test_failed_task_sandbox_cleanup.py` - - Update mocks to target the lifecycle service instead of `BaseSandboxManager` directly. - -## Task 1: Add Provider-Owned Sandbox Lifecycle API - -**Files:** - -- Create: `ergon_core/ergon_core/core/providers/sandbox/lifecycle.py` -- Test: `tests/unit/sandbox/test_sandbox_lifecycle_service.py` - -- [ ] **Step 1: Write failing lifecycle service tests** - -Create `tests/unit/sandbox/test_sandbox_lifecycle_service.py`: - -```python -from unittest.mock import AsyncMock, patch - -import pytest - -from ergon_core.core.providers.sandbox.lifecycle import ( - SandboxTerminationReason, - terminate_sandbox_by_id, -) - - -@pytest.mark.asyncio -async def test_terminate_sandbox_by_id_dispatches_real_ids() -> None: - with patch( - "ergon_core.core.providers.sandbox.manager.BaseSandboxManager.terminate_by_sandbox_id", - new=AsyncMock(return_value=True), - ) as terminate: - result = await terminate_sandbox_by_id("sbx-live-123") - - terminate.assert_awaited_once_with("sbx-live-123") - assert result.terminated is True - assert result.reason == SandboxTerminationReason.TERMINATED - - -@pytest.mark.asyncio -async def test_terminate_sandbox_by_id_handles_missing_id_explicitly() -> None: - result = await terminate_sandbox_by_id(None) - - assert result.terminated is False - assert result.reason == SandboxTerminationReason.MISSING_ID - assert result.sandbox_id is None -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/sandbox/test_sandbox_lifecycle_service.py -q -``` - -Expected: FAIL because `ergon_core.core.providers.sandbox.lifecycle` does not exist yet. - -- [ ] **Step 3: Add lifecycle service** - -Create `ergon_core/ergon_core/core/providers/sandbox/lifecycle.py`: - -```python -"""Provider-owned sandbox lifecycle helpers. - -Runtime orchestration code should not inspect sandbox ID sentinels. It should -delegate lifecycle operations here and let the provider layer terminate real -sandboxes. Test-support sentinel IDs are owned by test-support managers, not by -core runtime. -""" - -from __future__ import annotations - -import logging -from enum import StrEnum - -from pydantic import BaseModel - -logger = logging.getLogger(__name__) - - -class SandboxTerminationReason(StrEnum): - TERMINATED = "terminated" - NOT_FOUND_OR_ALREADY_CLOSED = "not_found_or_already_closed" - MISSING_ID = "missing_id" - ERROR = "error" - - -class SandboxTerminationResult(BaseModel): - sandbox_id: str | None - terminated: bool - reason: SandboxTerminationReason - - -async def terminate_sandbox_by_id(sandbox_id: str | None) -> SandboxTerminationResult: - """Terminate a sandbox ID behind one runtime-facing lifecycle boundary.""" - if sandbox_id is None: - return SandboxTerminationResult( - sandbox_id=None, - terminated=False, - reason=SandboxTerminationReason.MISSING_ID, - ) - - try: - from ergon_core.core.providers.sandbox.manager import BaseSandboxManager - - terminated = await BaseSandboxManager.terminate_by_sandbox_id(sandbox_id) - except Exception: # slopcop: ignore[no-broad-except] - logger.error("Failed to terminate sandbox %s", sandbox_id, exc_info=True) - return SandboxTerminationResult( - sandbox_id=sandbox_id, - terminated=False, - reason=SandboxTerminationReason.ERROR, - ) - - return SandboxTerminationResult( - sandbox_id=sandbox_id, - terminated=terminated, - reason=( - SandboxTerminationReason.TERMINATED - if terminated - else SandboxTerminationReason.NOT_FOUND_OR_ALREADY_CLOSED - ), - ) -``` - -- [ ] **Step 4: Run lifecycle tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/sandbox/test_sandbox_lifecycle_service.py -q -``` - -Expected: PASS. - -## Task 2: Move Stub Sandbox Manager Out of Core - -**Files:** - -- Modify: `ergon_core/ergon_core/core/providers/sandbox/manager.py` -- Modify: `ergon_core/ergon_core/core/providers/sandbox/__init__.py` -- Create: `ergon_core/ergon_core/test_support/sandbox/__init__.py` -- Create: `ergon_core/ergon_core/test_support/sandbox/stub_manager.py` -- Test: `tests/unit/sandbox/test_sandbox_lifecycle_service.py` -- Test: `tests/unit/smoke_base/test_smoke_sandbox_manager.py` - -- [ ] **Step 1: Update exports** - -Modify `ergon_core/ergon_core/core/providers/sandbox/__init__.py` to export the lifecycle API: - -```python -from ergon_core.core.providers.sandbox.lifecycle import ( - SandboxTerminationReason, - SandboxTerminationResult, - terminate_sandbox_by_id, -) -``` - -Add these names to `__all__`: - -```python - "SandboxTerminationReason", - "SandboxTerminationResult", - "terminate_sandbox_by_id", -``` - -- [ ] **Step 2: Move `StubSandboxManager` to test support** - -Create `ergon_core/ergon_core/test_support/sandbox/__init__.py`: - -```python -"""Test-support sandbox doubles.""" - -from ergon_core.test_support.sandbox.stub_manager import StubSandboxManager - -__all__ = ["StubSandboxManager"] -``` - -Create `ergon_core/ergon_core/test_support/sandbox/stub_manager.py`: - -```python -"""Sandbox manager test double. - -This class exists for unit tests and test harnesses that need a concrete -manager object without provisioning E2B. Production/core code must not import -this module. -""" - -from __future__ import annotations - -import logging -from uuid import UUID - -from ergon_core.core.providers.sandbox.manager import AsyncSandbox, BaseSandboxManager - -logger = logging.getLogger(__name__) - - -class _StubSandbox: - def __init__(self, sandbox_id: str) -> None: - self.sandbox_id = sandbox_id - - async def kill(self) -> None: - return None - - -class StubSandboxManager(BaseSandboxManager): - """No-op sandbox manager for tests. - - ``create`` returns a test-owned sentinel ID. Production/core code must not - create or inspect this ID format. - """ - - _PREFIX = "stub-sandbox-" - - async def create( - self, - sandbox_key: UUID, - run_id: UUID, - timeout_minutes: int = 30, - envs: dict[str, str] | None = None, - display_task_id: UUID | None = None, - ) -> str: - sandbox_id = f"{self._PREFIX}{sandbox_key}" - logger.info( - "Returning test stub sandbox id %s for task %s", - sandbox_id, - sandbox_key, - ) - self._ensure_registries(sandbox_key) - self._sandboxes[sandbox_key] = _StubSandbox(sandbox_id) # type: ignore[assignment] - self._run_ids[sandbox_key] = run_id - self._display_task_ids[sandbox_key] = display_task_id or sandbox_key - self._sandbox_manager_classes[sandbox_key] = type(self) - return sandbox_id - - async def _install_dependencies(self, sandbox: AsyncSandbox, task_id: UUID) -> None: - return None - - async def terminate(self, task_id: UUID, reason: str = "completed") -> None: - self._file_registries.pop(task_id, None) - self._created_files_registry.pop(task_id, None) - self._run_ids.pop(task_id, None) - self._display_task_ids.pop(task_id, None) - - async def reset_timeout(self, task_id: UUID, timeout_minutes: int = 30) -> bool: - return True -``` - -Then remove these symbols from `ergon_core/ergon_core/core/providers/sandbox/manager.py`: - -```python -_STUB_SANDBOX_PREFIX = "stub-sandbox-" - - -def is_stub_sandbox_id(sandbox_id: JsonValue) -> bool: - ... - - -class StubSandboxManager(BaseSandboxManager): - ... -``` - -- [ ] **Step 3: Remove core no-E2B fallback** - -Do not add any `DefaultSandboxManager.create` fallback. In `ergon_core/ergon_core/core/providers/sandbox/manager.py`, either delete the `DefaultSandboxManager.create` override entirely or reduce `DefaultSandboxManager` to dependency hooks only: - -```python -class DefaultSandboxManager(BaseSandboxManager): - """No custom dependencies. Used by benchmarks without specific sandbox setup.""" - - async def _install_dependencies(self, sandbox: AsyncSandbox, task_id: UUID) -> None: - pass -``` - -This intentionally preserves `BaseSandboxManager.create`'s existing loud failure when `E2B_API_KEY` is missing. - -- [ ] **Step 4: Run sandbox tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/sandbox/test_sandbox_lifecycle_service.py tests/unit/smoke_base/test_smoke_sandbox_manager.py -q -``` - -Expected: PASS. - -## Task 3: Remove Skipped-Task Placeholder Minting from Task Execution - -**Files:** - -- Modify: `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` -- Test: `tests/unit/runtime/test_child_function_payloads.py` -- Test: `tests/unit/runtime/test_worker_execute_output_failure.py` - -- [ ] **Step 1: Remove skipped-task stub manager import** - -In `ergon_core/ergon_core/core/runtime/inngest/execute_task.py`, replace: - -```python -from ergon_core.core.providers.sandbox.manager import StubSandboxManager -``` - -with no sandbox-manager import. `execute_task.py` should not import `StubSandboxManager`, `make_noop_sandbox_id`, or any test-support sandbox module. - -Then replace the skipped-task block: - -```python -if prepared.skipped: - logger.info( - "task-execute skipped task_id=%s reason=%s", - payload.task_id, - prepared.skip_reason, - ) - stub_sandbox_id = await StubSandboxManager().create( - prepared.node_id, - run_id=payload.run_id, - display_task_id=prepared.node_id, - ) - await _emit_task_completed(payload, prepared, stub_sandbox_id) - return TaskExecuteResult( - run_id=payload.run_id, - task_id=payload.task_id, - execution_id=prepared.execution_id, - success=True, - skipped=True, - skip_reason=prepared.skip_reason, - ) -``` - -with: - -```python -if prepared.skipped: - raise ContractViolationError( - "Skipped task execution cannot emit task/completed without a real sandbox_id. " - "Introduce a first-class task/skipped event before supporting skipped tasks." - ) -``` - -Rationale: production has no "no remote sandbox was provisioned" path. If skipped tasks become a real product feature, they need their own explicit event/propagation contract instead of fake sandbox IDs. - -- [ ] **Step 2: Add a regression test for skipped-task contract failure** - -Add a focused unit test for `execute_task_fn`'s skipped branch if there is an existing task-execution test harness. If no focused harness exists, add this behavior to the architecture guard: - -```python -def test_core_task_execution_does_not_mint_placeholder_sandbox_ids() -> None: - path = CORE / "runtime" / "inngest" / "execute_task.py" - text = path.read_text() - - assert "StubSandboxManager" not in text - assert "make_noop_sandbox_id" not in text - assert "stub_sandbox_id" not in text -``` - -- [ ] **Step 3: Compile task execution module** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run python -m py_compile ergon_core/ergon_core/core/runtime/inngest/execute_task.py -``` - -Expected: no output. - -- [ ] **Step 4: Run targeted runtime tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/runtime/test_child_function_payloads.py tests/unit/runtime/test_worker_execute_output_failure.py -q -``` - -Expected: PASS. - -## Task 4: Route All Runtime Teardown Through Lifecycle Service - -**Files:** - -- Modify: `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` -- Modify: `tests/unit/runtime/test_failed_task_sandbox_cleanup.py` - -- [ ] **Step 1: Update failed-task cleanup test** - -Change `tests/unit/runtime/test_failed_task_sandbox_cleanup.py` to patch the provider lifecycle API: - -```python -from unittest.mock import AsyncMock, patch - -import pytest - -from ergon_core.core.providers.sandbox.lifecycle import ( - SandboxTerminationReason, - SandboxTerminationResult, -) -from ergon_core.core.runtime.inngest.propagate_execution import _terminate_failed_task_sandbox - - -@pytest.mark.asyncio -async def test_failed_task_sandbox_cleanup_delegates_to_lifecycle_service() -> None: - result = SandboxTerminationResult( - sandbox_id="sbx-real", - terminated=True, - reason=SandboxTerminationReason.TERMINATED, - ) - with patch( - "ergon_core.core.runtime.inngest.propagate_execution.terminate_sandbox_by_id", - new=AsyncMock(return_value=result), - ) as terminate: - await _terminate_failed_task_sandbox("sbx-real") - - terminate.assert_awaited_once_with("sbx-real") -``` - -- [ ] **Step 2: Update failed-task cleanup implementation** - -In `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py`, replace: - -```python -from ergon_core.core.providers.sandbox.manager import BaseSandboxManager, is_stub_sandbox_id -``` - -with: - -```python -from ergon_core.core.providers.sandbox.lifecycle import terminate_sandbox_by_id -``` - -Replace `_terminate_failed_task_sandbox` with: - -```python -async def _terminate_failed_task_sandbox(sandbox_id: str | None) -> None: - result = await terminate_sandbox_by_id(sandbox_id) - if not result.terminated: - logger.info( - "failed-task sandbox cleanup did not terminate sandbox_id=%s reason=%s", - result.sandbox_id, - result.reason, - ) -``` - -- [ ] **Step 3: Update evaluator cleanup** - -In `ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py`, replace imports of `BaseSandboxManager` and `is_stub_sandbox_id` with: - -```python -from ergon_core.core.providers.sandbox.lifecycle import terminate_sandbox_by_id -``` - -Replace `_terminate_sandbox` with: - -```python -async def _terminate_sandbox(sandbox_id: str) -> None: - """Terminate the task's sandbox through the provider lifecycle boundary.""" - result = await terminate_sandbox_by_id(sandbox_id) - logger.info( - "Evaluator sandbox cleanup sandbox_id=%s terminated=%s reason=%s", - result.sandbox_id, - result.terminated, - result.reason, - ) -``` - -- [ ] **Step 4: Update run cleanup** - -In `ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py`, replace: - -```python -from ergon_core.core.providers.sandbox.manager import ( - BaseSandboxManager, - is_stub_sandbox_id, -) -``` - -with: - -```python -from ergon_core.core.providers.sandbox.lifecycle import terminate_sandbox_by_id -``` - -Replace the branch over `sandbox_id` with: - -```python -sandbox_result = await terminate_sandbox_by_id( - sandbox_id if isinstance(sandbox_id, str) else None -) -sandbox_terminated = sandbox_result.terminated - -if sandbox_id is not None and not isinstance(sandbox_id, str): - logger.warning( - "run-cleanup run_id=%s: sandbox_id has unexpected type %s, skipping termination", - run_id, - type(sandbox_id).__name__, - ) -``` - -- [ ] **Step 5: Run targeted teardown tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/runtime/test_failed_task_sandbox_cleanup.py tests/unit/sandbox/test_sandbox_lifecycle_service.py -q -``` - -Expected: PASS. - -## Task 5: Remove Test-Support Imports from App Bootstrap - -**Files:** - -- Modify: `ergon_core/ergon_core/core/api/app.py` -- Modify: `ergon_core/ergon_core/core/settings.py` -- Test: `tests/unit/test_test_harness.py` -- Test: `tests/unit/architecture/test_no_test_logic_in_core.py` - -- [ ] **Step 1: Add a generic startup-plugin setting** - -In `ergon_core/ergon_core/core/settings.py`, add a string setting that can hold comma-separated import specs: - -```python -startup_plugin_specs: str = Field( - default="", - validation_alias=AliasChoices("ERGON_STARTUP_PLUGINS"), -) -``` - -Add a helper property: - -```python -@property -def startup_plugins(self) -> tuple[str, ...]: - return tuple( - spec.strip() - for spec in self.startup_plugin_specs.split(",") - if spec.strip() - ) -``` - -Keep `enable_test_harness` for mounting the harness router. Treat `enable_smoke_fixtures` as compatibility only until callers are moved to `ERGON_STARTUP_PLUGINS`. - -- [ ] **Step 2: Add an app-local plugin loader** - -In `ergon_core/ergon_core/core/api/app.py`, add: - -```python -from importlib import import_module - - -def _run_startup_plugins(plugin_specs: tuple[str, ...]) -> None: - for spec in plugin_specs: - module_name, sep, attr_name = spec.partition(":") - if not sep or not module_name or not attr_name: - raise RuntimeError( - "Invalid ERGON_STARTUP_PLUGINS entry " - f"{spec!r}; expected 'module:function'" - ) - module = import_module(module_name) - plugin = getattr(module, attr_name) - plugin() -``` - -Then replace: - -```python -if settings.smoke_fixtures_enabled: - from ergon_core.test_support.smoke_fixtures import register_smoke_fixtures - - register_smoke_fixtures() -``` - -with: - -```python -_run_startup_plugins(settings.startup_plugins) -``` - -- [ ] **Step 3: Preserve local/CI smoke fixture registration through configuration** - -Update local and CI smoke environment setup to use: - -```bash -ERGON_STARTUP_PLUGINS=ergon_core.test_support.smoke_fixtures:register_smoke_fixtures -``` - -Candidate files to inspect and update: - -- `docker-compose.yml` -- `.github/workflows/e2e-benchmarks.yml` -- `scripts/smoke_local_up.sh` -- `scripts/smoke_local_run.sh` - -Do not keep a direct import of `ergon_core.test_support` in `core/api/app.py`. - -- [ ] **Step 4: Add tests for startup plugin loading** - -In `tests/unit/test_test_harness.py`, add a focused test for `_run_startup_plugins` using a standard-library callable that is safe to call, or a tiny in-test module fixture if one already exists. If a direct callable test is awkward, test invalid config instead: - -```python -import pytest - -from ergon_core.core.api.app import _run_startup_plugins - - -def test_startup_plugin_loader_rejects_invalid_specs() -> None: - with pytest.raises(RuntimeError, match="expected 'module:function'"): - _run_startup_plugins(("ergon_core.test_support.smoke_fixtures",)) -``` - -- [ ] **Step 5: Run app/test-harness tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/test_test_harness.py -q -``` - -Expected: PASS. - -## Task 6: Add Architecture Guard for Test Logic in Core - -**Files:** - -- Create: `tests/unit/architecture/test_no_test_logic_in_core.py` - -- [ ] **Step 1: Write architecture guard** - -Create `tests/unit/architecture/test_no_test_logic_in_core.py`: - -```python -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[3] -CORE = ROOT / "ergon_core" / "ergon_core" / "core" - -ALLOWED_FILES = { - CORE / "api" / "test_harness.py", - CORE / "settings.py", -} - -FORBIDDEN_IMPORT_SNIPPETS = ( - "ergon_core.test_support", - "tests.", -) - -FORBIDDEN_CORE_TEST_DOUBLE_TERMS = ( - "StubSandboxManager", - "is_stub_sandbox_id", - "stub-sandbox-", -) - - -def _core_python_files() -> list[Path]: - return [ - path - for path in CORE.rglob("*.py") - if path not in ALLOWED_FILES and "__pycache__" not in path.parts - ] - - -def test_core_does_not_import_test_support_or_tests() -> None: - offenders: list[str] = [] - for path in _core_python_files(): - text = path.read_text() - for snippet in FORBIDDEN_IMPORT_SNIPPETS: - if snippet in text: - offenders.append(f"{path.relative_to(ROOT)} contains {snippet!r}") - - assert offenders == [] - - -def test_core_does_not_define_or_branch_on_stub_sandbox_terms() -> None: - offenders: list[str] = [] - for path in _core_python_files(): - text = path.read_text() - for term in FORBIDDEN_CORE_TEST_DOUBLE_TERMS: - if term in text: - offenders.append(f"{path.relative_to(ROOT)} contains {term!r}") - - assert offenders == [] -``` - -- [ ] **Step 2: Run architecture guard** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/architecture/test_no_test_logic_in_core.py -q -``` - -Expected: PASS after Tasks 1-4. If it fails, move the offending logic behind provider/test-support boundaries instead of weakening the guard. - -## Task 7: Clean Comments and Event Contract Language - -**Files:** - -- Modify: `ergon_core/ergon_core/core/runtime/events/task_events.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/execute_task.py` -- Modify: `ergon_core/ergon_core/core/providers/sandbox/manager.py` - -- [ ] **Step 1: Remove "stub" and no-sandbox fallback terminology from core comments** - -Replace comments that describe no-E2B behavior as "stub mode" or "provider no-op sandbox ID" with loud production setup language. - -In `task_events.py`, replace the existing stub comment with: - -```python -# Production task execution emits real sandbox IDs. Test-support managers may -# use sentinel IDs, but core event consumers must not parse or branch on those -# sentinel formats. -``` - -- [ ] **Step 2: Search for remaining core stub terms** - -Run: - -```bash -rg "StubSandboxManager|is_stub_sandbox_id|stub-sandbox|stub mode|stub sandbox" ergon_core/ergon_core/core -``` - -Expected: no matches. If matches remain in core runtime/provider code, rewrite them or move the implementation to `test_support`. - -## Task 8: Run Focused Verification - -**Files:** - -- No source edits. - -- [ ] **Step 1: Compile touched Python files** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run python -m py_compile \ - ergon_core/ergon_core/core/providers/sandbox/lifecycle.py \ - ergon_core/ergon_core/core/providers/sandbox/manager.py \ - ergon_core/ergon_core/core/runtime/inngest/execute_task.py \ - ergon_core/ergon_core/core/runtime/inngest/check_evaluators.py \ - ergon_core/ergon_core/core/runtime/inngest/run_cleanup.py \ - ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py -``` - -Expected: no output. - -- [ ] **Step 2: Run unit tests** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest \ - tests/unit/sandbox/test_sandbox_lifecycle_service.py \ - tests/unit/runtime/test_failed_task_sandbox_cleanup.py \ - tests/unit/runtime/test_worker_execute_output_failure.py \ - tests/unit/smoke_base/test_smoke_sandbox_manager.py \ - tests/unit/architecture/test_no_test_logic_in_core.py \ - -q -``` - -Expected: PASS. - -- [ ] **Step 3: Run local canonical smoke e2es** - -Run: - -```bash -ERGON_DATABASE_URL=postgresql://ergon:ergon_dev@localhost:5433/ergon \ -INNGEST_API_BASE_URL=http://localhost:8289 \ -INNGEST_DEV=1 \ -INNGEST_EVENT_KEY=dev \ -ERGON_API_BASE_URL=http://127.0.0.1:9000 \ -PLAYWRIGHT_BASE_URL=http://127.0.0.1:3001 \ -ENABLE_TEST_HARNESS=1 \ -TEST_HARNESS_SECRET=local-dev \ -SCREENSHOT_DIR=/tmp/playwright \ -SMOKE_COHORT_SIZE=1 \ -PYTHONPATH="ergon_core:ergon_builtins" \ -uv run pytest tests/e2e/test_researchrubrics_smoke.py tests/e2e/test_minif2f_smoke.py tests/e2e/test_swebench_smoke.py -q -s --timeout=300 --tb=short -``` - -Expected: all three benchmark smoke tests pass. The sad-path shape should remain: - -- `l_2` fails. -- `l_3` is blocked and never starts. -- Independent leaves complete. -- Run status is `FAILED`. -- Sandbox lifecycle events are symmetric for created/closed sandbox IDs. - -## Task 9: Close Remaining Audit Findings - -**Files:** - -- Modify: `ergon_core/ergon_core/core/runtime/inngest/benchmark_run_start.py` -- Modify: `ergon_core/ergon_core/core/rl/eval_runner.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py` -- Update: `tests/unit/architecture/test_no_test_logic_in_core.py` - -- [ ] **Step 1: Re-run the audit search and compare against the inventory** - -Run: - -```bash -rg "test_support|tests\\.|smoke|fake|mock|stub|fixture|ENABLE_TEST|ENABLE_SMOKE" ergon_core/ergon_core/core -``` - -Expected remaining matches should be limited to the `Allowed / No Code Change` section above. Any match from `Must Fix` should be gone. - -- [ ] **Step 2: Fix request-contract defaults that read as test doubles** - -In `ergon_core/ergon_core/core/runtime/inngest/benchmark_run_start.py`, make worker/evaluator slugs explicit instead of defaulting to test-looking values: - -```python -class BenchmarkRunRequest(InngestEventContract): - """CLI sends this to request a full benchmark run.""" - - name: ClassVar[str] = "benchmark/run-request" - - benchmark_slug: str - model: str - worker_slug: str - evaluator_slug: str - cohort_name: str = "" # slopcop: ignore[no-str-empty-default] -``` - -Then update call sites/tests that construct `BenchmarkRunRequest` without `worker_slug` or `evaluator_slug` so they pass concrete slugs. - -- [ ] **Step 3: Require explicit RL eval runner inputs** - -In `ergon_core/ergon_core/core/rl/eval_runner.py`, replace `"stub-rubric"` defaults with required evaluator arguments: - -```python -async def watch_and_evaluate( - checkpoint_dir: str, - benchmark_type: str, - *, - evaluator_type: str, - model_base: str, - poll_interval_s: int = 60, - eval_limit: int | None = None, - on_checkpoint_cmd: str | None = None, - external_cmd_timeout_s: int = 600, -) -> None: -``` - -For `_run_local_eval`, make `model_base` required: - -```python -async def _run_local_eval( - ckpt: CheckpointInfo, - *, - benchmark_type: str, - evaluator_type: str, - model_base: str, - eval_limit: int | None, -) -> int: -``` - -Then replace: - -```python -model_target = f"vllm:{ckpt.path}" if model_base else "stub-worker" -``` - -with: - -```python -model_target = f"vllm:{ckpt.path}" -``` - -Apply the same required `evaluator_type` and `model_base` signatures to `evaluate_checkpoint`. Callers must pass concrete values. - -- [ ] **Step 4: Update cancelled-task cleanup comment** - -In `ergon_core/ergon_core/core/runtime/inngest/cleanup_cancelled_task.py`, replace: - -```python -2. release-sandbox — stub (pending sandbox management module) -``` - -with: - -```python -2. release-sandbox — routed through the sandbox lifecycle provider when an - execution has an associated sandbox. -``` - -- [ ] **Step 5: Keep the allowed list narrow** - -Do not add broad exemptions to `tests/unit/architecture/test_no_test_logic_in_core.py`. The allowed files should remain: - -```python -ALLOWED_FILES = { - CORE / "api" / "test_harness.py", - CORE / "settings.py", -} -``` - -If the architecture guard catches a new file, fix the dependency direction instead of adding the file to `ALLOWED_FILES`. - -- [ ] **Step 6: Fix any new offenders with the same pattern** - -For each offender: - -1. Move test-owned implementation into `ergon_core/ergon_core/test_support`. -2. Leave only a production abstraction in `ergon_core/ergon_core/core`. -3. Wire the test implementation from an explicitly gated bootstrap path. -4. Add the offender term to `tests/unit/architecture/test_no_test_logic_in_core.py` if it should never recur. - -- [ ] **Step 7: Re-run architecture guard** - -Run: - -```bash -PYTHONPATH="ergon_core:ergon_builtins" uv run pytest tests/unit/architecture/test_no_test_logic_in_core.py -q -``` - -Expected: PASS. - -## Self-Review - -- Spec coverage: covers the selected sandbox stub leak and widens scope to an architectural audit for test logic in core. -- Placeholder scan: no unresolved implementation placeholders are used as plan content; the audit task gives concrete classification and remediation rules. -- Type consistency: lifecycle names are consistent across tasks: `terminate_sandbox_by_id`, `SandboxTerminationResult`, and `SandboxTerminationReason`. -- Scope check: intentionally scoped to test-logic leakage in core, with sandbox lifecycle as the first concrete refactor and a guardrail test to prevent recurrence. - diff --git a/docs/superpowers/plans/2026-04-26-finish-agent-workflow-cli.md b/docs/superpowers/plans/2026-04-26-finish-agent-workflow-cli.md deleted file mode 100644 index ffb11efd0..000000000 --- a/docs/superpowers/plans/2026-04-26-finish-agent-workflow-cli.md +++ /dev/null @@ -1,99 +0,0 @@ -# Finish Agent Workflow CLI Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish `ergon workflow` as an agent-facing CLI for task editing and resource copying in one PR off `main`. - -**Architecture:** Extend the already-merged V1 instead of replacing it. Keep scoped reads and mutation policy in `WorkflowService`, command parsing/rendering in `ergon_cli.commands.workflow`, and model-facing scope injection in `workflow_cli_tool`. All commands stay current-run/current-node scoped unless an injected manager-capable context explicitly permits broader graph edits. - -**Tech Stack:** Python, argparse, SQLModel, existing `WorkflowGraphRepository`, existing run graph tables, pydantic DTOs, pytest. - ---- - -## Current Baseline - -Already merged: - -- `ergon workflow ...` top-level command. -- `WorkflowService` with task/resource inspection and `materialize_resource`. -- `workflow(command)` pydantic-ai wrapper with injected run/node/execution/sandbox scope. -- ResearchRubrics workflow worker registration. - -## Implementation Tasks - -### Task 1: Real Task Editing Commands - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_service.py` -- Modify: `ergon_cli/ergon_cli/commands/workflow.py` -- Test: `tests/unit/runtime/test_workflow_service.py` -- Test: `tests/unit/cli/test_workflow_cli.py` - -- [ ] Add a `WorkflowMutationRef` DTO with `action`, `dry_run`, `node`, `edge`, `message`, and `suggested_commands`. -- [ ] Add service methods for `add_task`, `add_edge`, `update_task_description`, `restart_task`, and `abandon_task`. -- [ ] Use `WorkflowGraphRepository` for graph writes and mutation logging. -- [ ] Keep `--dry-run` behavior identical to real command validation but without writes. -- [ ] Add CLI parser arguments for task slug, description, worker, source/target, and status fields. -- [ ] Add text and JSON renderers for mutation results. -- [ ] Verify with focused unit tests before moving on. - -### Task 2: Resource Copying Completion - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_service.py` -- Modify: `ergon_cli/ergon_cli/commands/workflow.py` -- Test: `tests/unit/runtime/test_workflow_service.py` -- Test: `tests/unit/cli/test_workflow_cli.py` - -- [ ] Add `inspect resource-location`. -- [ ] Add `inspect task-workspace`. -- [ ] Harden `materialize-resource` destination handling: reject absolute paths, `..`, and paths outside `/workspace`. -- [ ] Preserve source resource bytes and row unchanged. -- [ ] Ensure copied resource rows use `RunResourceKind.IMPORT`, `copied_from_resource_id`, and metadata with source resource, source task, and sandbox destination. -- [ ] Add JSON/text outputs for resource location and task workspace. -- [ ] Verify with unit tests and one integration-style sandbox-manager-injected test. - -### Task 3: Agent Wrapper Permissions - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Test: `tests/unit/state/test_workflow_cli_tool.py` - -- [ ] Add a permission mode to the wrapper: leaf agents can inspect and materialize visible resources; manager-capable agents can use graph edit commands. -- [ ] Reject user-supplied scope/context flags before command execution. -- [ ] Reject multiline commands. -- [ ] Return structured, model-readable failure strings instead of leaking tracebacks. -- [ ] Verify wrapper tests for allowed inspect, allowed materialize, denied graph edit, and allowed manager graph edit. - -### Task 4: Acceptance Coverage - -**Files:** -- Modify: existing smoke fixture workers only as needed. -- Modify: existing E2E assertions only as needed. -- Test: focused unit tests plus existing smoke tests. - -- [ ] Ensure one deterministic no-LLM smoke path calls `workflow("inspect task-tree")`. -- [ ] Ensure one deterministic no-LLM smoke path calls `workflow("inspect resource-list --scope input")`. -- [ ] Ensure one deterministic no-LLM smoke path dry-runs `manage materialize-resource`. -- [ ] Keep real-LLM rollout optional, using `researchrubrics-workflow-cli-react`. -- [ ] Run focused workflow tests, Python unit tests touched by runtime changes, frontend contract generation if schemas change, and CI-fast-compatible checks. - -## Verification Commands - -Run incrementally: - -```bash -uv run pytest tests/unit/runtime/test_workflow_service.py -v -uv run pytest tests/unit/cli/test_workflow_cli.py -v -uv run pytest tests/unit/state/test_workflow_cli_tool.py -v -``` - -Before PR: - -```bash -uv run pytest tests/unit/runtime/test_workflow_service.py tests/unit/cli/test_workflow_cli.py tests/unit/state/test_workflow_cli_tool.py -v -uv run pytest tests/unit/runtime tests/unit/cli tests/unit/state -q -pnpm --dir ergon-dashboard run typecheck -``` diff --git a/docs/superpowers/plans/2026-04-26-mas-navigation-cli.md b/docs/superpowers/plans/2026-04-26-mas-navigation-cli.md deleted file mode 100644 index 33d93d9ce..000000000 --- a/docs/superpowers/plans/2026-04-26-mas-navigation-cli.md +++ /dev/null @@ -1,1439 +0,0 @@ -# Workflow Agent CLI Implementation Plan - -> **Superseded:** this single-file plan has been split into [`workflow-agent-cli/`](workflow-agent-cli/). Start at [`workflow-agent-cli/README.md`](workflow-agent-cli/README.md). Keep this file only as historical context while reviewers migrate. - -> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` (recommended) or `superpowers:executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build `ergon workflow ...`, an agent-local command surface for the benchmark-task worker to navigate its current workflow topology/resources and explicitly materialize useful visible resources into its workspace without exposing raw SQL. - -**Architecture:** Put scoped Postgres queries and resource materialization policy in `ergon_core`, command parsing in `ergon_cli`, and the agent-callable wrapper in `ergon_builtins/tools`. This is not a general operator/debugging CLI: it is invoked by the benchmark agent, runs in the local worker/API process, reads local Postgres directly via `get_session()`, and uses the existing sandbox manager to copy approved resources into the current E2B workspace. - -**Tech Stack:** Python, argparse, SQLModel, existing `get_session()` / `ensure_db()`, pydantic-ai `Tool`, pytest. - ---- - -## Package Placement - -There is no `arcane_builtins` package in this workspace. The right homes are: - -- Core scoped read logic: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_service.py` -- Core materialization policy: `ergon_core/ergon_core/core/runtime/services/workflow_resource_materialization_service.py` -- DTOs: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_dto.py` -- CLI parser and handlers: `ergon_cli/ergon_cli/main.py` and `ergon_cli/ergon_cli/commands/workflow.py` -- Agent wrapper: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Optional worker wiring: initially a target ReAct/research worker, not all workers by default - -Keep `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` intact for now. It is resource-discovery-specific and research-named; `workflow` should be more general and command-shaped. - -## Resource Ownership and Copy Semantics - -Resources are immutable published artifacts. A task may read a visible resource from another task in the same run, including a resource from a different branch of the control DAG, but it must never mutate the source row or source bytes. - -The workflow CLI must treat copying as a fork: - -- **Read is context.** Reading `resource-content` does not change graph state. -- **Materialize is fork.** Copying a resource into the current agent workspace creates a new `RunResource` row owned by the current task execution. -- **Publish is ownership.** If the current task edits the copied file and publishes it later, the edited artifact is another new resource owned by the current task execution. -- **Lineage is evidence.** The copied resource row records `copied_from_resource_id=<source_resource_id>`, and later edited outputs should preserve provenance in metadata where practical. -- **Control edges schedule work; resource lineage explains information flow.** Materializing a resource from a divergent DAG branch must not add a control dependency edge. - -Example: - -```text -task_a publishes: - resource_id=res_a - task_execution_id=task_a_execution - name="paper.pdf" - content_hash=abc - -task_b materializes res_a: - resource_id=res_b_copy - task_execution_id=task_b_execution - name="paper (copy).pdf" - content_hash=abc - copied_from_resource_id=res_a - metadata.sandbox_destination="/workspace/imported/task-a/paper (copy).pdf" - -task_b edits and publishes: - resource_id=res_b_edited - task_execution_id=task_b_execution - name="paper_annotated.pdf" - content_hash=def - metadata.derived_from_resource_ids=["res_a", "res_b_copy"] -``` - -If task A later republishes a newer `paper.pdf`, task B's copy remains pinned to the old `resource_id` and `content_hash`. Rerun/staleness logic can use resource lineage to flag B as potentially stale, but A never mutates B's copy. - -## Code Write Locations - -Review this section before implementation. These are the proposed new and modified files. - -### New Core Runtime Files - -- Create `ergon_core/ergon_core/core/runtime/services/workflow_navigation_dto.py` - - Owns Pydantic DTOs returned by the workflow inspection service. - - Intended types: `WorkflowTaskRef`, `WorkflowExecutionRef`, `WorkflowResourceRef`, `WorkflowDependencyRef`, `WorkflowBlockerRef`, `WorkflowNextActionRef`, `WorkflowMaterializedResourceRef`. - -- Create `ergon_core/ergon_core/core/runtime/services/workflow_navigation_service.py` - - Owns scoped Postgres reads for the current run. - - Implements task listing, task tree traversal, dependency inspection, resource visibility, task blockers, next actions, and resource content reads. - - Must not query evaluation tables. - -- Create `ergon_core/ergon_core/core/runtime/services/workflow_resource_materialization_service.py` - - Owns the policy-checked "copy visible resource into my current E2B workspace" operation. - - Reads source `RunResource` metadata/bytes from local Postgres/blob store. - - Creates a current-task-owned copied `RunResource` row with a new ID. - - Writes the bytes to the current task sandbox under a controlled workspace path. - - Records provenance via `copied_from_resource_id` and metadata. - -### New Migration File - -- Create `ergon_core/migrations/versions/<revision>_add_copied_from_resource_id.py` - - Adds nullable `run_resources.copied_from_resource_id`. - - Adds a self-referential foreign key to `run_resources.id`. - - Adds an index for lineage queries. - -### Modified Persistence Files - -- Modify `ergon_core/ergon_core/core/persistence/telemetry/models.py` - - Adds `RunResourceKind.IMPORT`. - - Adds nullable `RunResource.copied_from_resource_id`. - -- Modify `ergon_core/ergon_core/core/persistence/queries.py` - - Extends `ResourcesQueries.append(...)` to accept `copied_from_resource_id`. - - Adds small lineage read helpers if needed by workflow inspection/tests. - -### New CLI Files - -- Create `ergon_cli/ergon_cli/commands/workflow.py` - - Owns all `ergon workflow inspect ...` and `ergon workflow manage ...` command handlers. - - Uses `WorkflowNavigationService` for reads. - - Uses `WorkflowResourceMaterializationService` for `manage materialize-resource`. - - Uses existing graph/task management services for mutations. - - Handles text/JSON rendering, `--explain`, output caps, and `--dry-run`. - -### Modified CLI Files - -- Modify `ergon_cli/ergon_cli/main.py` - - Imports `handle_workflow`. - - Registers the top-level `workflow` command. - - Registers nested `inspect` and `manage` subcommands. - - Dispatches `args.command == "workflow"` to `handle_workflow(args)`. - -### New Builtins Tool File - -- Create `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` - - Provides the local pydantic-ai `workflow(command=...)` tool. - - Injects `WorkerContext.run_id` and `WorkerContext.node_id`. - - Rejects user-supplied run/experiment/cohort scope arguments. - - Calls the CLI in-process rather than spawning a shell command. - - Enforces leaf vs manager command permissions. - -### New Proof-of-Concept Worker File - -- Create `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` - - Defines `ResearchRubricsWorkflowCliReActWorker`. - - Reuses the research-rubrics ReAct/toolkit behavior, but adds the local `workflow(command=...)` tool. - - Uses worker slug `researchrubrics-workflow-cli-react`. - - Adds prompt guidance that tells agents to start with `inspect task-tree`, `inspect task-workspace`, `inspect next-actions`, or `inspect resource-list --scope input`. - -### Modified Registry File - -- Modify `ergon_builtins/ergon_builtins/registry_data.py` - - Imports `ResearchRubricsWorkflowCliReActWorker`. - - Registers `"researchrubrics-workflow-cli-react": ResearchRubricsWorkflowCliReActWorker`. - - Leaves the existing `"researchrubrics-researcher"` worker unchanged. - -### New Tests - -- Create `tests/unit/runtime/test_workflow_navigation_service.py` - - Tests core service behavior: current-run reads, immediate-upstream resource semantics, task tree traversal, blockers, next actions, and cross-run resource rejection. - -- Create `tests/unit/runtime/test_workflow_resource_materialization_service.py` - - Tests materialize-resource semantics: same-run visibility, new copied resource ID, copied name, `copied_from_resource_id`, controlled destination path, import manifest, collision handling, and no mutation of the source resource. - -- Create `tests/unit/cli/test_workflow_cli.py` - - Tests parser/handler behavior for `inspect` and `manage` commands. - - Tests text and JSON output. - - Tests invalid UUIDs, duplicate slugs, and mutation `--dry-run`. - -- Create `tests/unit/state/test_workflow_cli_tool.py` - - Tests the pydantic-ai wrapper. - - Verifies scope injection, denial of user-supplied `--run-id`, denial of `inspect resource-list --scope run`, leaf vs manager permissions, multiline rejection, and structured failures. - -- Create `tests/unit/runtime/test_workflow_input_resource_semantics.py` - - Tests the canonical input-resource policy on a diamond and line graph. - - Ensures a task sees only immediate predecessor resources by default, not transitive ancestors. - -### Modified Existing Tests - -- Modify `tests/unit/state/test_research_rubrics_workers.py` - - Adds coverage for `ResearchRubricsWorkflowCliReActWorker`. - - Asserts the new worker exposes the `workflow` tool. - - Asserts the existing `ResearchRubricsResearcherWorker` behavior remains unchanged. - -- Modify `tests/real_llm/benchmarks/test_researchrubrics.py` - - Adds environment overrides or a dedicated rollout path for `researchrubrics-workflow-cli-react`. - - Supports running the final workflow-CLI acceptance rollout with `--limit 5`. - - Persists enough rollout artifacts to inspect whether and how the agent used `workflow(...)`. - -### Files Explicitly Not Planned For V1 - -- Do not expose eval tables. -- Do not modify dashboard UI files. -- Do not modify `ergon_builtins/ergon_builtins/tools/graph_toolkit.py`; keep it as the existing research/resource toolkit. -- Do not add E2B-side CLI execution or require E2B-to-localhost networking. - -## Command Surface - -Build two explicit command groups: - -- `workflow inspect ...`: read-only task topology, dependency, resource, and workspace inspection. -- `workflow manage ...`: state-changing task/dependency management commands that wrap the existing task lifecycle services. - -The command surface is intended for the currently executing benchmark-task agent. It reads Postgres directly through scoped services, but the model never receives a SQL console and never selects another run. - -Every command is scoped by injected runtime context: - -- `run_id`: injected from `WorkerContext.run_id` -- `node_id`: injected from `WorkerContext.node_id` -- `execution_id`: available to the wrapper from `WorkerContext.execution_id` if needed later - -For direct developer testing, `ergon workflow ...` may still accept explicit `--run-id` and `--node-id`, but that is test/debug plumbing. The agent wrapper strips or rejects user-supplied scope arguments and supplies the real values. - -Keep mutation commands out of `inspect`. If a command changes graph state, it belongs under `manage` and should return an explicit mutation result. - -All commands should support `--format text|json` (default `text`) and `--explain` (adds a short explanation of the scope/policy being applied). Commands that can produce large output should also support caps: `--limit`, `--max-chars`, or `--max-bytes` as appropriate. - -All `manage ...` commands must support `--dry-run`. Dry-run resolves slugs to node IDs, checks visibility and service preconditions, and prints the mutation that would happen without writing to Postgres or emitting events. - -## Inspect Commands - -### `workflow inspect task-list` - -Lists task nodes in a run. - -Examples: - -```bash -workflow("inspect task-list") -workflow("inspect task-list --children") -workflow("inspect task-list --level 2") -workflow("inspect task-list --under d_root --level 3") -workflow("inspect task-list --format json") -``` - -Behavior: - -- Reads `RunGraphNode` rows for the run. -- `--children` restricts to direct children of the current node. -- `--level N` returns tasks at absolute graph level `N`. -- `--under TASK_SLUG_OR_NODE_ID --level N` returns tasks at relative level `N` inside that subtree. -- `--status STATUS` filters by node status. -- Table columns: `node_id_short`, `task_slug`, `status`, `level`, `parent_node_id_short`, `worker`. -- JSON output returns full IDs and the same fields. -- Does not read evaluations or context events. - -### `workflow inspect task-tree` - -Shows a recursive task subtree, grouped by level. This should be the default orientation command for agents because it behaves like a mixture of `cd` and `ls`, but with a self-describing name. - -Examples: - -```bash -workflow("inspect task-tree") -workflow("inspect task-tree --from current --depth 2") -workflow("inspect task-tree --from root --depth 3") -workflow("inspect task-tree --from d_root --level 2") -workflow("inspect task-tree --from d_root --status pending") -``` - -Behavior: - -- `--from current` starts at the current node; this is the default. -- `--from root` starts at the root node for the current run. -- `--from TASK_SLUG_OR_NODE_ID` starts at that node. -- `--depth N` prints descendants up to `N` levels below the start node. -- `--level N` prints only descendants at relative level `N` below the start node. -- Uses `parent_node_id` containment, not dependency edges. - -Example output: - -```text -TREE from=parent depth=2 - -level +0 - parent running node=9af1c2aa worker=researchrubrics-smoke-worker - -level +1 - d_root completed node=8a31c4f2 worker=researchrubrics-smoke-leaf - l_1 completed node=0c72d1aa worker=researchrubrics-smoke-leaf - s_a completed node=f71e5a10 worker=researchrubrics-smoke-leaf - -level +2 - d_left completed node=19de72aa parent=d_root worker=researchrubrics-smoke-leaf - d_right completed node=34b7a901 parent=d_root worker=researchrubrics-smoke-leaf - l_2 pending node=aa10e821 parent=l_1 worker=researchrubrics-smoke-leaf -``` - -For "show every task on this level recursive down", the agent uses: - -```bash -workflow("inspect task-tree --from current --level 2") -``` - -### `workflow inspect task-details` - -Shows one task node plus latest execution summary. - -Examples: - -```bash -workflow("inspect task-details") -workflow("inspect task-details --task-slug d_left --include-output") -workflow("inspect task-details --task-slug d_left --format json") -``` - -Behavior: - -- With no selector, shows the current node. -- With `--task-slug`, resolves exactly one node in the current run. -- If `--task-slug` matches multiple nodes in a run, exits non-zero and prints the matching node IDs. -- Includes latest `RunTaskExecution` status, attempt number, timestamps, and resource count. -- `--include-output` includes a truncated `final_assistant_message`, default max 1200 chars, configurable with `--max-chars`. -- Does not expose evaluation feedback. - -### `workflow inspect task-dependencies` - -Shows graph dependencies for a task. - -Examples: - -```bash -workflow("inspect task-dependencies") -workflow("inspect task-dependencies --task-slug d_join --direction upstream") -workflow("inspect task-dependencies --task-slug d_left --direction downstream --format json") -``` - -Behavior: - -- Reads `RunGraphEdge`. -- `--direction upstream` lists incoming edges: source task -> current task. -- `--direction downstream` lists outgoing edges: current task -> target task. -- `--direction both` lists both. -- Table columns: `direction`, `edge_status`, `source_slug`, `source_status`, `target_slug`, `target_status`, `edge_id_short`. - -### `workflow inspect task-blockers` - -Explains why a task is not ready, not completed, or cannot proceed. - -Examples: - -```bash -workflow("inspect task-blockers") -workflow("inspect task-blockers --task-slug d_join") -workflow("inspect task-blockers --task-slug l_3 --format json") -``` - -Behavior: - -- Defaults to current node. -- Reports unsatisfied upstream dependencies, failed upstream dependencies, blocked/cancelled status, running children, and missing input resources if inferable. -- Does not mutate anything. -- Includes suggested next inspection commands. - -Example output: - -```text -Task blockers: d_join - -Readiness: - blocked: yes - reason: waiting_for_upstream - -Upstream dependencies: - d_left completed edge=satisfied resources=2 - d_right running edge=pending resources=0 - -Next useful commands: - workflow("inspect task-details --task-slug d_right") - workflow("inspect resource-list --scope input") -``` - -### `workflow inspect next-actions` - -Gives the agent a concise recovery/orientation summary for the current visible run scope. - -Examples: - -```bash -workflow("inspect next-actions") -workflow("inspect next-actions --include-completed") -``` - -Behavior: - -- Lists ready, pending, blocked, failed, and cancelled tasks visible to the current agent. -- Suggests concrete commands to inspect or manage the highest-priority items. -- For leaf agents, suggestions include only `inspect ...` commands. -- For manager-capable agents, suggestions may include `manage ... --dry-run` commands. - -Example output: - -```text -Next actions - -Blocked: - l_3 blocked because l_2 failed - inspect: workflow("inspect task-details --task-slug l_2 --include-output") - manager dry-run: workflow("manage restart-task --task-slug l_2 --dry-run") - -Ready: - d_join has all upstream inputs satisfied - inspect: workflow("inspect task-workspace --task-slug d_join") - -Input resources: - current task has 4 input resources - inspect: workflow("inspect resource-list --scope input") -``` - -### `workflow inspect resource-list` - -Lists visible resources. - -Examples: - -```bash -workflow("inspect resource-list --scope input") -workflow("inspect resource-list --scope upstream") -workflow("inspect resource-list --scope children") -workflow("inspect resource-list --scope descendants --max-depth 3") -workflow("inspect resource-list --scope visible --limit 20") -workflow("inspect resource-list --scope own --kind report") -workflow("inspect resource-list --scope input --format json") -``` - -Scopes: - -- `input`: resources produced by latest successful executions of immediate upstream dependency nodes. This is the default for task-scoped agents. -- `upstream`: same as `input` for v1; kept as a readable alias. -- `own`: resources produced by the current node's latest execution. -- `children`: resources produced by direct child task executions. -- `descendants`: resources produced by descendants up to `--max-depth`, default 3. -- `visible`: same-run resources the current profile is allowed to see, including resources from divergent DAG branches. This is needed for opportunistic collaboration, but it must still exclude eval/private/system resources and be capped by `--limit`. -- `run`: do not expose in v1. Even current-run-wide raw resources are broader than a benchmark-task agent needs by default. - -Table columns: `resource_id_short`, `kind`, `name`, `task_slug`, `size_bytes`, `mime_type`, `created_at`, `content_hash_short`. - -### `workflow inspect resource-content` - -Reads resource content from the blob path stored in `RunResource.file_path`. - -Examples: - -```bash -workflow("inspect resource-content --resource-id $RESOURCE_ID") -workflow("inspect resource-content --resource-id $RESOURCE_ID --max-bytes 20000") -``` - -Behavior: - -- Verifies the resource belongs to the injected current `run_id`. -- Verifies resource ID is visible under the active scope policy before reading bytes. -- Reads bytes from `RunResource.file_path`. -- If bytes decode as UTF-8, prints text. -- If not UTF-8, prints a short binary summary and exits 0 unless `--raw` is supplied later. -- Caps output with `--max-bytes`, default 64 KiB. - -### `workflow inspect resource-location` - -Returns metadata and local blob path for a resource without dumping content. - -Example: - -```bash -workflow("inspect resource-location --resource-id $RESOURCE_ID") -``` - -Behavior: - -- Useful for humans and tests. -- Agent wrapper may hide `file_path` if path leakage becomes a concern; v1 can expose it because it is a host blob path and already part of resource metadata. - -### `workflow inspect task-workspace` - -Shows the full task workspace snapshot: task, execution, upstream dependencies, downstream dependents, input resources, own resources, children, and suggested next commands. - -Examples: - -```bash -workflow("inspect task-workspace") -workflow("inspect task-workspace --task-slug d_join") -workflow("inspect task-workspace --task-slug d_join --include-output") -``` - -Behavior: - -- Defaults to current node. -- Uses only current-run data. -- No evaluation rows. -- Output should be compact and sectioned so the agent can orient in one call. - -## Manage Commands - -`manage` means "state-changing", not necessarily "manager-only". - -- Graph lifecycle commands require a manager-capable wrapper profile: create/restart/abandon/update task graph state. -- `manage materialize-resource` is a current-task workspace/import operation and should be available to ordinary task agents when the source resource is visible under policy. - -### `workflow manage create-task` - -Adds one dynamic subtask under the current node by wrapping `TaskManagementService.add_subtask`. - -Examples: - -```bash -workflow("manage create-task --task-slug summarize_left --worker researchrubrics-smoke-leaf --description 'Summarize left branch'") -workflow("manage create-task --task-slug join --worker researchrubrics-smoke-leaf --description 'Join summaries' --depends-on summarize_left --depends-on summarize_right") -workflow("manage create-task --task-slug join --worker researchrubrics-smoke-leaf --description 'Join summaries' --depends-on summarize_left --dry-run") -``` - -Behavior: - -- Parent is always the current node unless a privileged manager profile later allows `--parent`. -- Creates a `RunGraphNode` with `parent_node_id=current_node_id`. -- Creates dependency edges for `--depends-on` slugs/node IDs. -- Returns created node ID, slug, and status. -- With `--dry-run`, validates parent, dependency references, and worker slug, then prints the proposed node and edges without writing. - -### `workflow manage create-task-plan` - -Adds multiple dynamic subtasks in one transaction by wrapping `TaskManagementService.plan_subtasks`. - -Examples: - -```bash -workflow("manage create-task-plan --json '[{\"task_slug\":\"a\",\"description\":\"Do A\",\"assigned_worker_slug\":\"researchrubrics-smoke-leaf\"},{\"task_slug\":\"b\",\"description\":\"Do B\",\"assigned_worker_slug\":\"researchrubrics-smoke-leaf\",\"depends_on\":[\"a\"]}]'") -workflow("manage create-task-plan --json '[{\"task_slug\":\"a\",\"description\":\"Do A\",\"assigned_worker_slug\":\"researchrubrics-smoke-leaf\"}]' --dry-run") -``` - -Behavior: - -- This is the safest way to add a local DAG. -- Rejects cycles/duplicates through existing service validation. -- Returns created nodes and root slugs. -- With `--dry-run`, runs the same validation and returns the normalized plan without inserting nodes. - -### `workflow manage create-dependency` - -Adds a dependency edge between two existing sibling/visible tasks. - -Examples: - -```bash -workflow("manage create-dependency --source summarize_left --target join") -workflow("manage create-dependency --source summarize_left --target join --dry-run") -``` - -Behavior: - -- Uses `WorkflowGraphRepository.add_edge`. -- Source and target must resolve inside the current run and be visible to the current agent profile. -- Fails if the edge would create a cycle. -- New edge status starts as `pending`. -- With `--dry-run`, resolves source/target and checks cycle risk without adding the edge. - -### `workflow manage restart-task` - -Resets a terminal task back to pending by wrapping `TaskManagementService.restart_task`. - -Examples: - -```bash -workflow("manage restart-task --task-slug l_2") -workflow("manage restart-task --node-id aa10e821-...") -workflow("manage restart-task --task-slug l_2 --dry-run") -``` - -Behavior: - -- Only terminal tasks can be reset. -- Existing service handles downstream invalidation/reset behavior. -- Returns old status and invalidated downstream node IDs. -- With `--dry-run`, reports whether the task is restartable and which downstream nodes would be invalidated. - -### `workflow manage abandon-task` - -Abandons/cancels a task by wrapping `TaskManagementService.cancel_task`. - -Examples: - -```bash -workflow("manage abandon-task --task-slug l_3") -workflow("manage abandon-task --node-id 98db73a2-...") -workflow("manage abandon-task --task-slug l_3 --dry-run") -``` - -Behavior: - -- Transitions the target to cancelled when allowed. -- Emits the existing cancellation event through the service path. -- Returns old status and cascade count. -- With `--dry-run`, reports whether cancellation is allowed and the descendant cascade count without writing or emitting events. - -### `workflow manage update-task-description` - -Updates a non-running task description by wrapping `TaskManagementService.refine_task`. - -Examples: - -```bash -workflow("manage update-task-description --task-slug l_3 --description 'Retry with the corrected input file'") -workflow("manage update-task-description --task-slug l_3 --description 'Retry with the corrected input file' --dry-run") -``` - -Behavior: - -- Fails on running tasks. -- Returns old and new description. -- With `--dry-run`, validates mutability and prints the old/new description without writing. - -### `workflow manage materialize-resource` - -Copies one immutable, visible `RunResource` from local Postgres/blob storage into the current agent's E2B workspace and records the copy as a new current-task-owned resource. - -This is a fork operation, not a mutation of the source task's artifact. - -Examples: - -```bash -workflow("manage materialize-resource --resource-id $RESOURCE_ID") -workflow("manage materialize-resource --resource-id $RESOURCE_ID --destination imported/task-a/paper.pdf") -workflow("manage materialize-resource --resource-id $RESOURCE_ID --destination imported/task-a/paper.pdf --dry-run") -``` - -Behavior: - -- Resolves `--resource-id` to a source `RunResource` in the current run. -- Requires the source resource to be visible to the current agent profile. -- Rejects evaluation/private/system resources and cross-run resource IDs. -- Reads bytes from the source resource's content-addressed `file_path`. -- Writes bytes into the current E2B sandbox under `/workspace/<destination>`. -- If `--destination` is omitted, uses a collision-safe default like `/workspace/imported/<producer-task-slug>/<resource name (copy).ext>`. -- Rejects absolute destinations, `..`, symlink escapes, and paths outside `/workspace`. -- Creates a new `RunResource` row owned by the current task execution: - - new `id` - - `task_execution_id=current_execution_id` - - `kind=import` - - `name="<source name> (copy)<ext>"` unless an explicit destination name is provided - - same `file_path` and `content_hash` as the source resource - - `copied_from_resource_id=<source resource id>` - - metadata containing source task/node identifiers, source name/hash, sandbox destination, and materialized timestamp -- Updates `/workspace/.ergon/resource_imports.json` in the sandbox with the source resource ID, copied resource ID, content hash, and destination path so future tools/debuggers can reconstruct the local workspace import history. -- Does not add a control edge to the DAG. -- With `--dry-run`, validates source visibility, destination normalization, collision behavior, sandbox target, and the copied resource name without writing to Postgres or E2B. - -Ordering: - -- Validate source visibility and destination before side effects. -- Write bytes to the sandbox destination first. -- Append the copied `RunResource` row only after the sandbox write succeeds. -- Update the import manifest after the copied row exists so it can include the new copied resource ID. -- If the manifest update fails after the file copy/resource row succeeds, return a structured warning rather than pretending the source was not materialized. - -V1 lineage boundary: - -- The materialized copy row has strong lineage via `copied_from_resource_id`. -- Arbitrary later edits are B-owned outputs. They must never mutate A's row. -- Do not pretend to infer every arbitrary transformation automatically in v1. If the edited output is later published, it is enough that the run has the materialization row, the import manifest, and the tool-call context events to explain how B got the source bytes. A richer many-to-many `run_resource_lineage` table can come later if synthesis from multiple copied resources becomes central. - -Output: - -```text -materialized resource - source: res_a paper.pdf sha256:abc... - copy: res_b paper (copy).pdf kind=import - copied_from_resource_id: res_a - sandbox_path: /workspace/imported/task-a/paper (copy).pdf - note: source resource was not modified -``` - -JSON output should include at least: - -```json -{ - "source_resource_id": "res_a", - "copied_resource_id": "res_b", - "copied_from_resource_id": "res_a", - "source_content_hash": "abc", - "copied_content_hash": "abc", - "sandbox_path": "/workspace/imported/task-a/paper (copy).pdf", - "source_mutated": false -} -``` - -## Deferred Commands - -Do not build these in v1: - -- `ergon workflow messages send`: useful workflow primitive, but not needed for the first read/navigation surface. -- Arbitrary `manage remove-dependency`: higher risk than `create-dependency` because it can unexpectedly unblock or strand work; add after mutation auditing is clearer. - -## Agent Invocation Model - -The agent runs locally in the worker process. The environment runs in E2B. Therefore this command surface should read local Postgres directly, then let existing sandbox tools handle E2B environment actions. Do not put this command inside the E2B sandbox. - -The model must not pass, override, or discover `--run-id`; the wrapper injects the current `WorkerContext.run_id`. The model also must not inspect other runs in the same experiment, cohort, benchmark, or different experiments. - -### Single Tool Path - -Create `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` with: - -```python -def make_workflow_cli_tool( - *, - context: WorkerContext, - allowed_scopes: frozenset[str] = frozenset({"input", "own", "upstream", "children", "descendants", "visible"}), - manager_capable: bool = False, -) -> Callable[..., Awaitable[WorkflowCliToolResponse]]: - ... -``` - -The pydantic-ai tool signature should be: - -```python -async def workflow(command: str, timeout_s: int = 10) -> WorkflowCliToolResponse: - """Run a scoped workflow navigation command, for example: - `inspect task-tree`, `inspect task-details --task-slug d_left`, - `inspect task-dependencies --task-slug d_join`, - `inspect resource-list --scope input`, `inspect resource-content --resource-id ...`, - `manage materialize-resource --resource-id ...`, - or manager-profile commands like `manage restart-task --task-slug l_2`. - `run_id`, current `node_id`, current `execution_id`, and sandbox context - are injected automatically. - """ -``` - -Important behavior: - -- The model passes only the subcommand string, e.g. `inspect resource-list --scope input`. -- The wrapper prepends `workflow` and injects `--run-id <context.run_id>`, `--node-id <context.node_id>`, `--execution-id <context.execution_id>`, `--sandbox-id <context.sandbox_id>`, and `--sandbox-task-key <context.task_id or context.node_id>` when the command supports them. -- The wrapper rejects any user-supplied `--run-id`, `--node-id`, `--execution-id`, `--sandbox-id`, `--sandbox-task-key`, `--definition-id`, `--experiment-id`, or `--cohort-id` argument in v1. Current-run/current-task scope is an invariant, not a prompt instruction. -- The wrapper calls the CLI handler in-process via `ergon_cli.main._main(argv)` with stdout/stderr captured, not via subprocess. -- The wrapper rejects disallowed tokens in v1: shell metacharacters are irrelevant for in-process argv parsing, but still reject newlines and commands starting with `run`, `eval`, `doctor`, etc. -- The wrapper does not allow `inspect resource-list --scope run`. - -### Bash Path - -For local agents, the "bash" path should mean local command execution, not the current E2B sandbox bash tool. The existing `bash_sandbox_tool.py` runs inside E2B and cannot see local Postgres. - -Two options: - -- Preferred v1: use the single `workflow(command=...)` tool. -- Later: add a local bash-like tool that only whitelists `ergon workflow ...`, but do not mix it with E2B bash. - -System prompt guidance for ReAct agents should say: - -```text -Use the `workflow` tool to inspect the workflow run graph and resources. -Start with `inspect task-tree`, `inspect task-workspace`, `inspect next-actions`, or `inspect resource-list --scope input`. -Do not assume transitive dependencies are inputs; use `inspect task-dependencies` if unsure. -Use `manage materialize-resource --resource-id ... --dry-run` before importing a useful resource into your workspace. -Before mutating the task graph, run the same graph-lifecycle `manage ...` command with `--dry-run`. -``` - -## Permissions Model - -V1 policy is simple and explicit: - -- Agent wrapper: automatically scoped to one `run_id` and current `node_id`; the model cannot choose a run. -- Direct CLI invocation with explicit IDs exists only for developer tests/debugging, not as a model-facing capability. -- Agent wrapper default read scopes: `input`, `own`, `upstream`, `children`, `descendants`, `visible`. -- Agent wrapper same-run collaboration scope: `visible`, capped by `--limit`, for resources from divergent branches that are useful context but not control dependencies. -- Agent wrapper denied scope: `run`. -- Agent wrapper denied cross-run scope: other runs from the same experiment, cohort, benchmark, or other experiments. -- Graph mutation commands require a manager-capable profile. Manager agents may get `manage create-task`, `manage create-task-plan`, `manage create-dependency`, `manage restart-task`, `manage abandon-task`, and `manage update-task-description`. -- Resource materialization is allowed for leaf and manager agents, but only for visible same-run resources and only into the current task sandbox/workspace. -- Manager-capable agents should be instructed to use `--dry-run` before every non-trivial mutation. V1 enforces support for dry-run but does not require a confirm token. -- No eval tables are queried by any command. -- No raw SQL is accepted. - -Later profiles can be layered on top: - -- `leaf`: input, own, upstream summaries, and capped same-run visible resource discovery/materialization. -- `manager`: children, descendants, task lifecycle mutations. -- `evaluator`: target task outputs only. -- `cohort_observer` / `meta_analyst`: cross-run summaries without raw resource content and without evaluation leakage, only if explicitly assigned. - -Future safety layer: - -- `--confirm-token` for destructive mutations. A dry run would produce a short token that must be echoed back on the real command. Do not build this in v1 unless mutation behavior proves too risky in tests. - -## Implementation File Plan - -This is the master file/folder plan for implementation. - -### Added Files - -```text -ergon_core/ - migrations/ - versions/ - <revision>_add_copied_from_resource_id.py - ergon_core/ - core/ - runtime/ - services/ - workflow_navigation_dto.py - workflow_navigation_service.py - workflow_resource_materialization_service.py - -ergon_cli/ - ergon_cli/ - commands/ - workflow.py - -ergon_builtins/ - ergon_builtins/ - tools/ - workflow_cli_tool.py - workers/ - research_rubrics/ - workflow_cli_react_worker.py - -tests/ - unit/ - runtime/ - test_workflow_navigation_service.py - test_workflow_input_resource_semantics.py - test_workflow_resource_materialization_service.py - cli/ - test_workflow_cli.py - state/ - test_workflow_cli_tool.py -``` - -### Edited Files - -```text -ergon_cli/ - ergon_cli/ - main.py - -ergon_core/ - ergon_core/ - core/ - persistence/ - telemetry/ - models.py - queries.py - -ergon_builtins/ - ergon_builtins/ - registry_data.py - -tests/ - unit/ - state/ - test_research_rubrics_workers.py - real_llm/ - benchmarks/ - test_researchrubrics.py -``` - -`ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` is intentionally not edited for the POC path. - -### Deleted Files - -```text -(none) -``` - -### Optional Edited Files If We Include Automatic Sandbox Input Threading Now - -```text -ergon_core/ - ergon_core/ - api/ - worker_context.py - core/ - runtime/ - inngest/ - execute_task.py - services/ - orchestration_dto.py - task_execution_service.py -``` - -These optional edits are only for automatic input materialization at sandbox creation time. They are separate from `workflow manage materialize-resource`, which is an explicit on-demand copy/fork operation initiated by the current agent. - -## Task 0: Add Resource Copy Lineage Schema - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/models.py` -- Modify: `ergon_core/ergon_core/core/persistence/queries.py` -- Create: `ergon_core/migrations/versions/<revision>_add_copied_from_resource_id.py` - -- [ ] **Step 1: Add failing persistence tests** - -Add focused tests that create a source `RunResource`, append a copied resource, and assert: - -- copied row has a new `id` -- copied row keeps the same `content_hash` and `file_path` -- copied row has `kind="import"` -- copied row has `copied_from_resource_id=<source id>` -- source row is unchanged - -- [ ] **Step 2: Update `RunResourceKind` and `RunResource`** - -File path: `ergon_core/ergon_core/core/persistence/telemetry/models.py` - -```python -class RunResourceKind(StrEnum): - OUTPUT = "output" - REPORT = "report" - ARTIFACT = "artifact" - SEARCH_CACHE = "search_cache" - NOTE = "note" - IMPORT = "import" - """Copied snapshot materialized from another RunResource into a task workspace.""" - -class RunResource(SQLModel, table=True): - # ... - copied_from_resource_id: UUID | None = Field( - default=None, - foreign_key="run_resources.id", - index=True, - ) -``` - -- [ ] **Step 3: Update resource append helper** - -File path: `ergon_core/ergon_core/core/persistence/queries.py` - -Extend `ResourcesQueries.append(...)` with: - -```python -copied_from_resource_id: UUID | None = None -``` - -and pass it to `RunResource(...)`. - -- [ ] **Step 4: Add migration** - -File path: `ergon_core/migrations/versions/<revision>_add_copied_from_resource_id.py` - -Migration behavior: - -- `upgrade()` adds nullable `copied_from_resource_id` UUID column to `run_resources` -- creates a self-referential foreign key to `run_resources.id` -- creates an index on `run_resources.copied_from_resource_id` -- `downgrade()` drops index, foreign key, and column - -Run: - -```bash -pytest tests/unit/runtime/test_workflow_resource_materialization_service.py -v -``` - -Expected: schema/helper tests pass once the materialization service is implemented. - -## Task 1: Add Workflow Navigation DTOs and Service - -**Files:** - -- Create: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_dto.py` -- Create: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_service.py` -- Create: `ergon_core/ergon_core/core/runtime/services/workflow_resource_materialization_service.py` -- Test: `tests/unit/runtime/test_workflow_navigation_service.py` -- Test: `tests/unit/runtime/test_workflow_resource_materialization_service.py` - -- [ ] **Step 1: Write failing service tests** - -Add tests for: - -- `list_tasks(run_id)` returns all run nodes ordered by level/task_slug. -- `list_tasks(run_id, parent_node_id=...)` returns direct children only. -- `list_deps(..., direction="upstream")` returns incoming edges with source/target summaries. -- `list_resources(..., scope="input")` returns resources from latest completed executions of immediate upstream nodes only. -- `list_resources(..., scope="visible")` can include same-run resources from divergent branches while still excluding cross-run/eval/private resources. -- `get_resource_content` rejects a resource from a different run. -- `get_task_blockers(...)` reports pending upstream dependencies and failed upstream dependencies. -- `get_next_actions(...)` suggests inspect-only commands for leaf profiles and dry-run manage commands for manager profiles. -- `materialize_resource(...)` creates a new current-task-owned `kind=import` resource row with a new ID, copied name, same hash/blob path, and `copied_from_resource_id`. -- `materialize_resource(...)` writes or updates `/workspace/.ergon/resource_imports.json` in the sandbox with source/copy/destination metadata. -- `materialize_resource(..., dry_run=True)` validates source/destination without writing to Postgres or E2B. -- `materialize_resource(...)` rejects cross-run resources, invisible resources, absolute destinations, `..`, and destination collisions unless overwrite/versioning behavior is explicit. - -Use a tiny graph fixture: - -```text -a -> b -> c -x -``` - -Give `a`, `b`, and `x` one completed execution each; give `a` and `b` one resource each. Assert `c` input resources include only `b` resource, not `a`. - -- [ ] **Step 2: Implement DTOs** - -Define frozen Pydantic models: - -File path: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_dto.py` - -```python -class WorkflowTaskRef(BaseModel): - model_config = {"frozen": True} - node_id: UUID - task_slug: str - status: str - level: int - parent_node_id: UUID | None = None - assigned_worker_slug: str | None = None - -class WorkflowExecutionRef(BaseModel): - model_config = {"frozen": True} - execution_id: UUID - status: str - attempt_number: int - final_assistant_message: str | None = None - -class WorkflowResourceRef(BaseModel): - model_config = {"frozen": True} - resource_id: UUID - run_id: UUID - task_execution_id: UUID | None - node_id: UUID | None - task_slug: str | None - kind: str - name: str - mime_type: str - size_bytes: int - file_path: str - content_hash: str | None = None - copied_from_resource_id: UUID | None = None - created_at: datetime - -class WorkflowDependencyRef(BaseModel): - model_config = {"frozen": True} - edge_id: UUID - edge_status: str - source: WorkflowTaskRef - target: WorkflowTaskRef - -class WorkflowBlockerRef(BaseModel): - model_config = {"frozen": True} - task: WorkflowTaskRef - reason: str - details: list[str] = Field(default_factory=list) - suggested_commands: list[str] = Field(default_factory=list) - -class WorkflowNextActionRef(BaseModel): - model_config = {"frozen": True} - priority: str - task: WorkflowTaskRef | None = None - summary: str - suggested_commands: list[str] = Field(default_factory=list) - -class WorkflowMaterializedResourceRef(BaseModel): - model_config = {"frozen": True} - source_resource_id: UUID - copied_resource_id: UUID | None - copied_from_resource_id: UUID - source_name: str - copied_name: str - source_content_hash: str | None - copied_content_hash: str | None - sandbox_path: str - dry_run: bool = False - source_mutated: bool = False -``` - -- [ ] **Step 3: Implement read service** - -Implement `WorkflowNavigationService` methods: - -File path: `ergon_core/ergon_core/core/runtime/services/workflow_navigation_service.py` - -```python -class WorkflowNavigationService: - def list_tasks(self, session: Session, *, run_id: UUID, parent_node_id: UUID | None = None) -> list[WorkflowTaskRef]: ... - def get_task(self, session: Session, *, run_id: UUID, node_id: UUID | None, task_slug: str | None) -> WorkflowTaskRef: ... - def get_latest_execution(self, session: Session, *, node_id: UUID) -> RunTaskExecution | None: ... - def list_dependencies(self, session: Session, *, run_id: UUID, node_id: UUID, direction: Literal["upstream", "downstream", "both"]) -> list[WorkflowDependencyRef]: ... - def list_resources(self, session: Session, *, run_id: UUID, node_id: UUID | None, scope: Literal["input", "upstream", "own", "children", "descendants", "visible", "run"], kind: str | None = None, max_depth: int = 3, limit: int = 50) -> list[WorkflowResourceRef]: ... - def read_resource_bytes(self, session: Session, *, run_id: UUID, resource_id: UUID, max_bytes: int) -> bytes: ... - def get_task_blockers(self, session: Session, *, run_id: UUID, node_id: UUID) -> list[WorkflowBlockerRef]: ... - def get_next_actions(self, session: Session, *, run_id: UUID, node_id: UUID, manager_capable: bool) -> list[WorkflowNextActionRef]: ... -``` - -For `input` / `upstream`, use incoming edges to the current node, get each source node's latest completed execution, then collect `RunResource` rows for those execution IDs. - -Implement `WorkflowResourceMaterializationService` separately from read-only navigation: - -File path: `ergon_core/ergon_core/core/runtime/services/workflow_resource_materialization_service.py` - -```python -class WorkflowResourceMaterializationService: - async def materialize_resource( - self, - session: Session, - *, - run_id: UUID, - current_node_id: UUID, - current_execution_id: UUID, - sandbox_task_key: UUID, - benchmark_type: str, - resource_id: UUID, - destination: str | None, - dry_run: bool, - ) -> WorkflowMaterializedResourceRef: ... -``` - -The service should use the benchmark's sandbox manager class and existing `BaseSandboxManager.upload_file(...)` to write into the live E2B sandbox. It should not create a new low-level E2B upload primitive. - -- [ ] **Step 4: Run service tests** - -Run: - -```bash -pytest tests/unit/runtime/test_workflow_navigation_service.py tests/unit/runtime/test_workflow_resource_materialization_service.py -v -``` - -Expected: all tests pass. - -## Task 2: Add `ergon workflow` CLI Commands - -**Files:** - -- Create: `ergon_cli/ergon_cli/commands/workflow.py` -- Modify: `ergon_cli/ergon_cli/main.py` -- Test: `tests/unit/cli/test_workflow_cli.py` - -- [ ] **Step 1: Write failing parser/handler tests** - -Add tests that call `handle_workflow(args)` directly and at least one parser integration test using `build_parser().parse_args(...)`. - -Test cases: - -- `ergon workflow inspect task-list --run-id <id>` renders slugs. -- `ergon workflow inspect task-details --run-id <id> --task-slug b --include-output` renders latest output excerpt. -- `ergon workflow inspect task-dependencies --run-id <id> --task-slug c --direction upstream` renders `b -> c`. -- `ergon workflow inspect resource-list --run-id <id> --node-id <c> --scope input` includes `b` resource only. -- `ergon workflow inspect resource-content --run-id <id> --resource-id <rid>` prints file content. -- `ergon workflow inspect task-blockers --run-id <id> --node-id <blocked>` explains why a task is blocked. -- `ergon workflow inspect next-actions --run-id <id> --node-id <manager>` prints suggested commands. -- `ergon workflow manage materialize-resource --run-id <id> --node-id <b> --execution-id <exec_b> --sandbox-task-key <task-or-node> --resource-id <rid> --destination imported/a/report.pdf --dry-run` reports the copy/fork without DB or E2B writes. -- `ergon workflow manage materialize-resource ...` creates a new `kind=import` resource row with `copied_from_resource_id` and writes the file to `/workspace/imported/a/report (copy).pdf`. -- Every `manage ... --dry-run` command validates and reports the planned mutation without DB writes. -- invalid UUID returns exit code 1. -- duplicate `--task-slug` returns exit code 1 and helpful message. - -- [ ] **Step 2: Register parser** - -In `ergon_cli/ergon_cli/main.py`: - -- Import `handle_workflow`. -- Add top-level `workflow`. -- Add nested subcommands: - - `inspect task-list` - - `inspect task-tree` - - `inspect task-details` - - `inspect task-dependencies` - - `inspect task-blockers` - - `inspect next-actions` - - `inspect resource-list` - - `inspect resource-content` - - `inspect resource-location` - - `inspect task-workspace` - - `manage create-task` - - `manage create-task-plan` - - `manage create-dependency` - - `manage restart-task` - - `manage abandon-task` - - `manage update-task-description` - - `manage materialize-resource` -- Add branch: - -File path: `ergon_cli/ergon_cli/main.py` - -```python -elif args.command == "workflow": - return handle_workflow(args) -``` - -- [ ] **Step 3: Implement handler** - -In `commands/workflow.py`, implement: - -File path: `ergon_cli/ergon_cli/commands/workflow.py` - -```python -def handle_workflow(args: Namespace) -> int: - ensure_db() - ... -``` - -Use `WorkflowNavigationService`, `WorkflowResourceMaterializationService`, `get_session()`, `render_table`, and `json.dumps(..., default=str)` for `--format json`. - -Keep output agent-friendly: - -- Plain table by default. -- Compact JSON when requested. -- No rich formatting. -- Stable field names. - -- [ ] **Step 4: Run CLI tests** - -Run: - -```bash -pytest tests/unit/cli/test_workflow_cli.py -v -``` - -Expected: all tests pass. - -## Task 3: Add Agent-Facing Workflow CLI Tool - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Test: `tests/unit/state/test_workflow_cli_tool.py` - -- [ ] **Step 1: Write failing wrapper tests** - -Test cases: - -- `workflow("inspect task-list")` injects `--run-id` and returns stdout/exit code. -- `workflow("inspect resource-list --scope input")` injects `--node-id`. -- `workflow("inspect resource-list --scope visible --limit 20")` is allowed for same-run collaboration discovery. -- `workflow("inspect resource-list --scope run")` is denied by default. -- `workflow("manage materialize-resource --resource-id <rid> --destination imported/a/report.pdf --dry-run")` is allowed for leaf wrappers and injects current execution/sandbox context. -- `workflow("manage restart-task --task-slug l_2 --dry-run")` is allowed only for manager-capable wrappers. -- `workflow("manage restart-task --task-slug l_2")` is denied for leaf wrappers. -- User-supplied `--execution-id`, `--sandbox-id`, or `--sandbox-task-key` is rejected. -- `workflow("../bad")` or multiline input is rejected. -- Non-zero CLI exit returns a structured failure, not an exception. - -- [ ] **Step 2: Implement response DTOs** - -Use: - -File path: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` - -```python -class WorkflowCliToolSuccess(BaseModel): - kind: Literal["success"] = "success" - stdout: str - stderr: str - exit_code: int - -class WorkflowCliToolFailure(BaseModel): - kind: Literal["failure"] = "failure" - error: str - stdout: str = "" - stderr: str = "" - exit_code: int = 1 -``` - -- [ ] **Step 3: Implement `make_workflow_cli_tool`** - -Use `shlex.split(command)` to build argv. Capture stdout/stderr with `contextlib.redirect_stdout` and `redirect_stderr`. Call `await ergon_cli.main._main(argv)`. - -Do not spawn a subprocess. - -- [ ] **Step 4: Run wrapper tests** - -Run: - -```bash -pytest tests/unit/state/test_workflow_cli_tool.py -v -``` - -Expected: all tests pass. - -## Task 4: Add a ResearchRubrics Workflow CLI ReAct Worker - -**Files:** - -- New worker: `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` -- Registry edit: `ergon_builtins/ergon_builtins/registry_data.py` -- Test: update `tests/unit/state/test_research_rubrics_workers.py` or add a focused worker wiring test beside it. - -- [ ] **Step 1: Create the proof-of-concept worker** - -Create `ResearchRubricsWorkflowCliReActWorker` as the first consumer of the workflow CLI tool. - -Do not alter `ResearchRubricsResearcherWorker` for this proof of concept. Keeping the new behavior behind a separate worker slug makes it easy to compare workflow-CLI behavior against the existing research-rubrics agent. - -- [ ] **Step 2: Add the workflow tool to the new worker** - -File path: `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` - -```python -class ResearchRubricsWorkflowCliReActWorker(ResearchRubricsResearcherWorker): - """Research-rubrics ReAct worker with the workflow CLI tool enabled.""" -``` - -Inside `execute()`, mirror the existing research-rubrics runtime tool composition and add: - -```python -workflow_tool = make_workflow_cli_tool(context=context) -self.tools = [*rr_tools, *graph_tools, Tool(function=workflow_tool, takes_ctx=False)] -``` - -If pydantic-ai expects the callable directly rather than prewrapped `Tool`, mirror the existing toolkit pattern. - -- [ ] **Step 3: Update system prompt** - -Add a short instruction: - -File path: `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` - -```text -Use the `workflow` tool to inspect task topology and resources. Start with -`inspect task-tree`, `inspect task-workspace`, or -`inspect resource-list --scope input`. If a visible resource from another task -is useful, run `manage materialize-resource --resource-id ... --dry-run` -before importing it into your workspace. Use `--dry-run` before any graph -lifecycle `manage ...` mutation command. -``` - -- [ ] **Step 4: Register the new worker slug** - -File path: `ergon_builtins/ergon_builtins/registry_data.py` - -```python -from ergon_builtins.workers.research_rubrics.workflow_cli_react_worker import ( - ResearchRubricsWorkflowCliReActWorker, -) - -WORKERS: dict[str, Callable[..., Worker]] = { - "researchrubrics-researcher": ResearchRubricsResearcherWorker, - "researchrubrics-workflow-cli-react": ResearchRubricsWorkflowCliReActWorker, -} -``` - -- [ ] **Step 5: Run worker wiring test** - -Run: - -```bash -pytest tests/unit/state/test_research_rubrics_workers.py -v -``` - -Expected: - -- `researchrubrics-workflow-cli-react` is registered. -- `ResearchRubricsWorkflowCliReActWorker` exposes the `workflow` tool. -- The prompt recommends `inspect task-workspace`, `inspect resource-list --scope input`, `manage materialize-resource --dry-run`, and `--dry-run` before graph lifecycle `manage` commands. -- Existing `ResearchRubricsResearcherWorker` assertions remain unchanged and do not require `workflow`. - -## Task 5: Add Contract Tests Around Input Resource Semantics - -**Files:** - -- Test: `tests/unit/runtime/test_workflow_input_resource_semantics.py` -- Optional later implementation: thread computed IDs into `PreparedTaskExecution.input_resource_ids`. - -- [ ] **Step 1: Test the default policy** - -Build a graph: - -```text -d_root -> d_left -d_root -> d_right -d_left -> d_join -d_right -> d_join -l_1 -> l_2 -> l_3 -``` - -Assert: - -- `d_join` input resources are exactly latest resources from `d_left` and `d_right`. -- `l_3` input resources are exactly latest resources from `l_2`, not `l_1`. -- roots and singletons have empty input resources. - -- [ ] **Step 2: Decide whether to wire sandbox inputs now** - -If included in this implementation, add: - -- `input_resource_ids` to `PreparedTaskExecution` -- computation in `TaskExecutionService.prepare` -- pass through `_setup_sandbox` in `execute_task.py` -- optional `WorkerContext.input_resource_ids` - -If not included, keep the CLI/tool behavior independent and leave sandbox auto-materialization as the next implementation plan. - -## Task 6: Add Real-LLM Acceptance Rollout - -**Files:** - -- Modify: `tests/real_llm/benchmarks/test_researchrubrics.py` - -- [ ] **Step 1: Parameterize the rollout worker and sample limit** - -Keep the existing defaults for the current research-rubrics rollout, but allow the final acceptance run to choose the workflow-CLI worker and five samples: - -```python -model = os.environ.get("ERGON_REAL_LLM_MODEL", _DEFAULT_MODEL) -benchmark = os.environ.get("ERGON_REAL_LLM_BENCHMARK", "researchrubrics") -worker = os.environ.get("ERGON_REAL_LLM_WORKER", "researchrubrics-researcher") -evaluator = os.environ.get("ERGON_REAL_LLM_EVALUATOR", "research-rubric") -limit = os.environ.get("ERGON_REAL_LLM_LIMIT", "1") -``` - -Then pass `limit` into the CLI invocation instead of hard-coding `"1"`. - -- [ ] **Step 2: Preserve rollout artifact capture** - -Keep the existing artifact behavior: CLI stdout/stderr, table dumps, dashboard screenshots, manifest, and `report.md`. The final review should use these artifacts to inspect the agent's actual behavior rather than asserting a brittle exact tool-call sequence. - -- [ ] **Step 3: Run the final acceptance rollout** - -Run: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=5 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s -``` - -Expected: - -- The real-LLM test reaches a terminal run status: `completed`, `failed`, or `cancelled`. -- The rollout artifacts are written under `tests/real_llm/.rollouts/<timestamp>-<run_id>/`. -- The manifest records `worker=researchrubrics-workflow-cli-react` and `limit=5`. -- The generated `report.md` plus dumped persistence rows provide enough evidence to inspect whether the agent invoked `workflow(...)`, which workflow commands it chose, whether it materialized any copied resources, and whether those commands helped it orient around task topology/resources. - -This is the final acceptance criterion for the feature. Unit and focused integration tests remain the normal correctness gate; this rollout is the observational gate for whether the workflow CLI is useful to a real ResearchRubrics agent. - -## Verification - -Run focused tests: - -```bash -pytest tests/unit/runtime/test_workflow_navigation_service.py tests/unit/runtime/test_workflow_resource_materialization_service.py tests/unit/cli/test_workflow_cli.py tests/unit/state/test_workflow_cli_tool.py -v -``` - -Run affected worker tests: - -```bash -pytest tests/unit/state/test_research_rubrics_workers.py -v -``` - -If sandbox input threading is included, also run: - -```bash -pytest tests/unit/runtime/test_workflow_input_resource_semantics.py -v -``` - -Final acceptance rollout: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=5 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s -``` diff --git a/docs/superpowers/plans/2026-04-26-run-workspace-interaction-corrections.md b/docs/superpowers/plans/2026-04-26-run-workspace-interaction-corrections.md deleted file mode 100644 index 4ee687c70..000000000 --- a/docs/superpowers/plans/2026-04-26-run-workspace-interaction-corrections.md +++ /dev/null @@ -1,932 +0,0 @@ -# Run Workspace Interaction Corrections Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Bring the run workspace back into alignment with the design brief: tabbed right-hand task workspace, always-live graph with bottom timeline-driven snapshots, cleaner concurrent activity visualization, and no dead controls. - -**Architecture:** Keep `RunWorkspacePage` as the orchestration point, but replace the `live | timeline` mode split with a single `snapshotSequence: number | null`. A null snapshot means live; a selected sequence replays graph mutations with `replayToSequence()`. `TaskWorkspace` becomes a tabbed inspector, and `buildRunActivities()` stops mixing every event type into the concurrent activity stack. - -**Tech Stack:** Next.js App Router, React client components, React Flow, node:test via `tsx --test`, Playwright for browser verification. - ---- - -## File Structure - -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` - - Owns run-level live state, snapshot selection, mutation loading, graph replay, header controls, and rerun button state. -- Modify: `ergon-dashboard/src/components/workspace/TaskWorkspace.tsx` - - Converts the right drawer from stacked sections to tabs: `Overview`, `Actions`, `Communication`, `Outputs`, `State transitions`, `Evaluation`. -- Modify: `ergon-dashboard/src/features/activity/buildRunActivities.ts` - - Narrows the concurrent activity data model to execution/concurrency bars plus graph/key-event markers. -- Modify: `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` - - Removes play/pause/speed controls and uses click-only marker navigation. -- Modify: `ergon-dashboard/src/features/activity/components/ActivityBar.tsx` - - Keeps bar rendering, but supports clearer marker-vs-span styling. -- Create: `ergon-dashboard/src/features/activity/snapshotSequence.ts` - - Pure helper for resolving a clicked activity to the nearest replay mutation sequence. -- Create: `ergon-dashboard/src/features/activity/snapshotSequence.test.ts` - - Unit tests for direct sequence and timestamp-to-nearest-mutation behavior. -- Modify: `ergon-dashboard/src/features/activity/buildRunActivities.test.ts` - - Regression tests for “do not flood the stack with context/sandbox command detail”. -- Create/modify: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - - Browser-level checks for drawer tabs, timeline click rollback, no live/timeline toggle, no playback controls, no active dead rerun. - ---- - -## Current State Summary - -The backend and existing frontend data structures already support graph replay through: - -- `GET /api/runs/[runId]/mutations` -- `parseGraphMutationDtoArray()` -- `replayToSequence(mutations, currentSequence, emptyState, snapshotCache)` - -The current UI does not consistently use that support because: - -- `RunWorkspacePage` only fetches mutations after entering `timelineMode === "timeline"`. -- Activity clicks only rewind when `activity.sequence !== null`. -- Most visible activity bars have `sequence: null` because they represent execution/sandbox/context spans, not graph mutations. -- `buildRunActivities()` mixes execution spans, sandbox spans, sandbox commands, context events, event markers, and graph mutations into one stacked view. -- The right drawer is still a stacked section list, not tabs. -- The rerun button is visually active but has no `onClick`. - ---- - -## Task 1: Snapshot State Model In `RunWorkspacePage` - -**Files:** -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` -- Test: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - -- [ ] **Step 1: Write failing E2E test for removed mode controls** - -Create or extend `tests/e2e/run-workspace-interactions.spec.ts`: - -```ts -import { test, expect } from "@playwright/test"; - -const BASE = process.env.BASE_URL ?? "http://localhost:3001"; -const COHORT_ID = "a39ee959-376d-490c-8705-22f0c3e32d1e"; -const RUN_ID = "4028c6d2-d9db-4c5a-be21-d9223d46b4ca"; - -test("run workspace is always live and has no manual live/timeline or playback controls", async ({ page }) => { - await page.goto(`${BASE}/cohorts/${COHORT_ID}/runs/${RUN_ID}`); - await expect(page.locator('[data-testid="graph-canvas"]')).toBeVisible(); - - await expect(page.locator('[data-testid="mode-live"]')).toHaveCount(0); - await expect(page.locator('[data-testid="mode-timeline"]')).toHaveCount(0); - await expect(page.locator('[data-testid="activity-play-toggle"]')).toHaveCount(0); - await expect(page.locator('[data-testid^="speed-"]')).toHaveCount(0); -}); -``` - -- [ ] **Step 2: Run E2E test and verify it fails** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "always live" -``` - -Expected: FAIL because `mode-live`, `mode-timeline`, play/pause, and speed controls are currently rendered. - -- [ ] **Step 3: Replace mode state with snapshot state** - -In `RunWorkspacePage.tsx`, replace: - -```ts -const [timelineMode, setTimelineMode] = useState<"live" | "timeline">("live"); -const [currentSequence, setCurrentSequence] = useState(0); -const [isPlaying, setIsPlaying] = useState(false); -const [playbackSpeed, setPlaybackSpeed] = useState(1); -``` - -with: - -```ts -const [snapshotSequence, setSnapshotSequence] = useState<number | null>(null); -const currentSequence = snapshotSequence ?? 0; -``` - -- [ ] **Step 4: Always fetch mutations once per run** - -Replace the current mutation `useEffect()` guard: - -```ts -if (timelineMode !== "timeline") return; -``` - -with unconditional loading: - -```ts -useEffect(() => { - let cancelled = false; - fetch(`/api/runs/${runId}/mutations`) - .then((res) => res.json()) - .then((data) => { - if (cancelled) return; - const parsed = parseGraphMutationDtoArray(data); - setMutations(parsed); - snapshotCache.current.clear(); - const requestedSequence = requestedSequenceRef.current; - requestedSequenceRef.current = null; - if (requestedSequence !== null) { - setSnapshotSequence(nearestMutationAtOrBefore(parsed, requestedSequence)?.sequence ?? null); - } - }) - .catch(() => { - if (!cancelled) setMutations([]); - }); - return () => { - cancelled = true; - }; -}, [runId]); -``` - -- [ ] **Step 5: Replay only when `snapshotSequence !== null`** - -Change `displayState` to: - -```ts -const displayState = useMemo(() => { - if (snapshotSequence === null || mutations.length === 0) return runState; - if (!runState) return runState; - const emptyState: WorkflowRunState = { - ...runState, - tasks: new Map(), - totalTasks: 0, - totalLeafTasks: 0, - completedTasks: 0, - runningTasks: 0, - failedTasks: 0, - }; - return replayToSequence(mutations, snapshotSequence, emptyState, snapshotCache.current); -}, [runState, mutations, snapshotSequence]); -``` - -- [ ] **Step 6: Remove header mode toggle** - -Delete the whole `role="tablist"` block that renders `mode-live` and `mode-timeline`. - -Change header chip text from: - -```tsx -{timelineMode === "live" ? "live" : `seq ${currentSequence}`} · {formatSeconds(...)} -``` - -to: - -```tsx -{snapshotSequence === null ? "live" : `snapshot · seq ${snapshotSequence}`} · {formatSeconds(...)} -``` - -- [ ] **Step 7: Update keyboard behavior** - -Remove the `t` shortcut entirely. Change `Esc` behavior to: - -```ts -if (e.key === "Escape") { - if (selectedTaskId) { setSelectedTaskId(null); return; } - if (snapshotSequence !== null) { setSnapshotSequence(null); return; } - if (statusFilter) { setStatusFilter(null); return; } - return; -} -``` - -Change arrow stepping to use `snapshotSequence !== null`. - -- [ ] **Step 8: Run E2E test and verify it passes** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "always live" -``` - -Expected: PASS. - ---- - -## Task 2: Activity Click Resolves To Snapshot Sequence - -**Files:** -- Create: `ergon-dashboard/src/features/activity/snapshotSequence.ts` -- Create: `ergon-dashboard/src/features/activity/snapshotSequence.test.ts` -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` - -- [ ] **Step 1: Write failing unit tests** - -Create `src/features/activity/snapshotSequence.test.ts`: - -```ts -import assert from "node:assert/strict"; -import test from "node:test"; - -import type { GraphMutationDto } from "@/features/graph/contracts/graphMutations"; -import type { RunActivity } from "./types"; -import { resolveActivitySnapshotSequence } from "./snapshotSequence"; - -function mutation(sequence: number, createdAt: string): GraphMutationDto { - return { - id: `m-${sequence}`, - run_id: "run-1", - sequence, - mutation_type: "node.status_changed", - target_type: "node", - target_id: "task-1", - old_value: null, - new_value: { status: "running" }, - actor: "worker", - reason: null, - created_at: createdAt, - } as GraphMutationDto; -} - -function activity(overrides: Partial<RunActivity>): RunActivity { - return { - id: "a-1", - kind: "execution", - label: "activity", - taskId: "task-1", - sequence: null, - startAt: "2026-04-26T10:00:05.000Z", - endAt: "2026-04-26T10:00:08.000Z", - isInstant: false, - actor: "worker", - sourceKind: "execution.span", - metadata: {}, - ...overrides, - }; -} - -test("uses explicit activity sequence when present", () => { - assert.equal( - resolveActivitySnapshotSequence(activity({ sequence: 67 }), [ - mutation(1, "2026-04-26T10:00:00.000Z"), - ]), - 67, - ); -}); - -test("uses nearest mutation at or before activity start time when sequence is absent", () => { - assert.equal( - resolveActivitySnapshotSequence(activity({ startAt: "2026-04-26T10:00:05.000Z" }), [ - mutation(10, "2026-04-26T10:00:00.000Z"), - mutation(20, "2026-04-26T10:00:04.000Z"), - mutation(30, "2026-04-26T10:00:06.000Z"), - ]), - 20, - ); -}); - -test("returns null when no mutation can represent the activity time", () => { - assert.equal( - resolveActivitySnapshotSequence(activity({ startAt: "2026-04-26T09:59:00.000Z" }), [ - mutation(10, "2026-04-26T10:00:00.000Z"), - ]), - null, - ); -}); -``` - -- [ ] **Step 2: Run unit test and verify it fails** - -Run: - -```bash -cd ergon-dashboard -npx tsx --test src/features/activity/snapshotSequence.test.ts -``` - -Expected: FAIL because `snapshotSequence.ts` does not exist. - -- [ ] **Step 3: Implement helper** - -Create `src/features/activity/snapshotSequence.ts`: - -```ts -import type { GraphMutationDto } from "@/features/graph/contracts/graphMutations"; -import type { RunActivity } from "./types"; - -export function resolveActivitySnapshotSequence( - activity: RunActivity, - mutations: GraphMutationDto[], -): number | null { - if (activity.sequence !== null) return activity.sequence; - - const activityMs = Date.parse(activity.startAt); - if (!Number.isFinite(activityMs)) return null; - - let selected: GraphMutationDto | null = null; - for (const mutation of mutations) { - const mutationMs = Date.parse(mutation.created_at); - if (!Number.isFinite(mutationMs)) continue; - if (mutationMs > activityMs) break; - selected = mutation; - } - return selected?.sequence ?? null; -} -``` - -- [ ] **Step 4: Use helper in `handleActivityClick`** - -In `RunWorkspacePage.tsx`, import: - -```ts -import { resolveActivitySnapshotSequence } from "@/features/activity/snapshotSequence"; -``` - -Replace: - -```ts -if (activity.sequence !== null) { - requestedSequenceRef.current = activity.sequence; - if (timelineMode !== "timeline") setTimelineMode("timeline"); - handleSequenceChange(activity.sequence); -} -``` - -with: - -```ts -const snapshot = resolveActivitySnapshotSequence(activity, mutations); -if (snapshot !== null) { - setSnapshotSequence(snapshot); -} -``` - -- [ ] **Step 5: Run tests** - -Run: - -```bash -cd ergon-dashboard -npx tsx --test src/features/activity/snapshotSequence.test.ts -npm run build -``` - -Expected: unit test PASS and build PASS. - ---- - -## Task 3: Simplify Concurrent Activity Stack Data - -**Files:** -- Modify: `ergon-dashboard/src/features/activity/buildRunActivities.ts` -- Modify: `ergon-dashboard/src/features/activity/buildRunActivities.test.ts` -- Modify: `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` -- Modify: `ergon-dashboard/src/features/activity/components/ActivityBar.tsx` - -- [ ] **Step 1: Write failing test for reduced activity stack noise** - -Add to `src/features/activity/buildRunActivities.test.ts`: - -```ts -test("buildRunActivities keeps concurrent stack focused on execution spans and graph markers", () => { - const runState = makeRunStateWithOneExecutionAndSandboxCommand(); - const activities = buildRunActivities({ - runState, - events: [ - { - id: "message-1", - kind: "thread.message", - at: "2026-04-26T10:00:01.000Z", - taskId: "task-1", - actor: "worker", - preview: "verbose message", - sequence: null, - }, - ] as any, - mutations: [ - { - id: "mutation-1", - run_id: runState.id, - sequence: 1, - mutation_type: "node.status_changed", - target_type: "node", - target_id: "task-1", - old_value: null, - new_value: { status: "running" }, - actor: "worker", - reason: null, - created_at: "2026-04-26T10:00:00.000Z", - } as any, - ], - currentSequence: null, - }); - - assert.equal(activities.some((a) => a.sourceKind === "execution.span"), true); - assert.equal(activities.some((a) => a.sourceKind === "graph.mutation"), true); - assert.equal(activities.some((a) => a.sourceKind === "sandbox.command"), false); - assert.equal(activities.some((a) => a.sourceKind === "thread.message"), false); -}); -``` - -Define `makeRunStateWithOneExecutionAndSandboxCommand()` in the test file using the existing `WorkflowRunState` shape from nearby tests. It must include: - -```ts -tasks: new Map([["task-1", { id: "task-1", name: "task", status: TaskStatus.COMPLETED, parentId: null, childIds: [], dependsOnIds: [], isLeaf: true, level: 0, assignedWorkerId: "w1", assignedWorkerName: "worker", startedAt: "2026-04-26T10:00:00.000Z", completedAt: "2026-04-26T10:00:10.000Z" }]]) -executionsByTask: new Map([["task-1", [{ id: "exec-1", taskId: "task-1", attemptNumber: 1, status: TaskStatus.COMPLETED, agentId: "w1", agentName: "worker", startedAt: "2026-04-26T10:00:00.000Z", completedAt: "2026-04-26T10:00:10.000Z", finalAssistantMessage: null, outputResourceIds: [], errorMessage: null, score: null, evaluationDetails: {} }]]]) -sandboxesByTask: new Map([["task-1", { taskId: "task-1", sandboxId: "sandbox-1", status: "closed", template: "default", createdAt: "2026-04-26T10:00:00.000Z", closedAt: "2026-04-26T10:00:10.000Z", closeReason: null, commands: [{ command: "pytest", stdout: "", stderr: "", exitCode: 0, durationMs: 1000, timestamp: "2026-04-26T10:00:01.000Z" }] }]]) -``` - -- [ ] **Step 2: Run test and verify it fails** - -Run: - -```bash -cd ergon-dashboard -npx tsx --test src/features/activity/buildRunActivities.test.ts -``` - -Expected: FAIL because sandbox command and message activities are currently included. - -- [ ] **Step 3: Narrow `buildRunActivities()` output** - -Change: - -```ts -return [ - ...executionActivities(input.runState, selectedTime), - ...sandboxActivities(input.runState, selectedTime), - ...contextActivities(input.runState), - ...eventMarkerActivities(input.events), - ...graphMutationActivities(input.mutations), -].sort(compareActivity); -``` - -to: - -```ts -return [ - ...executionActivities(input.runState, selectedTime), - ...graphMutationActivities(input.mutations), -].sort(compareActivity); -``` - -Do not delete helper functions yet unless `npm run build` reports unused exports/imports. This keeps the diff small and allows future detail views to reuse them if needed. - -- [ ] **Step 4: Update activity copy** - -In `ActivityStackTimeline.tsx`, change: - -```tsx -<div className="font-semibold text-[var(--ink)]">Concurrent activity</div> -Bars stack only when they overlap. -``` - -to: - -```tsx -<div className="font-semibold text-[var(--ink)]">Concurrent execution</div> -Bars are task attempts; dots are graph snapshots. -``` - -Change footer hints: - -```tsx -<span>Bar = task execution</span> -<span>Dot = graph mutation snapshot</span> -<span>Click any item = inspect at that time</span> -``` - -- [ ] **Step 5: Verify visual density** - -Run: - -```bash -cd ergon-dashboard -npx tsx --test src/features/activity/buildRunActivities.test.ts -npm run build -``` - -Expected: tests PASS, build PASS. Browser should show fewer rows and fewer visual elements in the bottom stack. - ---- - -## Task 4: Remove Playback Controls From Activity Stack - -**Files:** -- Modify: `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` -- Test: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - -- [ ] **Step 1: Extend failing E2E test** - -Extend the Task 1 E2E test to assert: - -```ts -await expect(page.locator('[data-testid="activity-step-back"]')).toHaveCount(0); -await expect(page.locator('[data-testid="activity-step-forward"]')).toHaveCount(0); -await expect(page.locator('[data-testid="activity-play-toggle"]')).toHaveCount(0); -``` - -- [ ] **Step 2: Remove props from `ActivityStackTimelineProps`** - -Delete: - -```ts -isPlaying: boolean; -speed: number; -onTogglePlay: () => void; -onSpeedChange: (speed: number) => void; -``` - -Also delete: - -```ts -const SPEED_OPTIONS = [0.5, 1, 2, 4] as const; -const MIN_DELAY_MS = 50; -const MAX_DELAY_MS = 2000; -const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null); -const currentSequenceRef = useRef(currentSequence); -stepForward -stepBack -useEffect that schedules playback -``` - -- [ ] **Step 3: Delete playback UI** - -Remove the entire `isTimeline && (...)` button group containing `activity-step-back`, `activity-play-toggle`, `activity-step-forward`, and `speed-*`. - -- [ ] **Step 4: Update caller** - -In `RunWorkspacePage.tsx`, change: - -```tsx -<ActivityStackTimeline - activities={activities} - mutations={mutations} - currentSequence={currentSequence} - onSequenceChange={handleSequenceChange} - selectedTaskId={selectedTaskId} - selectedActivityId={selectedActivityId} - isPlaying={isPlaying} - onTogglePlay={() => setIsPlaying((p) => !p)} - speed={playbackSpeed} - onSpeedChange={setPlaybackSpeed} - onActivityClick={handleActivityClick} -/> -``` - -to: - -```tsx -<ActivityStackTimeline - activities={activities} - mutations={mutations} - currentSequence={currentSequence} - onSequenceChange={handleSequenceChange} - selectedTaskId={selectedTaskId} - selectedActivityId={selectedActivityId} - onActivityClick={handleActivityClick} -/> -``` - -- [ ] **Step 5: Run tests** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "always live" -npm run build -``` - -Expected: PASS. - ---- - -## Task 5: Tabbed Right-Hand Workspace Drawer - -**Files:** -- Modify: `ergon-dashboard/src/components/workspace/TaskWorkspace.tsx` -- Test: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - -- [ ] **Step 1: Write failing E2E test for drawer tabs and criteria visibility** - -Add: - -```ts -test("task workspace uses tabs and exposes evaluation criteria tab", async ({ page }) => { - await page.goto(`${BASE}/cohorts/${COHORT_ID}/runs/${RUN_ID}`); - await expect(page.locator('[data-testid="graph-canvas"]')).toBeVisible(); - await page.locator('[data-testid^="graph-node-"]').first().click(); - - await expect(page.locator('[data-testid="workspace-tab-overview"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-tab-actions"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-tab-communication"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-tab-outputs"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-tab-transitions"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-tab-evaluation"]')).toBeVisible(); - - await expect(page.locator('[data-testid="workspace-overview"]')).toBeVisible(); - await expect(page.locator('[data-testid="workspace-actions"]')).toHaveCount(0); - - await page.locator('[data-testid="workspace-tab-evaluation"]').click(); - await expect(page.locator('[data-testid="workspace-evaluation"]')).toBeVisible(); -}); -``` - -- [ ] **Step 2: Run E2E test and verify it fails** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "workspace uses tabs" -``` - -Expected: FAIL because drawer uses stacked sections. - -- [ ] **Step 3: Add tab state and tab metadata** - -In `TaskWorkspace.tsx`, import: - -```ts -import { useMemo, useState } from "react"; -``` - -Add: - -```ts -type WorkspaceTab = "overview" | "actions" | "communication" | "outputs" | "transitions" | "evaluation"; - -const WORKSPACE_TABS: { id: WorkspaceTab; label: string }[] = [ - { id: "overview", label: "Overview" }, - { id: "actions", label: "Actions" }, - { id: "communication", label: "Communication" }, - { id: "outputs", label: "Outputs" }, - { id: "transitions", label: "State transitions" }, - { id: "evaluation", label: "Evaluation" }, -]; -``` - -Inside component: - -```ts -const [activeTab, setActiveTab] = useState<WorkspaceTab>("overview"); -``` - -- [ ] **Step 4: Render tab strip below header** - -Insert after header metadata: - -```tsx -<nav className="flex shrink-0 border-b border-[var(--line)] bg-[var(--card)] px-3" aria-label="Task workspace sections"> - {WORKSPACE_TABS.map((tab) => { - const active = activeTab === tab.id; - return ( - <button - key={tab.id} - type="button" - onClick={() => setActiveTab(tab.id)} - className={`border-b-2 px-3 py-2 text-xs font-medium transition-colors ${ - active - ? "border-[var(--ink)] text-[var(--ink)]" - : "border-transparent text-[var(--muted)] hover:text-[var(--ink)]" - }`} - data-testid={`workspace-tab-${tab.id}`} - > - {tab.label} - {tab.id === "evaluation" && evaluation ? ( - <span className="ml-1 rounded-full bg-[var(--paper-2)] px-1.5 py-0.5 font-mono text-[10px]"> - {evaluation.criterionResults.length} - </span> - ) : null} - </button> - ); - })} -</nav> -``` - -- [ ] **Step 5: Replace stacked sections with single active panel** - -Replace the current scroll region contents with: - -```tsx -<div className="min-h-0 overflow-y-auto p-3" data-testid="workspace-scroll-region"> - {activeTab === "overview" && ( - <WorkspaceSection testId="workspace-overview" title="Overview"> - {/* existing dependency overview block */} - </WorkspaceSection> - )} - {activeTab === "actions" && ( - <WorkspaceSection testId="workspace-actions" title="Actions"> - <ContextEventLog events={filteredEvidence.contextEvents} /> - </WorkspaceSection> - )} - {activeTab === "communication" && ( - <WorkspaceSection testId="workspace-communication" title="Communication"> - <CommunicationPanel threads={filteredEvidence.threads} /> - </WorkspaceSection> - )} - {activeTab === "outputs" && ( - <WorkspaceSection testId="workspace-outputs" title="Outputs"> - <ResourcePanel resources={filteredEvidence.resources} runId={runState?.id ?? null} /> - </WorkspaceSection> - )} - {activeTab === "transitions" && ( - <WorkspaceSection testId="workspace-transitions" title="State transitions"> - <TaskTransitionLog task={task} onJumpToSequence={onJumpToSequence} /> - </WorkspaceSection> - )} - {activeTab === "evaluation" && ( - <WorkspaceSection testId="workspace-evaluation" title="Evaluation criteria"> - <EvaluationPanel evaluation={filteredEvidence.evaluation} /> - </WorkspaceSection> - )} -</div> -``` - -Move the existing overview dependency JSX into a local `overviewPanel` constant to avoid duplicating it. - -- [ ] **Step 6: Ensure evaluation panel shows criteria absence clearly** - -In `EvaluationPanel.tsx`, change the empty state text to: - -```tsx -<p>No evaluation criteria recorded yet</p> -<p className="text-sm">This task has no criterionResults in the persisted evaluation payload.</p> -``` - -If `evaluation` exists but `criterionResults.length === 0`, render: - -```tsx -<div data-testid="evaluation-criteria-empty" className="rounded-xl border border-dashed border-[var(--line)] p-4 text-sm text-[var(--muted)]"> - No criteria were recorded for this evaluation payload. -</div> -``` - -- [ ] **Step 7: Run tests** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "workspace uses tabs" -npm run build -``` - -Expected: PASS. - ---- - -## Task 6: Rerun Button Behavior - -**Files:** -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` -- Test: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - -- [ ] **Step 1: Write failing E2E test that rerun is not a dead active button** - -Add: - -```ts -test("rerun control is explicit about unavailable backend action", async ({ page }) => { - await page.goto(`${BASE}/cohorts/${COHORT_ID}/runs/${RUN_ID}`); - const rerun = page.locator('[data-testid="rerun-button"]'); - await expect(rerun).toBeVisible(); - await expect(rerun).toBeDisabled(); - await expect(rerun).toHaveAttribute("title", /not wired/i); -}); -``` - -- [ ] **Step 2: Run E2E and verify it fails** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "rerun control" -``` - -Expected: FAIL because current button has no `data-testid`, is enabled, and has no title explaining state. - -- [ ] **Step 3: Make rerun visibly disabled** - -Replace current rerun button: - -```tsx -<button - type="button" - className="rounded-[7px] border border-[var(--line)] bg-[var(--card)] px-3 py-1 text-xs font-medium text-[var(--ink)]" -> - Re-run -</button> -``` - -with: - -```tsx -<button - type="button" - disabled - title="Re-run is not wired yet: no dashboard API endpoint exists for cloning or dispatching a run." - className="cursor-not-allowed rounded-[7px] border border-[var(--line)] bg-[var(--paper)] px-3 py-1 text-xs font-medium text-[var(--muted)] opacity-70" - data-testid="rerun-button" -> - Re-run unavailable -</button> -``` - -- [ ] **Step 4: Run test** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "rerun control" -npm run build -``` - -Expected: PASS. - ---- - -## Task 7: End-To-End Snapshot Rollback Verification - -**Files:** -- Modify: `ergon-dashboard/tests/e2e/run-workspace-interactions.spec.ts` - -- [ ] **Step 1: Write E2E test that clicking bottom activity changes graph snapshot label** - -Add: - -```ts -test("clicking bottom activity marker locks graph to snapshot sequence", async ({ page }) => { - await page.goto(`${BASE}/cohorts/${COHORT_ID}/runs/${RUN_ID}`); - await expect(page.locator('[data-testid="graph-canvas"]')).toBeVisible(); - - const firstActivity = page.locator('[data-testid^="activity-bar-"]').first(); - await expect(firstActivity).toBeVisible(); - await firstActivity.click(); - - await expect(page.locator('[data-testid="snapshot-lock-label"]')).toBeVisible(); - await expect(page.locator('[data-testid="snapshot-pin"]')).toBeVisible(); - await expect(page.locator('[data-testid="run-header"]')).toContainText(/snapshot · seq|seq \d+/); -}); -``` - -If `ActivityBar` does not currently expose `data-testid^="activity-bar-"`, add it in `ActivityBar.tsx`: - -```tsx -data-testid={`activity-bar-${item.activity.id}`} -``` - -- [ ] **Step 2: Run E2E and verify failure before fixes** - -Run: - -```bash -cd ergon-dashboard -BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts -g "clicking bottom activity" -``` - -Expected: FAIL before Tasks 1-4; PASS after Tasks 1-4. - -- [ ] **Step 3: Manual visual check** - -Open: - -```text -http://localhost:3001/cohorts/a39ee959-376d-490c-8705-22f0c3e32d1e/runs/4028c6d2-d9db-4c5a-be21-d9223d46b4ca -``` - -Expected: - -- No Live/Timeline segmented control. -- No play/pause/speed controls. -- Bottom area is less dense. -- Clicking a bar/marker changes header chip to `snapshot · seq N`. -- `Esc` returns header chip to `live`. -- Graph node statuses/visibility match the selected sequence. - ---- - -## Verification Checklist - -- [ ] `npx tsx --test src/features/activity/snapshotSequence.test.ts` passes. -- [ ] `npx tsx --test src/features/activity/buildRunActivities.test.ts` passes. -- [ ] `npx tsx --test src/lib/timeFormat.test.ts` still passes. -- [ ] `npx tsx --test src/hooks/useRunState.socketHydration.test.ts` still passes. -- [ ] `npm run build` passes. -- [ ] `BASE_URL=http://localhost:3001 pnpm exec playwright test tests/e2e/run-workspace-interactions.spec.ts` passes. -- [ ] Browser smoke check shows no Next.js overlay, graph nodes render, drawer tabs render, activity stack is navigable. - ---- - -## Spec Coverage Review - -- Right drawer tabs: Task 5. -- Evaluation criteria visibility: Task 5, Step 6. -- Remove explicit live/timeline mode: Task 1. -- Bottom timeline drives graph replay: Tasks 1, 2, 7. -- Data structure support: Task 2 confirms mutation replay supports timestamp-to-sequence lookup. -- Concurrent activity clutter: Task 3. -- Remove pause/play/speed controls: Task 4. -- Dead rerun button: Task 6. - -Known follow-up outside this plan: implement a real rerun backend action if product wants rerun to work. This plan only makes the current dead button honest and non-interactive because no confirmed dashboard rerun API exists in the current frontend. diff --git a/docs/superpowers/plans/2026-04-26-trace-spans-ux-refinements.md b/docs/superpowers/plans/2026-04-26-trace-spans-ux-refinements.md deleted file mode 100644 index 2849f92e1..000000000 --- a/docs/superpowers/plans/2026-04-26-trace-spans-ux-refinements.md +++ /dev/null @@ -1,116 +0,0 @@ -# Trace Spans UX Refinements - -Date: 2026-04-26 - -## Context - -The immutable Trace Spans direction is right: the bottom trace should act as a fixed map of the completed run, while clicking or arrowing between events moves the cursor, replays the graph above, and updates the workspace detail. The next set of issues is about legibility: dense events overlap, hover metadata is hard to read, tooltips crop, and some events appear to be hidden or inaccessible. - -## 1. JSON Metadata Is Hard To Read - -### Problem - -The hover metadata is too raw. It technically exposes useful details, but the user has to parse JSON to answer the basic question: "what is this event?" - -### Proposed Fix - -Use a two-level hover card: - -- Summary header: event kind, label, task, sequence, and timestamp. -- Important fields table: fields such as `mutationType`, `targetId`, `actor`, `reason`, `status`, `toolName`, `exitCode`, or `score`. -- Raw JSON collapsed by default under a `Raw payload` disclosure. - -The raw JSON should remain available for debugging, but it should not be the first thing the user has to read. - -### Acceptance Criteria - -- Hovering an event answers "what is this?" without reading raw JSON. -- Raw JSON is still available on demand. -- The same summary fields are reused in the pinned workspace activity detail. - -## 2. Hover Cards Crop Off The Top - -### Problem - -Hover cards can be clipped when the event is near the top of the Trace Spans component. This likely happens because the tooltip is rendered inside an overflow-constrained timeline container. - -### Proposed Fix - -Make the tooltip viewport-aware: - -- Render the hover card as `position: fixed`, or via a small portal attached to `document.body`. -- Compute the hovered marker/bar bounding box. -- Prefer placing the card above the event when there is room. -- Flip below the event when there is not enough space above. -- Clamp left and right positions to the viewport. -- Give the card a `max-height` with internal scroll for larger payloads. - -### Acceptance Criteria - -- Hovering any visible event never clips the tooltip outside the viewport. -- The hover card remains readable for top-row, bottom-row, far-left, and far-right events. -- Keyboard focus should show the same preview behavior as mouse hover. - -## 3. Too Much Overlap - -### Problem - -Point events and duration spans currently compete for the same visual space. Dense regions become hard to read because markers pile up on top of bars or on top of each other. - -### Proposed Fix - -Separate the visual grammar: - -- Span rows show only duration bars: task executions and sandbox lifetimes. -- Point events render on marker rails, not directly as miniature bars inside the same span rows. -- Dense point events are clustered when they fall within a few pixels of each other. -- A cluster renders as a numbered bubble such as `+4`. -- Hovering or clicking a cluster opens a small list of the events inside that time window. -- Add optional kind filters so the user can hide/show `graph`, `context`, `message`, `artifact`, `evaluation`, and `sandbox` markers. - -### Acceptance Criteria - -- Overlapping work remains visible as stable bars. -- Dense point events remain inspectable without becoming a pile of dots. -- Markers do not change span row assignment. -- Clusters expose every hidden event through hover or click. - -## 4. Missing Dots / Possible Bottom Cropping - -### Problem - -When moving between examples, the UI reports multiple steps/events, but the corresponding dots are not always visible on the end swim lanes. This may mean markers are rendered below the visible component, hidden by overflow, or compressed into inaccessible rows. - -### Proposed Fix - -Audit and stabilize the timeline height and scroll behavior: - -- Ensure the timeline content height derives from the full layout, including marker rails and bottom padding. -- Add an assertion that every rendered `layout.item` is inside the scrollable timeline bounds. -- Make vertical overflow explicit: if the trace has more rows than fit, the panel should visibly scroll or offer an expand control. -- Add a trace status line, for example: `17 trace rows · 84 events · 0 hidden`. -- If filters are added, report hidden counts explicitly, for example: `12 hidden by filters`. -- Consider an "Expand trace" control for dense runs so users can inspect the full trace without fighting a 300px dock. - -### Acceptance Criteria - -- If the UI reports events at a point, the user can scroll or expand to see them. -- No event markers silently render outside the component. -- The component distinguishes between events that are hidden by filters, collapsed into clusters, or simply below the current scroll position. - -## Implementation Notes - -This should follow the immutable Trace Spans acceptance criterion: - -- Clicking or arrowing between point events must not change Trace Span bar lengths. -- Clicking or arrowing between point events must not change row assignments. -- Clicking or arrowing should only move the cursor/pin, update selected marker state, replay the top graph, and update the workspace detail. - -Suggested implementation order: - -1. Stabilize immutable trace derivation and layout. -2. Add marker rails and clustering. -3. Add the improved legend and trace status line. -4. Add viewport-aware hover cards. -5. Reuse the same event summary/debug payload in the workspace detail. -6. Add e2e coverage for no clipping, no relayout on marker click, and cluster inspection. diff --git a/docs/superpowers/plans/2026-04-27-frontend-evaluation-visibility.md b/docs/superpowers/plans/2026-04-27-frontend-evaluation-visibility.md deleted file mode 100644 index 76e79b86f..000000000 --- a/docs/superpowers/plans/2026-04-27-frontend-evaluation-visibility.md +++ /dev/null @@ -1,1390 +0,0 @@ -# Frontend Evaluation Visibility Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add the evaluation feature set from the design brief to the dashboard: cohort rubric status pips, graph node rubric cues, skipped/error states, rubric metadata, richer evaluation drawer details, container roll-ups, and an evaluation lens. - -**Implementation note:** The first implementation keeps the original API strategy: additive backend fields, frontend-derived run/container roll-ups, backend-owned cohort summaries, and stable `data-testid` coverage for cohort pips, graph rubric glyphs, the evaluation lens toggle, and criterion status details. - -**Architecture:** Keep the backend read model additive and make the frontend own presentation-specific selectors in a new `features/evaluations` domain. Enrich existing `GET /runs/{run_id}` and `GET /cohorts/{cohort_id}` payloads rather than introducing a new fetch path for the first implementation. Keep E2E assertions anchored to stable `data-testid` attributes and the backend harness DTO. - -**Tech Stack:** FastAPI, Pydantic DTOs, SQLModel persistence, Next.js App Router, React, TypeScript, Zod, React Flow, Playwright, pytest. - ---- - -## RFC - -### Problem - -The backend now produces enough evaluation data to validate task-level correctness, but the dashboard still treats evaluation as a narrow workspace tab. The design brief expects evaluation to be visible across the debugging loop: - -- Cohort rows show per-run rubric status pips and failure/skipped state at a glance. -- Graph nodes show which tasks have attached rubrics without requiring a click. -- Container nodes summarize evaluation status for their descendant tasks. -- The evaluation tab explains score composition, weights, skipped criteria, evaluator errors, input, feedback, and timing. -- Operators can switch the DAG into an evaluation lens that highlights evaluation-bearing tasks and dims unrelated work. - -### Non-Goals - -- Do not change evaluation execution semantics. -- Do not add interactive re-evaluation controls. -- Do not introduce a new standalone evaluation API service. -- Do not persist new relational tables unless the additive summary JSON fields prove insufficient. - -### Source Of Truth - -Use persisted `RunTaskEvaluation` rows and their typed `summary_json` as the source of truth. The frontend should not infer evaluation status from task status alone. It may derive roll-ups from evaluation rows and task parent/child relationships. - -### Nullability And Defaults Policy - -Avoid silent defaults at contract boundaries. If a field is owned by the backend and is required for rendering, make it required in the DTO and populate it explicitly in the builder. Use `None`/`null` only for genuinely absent data such as optional model reasoning, optional feedback, optional evaluation input, or optional error detail. In frontend derived state, represent "there is no evaluation evidence" as `null`, not as an all-zero roll-up object with a `"none"` sentinel. - -### API Strategy - -Use existing endpoints with additive fields: - -- `GET /runs/{run_id}` returns the enriched `RunSnapshotDto`. -- `GET /cohorts/{cohort_id}` returns enriched `CohortRunRowDto` rows with lightweight rubric status summaries. -- `GET /api/test/read/run/{run_id}/state` returns the expanded smoke harness fields used by Playwright. - -No existing response field should be removed or renamed. - -### Evaluation Status Semantics - -Use one canonical status vocabulary everywhere: - -```python -EvalCriterionStatus = Literal["passed", "failed", "errored", "skipped"] -RubricStatusSummaryStatus = Literal["passing", "failing", "errored", "skipped", "mixed", "none"] -``` - -Criterion status rules: - -- `errored`: `error` is non-null. -- `skipped`: criterion was part of the evaluator spec but did not execute because a prior gate failed or the attached task never reached the required lifecycle point. -- `passed`: criterion executed and `passed` is true. -- `failed`: criterion executed and `passed` is false. - -Roll-up status rules: - -- `none`: no evaluation rows or criteria. -- `errored`: at least one errored criterion. -- `failing`: at least one failed criterion and no errors. -- `mixed`: passed plus skipped criteria with no failed or errored criteria. -- `skipped`: all known criteria skipped. -- `passing`: all known criteria passed. - -### Backend Contract Additions - -Do not add parallel DTOs for data the run snapshot already exposes. The codebase already has: - -- `RunEvaluationCriterionDto` -- `RunTaskEvaluationDto` -- `RunSnapshotDto.evaluations_by_task` -- `CohortRunRowDto` - -The implementation should extend those existing DTOs in place. Graph glyphs, task roll-ups, container roll-ups, and run-level detail roll-ups should be derived in frontend selectors from `RunSnapshotDto.evaluations_by_task`. - -The only new backend DTO shape needed for the first implementation is a lightweight cohort-row rubric status summary, because the cohort page should show pips without fetching every run snapshot. The backend should own this summary, including counts and aggregate status. Keep the implementation direct: one compact builder over persisted `EvaluationSummary` rows, not a chain of helper functions or a second generic roll-up subsystem. - -Extend `ergon_core/ergon_core/core/api/schemas.py`: - -```python -from typing import Literal - -EvalCriterionStatus = Literal["passed", "failed", "errored", "skipped"] -``` - -Add fields to the existing `RunEvaluationCriterionDto` class: - -```python -class RunEvaluationCriterionDto(CamelModel): - # existing fields stay unchanged - criterion_name: str - status: EvalCriterionStatus - passed: bool - weight: float - contribution: float - model_reasoning: str | None = None - skipped_reason: str | None = None -``` - -Add fields to the existing `RunTaskEvaluationDto` class: - -```python -class RunTaskEvaluationDto(CamelModel): - # existing fields stay unchanged - evaluator_name: str - aggregation_rule: str -``` - -Add one lightweight DTO in `ergon_core/ergon_core/core/runtime/services/cohort_schemas.py`: - -```python -class CohortRubricStatusSummaryDto(BaseModel): - status: RubricStatusSummaryStatus - total_criteria: int - passed: int - failed: int - errored: int - skipped: int - criterion_statuses: list[str] - evaluator_names: list[str] - - -class CohortRunRowDto(BaseModel): - # existing fields stay unchanged - rubric_status_summary: CohortRubricStatusSummaryDto -``` - -### Frontend Contract Additions - -The generated REST contracts feed `ergon-dashboard/src/lib/contracts/rest.ts`. After regenerating contracts, normalize only fields that are genuinely optional on the backend contract. Do not use frontend defaults to hide missing required fields such as criterion `status`, criterion `weight`, evaluator name, aggregation rule, or cohort `rubric_status_summary`. - -Add frontend-only derived roll-up types in `ergon-dashboard/src/features/evaluations/contracts.ts`; do not mirror them as run-snapshot backend DTOs: - -```ts -export type EvalCriterionStatus = "passed" | "failed" | "errored" | "skipped"; -export type EvalRollupStatus = "passing" | "failing" | "errored" | "skipped" | "mixed"; -export type RubricStatusSummaryStatus = EvalRollupStatus | "none"; - -export interface EvaluationRollup { - status: EvalRollupStatus; - totalCriteria: number; - passed: number; - failed: number; - errored: number; - skipped: number; - normalizedScore: number; - maxScore: number; - evaluatorNames: string[]; - attachedTaskIds: string[]; - criterionStatuses: EvalCriterionStatus[]; -} -``` - -Extend existing normalized REST types in `ergon-dashboard/src/lib/contracts/rest.ts`: - -```ts -export interface RunEvaluationCriterion { - id: string; - stageNum: number; - stageName: string; - criterionNum: number; - criterionType: string; - criterionDescription: string; - criterionName: string; - status: EvalCriterionStatus; - passed: boolean; - weight: number; - contribution: number; - evaluationInput: string | null; - score: number; - maxScore: number; - feedback: string | null; - modelReasoning: string | null; - skippedReason: string | null; - evaluatedActionIds: string[]; - evaluatedResourceIds: string[]; - error: Record<string, unknown> | null; -} -``` - -### Frontend Domain Boundary - -Create a focused evaluation domain: - -```text -ergon-dashboard/src/features/evaluations/ - contracts.ts - status.ts - selectors.ts - selectors.test.ts - components/ - CriterionStatusPip.tsx - RubricStatusStrip.tsx - EvaluationNodeGlyph.tsx - EvaluationRollupBadge.tsx - EvaluationLensToggle.tsx - EvaluationCriterionCard.tsx - EvaluationMetadataSummary.tsx -``` - -Responsibilities: - -- `contracts.ts`: frontend-only types if the generated REST types are too broad for component props. -- `status.ts`: colors, labels, icons, and ordering for evaluation statuses. -- `selectors.ts`: pure roll-up helpers for run, task, container descendants, and cohort rows. -- `components/*`: small visual components with stable `data-testid` attributes. - -### UX Contract - -Use these stable test IDs: - -- `cohort-eval-strip-{run_id}` -- `cohort-eval-pip-{run_id}-{index}` -- `graph-eval-glyph-{task_id}` -- `graph-eval-rollup-{task_id}` -- `graph-eval-lens-toggle` -- `workspace-evaluation-metadata` -- `workspace-evaluation-criterion-{criterion_id}` -- `workspace-evaluation-criterion-status-{criterion_id}` -- `workspace-evaluation-input-{criterion_id}` -- `workspace-evaluation-reasoning-{criterion_id}` - -### Acceptance Criteria - -- Cohort run rows render a rubric status strip for runs with evaluations and an empty state for runs without evaluations. -- Graph task nodes with attached evaluations render a subtle diamond glyph using text or CSS, with an accessible label. -- Expanded graph containers render a roll-up badge computed from descendant task evaluations. -- Evaluation lens dims non-evaluated tasks and highlights tasks with direct or descendant evaluation evidence. -- Evaluation panel shows aggregation rule, weights, score contribution, status, input, feedback, model reasoning, skipped reasons, and error details. -- Existing smoke specs assert happy-path passing pips, sad-path failed/skipped/errored visibility, graph glyphs, and the evaluation drawer. - ---- - -## File Structure - -### Backend Files - -- Modify `ergon_core/ergon_core/core/api/schemas.py`: extend existing evaluation DTO fields only. -- Modify `ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py`: persist criterion `status`, optional `model_reasoning`, and optional `skipped_reason` in `summary_json`. -- Modify `ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py`: build criterion status, contribution, and model reasoning from `CriterionResult.metadata`. -- Modify `ergon_core/ergon_core/core/api/runs.py`: pass enriched criterion fields through existing `evaluations_by_task`. -- Modify `ergon_core/ergon_core/core/runtime/services/run_read_service.py`: keep using existing `evaluations_by_task`; no new run-snapshot roll-up fields. -- Modify `ergon_core/ergon_core/core/runtime/services/cohort_schemas.py`: add `rubric_status_summary` to cohort run rows. -- Modify `ergon_core/ergon_core/core/runtime/services/cohort_service.py`: query run evaluations and attach a backend-owned rubric status summary. -- Modify `ergon_core/ergon_core/core/api/test_harness.py`: expose criterion statuses and a lightweight run rubric status summary to Playwright smoke tests. -- Test `tests/unit/runtime/test_evaluation_summary_contracts.py`: assert enriched summary fields. -- Test `tests/unit/runtime/test_cohort_rubric_status_summary.py`: assert cohort row rubric status summary. - -### Frontend Files - -- Regenerate `ergon-dashboard/src/generated/rest/contracts.ts` after backend schema updates. -- Modify `ergon-dashboard/src/lib/contracts/rest.ts`: normalize additive evaluation fields. -- Modify `ergon-dashboard/src/lib/types.ts`: export enriched evaluation aliases only. -- Modify `ergon-dashboard/src/lib/runState.ts`: deserialize enriched existing evaluations only. -- Create `ergon-dashboard/src/features/evaluations/status.ts`: central status display mapping. -- Create `ergon-dashboard/src/features/evaluations/selectors.ts`: pure derived state helpers. -- Test `ergon-dashboard/src/features/evaluations/selectors.test.ts`: assert direct and container roll-ups. -- Create `ergon-dashboard/src/features/evaluations/components/CriterionStatusPip.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/RubricStatusStrip.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/EvaluationNodeGlyph.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/EvaluationRollupBadge.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/EvaluationLensToggle.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/EvaluationCriterionCard.tsx`. -- Create `ergon-dashboard/src/features/evaluations/components/EvaluationMetadataSummary.tsx`. -- Modify `ergon-dashboard/src/components/cohorts/CohortDetailView.tsx`: render cohort run rubric status strips. -- Modify `ergon-dashboard/src/components/dag/TaskNode.tsx`: pass evaluation roll-up props. -- Modify `ergon-dashboard/src/features/graph/components/LeafNode.tsx`: render glyph and roll-up badge. -- Modify `ergon-dashboard/src/features/graph/components/ContainerNode.tsx`: render container roll-up badge. -- Modify `ergon-dashboard/src/components/dag/DAGCanvas.tsx`: add evaluation lens toggle and graph dimming behavior. -- Modify `ergon-dashboard/src/components/panels/EvaluationPanel.tsx`: render richer metadata and criterion cards. -- Modify `ergon-dashboard/tests/helpers/backendHarnessClient.ts`: expand backend harness DTO. -- Modify `ergon-dashboard/tests/e2e/_shared/smoke.ts`: assert the visible evaluation features. - ---- - -## Implementation Tasks - -### Task 1: Backend Evaluation Read Contract - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py` -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py` -- Test: `tests/unit/runtime/test_evaluation_summary_contracts.py` - -- [ ] **Step 1: Write failing summary contract tests** - -Add tests that prove the persistence DTO carries status, weights, contribution, and optional reasoning: - -```python -def test_build_evaluation_summary_includes_status_weight_and_contribution() -> None: - result = _service_result( - criterion_score=0.5, - criterion_weight=2.0, - passed=False, - metadata={"model_reasoning": "missing supporting artifact"}, - ) - - summary = build_evaluation_summary(result, evaluation_input="task evidence") - - entry = summary.criterion_results[0] - assert entry.status == "failed" - assert entry.weight == 2.0 - assert entry.contribution == 0.5 - assert entry.model_reasoning == "missing supporting artifact" - assert entry.skipped_reason is None - - -def test_dashboard_evaluation_dto_includes_criterion_status_fields() -> None: - summary = EvaluationSummary( - evaluator_name="post-root", - max_score=1.0, - normalized_score=1.0, - stages_evaluated=1, - stages_passed=1, - criterion_results=[ - CriterionResultEntry( - criterion_name="timing", - criterion_type="smoke-post-root-timing-criterion", - criterion_description="post root timing", - status="passed", - score=1.0, - max_score=1.0, - passed=True, - weight=1.0, - contribution=1.0, - ) - ], - ) - - dto = build_dashboard_evaluation_dto( - evaluation_id=UUID("00000000-0000-0000-0000-000000000001"), - run_id=UUID("00000000-0000-0000-0000-000000000002"), - task_id=UUID("00000000-0000-0000-0000-000000000003"), - total_score=1.0, - created_at=datetime(2026, 4, 27, tzinfo=UTC), - summary=summary, - ) - - criterion = dto.criterion_results[0] - assert criterion.status == "passed" - assert criterion.passed is True - assert criterion.weight == 1.0 - assert criterion.contribution == 1.0 - assert dto.evaluator_name == "post-root" - assert dto.aggregation_rule == "weighted_sum" -``` - -- [ ] **Step 2: Run tests and verify failure** - -Run: `pytest tests/unit/runtime/test_evaluation_summary_contracts.py -q` - -Expected: failure mentioning missing fields such as `status`, `contribution`, or `evaluator_name`. - -- [ ] **Step 3: Add typed persistence fields** - -In `evaluation_summary.py`, extend `CriterionResultEntry`: - -```python -class CriterionResultEntry(BaseModel): - """One criterion result as stored in the evaluation summary.""" - - criterion_name: str - criterion_type: str - stage_num: int - stage_name: str - criterion_num: int - status: Literal["passed", "failed", "errored", "skipped"] - score: float - max_score: float - passed: bool - weight: float - contribution: float - criterion_description: str - feedback: str | None = None - model_reasoning: str | None = None - skipped_reason: str | None = None - evaluation_input: str | None = None - evaluated_action_ids: list[str] = Field(default_factory=list) - evaluated_resource_ids: list[str] = Field(default_factory=list) - error: dict | None = None -``` - -- [ ] **Step 4: Add DTO fields** - -In `schemas.py`, update `RunEvaluationCriterionDto` and `RunTaskEvaluationDto` with the RFC contract fields. - -- [ ] **Step 5: Build status and metadata in persistence** - -In `evaluation_persistence_service.py`, add a helper: - -```python -def _criterion_status(*, passed: bool, error: dict | None) -> str: - if error is not None: - return "errored" - return "passed" if passed else "failed" -``` - -Then populate the entry: - -```python -metadata = cr.metadata -model_reasoning = metadata.get("model_reasoning") -entries.append( - CriterionResultEntry( - criterion_name=cr.name, - criterion_type=spec.criterion.type_slug, - criterion_description=spec.criterion.name, - stage_num=spec.stage_idx, - stage_name=spec.stage_name, - criterion_num=spec.criterion_idx, - status=_criterion_status(passed=cr.passed, error=None), - score=cr.score, - max_score=spec.max_score, - passed=cr.passed, - weight=cr.weight, - contribution=cr.score, - feedback=cr.feedback, - model_reasoning=model_reasoning if isinstance(model_reasoning, str) else None, - evaluation_input=evaluation_input, - ) -) -``` - -- [ ] **Step 6: Run tests and verify pass** - -Run: `pytest tests/unit/runtime/test_evaluation_summary_contracts.py -q` - -Expected: all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py ergon_core/ergon_core/core/api/schemas.py ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py tests/unit/runtime/test_evaluation_summary_contracts.py -git commit -m "feat: enrich evaluation read contract" -``` - -### Task 2: Backend Cohort Rubric Status Summary - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/run_read_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/cohort_schemas.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/cohort_service.py` -- Modify: `ergon_core/ergon_core/core/api/test_harness.py` -- Test: `tests/unit/runtime/test_cohort_rubric_status_summary.py` - -- [ ] **Step 1: Write failing cohort rubric summary tests** - -Create `tests/unit/runtime/test_cohort_rubric_status_summary.py`: - -```python -def test_cohort_run_row_includes_rubric_status_summary(session: Session) -> None: - cohort, run, node = _persist_run_with_one_failed_evaluation(session) - - detail = experiment_cohort_service.get_detail(cohort.id) - - assert detail is not None - row = detail.runs[0] - assert row.rubric_status_summary.status == "failing" - assert row.rubric_status_summary.total_criteria == 1 - assert row.rubric_status_summary.failed == 1 - assert row.rubric_status_summary.criterion_statuses == ["failed"] -``` - -- [ ] **Step 2: Run tests and verify failure** - -Run: - -```bash -pytest tests/unit/runtime/test_cohort_rubric_status_summary.py -q -``` - -Expected: missing `rubric_status_summary` field or summary builder. - -- [ ] **Step 3: Implement one compact rubric summary builder** - -Add one private helper in `cohort_service.py`. Use `Counter` so the code says what it is doing without a separate status helper: - -```python -from collections import Counter - - -def _rubric_status_summary( - summaries: list[EvaluationSummary], -) -> CohortRubricStatusSummaryDto: - statuses = [ - criterion.status - for summary in summaries - for criterion in summary.criterion_results - ] - counts = Counter(statuses) - - if not statuses: - status = "none" - elif counts["errored"]: - status = "errored" - elif counts["failed"]: - status = "failing" - elif counts["passed"] and counts["skipped"]: - status = "mixed" - elif counts["skipped"] == len(statuses): - status = "skipped" - else: - status = "passing" - - return CohortRubricStatusSummaryDto( - status=status, - total_criteria=len(statuses), - passed=counts["passed"], - failed=counts["failed"], - errored=counts["errored"], - skipped=counts["skipped"], - criterion_statuses=statuses, - evaluator_names=sorted({summary.evaluator_name for summary in summaries}), - ) -``` - -- [ ] **Step 4: Attach cohort row rubric summary** - -In `cohort_service.py`, query `RunTaskEvaluation` for cohort runs, group by `run_id`, convert `summary_json` to `EvaluationSummary`, and pass `rubric_status_summary` into `_build_run_row`. - -- [ ] **Step 5: Expand test harness state** - -In `test_harness.py`, add these fields to the run state JSON: - -```json -{ - "rubric_status_summary": { - "status": "passing", - "total_criteria": 2, - "passed": 2, - "failed": 0, - "errored": 0, - "skipped": 0 - }, - "evaluations": [ - { - "task_id": "node-uuid", - "task_slug": "d_root", - "score": 1.0, - "reason": "root timing marker criterion ran", - "criterion_statuses": ["passed"], - "evaluator_name": "post-root" - } - ] -} -``` - -- [ ] **Step 6: Run backend tests** - -Run: - -```bash -pytest tests/unit/runtime/test_evaluation_summary_contracts.py tests/unit/runtime/test_cohort_rubric_status_summary.py -q -``` - -Expected: all selected tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/core/runtime/services/run_read_service.py ergon_core/ergon_core/core/runtime/services/cohort_schemas.py ergon_core/ergon_core/core/runtime/services/cohort_service.py ergon_core/ergon_core/core/api/test_harness.py tests/unit/runtime/test_cohort_rubric_status_summary.py -git commit -m "feat: expose cohort rubric status summary" -``` - -### Task 3: Frontend Contracts And Evaluation Selectors - -**Files:** -- Modify: `ergon-dashboard/src/generated/rest/contracts.ts` -- Modify: `ergon-dashboard/src/lib/contracts/rest.ts` -- Modify: `ergon-dashboard/src/lib/types.ts` -- Modify: `ergon-dashboard/src/lib/runState.ts` -- Create: `ergon-dashboard/src/features/evaluations/contracts.ts` -- Create: `ergon-dashboard/src/features/evaluations/status.ts` -- Create: `ergon-dashboard/src/features/evaluations/selectors.ts` -- Test: `ergon-dashboard/src/features/evaluations/selectors.test.ts` - -- [ ] **Step 1: Regenerate REST contracts** - -Run the repository's existing OpenAPI generation command. If the command is not documented, inspect `package.json` scripts and use the local script rather than hand-editing generated files. - -Expected: `src/generated/rest/contracts.ts` includes the new evaluation fields. - -- [ ] **Step 2: Write selector tests** - -Create `selectors.test.ts`: - -```ts -import { describe, expect, it } from "vitest"; -import { buildContainerEvaluationRollup, isEvaluationBearingTask } from "./selectors"; -import type { EvaluationRollup } from "./contracts"; -import type { TaskState, WorkflowRunState } from "@/lib/types"; - -function evaluation(status: "passed" | "failed" | "errored" | "skipped") { - return { - id: `evaluation-${status}`, - evaluatorName: "default", - totalScore: status === "passed" ? 1 : 0, - maxScore: 1, - normalizedScore: status === "passed" ? 1 : 0, - criterionResults: [{ id: `criterion-${status}`, status, score: status === "passed" ? 1 : 0, maxScore: 1 }], - }; -} - -it("detects tasks with direct evaluation evidence", () => { - const task = { id: "a", childIds: [] } as TaskState; - const state = { - evaluationsByTask: new Map([["a", evaluation("passed")]]), - } as unknown as WorkflowRunState; - - expect(isEvaluationBearingTask(state, task)).toBe(true); -}); - -it("rolls descendant evaluation failures up to a container", () => { - const state = { - tasks: new Map([ - ["root", { id: "root", childIds: ["a", "b"] }], - ["a", { id: "a", childIds: [] }], - ["b", { id: "b", childIds: [] }], - ]), - evaluationsByTask: new Map([ - ["a", evaluation("passed")], - ["b", evaluation("failed")], - ]), - } as unknown as WorkflowRunState; - - expect(buildContainerEvaluationRollup(state, "root").status).toBe("failing"); -}); -``` - -- [ ] **Step 3: Run selector tests and verify failure** - -Run: `cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts` - -Expected: failure because files/types are missing. - -- [ ] **Step 4: Add frontend evaluation contracts and status mapping** - -Create `contracts.ts`: - -```ts -export type EvalCriterionStatus = "passed" | "failed" | "errored" | "skipped"; -export type EvalRollupStatus = "passing" | "failing" | "errored" | "skipped" | "mixed"; -export type RubricStatusSummaryStatus = EvalRollupStatus | "none"; - -export interface EvaluationRollup { - status: EvalRollupStatus; - totalCriteria: number; - passed: number; - failed: number; - errored: number; - skipped: number; - normalizedScore: number | null; - maxScore: number | null; - evaluatorNames: string[]; - attachedTaskIds: string[]; - criterionStatuses: EvalCriterionStatus[]; -} -``` - -Create `status.ts`: - -```ts -import type { EvalCriterionStatus, EvalRollupStatus } from "./contracts"; - -export const EVALUATION_STATUS_LABEL: Record<EvalRollupStatus, string> = { - passing: "Passing", - failing: "Failing", - errored: "Errored", - skipped: "Skipped", - mixed: "Mixed", -}; - -export const CRITERION_STATUS_LABEL: Record<EvalCriterionStatus, string> = { - passed: "Passed", - failed: "Failed", - errored: "Errored", - skipped: "Skipped", -}; - -export function evaluationStatusTone(status: EvalRollupStatus): string { - switch (status) { - case "passing": - return "oklch(0.70 0.13 155)"; - case "failing": - return "oklch(0.68 0.18 22)"; - case "errored": - return "oklch(0.62 0.18 35)"; - case "skipped": - return "oklch(0.65 0.03 250)"; - case "mixed": - return "oklch(0.72 0.12 85)"; - } -} -``` - -- [ ] **Step 5: Add frontend selectors** - -Create `selectors.ts`: - -```ts -import type { TaskEvaluationState, TaskState, WorkflowRunState } from "@/lib/types"; -import type { EvalRollupStatus, EvaluationRollup } from "./contracts"; - -export function isEvaluationBearingTask(state: WorkflowRunState, task: TaskState): boolean { - return buildContainerEvaluationRollup(state, task.id) !== null; -} - -function combineStatus(statuses: EvalRollupStatus[]): EvalRollupStatus { - if (statuses.includes("errored")) return "errored"; - if (statuses.includes("failing")) return "failing"; - if (statuses.includes("mixed")) return "mixed"; - if (statuses.includes("skipped") && statuses.includes("passing")) return "mixed"; - if (statuses.every((status) => status === "skipped")) return "skipped"; - if (statuses.every((status) => status === "passing")) return "passing"; - return "mixed"; -} - -function evaluationToRollup(evaluation: TaskEvaluationState | undefined): EvaluationRollup | null { - if (!evaluation) return null; - const statuses = evaluation.criterionResults.map((criterion) => criterion.status); - if (statuses.length === 0) return null; - const passed = statuses.filter((status) => status === "passed").length; - const failed = statuses.filter((status) => status === "failed").length; - const errored = statuses.filter((status) => status === "errored").length; - const skipped = statuses.filter((status) => status === "skipped").length; - return { - status: combineStatus( - statuses.map((status) => - status === "passed" ? "passing" : status === "failed" ? "failing" : status === "errored" ? "errored" : "skipped", - ), - ), - totalCriteria: statuses.length, - passed, - failed, - errored, - skipped, - normalizedScore: evaluation.normalizedScore, - maxScore: evaluation.maxScore, - evaluatorNames: [evaluation.evaluatorName], - attachedTaskIds: evaluation.taskId ? [evaluation.taskId] : [], - criterionStatuses: statuses, - }; -} - -export function buildContainerEvaluationRollup(state: WorkflowRunState, taskId: string): EvaluationRollup | null { - const task = state.tasks.get(taskId); - if (!task) return null; - - const direct = evaluationToRollup(state.evaluationsByTask.get(taskId)); - const childRollups = task.childIds.map((childId) => buildContainerEvaluationRollup(state, childId)); - const rollups = [direct, ...childRollups].filter( - (rollup): rollup is EvaluationRollup => rollup !== null, - ); - - if (rollups.length === 0) return null; - - const totalCriteria = rollups.reduce((sum, rollup) => sum + rollup.totalCriteria, 0); - const maxScore = rollups.reduce((sum, rollup) => sum + rollup.maxScore, 0); - const weightedScore = rollups.reduce( - (sum, rollup) => sum + rollup.normalizedScore * rollup.maxScore, - 0, - ); - - return { - status: combineStatus(rollups.map((rollup) => rollup.status)), - totalCriteria, - passed: rollups.reduce((sum, rollup) => sum + rollup.passed, 0), - failed: rollups.reduce((sum, rollup) => sum + rollup.failed, 0), - errored: rollups.reduce((sum, rollup) => sum + rollup.errored, 0), - skipped: rollups.reduce((sum, rollup) => sum + rollup.skipped, 0), - normalizedScore: weightedScore / maxScore, - maxScore, - evaluatorNames: Array.from(new Set(rollups.flatMap((rollup) => rollup.evaluatorNames))).sort(), - attachedTaskIds: Array.from(new Set(rollups.flatMap((rollup) => rollup.attachedTaskIds))).sort(), - criterionStatuses: rollups.flatMap((rollup) => rollup.criterionStatuses), - }; -} -``` - -- [ ] **Step 6: Normalize contracts and run state** - -In `rest.ts`, require the enriched existing evaluation fields (`criterionName`, `status`, `passed`, `weight`, `contribution`, `evaluatorName`, `aggregationRule`) to be present after contract generation. Normalize only genuinely nullable fields (`modelReasoning`, `skippedReason`, `feedback`, `evaluationInput`, `error`) to `null`. In `runState.ts`, continue deserializing `evaluationsByTask`; do not add `taskEvaluationRollups` or `runEvaluationRollup` to `WorkflowRunState`. - -- [ ] **Step 7: Run frontend tests** - -Run: `cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts` - -Expected: tests pass. - -- [ ] **Step 8: Commit** - -```bash -git add ergon-dashboard/src/generated/rest/contracts.ts ergon-dashboard/src/lib/contracts/rest.ts ergon-dashboard/src/lib/types.ts ergon-dashboard/src/lib/runState.ts ergon-dashboard/src/features/evaluations/contracts.ts ergon-dashboard/src/features/evaluations/status.ts ergon-dashboard/src/features/evaluations/selectors.ts ergon-dashboard/src/features/evaluations/selectors.test.ts -git commit -m "feat: add frontend evaluation state domain" -``` - -### Task 4: Cohort Rubric Status Strips - -**Files:** -- Create: `ergon-dashboard/src/features/evaluations/components/CriterionStatusPip.tsx` -- Create: `ergon-dashboard/src/features/evaluations/components/RubricStatusStrip.tsx` -- Modify: `ergon-dashboard/src/components/cohorts/CohortDetailView.tsx` -- Test: `ergon-dashboard/tests/e2e/_shared/smoke.ts` - -- [ ] **Step 1: Add Playwright assertion first** - -In the cohort index test in `smoke.ts`, assert every run row has a strip: - -```ts -for (const { run_id } of cohort) { - await expect(page.getByTestId(`cohort-eval-strip-${run_id}`)).toBeVisible(); - await expect(page.locator(`[data-testid^="cohort-eval-pip-${run_id}-"]`).first()).toBeVisible(); -} -``` - -- [ ] **Step 2: Run Playwright smoke locally against an existing smoke stack** - -Run the narrow Playwright command used by the current E2E workflow for one benchmark. - -Expected: failure because the rubric status strip test IDs do not exist. - -- [ ] **Step 3: Create `CriterionStatusPip`** - -```tsx -import type { EvalCriterionStatus } from "@/features/evaluations/contracts"; -import { CRITERION_STATUS_LABEL, evaluationStatusTone } from "@/features/evaluations/status"; - -const rollupStatusByCriterion: Record<EvalCriterionStatus, Parameters<typeof evaluationStatusTone>[0]> = { - passed: "passing", - failed: "failing", - errored: "errored", - skipped: "skipped", -}; - -export function CriterionStatusPip({ - status, - testId, -}: { - status: EvalCriterionStatus; - testId?: string; -}) { - return ( - <span - data-testid={testId} - aria-label={`Criterion ${CRITERION_STATUS_LABEL[status]}`} - title={CRITERION_STATUS_LABEL[status]} - className="inline-block h-2.5 w-2.5 rounded-full ring-1 ring-white/80" - style={{ backgroundColor: evaluationStatusTone(rollupStatusByCriterion[status]) }} - /> - ); -} -``` - -- [ ] **Step 4: Create `RubricStatusStrip`** - -```tsx -import type { CohortRunRow } from "@/lib/types"; -import { CriterionStatusPip } from "./CriterionStatusPip"; - -export function RubricStatusStrip({ - runId, - summary, -}: { - runId: string; - summary: CohortRunRow["rubric_status_summary"]; -}) { - const statuses = summary.criterion_statuses; - - return ( - <div data-testid={`cohort-eval-strip-${runId}`} className="mt-2 flex items-center gap-1.5"> - <span className="text-[10px] font-medium uppercase tracking-[0.08em] text-[var(--faint)]"> - Rubric - </span> - {statuses.length === 0 ? ( - <span className="text-xs text-[var(--muted)]">No criteria</span> - ) : ( - <span className="flex items-center gap-1"> - {statuses.map((status, index) => ( - <CriterionStatusPip - key={`${status}-${index}`} - status={status} - testId={`cohort-eval-pip-${runId}-${index}`} - /> - ))} - </span> - )} - </div> - ); -} -``` - -- [ ] **Step 5: Render strip in cohort rows** - -In `CohortRunRowCard`, render: - -```tsx -<RubricStatusStrip runId={run.run_id} summary={run.rubric_status_summary} /> -``` - -Place it under the cohort/run ID metadata so it is visible without widening the grid. - -- [ ] **Step 6: Run frontend and E2E checks** - -Run: - -```bash -cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts -``` - -Then run the narrow Playwright smoke command. - -Expected: selector tests pass and Playwright sees cohort rubric status strips. - -- [ ] **Step 7: Commit** - -```bash -git add ergon-dashboard/src/features/evaluations/components/CriterionStatusPip.tsx ergon-dashboard/src/features/evaluations/components/RubricStatusStrip.tsx ergon-dashboard/src/components/cohorts/CohortDetailView.tsx ergon-dashboard/tests/e2e/_shared/smoke.ts -git commit -m "feat: show cohort rubric status" -``` - -### Task 5: Graph Glyphs, Container Roll-Ups, And Evaluation Lens - -**Files:** -- Create: `ergon-dashboard/src/features/evaluations/components/EvaluationNodeGlyph.tsx` -- Create: `ergon-dashboard/src/features/evaluations/components/EvaluationRollupBadge.tsx` -- Create: `ergon-dashboard/src/features/evaluations/components/EvaluationLensToggle.tsx` -- Modify: `ergon-dashboard/src/components/dag/TaskNode.tsx` -- Modify: `ergon-dashboard/src/features/graph/components/LeafNode.tsx` -- Modify: `ergon-dashboard/src/features/graph/components/ContainerNode.tsx` -- Modify: `ergon-dashboard/src/components/dag/DAGCanvas.tsx` -- Test: `ergon-dashboard/tests/e2e/_shared/smoke.ts` - -- [ ] **Step 1: Add Playwright graph assertions first** - -In `assertRunWorkspace`, after selecting an evaluated task: - -```ts -if (evaluatedTaskIds.has(selected.id)) { - await expect(page.getByTestId(`graph-eval-glyph-${selected.id}`)).toBeVisible(); -} -await expect(page.getByTestId("graph-eval-lens-toggle")).toBeVisible(); -await page.getByTestId("graph-eval-lens-toggle").click(); -await expect(page.getByTestId("graph-canvas")).toHaveAttribute("data-eval-lens", "on"); -``` - -- [ ] **Step 2: Run Playwright and verify failure** - -Expected: missing glyph/toggle test IDs. - -- [ ] **Step 3: Create graph evaluation components** - -`EvaluationNodeGlyph.tsx`: - -```tsx -import type { EvaluationRollup } from "@/features/evaluations/contracts"; -import { EVALUATION_STATUS_LABEL, evaluationStatusTone } from "@/features/evaluations/status"; - -export function EvaluationNodeGlyph({ - taskId, - rollup, -}: { - taskId: string; - rollup: EvaluationRollup; -}) { - return ( - <span - data-testid={`graph-eval-glyph-${taskId}`} - aria-label={`Evaluation ${EVALUATION_STATUS_LABEL[rollup.status]}`} - title={`Evaluation ${EVALUATION_STATUS_LABEL[rollup.status]}`} - className="inline-flex h-4 w-4 items-center justify-center rounded-full text-[10px] font-semibold" - style={{ color: evaluationStatusTone(rollup.status), backgroundColor: "rgba(255,255,255,0.8)" }} - > - ◇ - </span> - ); -} -``` - -`EvaluationRollupBadge.tsx`: - -```tsx -import type { EvaluationRollup } from "@/features/evaluations/contracts"; -import { EVALUATION_STATUS_LABEL, evaluationStatusTone } from "@/features/evaluations/status"; - -export function EvaluationRollupBadge({ - taskId, - rollup, -}: { - taskId: string; - rollup: EvaluationRollup; -}) { - return ( - <span - data-testid={`graph-eval-rollup-${taskId}`} - className="rounded-full px-1.5 py-0.5 text-[10px] font-medium" - style={{ - color: evaluationStatusTone(rollup.status), - backgroundColor: "rgba(255,255,255,0.75)", - border: `1px solid ${evaluationStatusTone(rollup.status)}`, - }} - > - {EVALUATION_STATUS_LABEL[rollup.status]} · {rollup.totalCriteria} - </span> - ); -} -``` - -`EvaluationLensToggle.tsx`: - -```tsx -export function EvaluationLensToggle({ - enabled, - onToggle, -}: { - enabled: boolean; - onToggle: () => void; -}) { - return ( - <button - type="button" - data-testid="graph-eval-lens-toggle" - aria-pressed={enabled} - onClick={onToggle} - className={`rounded px-2 py-1 text-xs font-medium ring-1 ${ - enabled - ? "bg-[var(--ink)] text-[var(--card)] ring-[var(--ink)]" - : "bg-[var(--card)] text-[var(--muted)] ring-[var(--line)]" - }`} - > - Eval lens - </button> - ); -} -``` - -- [ ] **Step 4: Pass roll-ups through React Flow node data** - -Extend `TaskNodeData`: - -```ts -evaluationRollup?: EvaluationRollup; -evalLensEnabled?: boolean; -``` - -When building React Flow nodes in `DAGCanvas.tsx`, set: - -```ts -const evaluationRollup = buildContainerEvaluationRollup(runState, task.id); -const evalBearing = evaluationRollup !== null; -data: { - task, - evaluationRollup, - evalLensEnabled, - dimmed: evalLensEnabled ? !evalBearing : isSearchDimmed, -} -``` - -- [ ] **Step 5: Render glyphs and roll-ups in nodes** - -In `LeafNode.tsx`, render `EvaluationNodeGlyph` near the title for direct task evaluations and `EvaluationRollupBadge` if there are multiple criteria. - -In `ContainerNode.tsx`, render `EvaluationRollupBadge` in the header row next to the child count. - -- [ ] **Step 6: Add lens toggle to DAG controls** - -In `DAGCanvas.tsx`, keep: - -```ts -const [evalLensEnabled, setEvalLensEnabled] = useState(false); -``` - -Render `EvaluationLensToggle` in the floating control card area and set: - -```tsx -<div data-testid="graph-canvas" data-eval-lens={evalLensEnabled ? "on" : "off"}> -``` - -- [ ] **Step 7: Run focused frontend tests and Playwright** - -Run: - -```bash -cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts -``` - -Run the narrow Playwright smoke command. - -Expected: graph glyph and lens assertions pass. - -- [ ] **Step 8: Commit** - -```bash -git add ergon-dashboard/src/features/evaluations/components/EvaluationNodeGlyph.tsx ergon-dashboard/src/features/evaluations/components/EvaluationRollupBadge.tsx ergon-dashboard/src/features/evaluations/components/EvaluationLensToggle.tsx ergon-dashboard/src/components/dag/TaskNode.tsx ergon-dashboard/src/features/graph/components/LeafNode.tsx ergon-dashboard/src/features/graph/components/ContainerNode.tsx ergon-dashboard/src/components/dag/DAGCanvas.tsx ergon-dashboard/tests/e2e/_shared/smoke.ts -git commit -m "feat: add evaluation graph lens" -``` - -### Task 6: Rich Evaluation Workspace Panel - -**Files:** -- Create: `ergon-dashboard/src/features/evaluations/components/EvaluationCriterionCard.tsx` -- Create: `ergon-dashboard/src/features/evaluations/components/EvaluationMetadataSummary.tsx` -- Modify: `ergon-dashboard/src/components/panels/EvaluationPanel.tsx` -- Test: `ergon-dashboard/tests/e2e/_shared/smoke.ts` - -- [ ] **Step 1: Add Playwright drawer assertions first** - -In `assertRunWorkspace`, inside the evaluation tab branch for evaluated tasks: - -```ts -await expect(page.getByTestId("workspace-evaluation-metadata")).toBeVisible(); -await expect(page.locator('[data-testid^="workspace-evaluation-criterion-"]').first()).toBeVisible(); -await expect(page.locator('[data-testid^="workspace-evaluation-criterion-status-"]').first()).toBeVisible(); -await expect(page.locator('[data-testid^="workspace-evaluation-input-"]').first()).toBeVisible(); -``` - -- [ ] **Step 2: Run Playwright and verify failure** - -Expected: metadata and criterion card test IDs missing. - -- [ ] **Step 3: Create `EvaluationMetadataSummary`** - -```tsx -import type { TaskEvaluationState } from "@/lib/types"; - -export function EvaluationMetadataSummary({ evaluation }: { evaluation: TaskEvaluationState }) { - return ( - <section data-testid="workspace-evaluation-metadata" className="rounded border border-[var(--line)] bg-[var(--paper)] p-3"> - <div className="grid grid-cols-2 gap-3 text-sm"> - <div> - <div className="text-xs text-[var(--faint)]">Evaluator</div> - <div className="font-medium text-[var(--ink)]">{evaluation.evaluatorName}</div> - </div> - <div> - <div className="text-xs text-[var(--faint)]">Aggregation</div> - <div className="font-medium text-[var(--ink)]">{evaluation.aggregationRule}</div> - </div> - <div> - <div className="text-xs text-[var(--faint)]">Score</div> - <div className="font-medium text-[var(--ink)]"> - {evaluation.totalScore.toFixed(2)} / {evaluation.maxScore.toFixed(2)} - </div> - </div> - <div> - <div className="text-xs text-[var(--faint)]">Stages</div> - <div className="font-medium text-[var(--ink)]"> - {evaluation.stagesPassed} / {evaluation.stagesEvaluated} passed - </div> - </div> - </div> - </section> - ); -} -``` - -- [ ] **Step 4: Create `EvaluationCriterionCard`** - -```tsx -import type { EvaluationCriterionState } from "@/lib/types"; -import { CRITERION_STATUS_LABEL, evaluationStatusTone } from "@/features/evaluations/status"; - -export function EvaluationCriterionCard({ criterion }: { criterion: EvaluationCriterionState }) { - const tone = evaluationStatusTone( - criterion.status === "passed" - ? "passing" - : criterion.status === "failed" - ? "failing" - : criterion.status === "errored" - ? "errored" - : "skipped", - ); - - return ( - <article - data-testid={`workspace-evaluation-criterion-${criterion.id}`} - className="rounded border border-[var(--line)] bg-[var(--card)] p-3" - > - <div className="flex items-start justify-between gap-3"> - <div> - <h4 className="font-medium text-[var(--ink)]">{criterion.criterionDescription}</h4> - <div className="mt-1 text-xs text-[var(--muted)]"> - {criterion.stageName} · weight {criterion.weight.toFixed(2)} · contribution {criterion.contribution.toFixed(2)} - </div> - </div> - <span - data-testid={`workspace-evaluation-criterion-status-${criterion.id}`} - className="rounded-full px-2 py-0.5 text-xs font-medium" - style={{ color: tone, border: `1px solid ${tone}` }} - > - {CRITERION_STATUS_LABEL[criterion.status]} - </span> - </div> - - {criterion.evaluationInput && ( - <div data-testid={`workspace-evaluation-input-${criterion.id}`} className="mt-3 rounded bg-[var(--paper)] p-2 text-xs text-[var(--muted)]"> - {criterion.evaluationInput} - </div> - )} - - {criterion.feedback && <p className="mt-3 text-sm text-[var(--ink)]">{criterion.feedback}</p>} - - {criterion.modelReasoning && ( - <div data-testid={`workspace-evaluation-reasoning-${criterion.id}`} className="mt-3 text-sm text-[var(--muted)]"> - {criterion.modelReasoning} - </div> - )} - - {criterion.skippedReason && <p className="mt-3 text-sm text-[var(--muted)]">{criterion.skippedReason}</p>} - - {criterion.error && ( - <pre className="mt-3 overflow-auto rounded bg-[var(--paper)] p-2 text-xs text-[var(--ink)]"> - {JSON.stringify(criterion.error, null, 2)} - </pre> - )} - </article> - ); -} -``` - -- [ ] **Step 5: Replace the current criterion map in `EvaluationPanel`** - -Keep existing empty state behavior, but render: - -```tsx -<EvaluationMetadataSummary evaluation={evaluation} /> -<div className="mt-3 space-y-3"> - {evaluation.criterionResults.map((criterion) => ( - <EvaluationCriterionCard key={criterion.id} criterion={criterion} /> - ))} -</div> -``` - -- [ ] **Step 6: Run frontend and E2E checks** - -Run: - -```bash -cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts -``` - -Run the narrow Playwright smoke command. - -Expected: evaluation workspace assertions pass. - -- [ ] **Step 7: Commit** - -```bash -git add ergon-dashboard/src/features/evaluations/components/EvaluationCriterionCard.tsx ergon-dashboard/src/features/evaluations/components/EvaluationMetadataSummary.tsx ergon-dashboard/src/components/panels/EvaluationPanel.tsx ergon-dashboard/tests/e2e/_shared/smoke.ts -git commit -m "feat: enrich evaluation workspace panel" -``` - -### Task 7: End-To-End Hardening - -**Files:** -- Modify: `ergon-dashboard/tests/helpers/backendHarnessClient.ts` -- Modify: `ergon-dashboard/tests/e2e/_shared/smoke.ts` -- Modify: `tests/e2e/_asserts.py` -- Modify: `docs/architecture/07_testing.md` - -- [ ] **Step 1: Expand backend harness TypeScript DTO** - -In `backendHarnessClient.ts`, add: - -```ts -export interface BackendEvaluationRollup { - status: "passing" | "failing" | "errored" | "skipped" | "mixed" | "none" | string; - total_criteria: number; - passed: number; - failed: number; - errored: number; - skipped: number; -} -``` - -Extend `BackendRunState`: - -```ts -rubric_status_summary: BackendEvaluationRollup; -evaluations: { - task_id: string; - task_slug: string | null; - score: number; - reason: string; - evaluator_name: string | null; - criterion_statuses: string[]; -}[]; -``` - -- [ ] **Step 2: Add backend E2E assertions** - -In `tests/e2e/_asserts.py`, assert happy runs expose: - -```python -assert len(root_evaluations) == 2 -assert {ev.parsed_summary().evaluator_name for ev in root_evaluations} >= {"default", "post-root"} -assert all( - cr.status == "passed" - for ev in root_evaluations - for cr in ev.parsed_summary().criterion_results -) -``` - -For sad runs, assert failed or skipped criterion state is exposed when a criterion does not pass. - -- [ ] **Step 3: Add UI assertions for each feature** - -In `smoke.ts`, assert: - -```ts -expect(state.rubric_status_summary.total_criteria).toBeGreaterThan(0); -await expect(page.getByTestId("graph-eval-lens-toggle")).toBeVisible(); -await expect(page.locator('[data-testid^="workspace-evaluation-criterion-"]').first()).toBeVisible(); -``` - -For happy runs: - -```ts -expect(state.rubric_status_summary.status).toBe("passing"); -``` - -For sad runs: - -```ts -expect(["failing", "errored", "mixed", "skipped"]).toContain(state.rubric_status_summary.status); -``` - -- [ ] **Step 4: Update testing docs** - -In `docs/architecture/07_testing.md`, add the frontend evaluation visibility surface to the E2E assertion table: - -```text -Evaluation visibility | Cohort pips, graph glyphs, container roll-ups, eval lens, workspace criterion cards | Playwright + backend harness DTO -``` - -- [ ] **Step 5: Run focused checks** - -Run: - -```bash -pytest tests/unit/runtime/test_evaluation_summary_contracts.py tests/unit/runtime/test_cohort_rubric_status_summary.py -q -cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts -``` - -Run the benchmark E2E smoke workflow locally for one benchmark if the stack is already available. - -Expected: unit and frontend tests pass; Playwright passes for the exercised benchmark. - -- [ ] **Step 6: Commit** - -```bash -git add ergon-dashboard/tests/helpers/backendHarnessClient.ts ergon-dashboard/tests/e2e/_shared/smoke.ts tests/e2e/_asserts.py docs/architecture/07_testing.md -git commit -m "test: cover evaluation visibility e2e" -``` - ---- - -## Rollout Notes - -1. Backend changes are additive and can ship before frontend rendering. -2. Generated REST contracts must be refreshed after backend DTO changes and before frontend contract normalization. -3. Cohort roll-ups intentionally stay lightweight to avoid loading full run snapshots for every row. -4. The evaluation lens is local UI state; it should not change the URL in the first implementation. -5. If skipped criteria require semantics not available in `summary_json`, extend `CriterionExecutor` to emit explicit skipped results in a later follow-up rather than inferring skipped state from missing rows. - -## Verification Matrix - -- Backend unit: `pytest tests/unit/runtime/test_evaluation_summary_contracts.py -q` -- Backend unit: `pytest tests/unit/runtime/test_cohort_rubric_status_summary.py -q` -- Frontend unit: `cd ergon-dashboard && npm test -- features/evaluations/selectors.test.ts` -- E2E: run the existing canonical smoke command for at least one happy/sad cohort. -- Lints: use `ReadLints` for edited files after each frontend and backend slice. - -## Self-Review - -- Spec coverage: cohort pips are covered in Task 4; graph glyphs, container roll-ups, and eval lens are covered in Task 5; richer drawer metadata and criterion detail are covered in Task 6; backend schemas/endpoints are covered in Tasks 1 and 2; E2E coverage is covered in Task 7. -- Placeholder scan: the plan contains concrete fields, commands, file paths, test IDs, and code shapes. Follow-up notes are explicitly scoped to future semantics rather than missing implementation steps. -- Type consistency: `EvalCriterionStatus`, `EvalRollupStatus`, `CohortRubricStatusSummaryDto`, and frontend-only `EvaluationRollup` names are used consistently across backend, frontend contracts, selectors, and components. diff --git a/docs/superpowers/plans/2026-04-27-react-worker-context-capture.md b/docs/superpowers/plans/2026-04-27-react-worker-context-capture.md deleted file mode 100644 index 6890dc1bb..000000000 --- a/docs/superpowers/plans/2026-04-27-react-worker-context-capture.md +++ /dev/null @@ -1,1116 +0,0 @@ -# ReAct Worker Context Capture Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make real LLM ReAct workers persist the full model-context transcript, including thinking blocks and tool observations, into `run_context_events`. - -**Architecture:** Keep `RunContextEvent` as the canonical durable context log. Move PydanticAI-specific transcript parsing out of `ReActWorker` into `ergon_builtins.common.llm_context`, add a small capture-settings helper there for provider-specific thinking/logprob settings, and keep runtime persistence framework-neutral by consuming `GenerationTurn`. - -**Tech Stack:** Python, PydanticAI, SQLModel, `GenerationTurn`, `ContextEventRepository`, pytest. - ---- - -## Scope - -This plan covers backend capture only: - -- Turn on reasoning/thinking capture for real ReAct workers where PydanticAI/provider support exists. -- Extract PydanticAI message history into `GenerationTurn` via a reusable utility. -- Persist `GenerationTurn.tool_results` as `tool_result` rows. -- Add unit tests around transcript extraction, model settings, and context event persistence. - -This plan does not redesign the workspace Actions UI. After this lands, the existing Actions tab should automatically receive richer `contextEventsByTask` data for new runs. - ---- - -## File Map - -The extraction and capture-settings code belongs in `ergon_builtins`, not `ergon_core`, because it depends on concrete worker/framework behavior. `ergon_core` should keep only stable contracts and persistence: `GenerationTurn` in, `RunContextEvent` out. - -Within `ergon_builtins`, this should live in `common/llm_context/`: shared code for built-in LLM workers that is not specific to MiniF2F, SWE-Bench, ResearchRubrics, or any one benchmark. Keep this domain narrow: - -- `capture_settings.py` decides what provider settings to pass when we want transcript capture. -- `adapters/base.py` defines the common transcript adapter interface in both directions. -- `adapters/pydantic_ai.py` adapts PydanticAI message history into Ergon's framework-neutral `GenerationTurn`, reconstructs PydanticAI messages from `RunContextEvent` rows, and owns PydanticAI response-metadata parsing such as logprobs. -- Benchmark toolkits, prompts, sandbox code, and worker output policy stay where they are. - -If this refactor also consolidates model resolution out of core, keep that under `ergon_builtins.models`, not under `llm_context`. Model resolution is about selecting a concrete model backend; `llm_context` is about transcript capture/replay. - -```text -ergon_builtins/ - ergon_builtins/ - common/ # add: shared builtins utility package - __init__.py # add - llm/ - structured_judge.py # optional move: core structured_judge helper if moving model resolution - llm_context/ # add: shared LLM context-capture domain for built-in workers - __init__.py # add - capture_settings.py # add: provider-specific thinking/logprob model_settings - adapters/ # add: framework transcript adapters - __init__.py # add - base.py # add: TranscriptAdapter protocol/base interface - pydantic_ai.py # add: PydanticAI <-> GenerationTurn/RunContextEvent adapter - langgraph.py # do not add yet: reserved for future framework adapter - openai_sdk.py # do not add yet: reserved for future direct-SDK adapter - prompts.py # do not add: benchmark prompts stay under workers/benchmarks - tools.py # do not add: benchmark toolkits stay under tools/ or benchmarks/ - workers/ - baselines/ - react_worker.py # modify: call shared capture/extraction helpers - # remove: _build_turns - # remove: _to_turn - # remove: _extract_request_parts - # remove: _extract_response_parts - # remove: _extract_tool_results - # remove: _make_json_safe - # remove: transcript-only imports for dataclasses, - # PydanticAI request/response parts, - # Ergon part classes, extract_logprobs, - # and LOGPROB_SETTINGS - react_prompts.py # leave alone: benchmark/system prompt definitions - research_rubrics/ - researcher_worker.py # leave alone unless it later adopts PydanticAI transcript capture - workflow_cli_react_worker.py # leave alone unless it later adopts PydanticAI transcript capture - models/ - resolution.py # optional move: ResolvedModel/register/resolve from core - openrouter_backend.py # leave alone: model resolution backend already exists - vllm_backend.py # leave alone: model resolution backend already exists - cloud_passthrough.py # leave alone: passthrough backend behavior unchanged - tools/ # leave alone: tool definitions are not transcript extraction - -ergon_core/ - ergon_core/ - api/ - generation.py # existing contract: GenerationTurn stays framework-neutral - core/ - rl/ - __init__.py # modify: remove PydanticAI-specific LOGPROB_SETTINGS if unused - providers/ - generation/ - model_resolution.py # optional remove: move to ergon_builtins.models.resolution - structured_judge.py # optional remove: move to ergon_builtins.common.llm.structured_judge - capture_settings.py # do not add here - adapters/ # do not add framework adapters here - pydantic_ai_format.py # remove or stop using: behavior moves to PydanticAI adapter - persistence/ - context/ - repository.py # modify: persist tool_result events from turn.tool_results - models.py # existing table model: RunContextEvent - event_payloads.py # existing payload union: tool_result/thinking/etc. - assembly.py # remove: PydanticAI-specific resume assembly moves to adapter - -tests/ - unit/ - builtins/ - common/ - test_capture_settings.py # add: provider settings contract - test_transcript_adapters.py # add: base interface + PydanticAI adapter contract - providers/ - test_capture_settings.py # do not add here - test_transcript_adapters.py # do not add here - persistence/ - test_context_event_repository.py # add: tool_results -> tool_result rows - state/ - test_generation_turn_build.py # modify: import new transcript adapter - test_context_assembly.py # remove or move assertions into test_transcript_adapters.py - workers/ - test_react_worker_contract.py # modify: ReActWorker no longer owns parser helpers -``` - -Import direction: - -- `ergon_builtins.common.llm_context.*` may import `ergon_core.api.generation` and, if moved, `ergon_builtins.models.resolution.ResolvedModel`. -- `ergon_builtins.workers.baselines.react_worker` may import `ergon_builtins.common.llm_context.*`. -- `ergon_core` must not import `ergon_builtins`. - -Additional core consolidation in scope: - -- Move `ergon_core/ergon_core/core/persistence/context/assembly.py` into `PydanticAITranscriptAdapter` because it imports `pydantic_ai.messages` directly. -- Move `ergon_core/ergon_core/core/providers/generation/pydantic_ai_format.py` behavior into `adapters/pydantic_ai.py` or a private sibling under `ergon_builtins.common.llm_context.adapters`; it is only useful for PydanticAI response dumps. -- Move `LOGPROB_SETTINGS` out of `ergon_core.core.rl.__init__` if no RL code imports it after this refactor; it is currently a PydanticAI model-settings constant, not an RL-domain primitive. -- Optional but coherent: move `ergon_core/ergon_core/core/providers/generation/model_resolution.py` to `ergon_builtins/ergon_builtins/models/resolution.py`. It imports PydanticAI and is populated by builtins model backends. -- Optional but coherent: move `ergon_core/ergon_core/core/providers/generation/structured_judge.py` to `ergon_builtins/ergon_builtins/common/llm/structured_judge.py`. It constructs a PydanticAI `Agent` and is currently used by builtins evaluator/benchmark code. -- Do not move `ergon_core/api/generation.py`, `event_payloads.py`, `models.py`, or `repository.py`; those are the framework-neutral core domain. -- Do not move model backends (`openrouter_backend.py`, `vllm_backend.py`, `cloud_passthrough.py`) in this refactor; they already live in `ergon_builtins.models`. - ---- - -## Provider Settings Contract - -Use one settings helper instead of scattering provider checks through workers. - -Expected behavior: - -- `vllm:*` keeps existing logprob settings: - -```python -{"openai_logprobs": True, "openai_top_logprobs": 1} -``` - -- `anthropic:*` asks Anthropic for thinking blocks: - -```python -{"anthropic_thinking": {"type": "enabled", "budget_tokens": 1024}} -``` - -- `openrouter:*` asks OpenRouter to include reasoning: - -```python -{"openrouter_reasoning": {"enabled": True, "exclude": False}} -``` - -- `google:*` asks Gemini to include thoughts: - -```python -{"gemini_thinking_config": {"include_thoughts": True}} -``` - -- Unknown providers return `None`; provider-specific capture behavior must be added explicitly with tests. - -If provider settings conflict with a model/output mode at runtime, the implementation should fail loudly in tests first. Do not silently suppress thinking capture unless a targeted fallback is added with a test. - ---- - -## Task 1: Add Capture Settings Helper - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/common/__init__.py` -- Create: `ergon_builtins/ergon_builtins/common/llm_context/__init__.py` -- Create: `ergon_builtins/ergon_builtins/common/llm_context/capture_settings.py` -- Create: `ergon_builtins/ergon_builtins/common/llm_context/adapters/__init__.py` -- Create: `ergon_builtins/ergon_builtins/common/llm_context/adapters/base.py` -- Test: `tests/unit/builtins/common/test_capture_settings.py` - -- [ ] **Step 1: Write tests for provider-specific model settings** - -Create `tests/unit/builtins/common/test_capture_settings.py`: - -```python -from ergon_builtins.common.llm_context.capture_settings import build_capture_model_settings -from ergon_core.core.providers.generation.model_resolution import ResolvedModel - - -def _resolved(*, supports_logprobs: bool = False) -> ResolvedModel: - return ResolvedModel(model="dummy", supports_logprobs=supports_logprobs) - - -def test_vllm_enables_logprobs() -> None: - assert build_capture_model_settings("vllm:http://localhost:8000", _resolved(supports_logprobs=True)) == { - "openai_logprobs": True, - "openai_top_logprobs": 1, - } - - -def test_anthropic_enables_thinking() -> None: - assert build_capture_model_settings("anthropic:claude-sonnet-4", _resolved()) == { - "anthropic_thinking": {"type": "enabled", "budget_tokens": 1024}, - } - - -def test_openrouter_includes_reasoning() -> None: - assert build_capture_model_settings("openrouter:anthropic/claude-sonnet-4.6", _resolved()) == { - "openrouter_reasoning": {"enabled": True, "exclude": False}, - } - - -def test_google_includes_thoughts() -> None: - assert build_capture_model_settings("google:gemini-2.5-pro", _resolved()) == { - "gemini_thinking_config": {"include_thoughts": True}, - } - - -def test_unknown_provider_without_capture_returns_none() -> None: - assert build_capture_model_settings("openai:gpt-4o", _resolved()) is None -``` - -- [ ] **Step 2: Run the focused test and verify it fails** - -Run: - -```bash -pytest tests/unit/builtins/common/test_capture_settings.py -q -``` - -Expected: FAIL because `capture_settings.py` does not exist. - -- [ ] **Step 3: Implement `capture_settings.py`** - -Create `ergon_builtins/ergon_builtins/common/__init__.py`: - -```python -"""Shared utilities for built-in Ergon workers.""" -``` - -Create `ergon_builtins/ergon_builtins/common/llm_context/__init__.py`: - -```python -"""Helpers for capturing LLM context from built-in worker frameworks.""" -``` - -Create `ergon_builtins/ergon_builtins/common/llm_context/adapters/__init__.py`: - -```python -"""Framework adapters for LLM transcript extraction and replay assembly.""" -``` - -Create `ergon_builtins/ergon_builtins/common/llm_context/adapters/base.py`: - -```python -"""Base interface for framework transcript adapters.""" - -from typing import Protocol, TypeVar - -from ergon_core.api.generation import GenerationTurn -from ergon_core.core.persistence.context.models import RunContextEvent - -TranscriptT = TypeVar("TranscriptT") -ReplayT = TypeVar("ReplayT") - - -class TranscriptAdapter(Protocol[TranscriptT, ReplayT]): - """Convert between framework-native transcripts and Ergon context events.""" - - def build_turns(self, transcript: TranscriptT) -> list[GenerationTurn]: - """Return ordered turns extracted from a complete transcript.""" - ... - - def assemble_replay(self, events: list[RunContextEvent]) -> ReplayT: - """Return framework-native replay context from ordered context events.""" - ... -``` - -Create `ergon_builtins/ergon_builtins/common/llm_context/capture_settings.py`: - -```python -"""Provider-specific settings for capturing model context events. - -Workers call this once before running an agent. The returned dictionary is -passed to PydanticAI as model_settings. -""" - -from ergon_core.api.json_types import JsonObject -from ergon_core.core.providers.generation.model_resolution import ResolvedModel -_ANTHROPIC_THINKING_BUDGET_TOKENS = 1024 -_OPENAI_COMPAT_LOGPROB_SETTINGS: JsonObject = { - "openai_logprobs": True, - "openai_top_logprobs": 1, -} - - -def _prefix(model_target: str | None) -> str: - target = model_target or "" - return target.split(":", 1)[0] if ":" in target else "" - - -def build_capture_model_settings( - model_target: str | None, - resolved_model: ResolvedModel, -) -> JsonObject | None: - """Return PydanticAI model_settings for transcript capture.""" - prefix = _prefix(model_target) - - if prefix == "vllm" and resolved_model.supports_logprobs: - return dict(_OPENAI_COMPAT_LOGPROB_SETTINGS) - - if prefix == "anthropic": - return { - "anthropic_thinking": { - "type": "enabled", - "budget_tokens": _ANTHROPIC_THINKING_BUDGET_TOKENS, - } - } - - if prefix == "openrouter": - return { - "openrouter_reasoning": { - "enabled": True, - "exclude": False, - } - } - - if prefix == "google": - return { - "gemini_thinking_config": { - "include_thoughts": True, - } - } - - return None -``` - -- [ ] **Step 4: Run the focused test and verify it passes** - -Run: - -```bash -pytest tests/unit/builtins/common/test_capture_settings.py -q -``` - -Expected: PASS. - ---- - -## Task 2: Extract PydanticAI Transcript Conversion - -**Files:** - -- Create: `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` -- Test: `tests/unit/builtins/common/test_transcript_adapters.py` -- Modify: `tests/unit/state/test_generation_turn_build.py` - -- [ ] **Step 1: Write tests for transcript extraction** - -Create `tests/unit/builtins/common/test_transcript_adapters.py`: - -```python -from ergon_core.api.generation import ( - GenerationTurn, - TextPart as ErgonTextPart, - ThinkingPart as ErgonThinkingPart, - ToolCallPart as ErgonToolCallPart, - ToolReturnPart as ErgonToolReturnPart, - UserPromptPart as ErgonUserPromptPart, -) -from ergon_builtins.common.llm_context.adapters.base import TranscriptAdapter -from ergon_builtins.common.llm_context.adapters.pydantic_ai import ( - PydanticAITranscriptAdapter, -) -from pydantic_ai.messages import ( - ModelRequest, - ModelResponse, - TextPart, - ThinkingPart, - ToolCallPart, - ToolReturnPart, - UserPromptPart, -) - - -def test_text_and_thinking_are_response_parts() -> None: - adapter: TranscriptAdapter[list[ModelRequest | ModelResponse], list[ModelRequest | ModelResponse]] = ( - PydanticAITranscriptAdapter() - ) - turns = adapter.build_turns( - [ - ModelRequest(parts=[UserPromptPart(content="hard question")]), - ModelResponse( - parts=[ - ThinkingPart(content="let me reason"), - TextPart(content="answer"), - ] - ), - ] - ) - - assert len(turns) == 1 - turn = turns[0] - assert isinstance(turn, GenerationTurn) - assert any(isinstance(part, ErgonUserPromptPart) for part in turn.messages_in) - assert any(isinstance(part, ErgonThinkingPart) for part in turn.response_parts) - assert any(isinstance(part, ErgonTextPart) for part in turn.response_parts) - - -def test_tool_return_is_attached_to_generating_turn() -> None: - adapter = PydanticAITranscriptAdapter() - turns = adapter.build_turns( - [ - ModelRequest(parts=[UserPromptPart(content="search")]), - ModelResponse( - parts=[ - ToolCallPart( - tool_name="search", - tool_call_id="call-1", - args={"query": "ergon"}, - ) - ] - ), - ModelRequest( - parts=[ - ToolReturnPart( - tool_name="search", - tool_call_id="call-1", - content={"result": "found"}, - ) - ] - ), - ModelResponse(parts=[TextPart(content="done")]), - ] - ) - - assert len(turns) == 2 - first = turns[0] - assert any(isinstance(part, ErgonToolCallPart) for part in first.response_parts) - assert len(first.tool_results) == 1 - result = first.tool_results[0] - assert isinstance(result, ErgonToolReturnPart) - assert result.tool_call_id == "call-1" - assert result.tool_name == "search" - assert result.content == '{"result": "found"}' -``` - -- [ ] **Step 2: Run the focused test and verify it fails** - -Run: - -```bash -pytest tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: FAIL because `adapters/pydantic_ai.py` does not exist. - -- [ ] **Step 3: Implement the transcript utility** - -Create `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` by moving the existing parsing helpers out of `react_worker.py`: - -```python -"""PydanticAI transcript adapter.""" - -import dataclasses # slopcop: ignore[no-dataclass] -import json -from typing import Any - -from ergon_core.api.generation import ( - GenerationTurn, - SystemPromptPart, - TextPart, - ThinkingPart, - TokenLogprob, - ToolCallPart, - ToolReturnPart, - UserPromptPart, -) -from ergon_core.core.persistence.context.event_payloads import ( - AssistantTextPayload, - SystemPromptPayload, - ThinkingPayload, - ToolCallPayload, - ToolResultPayload, - UserMessagePayload, -) -from ergon_core.core.persistence.context.models import RunContextEvent -from ergon_builtins.common.llm_context.adapters.base import TranscriptAdapter -from pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse -from pydantic_ai.messages import SystemPromptPart as PydanticSystemPromptPart -from pydantic_ai.messages import TextPart as PydanticTextPart -from pydantic_ai.messages import ThinkingPart as PydanticThinkingPart -from pydantic_ai.messages import ToolCallPart as PydanticToolCallPart -from pydantic_ai.messages import ToolReturnPart as PydanticToolReturnPart -from pydantic_ai.messages import UserPromptPart as PydanticUserPromptPart - - -class PydanticAITranscriptAdapter(TranscriptAdapter[list[ModelMessage], list[ModelMessage]]): - """Convert complete PydanticAI message history into Ergon turns.""" - - def build_turns(self, transcript: list[ModelMessage]) -> list[GenerationTurn]: - """Build turns from a complete PydanticAI message list. - - The full message history is required because tool returns appear in the - request after the response that created the tool call. - """ - turns: list[GenerationTurn] = [] - pending_response: ModelResponse | None = None - pending_request_in: ModelRequest | None = None - - for message in transcript: - if isinstance(message, ModelRequest): - if pending_response is not None: - turns.append( - _to_turn( - pending_request_in, - pending_response, - tool_result_request=message, - ) - ) - pending_response = None - pending_request_in = None - pending_request_in = message - elif isinstance(message, ModelResponse): - pending_response = message - - if pending_response is not None: - turns.append(_to_turn(pending_request_in, pending_response, tool_result_request=None)) - - return turns - - def assemble_replay(self, events: list[RunContextEvent]) -> list[ModelMessage]: - """Reconstruct PydanticAI messages from ordered context events.""" - messages: list[ModelMessage] = [] - current_request_parts: list[Any] = [] - current_response_parts: list[Any] = [] - - for event in events: - if event.event_type in ("system_prompt", "user_message"): - current_request_parts.append(_to_pydantic_request_part(event)) - elif event.event_type in ("thinking", "assistant_text", "tool_call"): - if current_request_parts and not current_response_parts: - messages.append(ModelRequest(parts=current_request_parts)) - current_request_parts = [] - current_response_parts.append(_to_pydantic_response_part(event)) - elif event.event_type == "tool_result": - if current_response_parts: - messages.append(ModelResponse(parts=current_response_parts)) - current_response_parts = [] - current_request_parts.append(_to_pydantic_request_part(event)) - - if current_response_parts: - messages.append(ModelResponse(parts=current_response_parts)) - - return messages - - -def _to_turn( - request_in: ModelRequest | None, - response: ModelResponse, - tool_result_request: ModelRequest | None, -) -> GenerationTurn: - raw_resp = _make_json_safe(dataclasses.asdict(response)) - return GenerationTurn( - messages_in=_extract_request_parts(request_in) if request_in else [], - response_parts=_extract_response_parts(response), - tool_results=_extract_tool_results(tool_result_request) if tool_result_request else [], - turn_logprobs=extract_logprobs(raw_resp), - ) - - -def extract_logprobs(raw: dict[str, Any]) -> list[TokenLogprob] | None: - """Extract per-token logprobs from a PydanticAI response dump.""" - details = raw.get("provider_details") - if not isinstance(details, dict): - return None - raw_logprobs = details.get("logprobs") - if not isinstance(raw_logprobs, list) or not raw_logprobs: - return None - return [ - TokenLogprob( - token=entry["token"], - logprob=entry["logprob"], - top_logprobs=entry.get("top_logprobs", []), - ) - for entry in raw_logprobs - if isinstance(entry, dict) and "token" in entry and "logprob" in entry - ] - - -def _to_pydantic_response_part(event: RunContextEvent) -> Any: # slopcop: ignore[no-typing-any] - parsed = event.parsed_payload() - if event.event_type == "thinking": - if not isinstance(parsed, ThinkingPayload): - raise ValueError(f"Expected ThinkingPayload for thinking event, got {type(parsed)}") - return PydanticThinkingPart(content=parsed.text) - if event.event_type == "assistant_text": - if not isinstance(parsed, AssistantTextPayload): - raise ValueError(f"Expected AssistantTextPayload for assistant_text event, got {type(parsed)}") - return PydanticTextPart(content=parsed.text) - if event.event_type == "tool_call": - if not isinstance(parsed, ToolCallPayload): - raise ValueError(f"Expected ToolCallPayload for tool_call event, got {type(parsed)}") - return PydanticToolCallPart( - tool_name=parsed.tool_name, - tool_call_id=parsed.tool_call_id, - args=parsed.args, - ) - raise ValueError(f"Unexpected response event_type: {event.event_type!r}") - - -def _to_pydantic_request_part(event: RunContextEvent) -> Any: # slopcop: ignore[no-typing-any] - parsed = event.parsed_payload() - if event.event_type == "system_prompt": - if not isinstance(parsed, SystemPromptPayload): - raise ValueError(f"Expected SystemPromptPayload for system_prompt event, got {type(parsed)}") - return PydanticSystemPromptPart(content=parsed.text) - if event.event_type == "user_message": - if not isinstance(parsed, UserMessagePayload): - raise ValueError(f"Expected UserMessagePayload for user_message event, got {type(parsed)}") - return PydanticUserPromptPart(content=parsed.text) - if event.event_type == "tool_result": - if not isinstance(parsed, ToolResultPayload): - raise ValueError(f"Expected ToolResultPayload for tool_result event, got {type(parsed)}") - return PydanticToolReturnPart( - tool_call_id=parsed.tool_call_id, - tool_name=parsed.tool_name, - content=str(parsed.result), - ) - raise ValueError(f"Unexpected request event_type: {event.event_type!r}") - - -def _extract_request_parts(request: ModelRequest) -> list[Any]: # slopcop: ignore[no-typing-any] - parts: list[Any] = [] # slopcop: ignore[no-typing-any] - for part in request.parts: - if isinstance(part, PydanticSystemPromptPart): - parts.append(SystemPromptPart(content=part.content)) - elif isinstance(part, PydanticUserPromptPart) and isinstance(part.content, str): - parts.append(UserPromptPart(content=part.content)) - return parts - - -def _extract_response_parts(response: ModelResponse) -> list[Any]: # slopcop: ignore[no-typing-any] - parts: list[Any] = [] # slopcop: ignore[no-typing-any] - for part in response.parts: - if isinstance(part, PydanticTextPart): - parts.append(TextPart(content=part.content)) - elif isinstance(part, PydanticToolCallPart): - parts.append( - ToolCallPart( - tool_name=part.tool_name, - tool_call_id=part.tool_call_id, - args=part.args_as_dict(), - ) - ) - elif isinstance(part, PydanticThinkingPart): - parts.append(ThinkingPart(content=part.content)) - return parts - - -def _extract_tool_results(request: ModelRequest) -> list[ToolReturnPart]: - results: list[ToolReturnPart] = [] - for part in request.parts: - if isinstance(part, PydanticToolReturnPart): - content = part.content - serialized = content if isinstance(content, str) else json.dumps(content, default=str) - results.append( - ToolReturnPart( - tool_call_id=part.tool_call_id, - tool_name=part.tool_name, - content=serialized, - ) - ) - return results - - -def _make_json_safe(obj: Any) -> Any: # slopcop: ignore[no-typing-any] - from datetime import datetime - - if isinstance(obj, dict): - return {k: _make_json_safe(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_make_json_safe(v) for v in obj] - if isinstance(obj, datetime): - return obj.isoformat() - if isinstance(obj, bytes): - return obj.decode("utf-8", errors="replace") - return obj -``` - -- [ ] **Step 4: Run the focused test and verify it passes** - -Run: - -```bash -pytest tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: PASS. - -- [ ] **Step 5: Update old generation-turn tests to import the new utility** - -Modify `tests/unit/state/test_generation_turn_build.py`: - -```python -from ergon_builtins.common.llm_context.adapters.pydantic_ai import PydanticAITranscriptAdapter - - -def _build_turns(messages): - return PydanticAITranscriptAdapter().build_turns(messages) -``` - -Remove the old import from `ergon_builtins.workers.baselines.react_worker`. - -- [ ] **Step 6: Run the old and new transcript tests together** - -Run: - -```bash -pytest tests/unit/state/test_generation_turn_build.py tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: PASS. - ---- - -## Task 3: Simplify `ReActWorker` - -**Files:** - -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Test: `tests/unit/workers/test_react_worker_contract.py` - -- [ ] **Step 1: Add a contract test that transcript helpers no longer live in `react_worker.py`** - -Modify `tests/unit/workers/test_react_worker_contract.py`: - -```python -def test_pydantic_ai_transcript_adapter_lives_outside_worker() -> None: - import ergon_builtins.workers.baselines.react_worker as react_worker - - assert not hasattr(react_worker, "_build_turns") - assert not hasattr(react_worker, "_extract_request_parts") - assert not hasattr(react_worker, "_extract_response_parts") - assert not hasattr(react_worker, "_extract_tool_results") -``` - -- [ ] **Step 2: Run the contract test and verify it fails** - -Run: - -```bash -pytest tests/unit/workers/test_react_worker_contract.py -q -``` - -Expected: FAIL because helper functions still exist in `react_worker.py`. - -- [ ] **Step 3: Update `ReActWorker` imports** - -In `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py`, remove imports that are only used by transcript parsing: - -```python -import dataclasses # remove -from ergon_core.api.generation import SystemPromptPart, TextPart, ThinkingPart, ToolCallPart, ToolReturnPart, UserPromptPart # remove -from ergon_core.api.json_types import JsonObject # remove if only used for model_settings type -from ergon_core.core.providers.generation.pydantic_ai_format import extract_logprobs # remove -from ergon_core.core.rl import LOGPROB_SETTINGS # remove -from pydantic_ai.messages import ModelRequest, ModelResponse # remove -from pydantic_ai.messages import SystemPromptPart as PydanticSystemPromptPart # remove -from pydantic_ai.messages import TextPart as PydanticTextPart # remove -from pydantic_ai.messages import ThinkingPart as PydanticThinkingPart # remove -from pydantic_ai.messages import ToolCallPart as PydanticToolCallPart # remove -from pydantic_ai.messages import ToolReturnPart as PydanticToolReturnPart # remove -from pydantic_ai.messages import UserPromptPart as PydanticUserPromptPart # remove -``` - -Add: - -```python -from ergon_builtins.common.llm_context.capture_settings import build_capture_model_settings -from ergon_builtins.common.llm_context.adapters.pydantic_ai import PydanticAITranscriptAdapter -``` - -- [ ] **Step 4: Update model settings and transcript extraction** - -Replace: - -```python -model_settings: JsonObject | None = None -if resolved.supports_logprobs and self.model and self.model.startswith("vllm:"): - model_settings = LOGPROB_SETTINGS -``` - -with: - -```python -model_settings = build_capture_model_settings(self.model, resolved) -``` - -Replace: - -```python -turns = _build_turns(run.ctx.state.message_history) -``` - -with: - -```python -turns = PydanticAITranscriptAdapter().build_turns(run.ctx.state.message_history) -``` - -- [ ] **Step 5: Delete transcript helper functions from `react_worker.py`** - -Delete the helper block that starts at: - -```python -# --------------------------------------------------------------------------- -# PydanticAI message → GenerationTurn -# --------------------------------------------------------------------------- -``` - -Keep `_format_task` and `_latest_final_result_message` in `react_worker.py` because they are worker behavior, not PydanticAI transcript parsing. - -- [ ] **Step 6: Run contract and worker tests** - -Run: - -```bash -pytest tests/unit/workers/test_react_worker_contract.py tests/unit/state/test_generation_turn_build.py tests/unit/builtins/common/test_capture_settings.py tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: PASS. - ---- - -## Task 4: Persist `GenerationTurn.tool_results` - -**Files:** - -- Modify: `ergon_core/ergon_core/core/persistence/context/repository.py` -- Test: `tests/unit/persistence/test_context_event_repository.py` - -- [ ] **Step 1: Write a failing repository test** - -Create `tests/unit/persistence/test_context_event_repository.py`: - -```python -from uuid import UUID - -import pytest -from ergon_core.api.generation import GenerationTurn, ToolCallPart, ToolReturnPart, UserPromptPart -from ergon_core.core.persistence.context.repository import ContextEventRepository -from ergon_core.core.persistence.telemetry.models import RunRecord, RunTaskExecution -from ergon_core.core.persistence.shared.ids import new_id -from sqlmodel import Session - - -@pytest.mark.asyncio -async def test_persist_turn_records_tool_results_from_tool_results(session: Session) -> None: - run_id = new_id() - execution_id = new_id() - - session.add(RunRecord(id=run_id, experiment_id=UUID(int=1), name="test", status="running")) - session.add( - RunTaskExecution( - id=execution_id, - run_id=run_id, - definition_task_id=UUID(int=2), - node_id=UUID(int=3), - attempt_number=1, - status="running", - ) - ) - session.commit() - - repo = ContextEventRepository() - events = await repo.persist_turn( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker", - turn=GenerationTurn( - messages_in=[UserPromptPart(content="search")], - response_parts=[ - ToolCallPart(tool_name="search", tool_call_id="call-1", args={"query": "ergon"}) - ], - tool_results=[ - ToolReturnPart(tool_name="search", tool_call_id="call-1", content="found") - ], - ), - ) - - assert [event.event_type for event in events] == ["user_message", "tool_call", "tool_result"] - tool_result = events[-1].parsed_payload() - assert tool_result.event_type == "tool_result" - assert tool_result.tool_name == "search" - assert tool_result.tool_call_id == "call-1" - assert tool_result.result == "found" -``` - -If the project uses a differently named DB fixture than `session`, adapt only the fixture name and setup rows to the existing test harness. Keep the assertion shape unchanged. - -- [ ] **Step 2: Run the focused repository test and verify it fails** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py -q -``` - -Expected: FAIL because `_events_from_tool_results` currently scans `turn.messages_in`, not `turn.tool_results`. - -- [ ] **Step 3: Update `_events_from_tool_results`** - -In `ergon_core/ergon_core/core/persistence/context/repository.py`, replace the loop source: - -```python -for part in turn.messages_in: -``` - -with a helper that prefers `turn.tool_results` and preserves compatibility with old/custom workers: - -```python -tool_result_parts = [ - *turn.tool_results, - *(part for part in turn.messages_in if isinstance(part, ToolReturnPart)), -] -for part in tool_result_parts: -``` - -Update the docstring to: - -```python -"""Produce tool_result events from GenerationTurn tool observations.""" -``` - -- [ ] **Step 4: Run the focused repository test and verify it passes** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py -q -``` - -Expected: PASS. - ---- - -## Task 5: Add End-to-End Unit Coverage for ReAct Capture Shape - -**Files:** - -- Modify or create: `tests/unit/builtins/common/test_transcript_adapters.py` -- Modify or create: `tests/unit/persistence/test_context_event_repository.py` - -- [ ] **Step 1: Add a combined transcript-to-event regression** - -Add a test that builds PydanticAI messages, converts them to `GenerationTurn`, persists the first turn, and asserts event types: - -```python -@pytest.mark.asyncio -async def test_pydantic_ai_tool_observation_becomes_context_event(session: Session) -> None: - from ergon_builtins.common.llm_context.adapters.pydantic_ai import PydanticAITranscriptAdapter - - turns = PydanticAITranscriptAdapter().build_turns( - [ - ModelRequest(parts=[UserPromptPart(content="search")]), - ModelResponse( - parts=[ - ToolCallPart( - tool_name="search", - tool_call_id="call-1", - args={"query": "ergon"}, - ) - ] - ), - ModelRequest( - parts=[ - ToolReturnPart( - tool_name="search", - tool_call_id="call-1", - content="found", - ) - ] - ), - ] - ) - - events = await repo.persist_turn( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker", - turn=turns[0], - ) - - assert [event.event_type for event in events] == ["user_message", "tool_call", "tool_result"] -``` - -Use the same DB setup helper from Task 4. This test is intentionally redundant: it protects the integration boundary where the current bug occurred. - -- [ ] **Step 2: Add a thinking regression** - -Add a test with `ThinkingPart(content="let me think")` in the PydanticAI response, then persist the resulting turn and assert a `thinking` context event appears before `assistant_text`: - -```python -assert [event.event_type for event in events] == ["user_message", "thinking", "assistant_text"] -``` - -- [ ] **Step 3: Run the combined tests** - -Run: - -```bash -pytest tests/unit/builtins/common/test_transcript_adapters.py tests/unit/persistence/test_context_event_repository.py -q -``` - -Expected: PASS. - ---- - -## Task 6: Verification Against a Real LLM Smoke Run - -**Files:** - -- No code changes required. -- Optional inspection command only. - -- [ ] **Step 1: Run a small real LLM benchmark using a reasoning-capable model** - -Use the repo's existing real-LLM harness or CLI with a cheap one-task run. Prefer a model target already used by the repo, such as: - -```bash -pytest tests/real_llm/benchmarks/test_researchrubrics.py -q -``` - -If the real-LLM test is intentionally skipped because credentials or budget are unavailable, record that skip in the implementation summary. - -- [ ] **Step 2: Inspect the run snapshot for richer context events** - -For a known run id, inspect event counts: - -```bash -RUN_ID=<run-id-from-smoke-run> python - <<'PY' -import json, urllib.request -import os - -run_id = os.environ["RUN_ID"] -with urllib.request.urlopen(f"http://127.0.0.1:3002/api/runs/{run_id}", timeout=5) as r: - data = json.load(r) - -counts = {} -for events in (data.get("contextEventsByTask") or {}).values(): - for event in events: - counts[event.get("eventType")] = counts.get(event.get("eventType"), 0) + 1 - -print(counts) -PY -``` - -Expected for a tool-using run: `tool_result` count is non-zero. Expected for a provider/model that returns thinking: `thinking` count is non-zero. - -Do not fail the implementation if `thinking` is zero for a provider that does not return thoughts despite the request. Do fail if tool-using ReAct runs still have zero `tool_result` events. - ---- - -## Task 7: Final Test Pass - -**Files:** - -- No code changes unless tests reveal a regression. - -- [ ] **Step 1: Run focused backend tests** - -Run: - -```bash -pytest tests/unit/builtins/common/test_capture_settings.py tests/unit/builtins/common/test_transcript_adapters.py tests/unit/persistence/test_context_event_repository.py tests/unit/state/test_generation_turn_build.py tests/unit/workers/test_react_worker_contract.py -q -``` - -Expected: PASS. - -- [ ] **Step 2: Run lints for edited Python files** - -Run the repo's standard Python lint/type command if available. If the repo does not expose a single lint command, at minimum run: - -```bash -python -m compileall ergon_builtins/ergon_builtins/common ergon_builtins/ergon_builtins/workers/baselines ergon_core/ergon_core/core/persistence/context -``` - -Expected: PASS. - -- [ ] **Step 3: Record implementation notes** - -In the implementation summary, include: - -- Whether `tool_result` is now persisted from `GenerationTurn.tool_results`. -- Which provider settings were added for thinking/reasoning. -- Whether real-LLM verification produced `thinking` events or only verified `tool_result`. -- Any provider-specific caveat, especially Anthropic thinking plus structured output behavior. - ---- - -## Acceptance Criteria - -- ReAct worker no longer owns PydanticAI message parsing internals. -- PydanticAI transcript extraction is reusable by other PydanticAI-based workers. -- Real ReAct workers pass capture-oriented model settings when the provider supports thinking/reasoning/logprobs. -- `ContextEventRepository.persist_turn` writes `tool_result` rows from `GenerationTurn.tool_results`. -- A tool-using ReAct run can be inspected through `GET /api/runs/{run_id}` and shows non-zero `tool_result` events. -- Thinking blocks are persisted as `thinking` events when the provider returns PydanticAI `ThinkingPart` objects. - diff --git a/docs/superpowers/plans/2026-04-27-react-worker-failure-context-capture.md b/docs/superpowers/plans/2026-04-27-react-worker-failure-context-capture.md deleted file mode 100644 index e730dc6f5..000000000 --- a/docs/superpowers/plans/2026-04-27-react-worker-failure-context-capture.md +++ /dev/null @@ -1,650 +0,0 @@ -# ReAct Worker Failure Context Capture Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Preserve partial PydanticAI ReAct transcript history when `agent.iter(...)` raises before `ReActWorker._run_agent()` reaches its normal post-run transcript extraction. - -**Architecture:** Keep runtime persistence ownership in `worker_execute_fn()`: workers yield `GenerationTurn`, runtime persists `RunContextEvent`. Add an incremental/cursor-based extraction API to `PydanticAITranscriptAdapter` so `ReActWorker` can yield completed turns during normal iteration and flush any remaining partial turn in an exception path before re-raising. This keeps failure semantics intact while eliminating the current zero-context failure gap for failed ReAct/CLI child workers. - -**Tech Stack:** Python, PydanticAI `Agent.iter`, `GenerationTurn`, `PydanticAITranscriptAdapter`, `ContextEventRepository`, pytest. - ---- - -## Root Cause - -Current `ReActWorker._run_agent()` only converts PydanticAI messages into `GenerationTurn`s after the `agent.iter(...)` context exits normally: - -```python -async with agent.iter(...) as run: - async for _node in run: - ... - -turns = PydanticAITranscriptAdapter().build_turns(run.ctx.state.message_history) -for turn in turns: - yield turn -``` - -If PydanticAI raises inside `async for _node in run`, control jumps out of `_run_agent()` before `build_turns(...)` runs. Then `worker_execute_fn()` catches the exception before it has received any turns to persist. That explains executions with an error stack but `0` `RunContextEvent` rows. - -The ResearchRubrics workflow CLI worker is affected because it subclasses `ReActWorker`: - -```python -async for turn in super().execute(task, context=context): - yield turn -``` - -Successful CLI runs use the shared adapter; failed CLI runs can still lose partial transcript history. - ---- - -## Desired Behavior - -- Successful ReAct runs keep capturing the same full transcript as today. -- Failed ReAct runs yield/persist every turn that can be reconstructed from `run.ctx.state.message_history` before re-raising the original exception. -- Runtime failure semantics do not change: `worker_execute_fn()` still returns the failure result and task status remains failed. -- Workers do not call `ContextEventRepository` directly. -- No duplicate context events are emitted when incremental extraction is called multiple times. -- Partial trailing responses can be flushed on final success or failure, but not emitted prematurely while a tool call may still receive a following `ToolReturnPart`. - ---- - -## File Map - -```text -ergon_builtins/ - ergon_builtins/ - common/ - llm_context/ - adapters/ - pydantic_ai.py # modify: replace post-run-only turn extraction with cursor API - workers/ - baselines/ - react_worker.py # modify: yield incremental turns and flush on exception - -tests/ - unit/ - builtins/ - common/ - test_transcript_adapters.py # modify: cursor extraction + trailing flush tests - workers/ - test_react_worker_contract.py # modify or add tests for failure transcript yield/re-raise -``` - -Do not modify `worker_execute_fn()` for this fix unless tests prove it cannot persist turns yielded immediately before an async generator raises. The existing `async for turn in worker.execute(...)` loop already persists each yielded turn before requesting the next one. - ---- - -## Closure And Removals - -This is not an additive second serialization path. Close the old behavior explicitly: - -- Remove `ReActWorker._run_agent()`'s post-run-only extraction pattern: - -```python -turns = PydanticAITranscriptAdapter().build_turns(run.ctx.state.message_history) -for turn in turns: - yield turn -``` - -Replace it with cursor extraction during the loop plus final/failure flush. - -- Do not add a new repository or direct DB writer for failure capture. `ContextEventRepository` remains the only `GenerationTurn` -> `RunContextEvent` serializer, and it remains called by `worker_execute_fn()`. -- Do not restore the old core PydanticAI serializers removed in the previous refactor: `ergon_core/core/persistence/context/assembly.py` and `ergon_core/core/providers/generation/pydantic_ai_format.py`. -- Do not add any new `ergon_core` PydanticAI transcript code. All PydanticAI transcript extraction/replay stays in `ergon_builtins.common.llm_context.adapters.pydantic_ai`. -- Treat the cursor API as the runtime extraction surface. If a batch `build_turns(...)` helper remains for tests or protocol compatibility, implement it as a wrapper around the same cursor extraction logic, not as a second independent serializer. -- Update tests that assert the worker no longer owns parser helpers so they also assert `ReActWorker` does not call a post-run-only extraction helper directly. - -There is no separate old "turn serialization repository" to delete after the previous refactor. The durable serialization repository is still `ContextEventRepository`, and that should stay. The old thing to remove here is the worker's post-run-only transcript extraction path, because it is the failure gap. - ---- - -## Design - -Use a small cursor object in the PydanticAI adapter: - -```python -from pydantic import BaseModel - - -class TranscriptTurnCursor(BaseModel): - model_config = {"validate_assignment": True} - - emitted_turn_count: int = 0 -``` - -Make cursor extraction the runtime API: - -```python -class PydanticAITranscriptAdapter(...): - def build_new_turns( - self, - transcript: list[ModelMessage], - cursor: TranscriptTurnCursor, - *, - flush_pending: bool = False, - ) -> list[GenerationTurn]: - turns = _build_turns_from_transcript(transcript, flush_pending=flush_pending) - new_turns = turns[cursor.emitted_turn_count :] - cursor.emitted_turn_count = len(turns) - return new_turns -``` - -If `build_turns(...)` remains public because `TranscriptAdapter` currently declares it, it should delegate to the same internal implementation used by `build_new_turns(...)`. Do not keep two independent conversion implementations. - -Change current trailing-response behavior in `build_turns()` so it is explicit: - -```python -if pending_response is not None and flush_pending: - turns.append(_to_turn(pending_request_in, pending_response, tool_result_request=None)) -``` - -`flush_pending=False` is important during the live `agent.iter(...)` loop. It prevents emitting a tool-call response before the following `ModelRequest` has a chance to include the `ToolReturnPart`. On final success or failure, use `flush_pending=True` so partial model output is not lost. - -Update `ReActWorker._run_agent()`: - -```python -adapter = PydanticAITranscriptAdapter() -cursor = TranscriptTurnCursor() -run = None - -try: - async with agent.iter(...) as active_run: - run = active_run - async for _node in run: - node_count += 1 - - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=False, - ): - yield turn - - if node_count >= self.max_iterations: - logger.warning(...) - break -except Exception: - if run is not None: - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=True, - ): - yield turn - raise - -if run is not None: - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=True, - ): - yield turn -``` - -This is extraction-as-iterator in practice: the cursor marks what has already been yielded, and `build_new_turns(...)` can be called repeatedly as message history grows. - -Do not swallow exceptions. The final `raise` is required so `worker_execute_fn()` still records failure. - ---- - -## Task 1: Adapter Cursor API - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` -- Modify: `tests/unit/builtins/common/test_transcript_adapters.py` - -- [ ] **Step 1: Write failing test for no premature trailing response** - -Add to `tests/unit/builtins/common/test_transcript_adapters.py`: - -```python -from ergon_builtins.common.llm_context.adapters.pydantic_ai import TranscriptTurnCursor - - -def test_incremental_extraction_does_not_emit_pending_tool_call_response() -> None: - adapter = PydanticAITranscriptAdapter() - cursor = TranscriptTurnCursor() - transcript = [ - ModelRequest(parts=[UserPromptPart(content="search")]), - ModelResponse( - parts=[ - ToolCallPart( - tool_name="search", - tool_call_id="call-1", - args={"query": "ergon"}, - ) - ] - ), - ] - - assert adapter.build_new_turns(transcript, cursor, flush_pending=False) == [] - - flushed = adapter.build_new_turns(transcript, cursor, flush_pending=True) - assert len(flushed) == 1 - assert any(isinstance(part, ErgonToolCallPart) for part in flushed[0].response_parts) -``` - -- [ ] **Step 2: Write failing test for no duplicate new turns** - -Add: - -```python -def test_incremental_extraction_tracks_emitted_turns() -> None: - adapter = PydanticAITranscriptAdapter() - cursor = TranscriptTurnCursor() - transcript = [ - ModelRequest(parts=[UserPromptPart(content="search")]), - ModelResponse( - parts=[ - ToolCallPart( - tool_name="search", - tool_call_id="call-1", - args={"query": "ergon"}, - ) - ] - ), - ModelRequest( - parts=[ - ToolReturnPart( - tool_name="search", - tool_call_id="call-1", - content={"result": "found"}, - ) - ] - ), - ] - - first = adapter.build_new_turns(transcript, cursor, flush_pending=False) - second = adapter.build_new_turns(transcript, cursor, flush_pending=False) - - assert len(first) == 1 - assert second == [] -``` - -- [ ] **Step 3: Run red tests** - -Run: - -```bash -uv run pytest tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: FAIL because `TranscriptTurnCursor` and `build_new_turns()` do not exist. - -- [ ] **Step 4: Replace batch extraction internals with cursor-backed extraction** - -In `pydantic_ai.py`, add: - -```python -from pydantic import BaseModel - - -class TranscriptTurnCursor(BaseModel): - model_config = {"validate_assignment": True} - - emitted_turn_count: int = 0 -``` - -Move the existing `build_turns(...)` body into a private helper that takes `flush_pending`: - -```python -def _build_turns_from_transcript( - transcript: list[ModelMessage], - *, - flush_pending: bool, -) -> list[GenerationTurn]: - ... -``` - -Keep `build_turns(...)` only as compatibility with the existing `TranscriptAdapter` protocol and any batch tests: - -```python -def build_turns( - self, - transcript: list[ModelMessage], - *, - flush_pending: bool = True, -) -> list[GenerationTurn]: - return _build_turns_from_transcript(transcript, flush_pending=flush_pending) -``` - -Do not call `build_turns(...)` from `ReActWorker`. Runtime extraction should use the cursor API only. - -Change trailing append: - -```python -if pending_response is not None: - turns.append(_to_turn(pending_request_in, pending_response, tool_result_request=None)) -``` - -to: - -```python -if pending_response is not None and flush_pending: - turns.append(_to_turn(pending_request_in, pending_response, tool_result_request=None)) -``` - -Add: - -```python -def build_new_turns( - self, - transcript: list[ModelMessage], - cursor: TranscriptTurnCursor, - *, - flush_pending: bool = False, -) -> list[GenerationTurn]: - turns = _build_turns_from_transcript(transcript, flush_pending=flush_pending) - new_turns = turns[cursor.emitted_turn_count :] - cursor.emitted_turn_count = len(turns) - return new_turns -``` - -After this change, there is one conversion implementation: `_build_turns_from_transcript(...)`. `build_turns(...)` and `build_new_turns(...)` are wrappers with different calling semantics. - -- [ ] **Step 5: Run green tests** - -Run: - -```bash -uv run pytest tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: PASS. - ---- - -## Task 2: ReActWorker Failure Flush - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Modify: `tests/unit/workers/test_react_worker_contract.py` - -- [ ] **Step 1: Write failing test for partial yield then re-raise** - -Add a fake `Agent` to `tests/unit/workers/test_react_worker_contract.py`: - -```python -from pydantic_ai.messages import ModelRequest, ModelResponse, TextPart, UserPromptPart - - -class _FakeRunState: - def __init__(self): - self.message_history = [ - ModelRequest(parts=[UserPromptPart(content="question")]), - ModelResponse(parts=[TextPart(content="partial answer")]), - ] - - -class _FakeRunContext: - def __init__(self): - self.state = _FakeRunState() - - -class _FailingAgentRun: - def __init__(self): - self.ctx = _FakeRunContext() - - def __aiter__(self): - return self - - async def __anext__(self): - raise RuntimeError("tool validation failed") - - -class _FailingAgentIter: - async def __aenter__(self): - return _FailingAgentRun() - - async def __aexit__(self, exc_type, exc, tb): - return False - - -class _FailingAgent: - def __init__(self, **kwargs): - pass - - def iter(self, *args, **kwargs): - return _FailingAgentIter() -``` - -Then add: - -```python -@pytest.mark.asyncio -async def test_react_worker_yields_partial_turn_before_reraising_agent_iter_failure(monkeypatch) -> None: - import ergon_builtins.workers.baselines.react_worker as react_worker - - monkeypatch.setattr(react_worker, "Agent", _FailingAgent) - monkeypatch.setattr( - react_worker, - "resolve_model_target", - lambda model: type( - "Resolved", - (), - {"model": "stub:constant", "capture_model_settings": None}, - )(), - ) - - worker = ReActWorker( - name="unit", - model=None, - task_id=UUID(int=1), - sandbox_id="test-sandbox", - tools=[], - system_prompt=None, - max_iterations=10, - ) - task = _minimal_task() - - turns = [] - with pytest.raises(RuntimeError, match="tool validation failed"): - async for turn in worker.execute(task, context=_minimal_context()): - turns.append(turn) - - assert len(turns) == 1 - assert any(part.content == "partial answer" for part in turns[0].response_parts) -``` - -Add small local helpers if this test file does not already have task/context fixtures: - -```python -from ergon_core.api.task_types import BenchmarkTask, EmptyTaskPayload -from ergon_core.api.worker_context import WorkerContext - - -def _minimal_task() -> BenchmarkTask: - return BenchmarkTask( - task_id=UUID(int=2), - task_slug="unit-task", - description="Unit task", - task_payload=EmptyTaskPayload(), - ) - - -def _minimal_context() -> WorkerContext: - return WorkerContext( - run_id=UUID(int=3), - definition_id=UUID(int=4), - task_id=UUID(int=2), - execution_id=UUID(int=5), - sandbox_id="test-sandbox", - node_id=UUID(int=6), - ) -``` - -- [ ] **Step 2: Run red test** - -Run: - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py::test_react_worker_yields_partial_turn_before_reraising_agent_iter_failure -q -``` - -Expected: FAIL because `_run_agent()` currently re-raises before yielding the partial transcript. - -- [ ] **Step 3: Implement failure flush in `_run_agent()`** - -Modify `ReActWorker._run_agent()`: - -```python -adapter = PydanticAITranscriptAdapter() -cursor = TranscriptTurnCursor() -run = None - -try: - async with agent.iter( - task_prompt, - model_settings=resolved.capture_model_settings, - message_history=self._seed_messages, - ) as active_run: - run = active_run - async for _node in run: - node_count += 1 - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=False, - ): - yield turn - if node_count >= self.max_iterations: - logger.warning(...) - break -except Exception: - if run is not None: - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=True, - ): - yield turn - raise - -if run is not None: - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=True, - ): - yield turn -``` - -Keep the existing warning text for `max_iterations`. - -- [ ] **Step 4: Run worker test** - -Run: - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py -q -``` - -Expected: PASS. - ---- - -## Task 3: Runtime Persistence Regression - -**Files:** -- Modify: `tests/unit/runtime/test_failure_error_json.py` or add `tests/unit/runtime/test_worker_execute_partial_failure_context.py` - -- [ ] **Step 1: Add runtime-level regression if feasible** - -Add a unit test around `worker_execute_fn()` with a fake registered worker whose `execute()` yields one `GenerationTurn` and then raises. Assert that `ContextEventRepository.persist_turn()` is called before the failure result is returned. - -If existing `worker_execute_fn()` setup makes this too fixture-heavy, keep the worker-level test from Task 2 as the required regression and add a short comment in the test explaining why it is sufficient: - -```python -# worker_execute_fn persists each yielded turn before requesting the next item -# from the async generator, so this test covers the failure-capture contract at -# the worker boundary without rebuilding Inngest context fixtures. -``` - -- [ ] **Step 2: Run focused runtime/worker tests** - -Run: - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py tests/unit/persistence/test_context_event_repository.py -q -``` - -Expected: PASS. - ---- - -## Task 4: Verification - -**Files:** -- No production edits. - -- [ ] **Step 1: Run affected capture suite** - -Run: - -```bash -uv run pytest \ - tests/unit/builtins/common/test_transcript_adapters.py \ - tests/unit/persistence/test_context_event_repository.py \ - tests/unit/workers/test_react_worker_contract.py \ - tests/unit/state/test_generation_turn_build.py \ - tests/unit/state/test_context_assembly.py \ - -q -``` - -Expected: PASS. - -- [ ] **Step 2: Run lint/compile** - -Run: - -```bash -uv run ruff check \ - ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py \ - ergon_builtins/ergon_builtins/workers/baselines/react_worker.py \ - tests/unit/builtins/common/test_transcript_adapters.py \ - tests/unit/workers/test_react_worker_contract.py -uv run slopcop \ - ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py \ - ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -uv run python -m compileall -q \ - ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py \ - ergon_builtins/ergon_builtins/workers/baselines/react_worker.py -``` - -Expected: PASS. - -- [ ] **Step 3: Optional real-run validation** - -Trigger a ReAct/CLI worker failure after the PydanticAI run has started, then inspect: - -```bash -RUN_ID=<run-id> python - <<'PY' -from uuid import UUID -from sqlmodel import select -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.context.models import RunContextEvent - -run_id = UUID(__import__("os").environ["RUN_ID"]) -with get_session() as session: - rows = session.exec( - select(RunContextEvent) - .where(RunContextEvent.run_id == run_id) - .order_by(RunContextEvent.task_execution_id, RunContextEvent.sequence) - ).all() - for row in rows: - print(row.task_execution_id, row.sequence, row.event_type) -PY -``` - -Expected: the failed child execution has at least the partial model request/response/tool-call events that existed before the exception. - ---- - -## Self-Review - -- Spec coverage: The plan addresses the observed gap where `agent.iter(...)` raises before post-run extraction, including CLI workers through `ReActWorker` inheritance. -- Iterator question: The plan proposes cursor-based incremental extraction from growing `message_history`, which is the appropriate iterator shape for PydanticAI histories. -- Persistence boundary: The plan keeps `ContextEventRepository` in the runtime path and does not make workers write directly to the DB. -- Failure semantics: The original exception is re-raised after partial turns are yielded. -- Known limitation: If `agent.iter(...)` fails during `__aenter__` before a `run` object exists, there is no PydanticAI `message_history` to flush. That case should still produce normal task failure metadata, but cannot produce transcript events. diff --git a/docs/superpowers/plans/2026-04-28-agent-tool-budget-harness.md b/docs/superpowers/plans/2026-04-28-agent-tool-budget-harness.md deleted file mode 100644 index c611f7314..000000000 --- a/docs/superpowers/plans/2026-04-28-agent-tool-budget-harness.md +++ /dev/null @@ -1,810 +0,0 @@ -# Agent Tool Budget Harness Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a simple, reusable tool-budget harness that prevents agent rollouts from looping indefinitely by counting `workflow` tool calls separately from all other tool calls and returning explicit budget-exhausted messages when either limit is reached. - -**Architecture:** Use Pydantic AI dependency injection. `ReActWorker` passes an optional deps object into `Agent.iter(...)`; tools that participate in the budget accept `RunContext[AgentToolBudgetDeps]` and call `ctx.deps.tool_budget.check(...)` before doing work. The budget system is generic and benchmark-agnostic: it knows only `workflow` vs `other`, not ResearchRubrics, Exa, or rubric-specific concepts. Reference: [Pydantic AI dependencies](https://pydantic.dev/docs/ai/core-concepts/dependencies/). - -**Tech Stack:** Python 3.13, pydantic-ai `RunContext`, Ergon `ReActWorker`, existing tool callables, pytest smoke checks, real-LLM rollout artifacts, Logfire. - ---- - -## Design - -The harness should enforce two counters per agent execution: - -```python -workflow_tool_calls <= max_workflow_tool_calls -other_tool_calls <= max_other_tool_calls -``` - -Initial defaults: - -```python -AgentToolBudgetPolicy( - max_workflow_tool_calls=12, - max_other_tool_calls=12, - warning_at_remaining=3, -) -``` - -The budget does not decide which benchmark is running and does not know about Exa. It only sees: - -- `workflow` calls: the workflow CLI tool. -- `other` calls: context-gathering and workspace-inspection tools other than `workflow`. -- `finalization` calls: tools that produce final output artifacts, such as report writing. These count for observability but are not blocked, because the budget should push the agent into finalization rather than prevent it. - -When a limit is reached, the tool returns a normal structured tool result: - -```python -AgentToolBudgetExhaustedResult( - status="TOOL_BUDGET_EXHAUSTED", - reason="workflow tool budget reached", - message="Stop calling workflow. Use currently visible context/resources and produce the best possible final output.", - budget_state={...}, -) -``` - -or: - -```python -AgentToolBudgetExhaustedResult( - status="TOOL_BUDGET_EXHAUSTED", - reason="non-workflow tool budget reached", - message="Stop calling tools. Produce the final answer from the context already gathered.", - budget_state={...}, -) -``` - -This is intentionally not a Python exception. The model gets a final chance to converge. The outer `max_iterations` guard still raises a real error if the agent keeps looping after exhausted tool responses. - -## Package Placement - -- Generic budget state: `ergon_builtins/ergon_builtins/workers/baselines/tool_budget.py` -- Base agent execution hook: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Budgeted workflow command tool: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Budgeted non-workflow tools for this rollout: `ergon_builtins/ergon_builtins/tools/research_rubrics_toolkit.py` and `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` -- Worker-specific budget policy wiring: `ergon_builtins/ergon_builtins/workers/research_rubrics/` -- Rollout diagnostics: `tests/real_llm/` - -## Added Files - -```text -ergon_builtins/ - ergon_builtins/ - workers/ - baselines/ - tool_budget.py -``` - -`tool_budget.py` owns the generic Pydantic models for budget policy, mutable per-execution budget state, deps passed into pydantic-ai, and helper logic for attaching warning text to tool results. - -## Edited Files - -```text -ergon_builtins/ - ergon_builtins/ - tools/ - graph_toolkit.py - research_rubrics_toolkit.py - workflow_cli_tool.py - workers/ - baselines/ - react_worker.py - research_rubrics/ - researcher_worker.py - workflow_cli_react_worker.py - -tests/ - real_llm/ - artifact_health.py - rollout.py -``` - -Edit responsibilities: - -- `react_worker.py`: add an optional deps hook, pass deps into `Agent.iter(...)`, and raise when `max_iterations` is hit. -- `workflow_cli_tool.py`: edit the existing workflow tool function path to support a ctx-taking budgeted mode for `workflow` calls. -- `research_rubrics_toolkit.py`: convert participating tools to ctx-taking functions and count context-gathering tools as `other`, while allowing report-writing as `finalization`. -- `graph_toolkit.py`: convert graph/resource tools to ctx-taking functions and count them as `other`. -- `researcher_worker.py`: provide generic budget deps to `ReActWorker` and steer the prompt toward quick convergence. -- `workflow_cli_react_worker.py`: provide generic budget deps, use budgeted workflow tool mode, and steer the prompt toward deliberate workflow use and subagent coordination. -- `artifact_health.py`: derive `workflow_tool_calls`, `other_tool_calls`, `budget_exhausted`, and `missing_final_report` from existing rollout artifacts. -- `rollout.py`: include those derived counters in `report.md`. - -## Deleted Files - -```text -(none) -``` - -## Optional Later Files - -If other benchmarks start showing the same loop behavior, apply the same `RunContext[AgentToolBudgetDeps]` pattern to their toolkits: - -```text -ergon_builtins/ - ergon_builtins/ - benchmarks/ - gdpeval/ - toolkit.py - minif2f/ - toolkit.py - swebench_verified/ - toolkit.py -``` - ---- - -## Task 1: Add Generic Tool Budget State - -**Files:** -- Create: `ergon_builtins/ergon_builtins/workers/baselines/tool_budget.py` - -- [ ] **Step 1: Create generic budget types** - -Create `tool_budget.py`: - -```python -from __future__ import annotations - -from typing import Any, Literal - -from pydantic import BaseModel, Field - -ToolBudgetKind = Literal["workflow", "other", "finalization"] -ToolBudgetExhaustedStatus = Literal["TOOL_BUDGET_EXHAUSTED"] - - -class AgentToolBudgetExhaustedResult(BaseModel): - status: ToolBudgetExhaustedStatus = "TOOL_BUDGET_EXHAUSTED" - reason: str - message: str - budget_state: dict[str, Any] # slopcop: ignore[no-typing-any] - - -class AgentToolBudgetPolicy(BaseModel): - model_config = {"frozen": True} - - max_workflow_tool_calls: int = 12 - max_other_tool_calls: int = 12 - warning_at_remaining: int = 3 - - -class AgentToolBudgetDecision(BaseModel): - model_config = {"frozen": True} - - allowed: bool - warning: str | None = None - exhausted: AgentToolBudgetExhaustedResult | None = None - - -class AgentToolBudgetState(BaseModel): - policy: AgentToolBudgetPolicy = Field(default_factory=AgentToolBudgetPolicy) - workflow_tool_calls: int = 0 - other_tool_calls: int = 0 - finalization_tool_calls: int = 0 - calls_by_tool: dict[str, int] = Field(default_factory=dict) - - def check(self, tool_name: str, kind: ToolBudgetKind) -> AgentToolBudgetDecision: - self.calls_by_tool[tool_name] = self.calls_by_tool.get(tool_name, 0) + 1 - - if kind == "workflow": - self.workflow_tool_calls += 1 - if self.workflow_tool_calls > self.policy.max_workflow_tool_calls: - return AgentToolBudgetDecision( - allowed=False, - exhausted=self.exhausted_result("workflow tool budget reached"), - ) - remaining = self.policy.max_workflow_tool_calls - self.workflow_tool_calls - elif kind == "finalization": - self.finalization_tool_calls += 1 - return AgentToolBudgetDecision(allowed=True) - else: - self.other_tool_calls += 1 - if self.other_tool_calls > self.policy.max_other_tool_calls: - return AgentToolBudgetDecision( - allowed=False, - exhausted=self.exhausted_result("non-workflow tool budget reached"), - ) - remaining = self.policy.max_other_tool_calls - self.other_tool_calls - - if remaining <= self.policy.warning_at_remaining: - return AgentToolBudgetDecision( - allowed=True, - warning=( - f"TOOL_BUDGET_WARNING: {remaining} {kind} tool calls remain. " - "Converge now using the context already gathered." - ), - ) - return AgentToolBudgetDecision(allowed=True) - - def snapshot(self) -> dict[str, Any]: # slopcop: ignore[no-typing-any] - return { - "workflow_tool_calls": self.workflow_tool_calls, - "max_workflow_tool_calls": self.policy.max_workflow_tool_calls, - "other_tool_calls": self.other_tool_calls, - "max_other_tool_calls": self.policy.max_other_tool_calls, - "finalization_tool_calls": self.finalization_tool_calls, - "calls_by_tool": dict(sorted(self.calls_by_tool.items())), - } - - def exhausted_result(self, reason: str) -> AgentToolBudgetExhaustedResult: - return AgentToolBudgetExhaustedResult( - reason=reason, - message=( - "Stop calling tools in this category. Use the context/resources already " - "available and produce the best possible final output. If the output is " - "incomplete, state what context or resource was missing." - ), - budget_state=self.snapshot(), - ) - - -class AgentToolBudgetDeps(BaseModel): - tool_budget: AgentToolBudgetState - - -def with_budget_warning(result: Any, warning: str | None) -> Any: # slopcop: ignore[no-typing-any] - if warning is None: - return result - if isinstance(result, str): - return f"{result}\n\n{warning}" - if isinstance(result, dict): - updated = dict(result) - updated["tool_budget_warning"] = warning - return updated - return result -``` - -- [ ] **Step 2: Run import smoke check** - -Run: - -```bash -uv run python - <<'PY' -from ergon_builtins.workers.baselines.tool_budget import ( - AgentToolBudgetDeps, - AgentToolBudgetPolicy, - AgentToolBudgetState, -) - -state = AgentToolBudgetState( - policy=AgentToolBudgetPolicy(max_workflow_tool_calls=1, max_other_tool_calls=2), -) -deps = AgentToolBudgetDeps(tool_budget=state) -print(deps.tool_budget.check("workflow", "workflow").allowed) -print(deps.tool_budget.check("workflow", "workflow").allowed) -print(deps.tool_budget.snapshot()) -PY -``` - -Expected: first line `True`, second line `False`, then a snapshot dictionary. - ---- - -## Task 2: Pass Deps Through ReActWorker - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` - -- [ ] **Step 1: Add a deps hook** - -Add to `ReActWorker`: - -```python - def build_agent_deps(self, context: WorkerContext) -> Any | None: # slopcop: ignore[no-typing-any] - return None -``` - -- [ ] **Step 2: Pass context into `_run_agent`** - -Change: - -```python -async for turn in self._run_agent(task): -``` - -to: - -```python -async for turn in self._run_agent(task, context): -``` - -Change `_run_agent` signature: - -```python - async def _run_agent( - self, - task: BenchmarkTask, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: -``` - -- [ ] **Step 3: Pass deps to pydantic-ai** - -Before `Agent(...)`: - -```python - agent_deps = self.build_agent_deps(context) - deps_type = type(agent_deps) if agent_deps is not None else None -``` - -Change the agent construction to include: - -```python - deps_type=deps_type, -``` - -Change `agent.iter(...)` to include: - -```python - deps=agent_deps, -``` - -- [ ] **Step 4: Make max-iteration exhaustion visible** - -Replace the current `break` on `max_iterations` with: - -```python - for turn in adapter.build_new_turns( - run.ctx.state.message_history, - cursor, - flush_pending=True, - ): - yield turn - raise RuntimeError( - f"ReActWorker exceeded max_iterations={self.max_iterations}" - ) -``` - -- [ ] **Step 5: Run existing focused tests** - -Run: - -```bash -uv run pytest tests/unit/workers/test_react_worker_contract.py tests/unit/builtins/common/test_transcript_adapters.py -q -``` - -Expected: PASS. - ---- - -## Task 3: Budget the Workflow Tool - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Existing test: `tests/unit/state/test_workflow_cli_tool.py` - -- [ ] **Step 1: Add ctx-aware mode** - -Import: - -```python -from pydantic_ai import RunContext -from ergon_builtins.workers.baselines.tool_budget import ( - AgentToolBudgetDeps, - AgentToolBudgetExhaustedResult, - with_budget_warning, -) -``` - -Add parameter to `make_workflow_cli_tool`: - -```python - budgeted: bool = False, -``` - -Edit the existing function body directly. Do not add a separate wrapper around workflow execution. Because pydantic-ai needs a clear callable signature, use two function definitions inside `make_workflow_cli_tool`: one ctx-taking definition for `budgeted=True`, and the existing no-ctx definition for `budgeted=False`. - -```python - if budgeted: - async def workflow( - ctx: RunContext[AgentToolBudgetDeps], - command: str, - ) -> str | AgentToolBudgetExhaustedResult: - decision = ctx.deps.tool_budget.check("workflow", "workflow") - if not decision.allowed: - assert decision.exhausted is not None - return decision.exhausted - - if worker_context.node_id is None: - raise ValueError("workflow tool requires WorkerContext.node_id") - - output = await asyncio.to_thread( - execute_command, - command, - context=WorkflowCommandContext( - run_id=worker_context.run_id, - node_id=worker_context.node_id, - execution_id=worker_context.execution_id, - sandbox_task_key=sandbox_task_key, - benchmark_type=benchmark_type, - ), - session_factory=session_factory, - service=service_factory(), - ) - if output.exit_code != 0: - detail = output.stderr or output.stdout - result = f"workflow exited {output.exit_code}: {detail}".strip() - elif output.stderr: - result = f"{output.stdout}\n\nstderr:\n{output.stderr}".strip() - else: - result = output.stdout - return with_budget_warning(result, decision.warning) - - return workflow -``` - -Keep the existing no-ctx `workflow(command: str)` function as the `budgeted=False` branch: - -```python - async def workflow(command: str) -> str: - if worker_context.node_id is None: - raise ValueError("workflow tool requires WorkerContext.node_id") - - output = await asyncio.to_thread( - execute_command, - command, - context=WorkflowCommandContext( - run_id=worker_context.run_id, - node_id=worker_context.node_id, - execution_id=worker_context.execution_id, - sandbox_task_key=sandbox_task_key, - benchmark_type=benchmark_type, - ), - session_factory=session_factory, - service=service_factory(), - ) - if output.exit_code != 0: - detail = output.stderr or output.stdout - return f"workflow exited {output.exit_code}: {detail}".strip() - if output.stderr: - return f"{output.stdout}\n\nstderr:\n{output.stderr}".strip() - return output.stdout - - return workflow -``` - -- [ ] **Step 2: Preserve existing behavior** - -Run: - -```bash -uv run pytest tests/unit/state/test_workflow_cli_tool.py -q -``` - -Expected: PASS. Existing tests use `budgeted=False`. - ---- - -## Task 4: Budget Other Tools Used by This Harness - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/tools/research_rubrics_toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` - -- [ ] **Step 1: Convert ResearchRubrics tools to ctx-taking functions** - -In `research_rubrics_toolkit.py`, import: - -```python -from pydantic_ai import RunContext -from ergon_builtins.workers.baselines.tool_budget import ( - AgentToolBudgetDeps, - AgentToolBudgetExhaustedResult, - with_budget_warning, -) -``` - -For each tool function, add `ctx` as the first arg: - -```python -ctx: RunContext[AgentToolBudgetDeps], -``` - -At the top of each context-gathering tool: - -```python -decision = ctx.deps.tool_budget.check("<actual_tool_name>", "other") -if not decision.allowed: - assert decision.exhausted is not None - return decision.exhausted -``` - -For final-output tools such as `write_report_draft` and `edit_report_draft`, use: - -```python -decision = ctx.deps.tool_budget.check("<actual_tool_name>", "finalization") -``` - -Do not block finalization tools after `other` is exhausted. The budget exists to force convergence into these tools. - -Use the actual function/tool name for each function so `calls_by_tool` remains useful in artifacts. - -After the existing result is produced: - -```python -return cast(<ResponseType> | AgentToolBudgetExhaustedResult, with_budget_warning(resp, decision.warning)) -``` - -For response types that are Pydantic models, returning `AgentToolBudgetExhaustedResult` on exhaustion is acceptable because the tool result is serialized back to the model. Keep type annotations broad enough, for example: - -```python -) -> SearchResponse | AgentToolBudgetExhaustedResult: -``` - -Change each `Tool(..., takes_ctx=False)` to: - -```python -Tool(function=..., takes_ctx=True) -``` - -- [ ] **Step 2: Convert graph/resource tools to ctx-taking functions** - -In `graph_toolkit.py`, apply the same pattern: - -```python -decision = ctx.deps.tool_budget.check("list_child_resources", "other") -if not decision.allowed: - assert decision.exhausted is not None - return decision.exhausted -``` - -Update all graph tools to `takes_ctx=True`. - -- [ ] **Step 3: Run import smoke checks** - -Run: - -```bash -uv run python - <<'PY' -from ergon_builtins.tools.research_rubrics_toolkit import ResearchRubricsToolkit -from ergon_builtins.tools.graph_toolkit import ResearchGraphToolkit -print(ResearchRubricsToolkit) -print(ResearchGraphToolkit) -PY -``` - -Expected: imports cleanly. - ---- - -## Task 5: Wire Budget Deps Into Current ResearchRubrics Workers - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` - -- [ ] **Step 1: Add policy imports** - -In both workers: - -```python -from ergon_builtins.workers.baselines.tool_budget import ( - AgentToolBudgetDeps, - AgentToolBudgetPolicy, - AgentToolBudgetState, -) -``` - -- [ ] **Step 2: Add a shared policy** - -Use the same generic policy in both files: - -```python -_TOOL_BUDGET_POLICY = AgentToolBudgetPolicy( - max_workflow_tool_calls=12, - max_other_tool_calls=12, - warning_at_remaining=3, -) -``` - -- [ ] **Step 3: Create deps per execution** - -In each `execute(...)`, before calling `super().execute(...)`: - -```python -self._agent_deps = AgentToolBudgetDeps( - AgentToolBudgetState(_TOOL_BUDGET_POLICY), -) -``` - -Add method: - -```python -def build_agent_deps(self, context: WorkerContext) -> AgentToolBudgetDeps: - return self._agent_deps -``` - -These worker instances are currently execution-scoped. If that changes later, move deps creation into a base-class execution context instead of storing on `self`. - -- [ ] **Step 4: Use budgeted workflow tool in manager** - -In `workflow_cli_react_worker.py`, change: - -```python -workflow_tool = make_workflow_cli_tool(...) -``` - -to: - -```python -workflow_tool = make_workflow_cli_tool(..., budgeted=True) -``` - -- [ ] **Step 5: Tighten prompts, but keep them generic** - -Researcher prompt: - -```text -You have a limited non-workflow tool budget. Gather enough context, then stop using tools and write final_output/report.md. If any tool returns TOOL_BUDGET_WARNING or TOOL_BUDGET_EXHAUSTED, immediately produce the best possible final report from the context already gathered. -``` - -Manager prompt: - -```text -For multi-step work, divide and conquer with focused subagents to manage context. Workflow calls are limited, so inspect deliberately, create focused children, avoid duplicate research, and converge after child resources are visible. If any tool returns TOOL_BUDGET_WARNING or TOOL_BUDGET_EXHAUSTED, stop polling/searching and produce the best possible final output from current context/resources. -``` - -- [ ] **Step 6: Run focused worker import** - -Run: - -```bash -uv run python - <<'PY' -from ergon_builtins.workers.research_rubrics.researcher_worker import ResearchRubricsResearcherWorker -from ergon_builtins.workers.research_rubrics.workflow_cli_react_worker import ResearchRubricsWorkflowCliReActWorker -print(ResearchRubricsResearcherWorker.type_slug) -print(ResearchRubricsWorkflowCliReActWorker.type_slug) -PY -``` - -Expected: prints both type slugs. - ---- - -## Task 6: Add Lightweight Rollout Reporting - -**Files:** -- Modify: `tests/real_llm/artifact_health.py` -- Modify: `tests/real_llm/rollout.py` - -- [ ] **Step 1: Count budget signals from existing events** - -In `artifact_health.py`, derive: - -```python -workflow_tool_calls -other_tool_calls -budget_exhausted -missing_final_report -``` - -Implementation rule: - -- If `tool_name == "workflow"`, increment `workflow_tool_calls`. -- Else if event type is `tool_call`, increment `other_tool_calls`. -- If any event payload has `status == "TOOL_BUDGET_EXHAUSTED"`, set `budget_exhausted=True`. -- If no resource path is `final_output/report.md`, set `missing_final_report=True`. - -- [ ] **Step 2: Show counters in rollout report** - -In `rollout.py`, add lines: - -```python -f"- workflow tool calls: {health.workflow_tool_calls}", -f"- other tool calls: {health.other_tool_calls}", -f"- budget exhausted: {health.budget_exhausted}", -f"- missing final report: {health.missing_final_report}", -``` - -- [ ] **Step 3: Run collection smoke** - -Run: - -```bash -uv run pytest tests/real_llm -q --collect-only -``` - -Expected: collection succeeds. - ---- - -## Task 7: Verify With One Real Sample - -**Files:** -- No new source files. - -- [ ] **Step 1: Run focused checks** - -Run: - -```bash -uv run pytest \ - tests/unit/state/test_workflow_cli_tool.py \ - tests/unit/workers/test_react_worker_contract.py \ - tests/unit/builtins/common/test_transcript_adapters.py \ - -q -``` - -Expected: PASS. - -- [ ] **Step 2: Run lint on changed files** - -Run: - -```bash -uv run ruff check \ - ergon_builtins/ergon_builtins/workers/baselines/tool_budget.py \ - ergon_builtins/ergon_builtins/workers/baselines/react_worker.py \ - ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py \ - ergon_builtins/ergon_builtins/tools/research_rubrics_toolkit.py \ - ergon_builtins/ergon_builtins/tools/graph_toolkit.py \ - ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py \ - ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py \ - tests/real_llm/artifact_health.py \ - tests/real_llm/rollout.py -``` - -Expected: `All checks passed!` - -- [ ] **Step 3: Rebuild and run one sample** - -Run: - -```bash -POSTGRES_PASSWORD=ergon_dev \ -TEST_HARNESS_SECRET=real-llm-secret \ -ENABLE_TEST_HARNESS=1 \ -ENABLE_SMOKE_FIXTURES=0 \ -ERGON_STARTUP_PLUGINS= \ -ERGON_LOGFIRE_PYDANTIC_AI=1 \ -ERGON_LOGFIRE_SERVICE_NAME=ergon-builtins \ -ERGON_LOGFIRE_ENVIRONMENT=real-llm \ -docker compose build api -``` - -Then: - -```bash -POSTGRES_PASSWORD=ergon_dev \ -TEST_HARNESS_SECRET=real-llm-secret \ -ENABLE_TEST_HARNESS=1 \ -ENABLE_SMOKE_FIXTURES=0 \ -ERGON_STARTUP_PLUGINS= \ -ERGON_LOGFIRE_PYDANTIC_AI=1 \ -ERGON_LOGFIRE_SERVICE_NAME=ergon-builtins \ -ERGON_LOGFIRE_ENVIRONMENT=real-llm \ -docker compose up -d --no-build --force-recreate --wait api -``` - -Then: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=1 \ -ERGON_REAL_LLM_BUDGET_USD=5 \ -TEST_HARNESS_SECRET=real-llm-secret \ -ENABLE_TEST_HARNESS=1 \ -ENABLE_SMOKE_FIXTURES=0 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -q -s --assume-stack-up -``` - -Expected improvement: - -- no silent runaway loop. -- report shows `workflow tool calls <= 12`, or budget exhaustion is visible. -- report shows `other tool calls <= 12`, or budget exhaustion is visible. -- if the run fails, it fails with persisted transcript/error context that explains whether the budget was exhausted. - ---- - -## Notes - -- This is intentionally simpler than per-tool caps. No Exa-specific budget, no rubric-specific budget, no child-poll-specific budget. -- This still supports better prompt steering, but prompt steering is advisory. The two counters are enforcement. -- We should not add broad unit tests for every tool. Existing workflow tests, import smoke checks, lint, and the one-sample real rollout are enough for this change. -- Do not commit unless explicitly asked. diff --git a/docs/superpowers/plans/2026-04-28-context-part-chunk-stream.md b/docs/superpowers/plans/2026-04-28-context-part-chunk-stream.md deleted file mode 100644 index d4f00e7af..000000000 --- a/docs/superpowers/plans/2026-04-28-context-part-chunk-stream.md +++ /dev/null @@ -1,1359 +0,0 @@ -# Context Part Chunk Stream Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the parallel `GenerationTurn` and context-event payload model with one canonical context-part stream emitted by workers and enriched by core before persistence. - -**Architecture:** Define a single discriminated `ContextPart` union for things that appear in an LLM context/action stream: system prompts, user messages, assistant text, tool calls, tool results, and thinking. Workers yield `ContextPartChunk` values containing a `part` plus optional token metadata; core normalizes and enriches those chunks into persisted `RunContextEvent` rows with sequence, turn id, timestamps, worker key, and run/execution ids. Keep database rows flat enough for SQLModel/JSONB, but make API, dashboard, replay, and RL consumers use typed chunk/log schemas instead of duplicate payload unions. This is a clean-break migration: old `*Payload`, `GenerationTurn`, request/response part aliases, and old discriminator names must be gone by the final task. - -**Tech Stack:** Python 3.13, Pydantic v2 discriminated unions, SQLModel JSON columns, pytest, existing Ergon worker/runtime/persistence packages. - ---- - -## Source Of Truth - -The canonical worker-facing stream type should live in `ergon_core.core.generation` or a renamed module such as `ergon_core.core.context_stream`. To avoid a large import churn in the first slice, start in `ergon_core.core.generation`. - -Use these names: - -```python -ContextPart -ContextPartChunk -ContextPartChunkLog -WorkerYield -``` - -`ContextPart` is the only union for LLM context/action parts. - -`ContextPartChunk` is the de facto worker generator type. - -`ContextPartChunkLog` is the core-enriched durable event shape. It is not the database ORM model; it is the typed payload/envelope used when projecting a stored `RunContextEvent`. - -`RunContextEvent` remains the SQLModel row with JSON storage and relational ids. - ---- - -## Change Tree - -```text -ergon/ - ergon_core/ - ergon_core/ - core/ - generation.py # modify: canonical ContextPart/ContextPartChunk/ContextPartChunkLog - api/ - schemas.py # modify: typed REST context event payloads - runs.py # modify: project parsed chunk logs - dashboard/ - event_contracts.py # modify: dashboard context event payload uses chunk log - emitter.py # modify: emit parsed chunk logs - persistence/ - context/ - event_payloads.py # modify/delete duplicate payload union; no final old aliases - models.py # modify: validate JSON as ContextPartChunkLog - repository.py # modify: add persist_chunk enrichment; later delete persist_turn - rl/ - extraction.py # modify: consume chunk-log parts - runtime/ - services/ - task_execution_service.py # modify: persist worker chunks instead of turns - test_support/ - smoke_fixtures/ - smoke_base/ - leaf_base.py # modify: yield ContextPartChunk - recursive.py # modify: yield ContextPartChunk - worker_base.py # modify: yield ContextPartChunk - tests/ - unit/ - architecture/ - test_core_schema_sources.py # modify: guard single context part union - test_model_field_descriptions.py # modify: check chunk-log field descriptions - builtins/ - common/ - test_transcript_adapters.py # modify: assert chunk extraction/replay - dashboard/ - test_event_contract_types.py # modify: assert typed chunk-log dashboard payload - persistence/ - test_context_event_repository.py # modify: persist_chunk tests - state/ - test_context_part_stream.py # add: canonical part/chunk serialization tests - test_context_assembly.py # modify: replay from ContextPartChunkLog - test_generation_turn_build.py # modify/delete after GenerationTurn compatibility removal - workers/ - test_react_worker_contract.py # modify: worker yields chunks - ergon_builtins/ - ergon_builtins/ - common/ - llm_context/ - adapters/ - pydantic_ai.py # modify: build_chunks/build_new_chunks and replay chunk logs - workers/ - baselines/ - react_worker.py # modify: inspect ContextPartChunkLog.part - training_stub_worker.py # modify: yield ContextPartChunk - research_rubrics/ - researcher_worker.py # modify if still yielding GenerationTurn - workflow_cli_react_worker.py # modify if still yielding GenerationTurn -``` - ---- - -## File Structure - -**Modify:** -- `ergon_core/ergon_core/core/generation.py` — replace request/response-specific part model as the canonical context stream model while preserving temporary aliases during migration. -- `ergon_core/ergon_core/core/persistence/context/event_payloads.py` — replace the duplicate payload union with canonical context-event type exports only; do not keep old payload aliases in the final state. -- `ergon_core/ergon_core/core/persistence/context/models.py` — validate stored JSON as `ContextPartChunkLog` or the log payload shape. -- `ergon_core/ergon_core/core/persistence/context/repository.py` — replace `persist_turn()` decomposition with `persist_chunk()` enrichment; keep a temporary `persist_turn()` adapter if needed for staged migration. -- `ergon_core/ergon_core/core/api/schemas.py` — type REST context-event DTOs with `ContextPartChunkLog` instead of `dict[str, Any]`. -- `ergon_core/ergon_core/core/api/runs.py` — project stored context events through typed log validation. -- `ergon_core/ergon_core/core/dashboard/event_contracts.py` — use the same typed log schema as REST for context events. -- `ergon_core/ergon_core/core/dashboard/emitter.py` — emit typed enriched context logs. -- `ergon_core/ergon_core/core/rl/extraction.py` — read `event.part` instead of payload-specific classes. -- `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` — convert PydanticAI messages into `ContextPartChunk` streams and replay logs back into PydanticAI messages. -- `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` — consume the new typed context stream. -- `ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py` — yield chunks instead of `GenerationTurn`. -- `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/*.py` — yield chunks in smoke workers. - -**Tests:** -- `tests/unit/state/test_context_part_stream.py` — new focused tests for canonical union and chunk serialization. -- `tests/unit/persistence/test_context_event_repository.py` — rewrite around `persist_chunk()`. -- `tests/unit/builtins/common/test_transcript_adapters.py` — update PydanticAI adapter tests to assert chunk/log behavior. -- `tests/unit/state/test_context_assembly.py` — update replay tests around `ContextPartChunkLog`. -- `tests/unit/architecture/test_core_schema_sources.py` — add architecture guard against reintroducing duplicate context payload unions. -- Existing focused tests: `tests/unit/state/test_generation_turn_build.py`, `tests/unit/workers/test_react_worker_contract.py`, `tests/unit/dashboard/test_event_contract_types.py`, `tests/unit/architecture/test_model_field_descriptions.py`. - ---- - -### Task 1: Introduce Canonical Context Parts - -**Files:** -- Modify: `ergon_core/ergon_core/core/generation.py` -- Create: `tests/unit/state/test_context_part_stream.py` - -- [ ] **Step 1: Write failing tests for the canonical part union** - -Create `tests/unit/state/test_context_part_stream.py` with: - -```python -from pydantic import TypeAdapter - -from ergon_core.core.generation import ( - AssistantTextPart, - ContextPart, - ContextPartChunk, - ContextPartChunkLog, - SystemPromptPart, - ThinkingPart, - TokenLogprob, - ToolCallPart, - ToolResultPart, - UserMessagePart, -) - - -def test_context_part_discriminates_all_part_kinds() -> None: - adapter = TypeAdapter(ContextPart) - - cases = [ - SystemPromptPart(content="sys"), - UserMessagePart(content="hi"), - AssistantTextPart(content="hello"), - ToolCallPart(tool_call_id="call-1", tool_name="search", args={"q": "x"}), - ToolResultPart(tool_call_id="call-1", tool_name="search", content="ok"), - ThinkingPart(content="reasoning"), - ] - - for part in cases: - dumped = part.model_dump(mode="json") - parsed = adapter.validate_python(dumped) - assert parsed == part - - -def test_context_part_chunk_wraps_part_with_optional_token_metadata() -> None: - chunk = ContextPartChunk( - part=AssistantTextPart(content="answer"), - token_ids=[1, 2], - logprobs=[TokenLogprob(token="answer", logprob=-0.1)], - ) - - dumped = chunk.model_dump(mode="json") - - assert dumped["part"]["part_kind"] == "assistant_text" - assert dumped["token_ids"] == [1, 2] - assert dumped["logprobs"][0]["token"] == "answer" - - -def test_context_part_chunk_log_adds_core_enrichment() -> None: - log = ContextPartChunkLog( - part=ThinkingPart(content="hmm"), - sequence=7, - worker_binding_key="researcher", - turn_id="turn-1", - token_ids=None, - logprobs=None, - ) - - dumped = log.model_dump(mode="json") - - assert dumped["part"]["part_kind"] == "thinking" - assert dumped["sequence"] == 7 - assert dumped["worker_binding_key"] == "researcher" - assert dumped["turn_id"] == "turn-1" -``` - -- [ ] **Step 2: Run the failing tests** - -Run: - -```bash -pytest tests/unit/state/test_context_part_stream.py -v -``` - -Expected: FAIL because `AssistantTextPart`, `UserMessagePart`, `ToolResultPart`, `ContextPartChunk`, and `ContextPartChunkLog` do not exist yet. - -- [ ] **Step 3: Implement canonical context stream types** - -Modify `ergon_core/ergon_core/core/generation.py` to define the canonical names. This task may keep request/response subset aliases only if needed to keep the next migration task small; those aliases must be deleted in Task 7 before the plan is complete. - -```python -"""Core model context-stream types. - -These types are used by worker APIs, transcript adapters, persistence, replay, -and RL extraction. Keep them in core so persistence can import them without -loading ``ergon_core.api``. -""" - -from datetime import datetime -from typing import Annotated, Any, Literal - -from ergon_core.core.json_types import JsonObject -from pydantic import BaseModel, Field - - -class TokenLogprob(BaseModel): - """Per-token log probability from the serving backend.""" - - model_config = {"frozen": True} - - token: str - logprob: float - top_logprobs: list[JsonObject] = Field(default_factory=list) - - -class SystemPromptPart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["system_prompt"] = "system_prompt" - content: str - - -class UserMessagePart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["user_message"] = "user_message" - content: str - - -class AssistantTextPart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["assistant_text"] = "assistant_text" - content: str - - -class ToolCallPart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["tool_call"] = "tool_call" - tool_name: str - tool_call_id: str - args: dict[str, Any] # slopcop: ignore[no-typing-any] - - -class ToolResultPart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["tool_result"] = "tool_result" - tool_call_id: str - tool_name: str - content: str - is_error: bool = False - - -class ThinkingPart(BaseModel): - model_config = {"frozen": True} - part_kind: Literal["thinking"] = "thinking" - content: str - - -ContextPart = Annotated[ - SystemPromptPart - | UserMessagePart - | AssistantTextPart - | ToolCallPart - | ToolResultPart - | ThinkingPart, - Field(discriminator="part_kind"), -] - - -class ContextPartChunk(BaseModel): - """One worker-emitted context/action stream item. - - Core adds run/execution/sequence/timing metadata before persistence. - """ - - model_config = {"frozen": True} - - part: ContextPart - token_ids: list[int] | None = None - logprobs: list[TokenLogprob] | None = None - - -class ContextPartChunkLog(ContextPartChunk): - """Core-enriched context stream item suitable for API/dashboard projection.""" - - sequence: int - worker_binding_key: str - turn_id: str | None = None - started_at: datetime | None = None - completed_at: datetime | None = None - policy_version: str | None = None - - -WorkerYield = ContextPartChunk - -# Temporary migration-only aliases. Task 7 must remove these before completion. -UserPromptPart = UserMessagePart -TextPart = AssistantTextPart -ToolReturnPart = ToolResultPart - -ModelRequestPart = Annotated[ - SystemPromptPart | UserMessagePart | ToolResultPart, - Field(discriminator="part_kind"), -] -ModelResponsePart = Annotated[ - AssistantTextPart | ToolCallPart | ThinkingPart, - Field(discriminator="part_kind"), -] - - -class GenerationTurn(BaseModel): - """Deprecated: use ContextPartChunk streams instead.""" - - model_config = {"frozen": True} - - messages_in: list[ModelRequestPart] = Field(default_factory=list) - response_parts: list[ModelResponsePart] = Field(default_factory=list) - tool_results: list[ToolResultPart] = Field(default_factory=list) - turn_token_ids: list[int] | None = None - turn_logprobs: list[TokenLogprob] | None = None - policy_version: str | None = None - started_at: datetime | None = None - completed_at: datetime | None = None -``` - -- [ ] **Step 4: Run the focused tests** - -Run: - -```bash -pytest tests/unit/state/test_context_part_stream.py -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run generation-related tests to expose compatibility fallout** - -Run: - -```bash -pytest tests/unit/state/test_generation_turn_build.py tests/unit/builtins/common/test_transcript_adapters.py -v -``` - -Expected: likely FAIL because existing tests assert old discriminator values such as `tool-call` and old constructor names such as `ToolReturnPart`. - ---- - -### Task 2: Replace Payload Union With Enriched Chunk Log - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/context/event_payloads.py` -- Modify: `ergon_core/ergon_core/core/persistence/context/models.py` -- Modify: `tests/unit/architecture/test_model_field_descriptions.py` - -- [ ] **Step 1: Write failing compatibility tests for typed log payload validation** - -Update or add tests that assert the context event row validates its JSON as `ContextPartChunkLog`: - -```python -from ergon_core.core.generation import AssistantTextPart, ContextPartChunkLog -from ergon_core.core.persistence.context.models import RunContextEvent - - -def test_run_context_event_parsed_payload_is_context_part_chunk_log() -> None: - log = ContextPartChunkLog( - part=AssistantTextPart(content="hello"), - sequence=3, - worker_binding_key="worker-a", - turn_id="turn-1", - ) - event = RunContextEvent( - run_id="00000000-0000-0000-0000-000000000001", - task_execution_id="00000000-0000-0000-0000-000000000002", - worker_binding_key="worker-a", - sequence=3, - event_type="assistant_text", - payload=log.model_dump(mode="json"), - ) - - parsed = event.parsed_payload() - - assert isinstance(parsed, ContextPartChunkLog) - assert parsed.part == AssistantTextPart(content="hello") -``` - -If UUID strings are not accepted by SQLModel in this test, use `uuid.UUID(...)` values instead. - -- [ ] **Step 2: Run the failing test** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py::test_run_context_event_parsed_payload_is_context_part_chunk_log -v -``` - -Expected: FAIL until `RunContextEvent.parsed_payload()` validates the new log shape. - -- [ ] **Step 3: Collapse `event_payloads.py` into canonical exports** - -Modify `ergon_core/ergon_core/core/persistence/context/event_payloads.py` so the canonical payload is `ContextPartChunkLog`. Do not define `SystemPromptPayload`, `UserMessagePayload`, `AssistantTextPayload`, `ToolCallPayload`, `ToolResultPayload`, or `ThinkingPayload`; callers must migrate to `ContextPartChunkLog.part` and the canonical part classes. - -```python -"""Typed context event payload exports. - -The canonical context payload is an enriched ContextPartChunkLog. Event-specific -payload classes were removed in favor of ContextPartChunkLog.part. -""" - -from typing import Literal - -from ergon_core.core.generation import ( - ContextPart, - ContextPartChunk, - ContextPartChunkLog, -) - -ContextEventType = Literal[ - "system_prompt", - "user_message", - "assistant_text", - "tool_call", - "tool_result", - "thinking", -] - -ContextEventPayload = ContextPartChunkLog -``` - -- [ ] **Step 4: Update `RunContextEvent` validation** - -Modify `ergon_core/ergon_core/core/persistence/context/models.py`: - -```python -from ergon_core.core.generation import ContextPartChunkLog -from pydantic import TypeAdapter - -_PAYLOAD_ADAPTER: TypeAdapter[ContextPartChunkLog] = TypeAdapter(ContextPartChunkLog) - - -class RunContextEvent(SQLModel, table=True): - ... - - def parsed_payload(self) -> ContextPartChunkLog: - return _PAYLOAD_ADAPTER.validate_python(self.payload) -``` - -Keep `event_type: str` and `payload: dict[str, Any]` on the SQLModel row because the database stores JSON and indexes `event_type`. - -- [ ] **Step 5: Replace field-description architecture tests** - -Update `tests/unit/architecture/test_model_field_descriptions.py` to check descriptions on `ContextPartChunkLog` if the project requires descriptions for public fields. Do not keep tests against the old payload classes once they are aliases. - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py tests/unit/architecture/test_model_field_descriptions.py -v -``` - -Expected: repository tests still fail until Task 3 replaces `persist_turn()` behavior. - ---- - -### Task 3: Persist Worker Chunks With Core Enrichment - -**Files:** -- Modify: `ergon_core/ergon_core/core/persistence/context/repository.py` -- Modify: `tests/unit/persistence/test_context_event_repository.py` - -- [ ] **Step 1: Write repository tests for `persist_chunk()`** - -Replace turn-oriented tests with chunk-oriented tests: - -```python -from uuid import uuid4 - -from ergon_core.core.generation import ( - AssistantTextPart, - ContextPartChunk, - ThinkingPart, - ToolCallPart, - ToolResultPart, - UserMessagePart, -) - - -async def test_persist_chunk_records_prompt_and_model_output_in_order(session): - repo = ContextEventRepository() - run_id = uuid4() - execution_id = uuid4() - - await repo.persist_chunk( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker-a", - chunk=ContextPartChunk(part=UserMessagePart(content="question")), - ) - await repo.persist_chunk( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker-a", - chunk=ContextPartChunk(part=ThinkingPart(content="think")), - ) - await repo.persist_chunk( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker-a", - chunk=ContextPartChunk(part=AssistantTextPart(content="answer")), - ) - - events = repo.get_for_execution(session, execution_id) - - assert [event.sequence for event in events] == [0, 1, 2] - assert [event.event_type for event in events] == [ - "user_message", - "thinking", - "assistant_text", - ] - assert events[1].parsed_payload().turn_id == events[2].parsed_payload().turn_id - - -async def test_persist_chunk_tool_result_closes_current_turn(session): - repo = ContextEventRepository() - run_id = uuid4() - execution_id = uuid4() - - await repo.persist_chunk( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker-a", - chunk=ContextPartChunk( - part=ToolCallPart(tool_call_id="call-1", tool_name="search", args={"q": "x"}) - ), - ) - await repo.persist_chunk( - session, - run_id=run_id, - execution_id=execution_id, - worker_binding_key="worker-a", - chunk=ContextPartChunk( - part=ToolResultPart(tool_call_id="call-1", tool_name="search", content="ok") - ), - ) - - events = repo.get_for_execution(session, execution_id) - - assert [event.event_type for event in events] == ["tool_call", "tool_result"] - assert events[0].parsed_payload().turn_id is not None - assert events[1].parsed_payload().turn_id is None -``` - -Adjust fixture names to match the existing `test_context_event_repository.py` session fixture. - -- [ ] **Step 2: Run repository tests to verify failure** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py -v -``` - -Expected: FAIL because `persist_chunk()` does not exist. - -- [ ] **Step 3: Implement event type derivation** - -In `ergon_core/ergon_core/core/persistence/context/repository.py`, add: - -```python -from ergon_core.core.generation import ( - AssistantTextPart, - ContextPartChunk, - ContextPartChunkLog, - SystemPromptPart, - ThinkingPart, - ToolCallPart, - ToolResultPart, - UserMessagePart, -) - - -def _event_type_for_part(part: ContextPart) -> str: - return part.part_kind -``` - -If type checkers object to `ContextPart` as an `Annotated` alias in the helper signature, use the explicit union type or accept `object` and narrow via `isinstance`. - -- [ ] **Step 4: Implement turn-id state machine** - -Add private state to the repository: - -```python -def __init__(self) -> None: - self._listeners: list[Callable[[RunContextEvent], Awaitable[None]]] = [] - self._sequence_counters: dict[UUID, int] = {} - self._active_turn_ids: dict[UUID, str] = {} -``` - -Add helpers: - -```python -def _turn_id_for_chunk(self, execution_id: UUID, chunk: ContextPartChunk) -> str | None: - part = chunk.part - if isinstance(part, (AssistantTextPart, ThinkingPart, ToolCallPart)): - turn_id = self._active_turn_ids.get(execution_id) - if turn_id is None: - turn_id = str(uuid4()) - self._active_turn_ids[execution_id] = turn_id - return turn_id - if isinstance(part, ToolResultPart): - self._active_turn_ids.pop(execution_id, None) - return None - if isinstance(part, (SystemPromptPart, UserMessagePart)): - return None - return None -``` - -This deliberately associates `thinking`, `assistant_text`, and `tool_call` chunks emitted contiguously with the same model-output turn. A following `tool_result` closes the active turn. - -- [ ] **Step 5: Implement `persist_chunk()`** - -Add: - -```python -async def persist_chunk( - self, - session: Session, - *, - run_id: UUID, - execution_id: UUID, - worker_binding_key: str, - chunk: ContextPartChunk, -) -> RunContextEvent: - seq = self._next_sequence(execution_id) - turn_id = self._turn_id_for_chunk(execution_id, chunk) - event_type = chunk.part.part_kind - now = datetime.now(UTC) - payload = ContextPartChunkLog( - part=chunk.part, - token_ids=chunk.token_ids, - logprobs=chunk.logprobs, - sequence=seq, - worker_binding_key=worker_binding_key, - turn_id=turn_id, - started_at=now, - completed_at=now, - ) - event = self._make_event( - run_id, - execution_id, - worker_binding_key, - seq, - payload, - started_at=payload.started_at, - completed_at=payload.completed_at, - policy_version=payload.policy_version, - ) - self._sequence_counters[execution_id] = seq + 1 - - session.add(event) - session.commit() - - for listener in self._listeners: - try: - await listener(event) - except Exception: # slopcop: ignore[no-broad-except] - logger.warning("Context event listener failed", exc_info=True) - - return event -``` - -Update `_make_event()` to accept `payload: ContextPartChunkLog` and store `payload.model_dump(mode="json")`. - -- [ ] **Step 6: Keep a temporary `persist_turn()` adapter** - -During migration only, keep `persist_turn()` by decomposing old `GenerationTurn` into chunks: - -```python -async def persist_turn(..., turn: GenerationTurn) -> list[RunContextEvent]: - events: list[RunContextEvent] = [] - for part in turn.messages_in: - events.append(await self.persist_chunk(..., chunk=ContextPartChunk(part=part))) - for part in turn.response_parts: - events.append( - await self.persist_chunk( - ..., - chunk=ContextPartChunk( - part=part, - token_ids=turn.turn_token_ids, - logprobs=turn.turn_logprobs, - ), - ) - ) - for part in turn.tool_results: - events.append(await self.persist_chunk(..., chunk=ContextPartChunk(part=part))) - return events -``` - -This keeps old workers running while the execution service migrates to chunks. - -- [ ] **Step 7: Run persistence tests** - -Run: - -```bash -pytest tests/unit/persistence/test_context_event_repository.py -v -``` - -Expected: PASS after updating any old assertions to inspect `event.parsed_payload().part`. - ---- - -### Task 4: Migrate PydanticAI Adapter To Chunk Streams - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` -- Modify: `tests/unit/builtins/common/test_transcript_adapters.py` -- Modify: `tests/unit/state/test_generation_turn_build.py` -- Modify: `tests/unit/state/test_context_assembly.py` - -- [ ] **Step 1: Write adapter tests for chunk extraction** - -Update `tests/unit/builtins/common/test_transcript_adapters.py` so PydanticAI transcript extraction returns chunks: - -```python -def test_text_and_thinking_are_context_part_chunks() -> None: - adapter = PydanticAITranscriptAdapter() - - chunks = adapter.build_chunks( - [ - ModelRequest(parts=[UserPromptPart(content="hard question")]), - ModelResponse( - parts=[ - ThinkingPart(content="let me reason"), - TextPart(content="answer"), - ] - ), - ] - ) - - assert [chunk.part.part_kind for chunk in chunks] == [ - "user_message", - "thinking", - "assistant_text", - ] -``` - -Add a tool-call/tool-result test: - -```python -def test_tool_call_and_return_become_context_part_chunks() -> None: - adapter = PydanticAITranscriptAdapter() - - chunks = adapter.build_chunks( - [ - ModelRequest(parts=[UserPromptPart(content="search")]), - ModelResponse( - parts=[ - ToolCallPart( - tool_name="search", - tool_call_id="call-1", - args={"query": "ergon"}, - ) - ] - ), - ModelRequest( - parts=[ - ToolReturnPart( - tool_name="search", - tool_call_id="call-1", - content={"result": "found"}, - ) - ] - ), - ] - ) - - assert [chunk.part.part_kind for chunk in chunks] == [ - "user_message", - "tool_call", - "tool_result", - ] -``` - -- [ ] **Step 2: Run adapter tests to verify failure** - -Run: - -```bash -pytest tests/unit/builtins/common/test_transcript_adapters.py -v -``` - -Expected: FAIL because `build_chunks()` does not exist. - -- [ ] **Step 3: Implement `build_chunks()` and `build_new_chunks()`** - -In `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py`, add methods parallel to the existing turn methods: - -```python -def build_chunks( - self, - transcript: list[ModelMessage], - *, - flush_pending: bool = True, -) -> list[ContextPartChunk]: - return _build_chunks_from_transcript(transcript, flush_pending=flush_pending) - - -def build_new_chunks( - self, - transcript: list[ModelMessage], - cursor: TranscriptTurnCursor, - *, - flush_pending: bool = False, -) -> list[ContextPartChunk]: - chunks = _build_chunks_from_transcript(transcript, flush_pending=flush_pending) - new_chunks = chunks[cursor.emitted_turn_count :] - cursor.emitted_turn_count = len(chunks) - return new_chunks -``` - -Rename `TranscriptTurnCursor.emitted_turn_count` to `emitted_chunk_count` only if the migration can update all callers in one task. Otherwise leave the field name temporarily and add a follow-up cleanup task. - -- [ ] **Step 4: Implement PydanticAI part conversion** - -Replace old `_extract_request_parts`, `_extract_response_parts`, and `_extract_tool_results` internals with chunk builders: - -```python -def _chunks_from_request(request: ModelRequest) -> list[ContextPartChunk]: - chunks: list[ContextPartChunk] = [] - for part in request.parts: - if isinstance(part, PydanticSystemPromptPart): - chunks.append(ContextPartChunk(part=SystemPromptPart(content=part.content))) - elif isinstance(part, PydanticUserPromptPart) and isinstance(part.content, str): - chunks.append(ContextPartChunk(part=UserMessagePart(content=part.content))) - elif isinstance(part, PydanticToolReturnPart): - chunks.append( - ContextPartChunk( - part=ToolResultPart( - tool_call_id=part.tool_call_id, - tool_name=part.tool_name, - content=_serialize_tool_content(part.content), - ) - ) - ) - return chunks - - -def _chunks_from_response(response: ModelResponse) -> list[ContextPartChunk]: - logprobs = extract_logprobs(response) - chunks: list[ContextPartChunk] = [] - for part in response.parts: - if isinstance(part, PydanticTextPart): - chunks.append( - ContextPartChunk(part=AssistantTextPart(content=part.content), logprobs=logprobs) - ) - logprobs = None - elif isinstance(part, PydanticToolCallPart): - chunks.append( - ContextPartChunk( - part=ToolCallPart( - tool_name=part.tool_name, - tool_call_id=part.tool_call_id, - args=part.args_as_dict(), - ), - logprobs=logprobs, - ) - ) - logprobs = None - elif isinstance(part, PydanticThinkingPart): - chunks.append( - ContextPartChunk(part=ThinkingPart(content=part.content), logprobs=logprobs) - ) - logprobs = None - return chunks -``` - -Only attach turn-level logprobs to the first model-output chunk. This matches the current persisted behavior where sibling events omit the shared token stream after the first model-output event. - -- [ ] **Step 5: Implement replay from chunk logs** - -Update `assemble_replay()` to consume `RunContextEvent.parsed_payload()` as `ContextPartChunkLog`, then switch on `log.part`. - -```python -payload = event.parsed_payload() -part = payload.part -``` - -Map: -- `SystemPromptPart` -> `PydanticSystemPromptPart` -- `UserMessagePart` -> `PydanticUserPromptPart` -- `ToolResultPart` -> `PydanticToolReturnPart` -- `ThinkingPart` -> `PydanticThinkingPart` -- `AssistantTextPart` -> `PydanticTextPart` -- `ToolCallPart` -> `PydanticToolCallPart` - -- [ ] **Step 6: Keep old adapter methods as wrappers** - -Keep `build_turns()` and `build_new_turns()` temporarily by grouping chunks into a deprecated `GenerationTurn` only if old callers still exist at this point. Add comments marking them as migration-only. Task 7 must delete these wrappers; the final codebase must not expose the old turn API. - -- [ ] **Step 7: Run adapter and replay tests** - -Run: - -```bash -pytest tests/unit/builtins/common/test_transcript_adapters.py tests/unit/state/test_context_assembly.py tests/unit/state/test_generation_turn_build.py -v -``` - -Expected: PASS after old tests are rewritten or any migration-only wrappers are correct. These wrappers are not allowed to remain after Task 7. - ---- - -### Task 5: Migrate Worker Interface And Execution Persistence - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/task_execution_service.py` -- Modify: `ergon_core/ergon_core/api/results.py` -- Modify: worker base API files that type `execute()` return values. -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/react_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/baselines/training_stub_worker.py` -- Modify: smoke fixture workers under `ergon_core/ergon_core/test_support/smoke_fixtures/smoke_base/` -- Modify: `tests/unit/workers/test_react_worker_contract.py` -- Modify: `tests/unit/state/test_research_rubrics_workers.py` - -- [ ] **Step 1: Find all `AsyncGenerator[GenerationTurn` callers** - -Run: - -```bash -rg "AsyncGenerator\\[GenerationTurn|GenerationTurn" ergon_core ergon_builtins tests -n -``` - -Expected: a finite list including builtins workers, smoke fixtures, test support, and execution persistence. - -- [ ] **Step 2: Update worker API type hints** - -Replace worker `execute()` signatures from: - -```python -) -> AsyncGenerator[GenerationTurn, None]: -``` - -to: - -```python -) -> AsyncGenerator[ContextPartChunk, None]: -``` - -Import `ContextPartChunk` from `ergon_core.core.generation`. - -- [ ] **Step 3: Update task execution persistence loop** - -In `task_execution_service.py`, replace the turn persistence call: - -```python -async for turn in worker.execute(task, context=context): - await context_event_repository.persist_turn( - session, - run_id=run_id, - execution_id=execution.id, - worker_binding_key=worker_binding_key, - turn=turn, - ) -``` - -with: - -```python -async for chunk in worker.execute(task, context=context): - await context_event_repository.persist_chunk( - session, - run_id=run_id, - execution_id=execution.id, - worker_binding_key=worker_binding_key, - chunk=chunk, - ) -``` - -Keep exact local variable names consistent with the existing file. - -- [ ] **Step 4: Update simple text-yielding workers** - -For smoke workers that currently yield: - -```python -yield GenerationTurn(response_parts=[TextPart(content="...")]) -``` - -replace with: - -```python -yield ContextPartChunk(part=AssistantTextPart(content="...")) -``` - -For user prompt setup chunks, emit: - -```python -yield ContextPartChunk(part=UserMessagePart(content="...")) -``` - -Only emit prompt chunks if the worker previously included them in `messages_in`; do not invent additional prompt events. - -- [ ] **Step 5: Update `training_stub_worker.py`** - -Replace synthetic `GenerationTurn` creation with chunk lists: - -```python -chunks: list[ContextPartChunk] = [] -chunks.append(ContextPartChunk(part=UserMessagePart(content=f"Task: Synthetic task {task_slug}"))) -chunks.append( - ContextPartChunk( - part=ToolCallPart( - tool_name="stub_tool", - tool_call_id=f"call_{i}", - args={"turn": i, "task": task_slug}, - ), - logprobs=logprobs, - ) -) -chunks.append( - ContextPartChunk( - part=ToolResultPart( - tool_call_id=f"call_{i}", - tool_name="stub_tool", - content=f"Tool result for turn {i} of {task_slug}", - ) - ) -) -``` - -For final assistant output: - -```python -ContextPartChunk( - part=AssistantTextPart(content=f"Synthetic response turn {i}"), - logprobs=logprobs, -) -``` - -- [ ] **Step 6: Update `react_worker.py`** - -Where the worker previously handled `GenerationTurn` outputs or inspected payload classes, switch to chunk/log parts: - -```python -payload = event.parsed_payload() -part = payload.part -if isinstance(part, AssistantTextPart): - ... -``` - -For final assistant message extraction, replace `AssistantTextPayload` checks with `AssistantTextPart`. - -- [ ] **Step 7: Run worker contract tests** - -Run: - -```bash -pytest tests/unit/workers/test_react_worker_contract.py tests/unit/state/test_research_rubrics_workers.py -v -``` - -Expected: PASS after signatures and assertions are migrated. - ---- - -### Task 6: Update REST, Dashboard, And RL Consumers - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/dashboard/event_contracts.py` -- Modify: `ergon_core/ergon_core/core/dashboard/emitter.py` -- Modify: `ergon_core/ergon_core/core/rl/extraction.py` -- Modify: dashboard generated contracts if this repo checks them in. -- Modify: `tests/unit/dashboard/test_event_contract_types.py` - -- [ ] **Step 1: Type REST context event DTOs with chunk logs** - -Modify `RunContextEventDto`: - -```python -from ergon_core.core.generation import ContextPartChunkLog -from ergon_core.core.persistence.context.event_payloads import ContextEventType - - -class RunContextEventDto(CamelModel): - id: str - task_execution_id: str - task_node_id: str - worker_binding_key: str - sequence: int - event_type: ContextEventType - payload: ContextPartChunkLog - created_at: str - started_at: str | None = None - completed_at: str | None = None -``` - -- [ ] **Step 2: Project typed payloads in REST snapshots** - -In `_context_events_by_task()`, change: - -```python -payload=event.payload, -``` - -to: - -```python -payload=event.parsed_payload(), -``` - -Keep `event_type=cast(ContextEventType, event.event_type)` if type checking requires it. - -- [ ] **Step 3: Type dashboard event contracts with the same payload** - -In `event_contracts.py`, ensure: - -```python -payload: ContextPartChunkLog -``` - -instead of the old `ContextEventPayload` union alias if that alias is still confusing. - -- [ ] **Step 4: Update dashboard emitter payload validation** - -In `emitter.py`, validate as: - -```python -payload=event.parsed_payload() -``` - -instead of constructing a separate TypeAdapter in the emitter. - -- [ ] **Step 5: Update RL extraction** - -Change event handling from payload-class checks to part-class checks: - -```python -payload = event.parsed_payload() -part = payload.part - -if isinstance(part, (SystemPromptPart, UserMessagePart)): - ... -elif isinstance(part, (AssistantTextPart, ToolCallPart, ThinkingPart)): - token_ids = _get_token_ids(payload, tokenizer) -elif isinstance(part, ToolResultPart): - result_tokens = tokenizer.encode(str(part.content)) -``` - -Update `_get_token_ids()` to accept `ContextPartChunkLog` and inspect `payload.part`. - -- [ ] **Step 6: Run REST/dashboard/RL tests** - -Run: - -```bash -pytest tests/unit/dashboard/test_event_contract_types.py tests/unit/state/test_context_assembly.py tests/unit/persistence/test_context_event_repository.py -v -``` - -Expected: PASS after DTOs and consumers use `ContextPartChunkLog`. - ---- - -### Task 7: Add Architecture Guards And Remove Deprecated Turn API - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `ergon_core/ergon_core/core/generation.py` -- Modify: any remaining files found by `rg`. - -- [ ] **Step 1: Add architecture guard against duplicate context payload unions** - -Add to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -from pathlib import Path - - -def test_context_stream_has_single_discriminated_part_union() -> None: - root = Path(__file__).resolve().parents[3] - generation = root / "ergon_core" / "ergon_core" / "core" / "generation.py" - event_payloads = ( - root - / "ergon_core" - / "ergon_core" - / "core" - / "persistence" - / "context" - / "event_payloads.py" - ) - - generation_text = generation.read_text() - event_payloads_text = event_payloads.read_text() - - assert "ContextPart = Annotated[" in generation_text - assert "SystemPromptPayload |" not in event_payloads_text - assert "AssistantTextPayload |" not in event_payloads_text - assert "ToolCallPayload |" not in event_payloads_text -``` - -- [ ] **Step 2: Run the architecture test** - -Run: - -```bash -pytest tests/unit/architecture/test_core_schema_sources.py -v -``` - -Expected: PASS only after `event_payloads.py` no longer owns a duplicate payload union. - -- [ ] **Step 3: Remove deprecated `GenerationTurn` compatibility** - -Run: - -```bash -rg "GenerationTurn|ModelRequestPart|ModelResponsePart|ToolReturnPart|TextPart|UserPromptPart" ergon_core ergon_builtins tests -n -``` - -Remove remaining old names where possible. Keep `TextPart` only when it refers to `pydantic_ai.messages.TextPart`, and alias it as `PydanticTextPart` in imports to avoid confusion. - -- [ ] **Step 4: Delete compatibility aliases** - -From `generation.py`, remove: - -```python -UserPromptPart = UserMessagePart -TextPart = AssistantTextPart -ToolReturnPart = ToolResultPart -ModelRequestPart = ... -ModelResponsePart = ... -class GenerationTurn(...) -``` - -Only do this once `rg` confirms no production caller depends on those names. - -- [ ] **Step 5: Verify no old payload classes or aliases exist in `event_payloads.py`** - -Run: - -```bash -rg "SystemPromptPayload|UserMessagePayload|AssistantTextPayload|ToolCallPayload|ToolResultPayload|ThinkingPayload" ergon_core ergon_builtins tests -n -``` - -Expected: no production matches. Test matches should be migrated to `ContextPartChunkLog` and canonical part classes. - -Confirm `event_payloads.py` does not define or export: - -```python -SystemPromptPayload -UserMessagePayload -AssistantTextPayload -ToolCallPayload -ToolResultPayload -ThinkingPayload -``` - -Keep: - -```python -ContextEventType -ContextEventPayload = ContextPartChunkLog -``` - -or rename `ContextEventPayload` to `ContextPartChunkLog` everywhere if the alias is no longer useful. - -- [ ] **Step 6: Run full focused suite** - -Run: - -```bash -pytest \ - tests/unit/state/test_context_part_stream.py \ - tests/unit/persistence/test_context_event_repository.py \ - tests/unit/builtins/common/test_transcript_adapters.py \ - tests/unit/state/test_context_assembly.py \ - tests/unit/workers/test_react_worker_contract.py \ - tests/unit/dashboard/test_event_contract_types.py \ - tests/unit/architecture/test_core_schema_sources.py \ - -v -``` - -Expected: PASS. - -- [ ] **Step 7: Run broader unit smoke** - -Run: - -```bash -pytest tests/unit -q -``` - -Expected: PASS, or only unrelated pre-existing failures. Investigate any failures mentioning context events, generation turns, workers, dashboard contracts, replay, or RL extraction. - ---- - -## Migration Notes - -This is a schema/API clean break. Do not preserve backwards compatibility with the old schemas in the final state. - -Temporary adapters are allowed only inside intermediate tasks to make the migration reviewable: -- `GenerationTurn` can exist only until worker execution is moved to chunks. -- request/response subset aliases can exist only until all worker and adapter callers move to `ContextPartChunk`. -- old `*Payload` event classes should not be reintroduced as aliases; migrate those callers directly to `ContextPartChunkLog.part`. - -After Task 7, the only canonical stream type should be `ContextPart`, the worker generator type should be `ContextPartChunk`, and the enriched log type should be `ContextPartChunkLog`. - -Do not add a second new union in `event_payloads.py`. Do not leave compatibility exports for the old payload classes. Either outcome recreates the drift this plan is removing. - ---- - -## Self-Review - -**Spec coverage:** The plan implements the requested model: `ContextPart` as the single discriminated union, `ContextPartChunk` as the worker generator type, and `ContextPartChunkLog` as the core-enriched persistence/API shape. - -**Placeholder scan:** No steps rely on `TBD`, unspecified tests, or unnamed files. Commands and expected outcomes are included for each task. - -**Type consistency:** The plan consistently uses `content` for text-bearing parts, `part_kind` for the part discriminator, `token_ids`/`logprobs` for worker-provided token metadata, and `sequence`/`worker_binding_key`/`turn_id` for core-enriched log metadata. - ---- - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-04-28-context-part-chunk-stream.md`. Two execution options: - -**1. Subagent-Driven (recommended)** - dispatch a fresh subagent per task, review between tasks, fast iteration. - -**2. Inline Execution** - execute tasks in this session using executing-plans, batch execution with checkpoints. - -Which approach? diff --git a/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout-implementation.md b/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout-implementation.md deleted file mode 100644 index a26fa51b5..000000000 --- a/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout-implementation.md +++ /dev/null @@ -1,1259 +0,0 @@ -# Core Hybrid Domain Layout Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move `ergon_core.core` to the approved hybrid layout: thin `rest_api`, product use cases under `application`, pure objects under `domain`, adapters under `infrastructure`, SQL rows under `persistence`, and `rl` kept as a separate bounded context. - -**Architecture:** This is a mechanical package migration with architecture guards. Each slice moves one cluster, bulk-renames imports, runs focused tests, and preserves behavior. A temporary exact-folder-structure test is added first and deleted at the end after durable architecture tests cover the important constraints. - -**Tech Stack:** Python, pytest, ruff, SQLModel, FastAPI, Inngest, Pydantic. - -**Commit Policy:** Do not create git commits unless the user explicitly asks. Treat each task's test run as the checkpoint. - ---- - -## Target Clusters - -The implementation follows `docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md`. - -```text -core/ - rest_api/ - application/ - experiments/ - workflows/ - graph/ - tasks/ - evaluation/ - read_models/ - communication/ - context/ - jobs/ - resources/ - events/ - domain/ - experiments/ - generation/ - persistence/ - infrastructure/ - inngest/ - handlers/ - sandbox/ - dashboard/ - tracing/ - dependencies.py - rl/ - shared/ -``` - -## Task 1: Add Temporary Exact Layout Guard - -**Files:** -- Create: `tests/unit/architecture/test_core_hybrid_layout_temporary.py` -- Modify: none -- Test: `tests/unit/architecture/test_core_hybrid_layout_temporary.py` - -- [ ] **Step 1: Add the temporary failing test** - -Create `tests/unit/architecture/test_core_hybrid_layout_temporary.py`: - -```python -"""Temporary guard for the core hybrid layout migration. - -Delete this file in the final migration task. It intentionally asserts the -exact file tree so each migration slice has a visible end state. -""" - -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[3] -CORE = ROOT / "ergon_core/ergon_core/core" - -EXPECTED_FILES = { - "__init__.py", - "rest_api/__init__.py", - "rest_api/app.py", - "rest_api/cohorts.py", - "rest_api/experiments.py", - "rest_api/rollouts.py", - "rest_api/runs.py", - "rest_api/test_harness.py", - "application/__init__.py", - "application/experiments/__init__.py", - "application/experiments/service.py", - "application/experiments/models.py", - "application/experiments/repository.py", - "application/experiments/definition_writer.py", - "application/experiments/launch.py", - "application/workflows/__init__.py", - "application/workflows/service.py", - "application/workflows/orchestration.py", - "application/workflows/runs.py", - "application/workflows/models.py", - "application/workflows/errors.py", - "application/graph/__init__.py", - "application/graph/repository.py", - "application/graph/propagation.py", - "application/graph/traversal.py", - "application/graph/lookup.py", - "application/graph/models.py", - "application/graph/errors.py", - "application/tasks/__init__.py", - "application/tasks/service.py", - "application/tasks/execution.py", - "application/tasks/management.py", - "application/tasks/inspection.py", - "application/tasks/cleanup.py", - "application/tasks/repository.py", - "application/tasks/models.py", - "application/tasks/errors.py", - "application/evaluation/__init__.py", - "application/evaluation/service.py", - "application/evaluation/executors.py", - "application/evaluation/inngest_executor.py", - "application/evaluation/criterion_runtime.py", - "application/evaluation/scoring.py", - "application/evaluation/protocols.py", - "application/evaluation/models.py", - "application/evaluation/errors.py", - "application/read_models/__init__.py", - "application/read_models/runs.py", - "application/read_models/run_snapshot.py", - "application/read_models/experiments.py", - "application/read_models/cohorts.py", - "application/read_models/resources.py", - "application/read_models/models.py", - "application/read_models/errors.py", - "application/communication/__init__.py", - "application/communication/service.py", - "application/communication/models.py", - "application/communication/errors.py", - "application/context/__init__.py", - "application/context/events.py", - "application/context/output_extraction.py", - "application/jobs/__init__.py", - "application/jobs/cancel_orphan_subtasks.py", - "application/jobs/check_evaluators.py", - "application/jobs/cleanup_cancelled_task.py", - "application/jobs/complete_workflow.py", - "application/jobs/evaluate_task_run.py", - "application/jobs/execute_task.py", - "application/jobs/fail_workflow.py", - "application/jobs/persist_outputs.py", - "application/jobs/propagate_execution.py", - "application/jobs/run_cleanup.py", - "application/jobs/sandbox_setup.py", - "application/jobs/start_workflow.py", - "application/jobs/worker_execute.py", - "application/jobs/models.py", - "application/resources/__init__.py", - "application/resources/repository.py", - "application/resources/models.py", - "application/events/__init__.py", - "application/events/base.py", - "application/events/task_events.py", - "application/events/infrastructure_events.py", - "domain/__init__.py", - "domain/experiments/__init__.py", - "domain/experiments/experiment.py", - "domain/experiments/handles.py", - "domain/experiments/worker_spec.py", - "domain/experiments/validation.py", - "domain/generation/__init__.py", - "domain/generation/context_parts.py", - "persistence/shared/__init__.py", - "persistence/shared/db.py", - "persistence/shared/enums.py", - "persistence/shared/ids.py", - "persistence/shared/types.py", - "persistence/definitions/__init__.py", - "persistence/definitions/models.py", - "persistence/telemetry/__init__.py", - "persistence/telemetry/models.py", - "persistence/telemetry/repositories.py", - "persistence/telemetry/evaluation_summary.py", - "persistence/graph/__init__.py", - "persistence/graph/models.py", - "persistence/graph/status_conventions.py", - "persistence/context/__init__.py", - "persistence/context/models.py", - "persistence/context/event_payloads.py", - "persistence/saved_specs/__init__.py", - "persistence/saved_specs/models.py", - "infrastructure/__init__.py", - "infrastructure/inngest/__init__.py", - "infrastructure/inngest/client.py", - "infrastructure/inngest/registry.py", - "infrastructure/inngest/contracts.py", - "infrastructure/inngest/errors.py", - "infrastructure/inngest/handlers/__init__.py", - "infrastructure/inngest/handlers/cancel_orphan_subtasks.py", - "infrastructure/inngest/handlers/check_evaluators.py", - "infrastructure/inngest/handlers/cleanup_cancelled_task.py", - "infrastructure/inngest/handlers/complete_workflow.py", - "infrastructure/inngest/handlers/evaluate_task_run.py", - "infrastructure/inngest/handlers/execute_task.py", - "infrastructure/inngest/handlers/fail_workflow.py", - "infrastructure/inngest/handlers/persist_outputs.py", - "infrastructure/inngest/handlers/propagate_execution.py", - "infrastructure/inngest/handlers/run_cleanup.py", - "infrastructure/inngest/handlers/sandbox_setup.py", - "infrastructure/inngest/handlers/start_workflow.py", - "infrastructure/inngest/handlers/worker_execute.py", - "infrastructure/sandbox/__init__.py", - "infrastructure/sandbox/manager.py", - "infrastructure/sandbox/lifecycle.py", - "infrastructure/sandbox/resource_publisher.py", - "infrastructure/sandbox/instrumentation.py", - "infrastructure/sandbox/event_sink.py", - "infrastructure/sandbox/errors.py", - "infrastructure/sandbox/utils.py", - "infrastructure/dashboard/__init__.py", - "infrastructure/dashboard/emitter.py", - "infrastructure/dashboard/provider.py", - "infrastructure/dashboard/event_contracts.py", - "infrastructure/tracing/__init__.py", - "infrastructure/tracing/attributes.py", - "infrastructure/tracing/contexts.py", - "infrastructure/tracing/ids.py", - "infrastructure/tracing/noop.py", - "infrastructure/tracing/otel.py", - "infrastructure/tracing/sinks.py", - "infrastructure/tracing/types.py", - "infrastructure/dependencies.py", - "rl/__init__.py", - "rl/rollout_service.py", - "rl/eval_runner.py", - "rl/extraction.py", - "rl/rewards.py", - "rl/checkpoint.py", - "rl/rollout_types.py", - "rl/vllm_manager.py", - "shared/__init__.py", - "shared/json_types.py", - "shared/settings.py", - "shared/utils.py", -} - -REMOVED_DIRS = { - "api", - "definitions", - "composition", - "runtime", - "sandbox", - "dashboard", -} - -REMOVED_ROOT_FILES = { - "generation.py", - "json_types.py", - "settings.py", - "utils.py", -} - - -def test_core_has_exact_target_layout_during_migration() -> None: - actual_files = { - str(path.relative_to(CORE)) - for path in CORE.rglob("*.py") - if "__pycache__" not in path.parts - } - missing = sorted(EXPECTED_FILES - actual_files) - unexpected = sorted(actual_files - EXPECTED_FILES) - - assert missing == [] - assert unexpected == [] - - -def test_old_core_roots_are_removed_during_migration() -> None: - restored_dirs = sorted(name for name in REMOVED_DIRS if (CORE / name).exists()) - restored_files = sorted(name for name in REMOVED_ROOT_FILES if (CORE / name).exists()) - - assert restored_dirs == [] - assert restored_files == [] -``` - -- [ ] **Step 2: Run the temporary test and confirm it fails** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_core_hybrid_layout_temporary.py -q -``` - -Expected: FAIL because the target directories do not exist yet and old roots still exist. - -## Task 2: Rename HTTP Layer To `core/rest_api` - -**Files:** -- Move: `ergon_core/ergon_core/core/api/*` -> `ergon_core/ergon_core/core/rest_api/*` -- Modify: imports in `ergon_core/ergon_core/core/rest_api/*.py` -- Modify: imports across `ergon_core`, `ergon_cli`, `ergon_builtins`, and `tests` -- Test: `tests/unit/architecture/test_public_api_boundaries.py` -- Test: `tests/unit/architecture/test_core_schema_sources.py` - -- [ ] **Step 1: Move the package** - -Move files: - -```bash -mkdir -p ergon_core/ergon_core/core/rest_api -mv ergon_core/ergon_core/core/api/*.py ergon_core/ergon_core/core/rest_api/ -rmdir ergon_core/ergon_core/core/api -``` - -- [ ] **Step 2: Bulk update imports** - -Replace every `ergon_core.core.api` import with `ergon_core.core.rest_api`. - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text.replace("ergon_core.core.api", "ergon_core.core.rest_api") - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 3: Add a durable architecture guard** - -In `tests/unit/architecture/test_public_api_boundaries.py`, add: - -```python -def test_internal_http_api_is_named_rest_api_not_core_api() -> None: - core_root = ROOT / "ergon_core" / "ergon_core" / "core" - - assert not (core_root / "api").exists() - assert (core_root / "rest_api").exists() -``` - -- [ ] **Step 4: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_public_api_boundaries.py tests/unit/architecture/test_core_schema_sources.py -q -``` - -Expected: PASS for durable architecture tests. The temporary exact-layout test still fails until the full migration finishes. - -## Task 3: Move Shared Primitives And Pure Domain Objects - -**Files:** -- Move: `core/json_types.py` -> `core/shared/json_types.py` -- Move: `core/settings.py` -> `core/shared/settings.py` -- Move: `core/utils.py` -> `core/shared/utils.py` -- Move: `core/generation.py` -> `core/domain/generation/context_parts.py` -- Move: `core/composition/*` -> `core/domain/experiments/*` -- Create: `core/shared/__init__.py` -- Create: `core/domain/__init__.py` -- Create: `core/domain/generation/__init__.py` -- Modify: imports across source and tests -- Test: `tests/unit/architecture/test_public_api_boundaries.py` -- Test: `tests/unit/architecture/test_core_schema_sources.py` - -- [ ] **Step 1: Move shared files** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/shared -mv ergon_core/ergon_core/core/json_types.py ergon_core/ergon_core/core/shared/json_types.py -mv ergon_core/ergon_core/core/settings.py ergon_core/ergon_core/core/shared/settings.py -mv ergon_core/ergon_core/core/utils.py ergon_core/ergon_core/core/shared/utils.py -touch ergon_core/ergon_core/core/shared/__init__.py -``` - -- [ ] **Step 2: Move generation primitives** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/domain/generation -mv ergon_core/ergon_core/core/generation.py ergon_core/ergon_core/core/domain/generation/context_parts.py -touch ergon_core/ergon_core/core/domain/__init__.py -touch ergon_core/ergon_core/core/domain/generation/__init__.py -``` - -- [ ] **Step 3: Move experiment composition domain** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/domain/experiments -mv ergon_core/ergon_core/core/composition/*.py ergon_core/ergon_core/core/domain/experiments/ -rmdir ergon_core/ergon_core/core/composition -``` - -- [ ] **Step 4: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.json_types": "ergon_core.core.shared.json_types", - "ergon_core.core.settings": "ergon_core.core.shared.settings", - "ergon_core.core.utils": "ergon_core.core.shared.utils", - "ergon_core.core.generation": "ergon_core.core.domain.generation.context_parts", - "ergon_core.core.composition": "ergon_core.core.domain.experiments", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 5: Restore domain exports** - -Ensure `ergon_core/ergon_core/core/domain/experiments/__init__.py` exports the same names previously exported by `core/composition/__init__.py`: - -```python -from ergon_core.core.domain.experiments.experiment import Experiment -from ergon_core.core.domain.experiments.handles import DefinitionHandle -from ergon_core.core.domain.experiments.worker_spec import WorkerSpec - -__all__ = ["DefinitionHandle", "Experiment", "WorkerSpec"] -``` - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_public_api_boundaries.py tests/unit/architecture/test_core_schema_sources.py tests/unit/api/test_public_api_imports.py -q -``` - -Expected: PASS. - -## Task 4: Move Experiment Application Cluster - -**Files:** -- Move: `core/definitions/service.py` -> `core/application/experiments/service.py` -- Move: `core/definitions/schemas.py` -> `core/application/experiments/models.py` -- Move: `core/definitions/repository.py` -> `core/application/experiments/repository.py` -- Move: `core/definitions/persistence.py` -> `core/application/experiments/definition_writer.py` -- Move: `core/runtime/workflows/launch.py` -> `core/application/experiments/launch.py` -- Create: `core/application/__init__.py` -- Create: `core/application/experiments/__init__.py` -- Delete: `core/definitions/` -- Test: `tests/unit/runtime/test_experiment_definition_service.py` -- Test: `tests/unit/runtime/test_experiment_launch_service.py` -- Test: `tests/unit/cli/test_experiment_cli.py` - -- [ ] **Step 1: Move files** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/experiments -mv ergon_core/ergon_core/core/definitions/service.py ergon_core/ergon_core/core/application/experiments/service.py -mv ergon_core/ergon_core/core/definitions/schemas.py ergon_core/ergon_core/core/application/experiments/models.py -mv ergon_core/ergon_core/core/definitions/repository.py ergon_core/ergon_core/core/application/experiments/repository.py -mv ergon_core/ergon_core/core/definitions/persistence.py ergon_core/ergon_core/core/application/experiments/definition_writer.py -mv ergon_core/ergon_core/core/runtime/workflows/launch.py ergon_core/ergon_core/core/application/experiments/launch.py -touch ergon_core/ergon_core/core/application/__init__.py -touch ergon_core/ergon_core/core/application/experiments/__init__.py -rm ergon_core/ergon_core/core/definitions/__init__.py -rmdir ergon_core/ergon_core/core/definitions -``` - -- [ ] **Step 2: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.definitions.service": "ergon_core.core.application.experiments.service", - "ergon_core.core.definitions.schemas": "ergon_core.core.application.experiments.models", - "ergon_core.core.definitions.repository": "ergon_core.core.application.experiments.repository", - "ergon_core.core.definitions.persistence": "ergon_core.core.application.experiments.definition_writer", - "ergon_core.core.runtime.workflows.launch": "ergon_core.core.application.experiments.launch", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 3: Ensure experiment package exports the front door** - -Set `ergon_core/ergon_core/core/application/experiments/__init__.py` to: - -```python -from ergon_core.core.application.experiments.service import ExperimentService - -__all__ = ["ExperimentService"] -``` - -- [ ] **Step 4: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/runtime/test_experiment_definition_service.py tests/unit/runtime/test_experiment_launch_service.py tests/unit/cli/test_experiment_cli.py -q -``` - -Expected: PASS. - -## Task 5: Move Workflow, Graph, Task, And Evaluation Application Clusters - -**Files:** -- Move: `core/runtime/workflows/{service,orchestration,runs,models,errors}.py` -> `core/application/workflows/` -- Move: `core/runtime/graph/{repository,propagation,traversal,lookup,dto,errors}.py` -> `core/application/graph/` -- Rename: `core/application/graph/dto.py` -> `core/application/graph/models.py` -- Move: `core/runtime/tasks/*` -> `core/application/tasks/` -- Rename: `core/application/tasks/management.py` remains `management.py` -- Create: `core/application/tasks/service.py` if needed as a package front door -- Move: `core/runtime/evaluation/*` -> `core/application/evaluation/` -- Modify: imports across source and tests -- Test: runtime workflow/task/evaluation tests - -- [ ] **Step 1: Move workflows** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/workflows -mv ergon_core/ergon_core/core/runtime/workflows/service.py ergon_core/ergon_core/core/application/workflows/service.py -mv ergon_core/ergon_core/core/runtime/workflows/orchestration.py ergon_core/ergon_core/core/application/workflows/orchestration.py -mv ergon_core/ergon_core/core/runtime/workflows/runs.py ergon_core/ergon_core/core/application/workflows/runs.py -mv ergon_core/ergon_core/core/runtime/workflows/models.py ergon_core/ergon_core/core/application/workflows/models.py -mv ergon_core/ergon_core/core/runtime/workflows/errors.py ergon_core/ergon_core/core/application/workflows/errors.py -touch ergon_core/ergon_core/core/application/workflows/__init__.py -rm -f ergon_core/ergon_core/core/runtime/workflows/__init__.py -rmdir ergon_core/ergon_core/core/runtime/workflows -``` - -- [ ] **Step 2: Move graph** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/graph -mv ergon_core/ergon_core/core/runtime/graph/repository.py ergon_core/ergon_core/core/application/graph/repository.py -mv ergon_core/ergon_core/core/runtime/graph/propagation.py ergon_core/ergon_core/core/application/graph/propagation.py -mv ergon_core/ergon_core/core/runtime/graph/traversal.py ergon_core/ergon_core/core/application/graph/traversal.py -mv ergon_core/ergon_core/core/runtime/graph/lookup.py ergon_core/ergon_core/core/application/graph/lookup.py -mv ergon_core/ergon_core/core/runtime/graph/dto.py ergon_core/ergon_core/core/application/graph/models.py -mv ergon_core/ergon_core/core/runtime/graph/errors.py ergon_core/ergon_core/core/application/graph/errors.py -touch ergon_core/ergon_core/core/application/graph/__init__.py -rm -f ergon_core/ergon_core/core/runtime/graph/__init__.py -rmdir ergon_core/ergon_core/core/runtime/graph -``` - -- [ ] **Step 3: Move tasks** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/tasks -mv ergon_core/ergon_core/core/runtime/tasks/*.py ergon_core/ergon_core/core/application/tasks/ -touch ergon_core/ergon_core/core/application/tasks/service.py -rmdir ergon_core/ergon_core/core/runtime/tasks -``` - -Set `ergon_core/ergon_core/core/application/tasks/service.py` to: - -```python -"""Task application package front door. - -Task lifecycle behavior currently lives in focused modules: -`execution`, `management`, `inspection`, and `cleanup`. -""" - -from ergon_core.core.application.tasks.execution import TaskExecutionService -from ergon_core.core.application.tasks.management import TaskManagementService - -__all__ = ["TaskExecutionService", "TaskManagementService"] -``` - -- [ ] **Step 4: Move evaluation** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/evaluation -mv ergon_core/ergon_core/core/runtime/evaluation/*.py ergon_core/ergon_core/core/application/evaluation/ -touch ergon_core/ergon_core/core/application/evaluation/__init__.py -rmdir ergon_core/ergon_core/core/runtime/evaluation -``` - -- [ ] **Step 5: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.runtime.workflows": "ergon_core.core.application.workflows", - "ergon_core.core.runtime.graph.dto": "ergon_core.core.application.graph.models", - "ergon_core.core.runtime.graph": "ergon_core.core.application.graph", - "ergon_core.core.runtime.tasks": "ergon_core.core.application.tasks", - "ergon_core.core.runtime.evaluation": "ergon_core.core.application.evaluation", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/runtime/test_workflow_service.py tests/unit/runtime/test_graph_mutation_contracts.py tests/unit/runtime/test_graph_worker_identity.py tests/unit/runtime/test_task_execution_repository.py tests/unit/runtime/test_inngest_criterion_executor.py tests/unit/runtime/test_dynamic_task_evaluation_mapping.py -q -``` - -Expected: PASS. - -## Task 6: Move Read Models, Communication, Context, And Resources - -**Files:** -- Move: `core/runtime/read_models/{runs,run_snapshot,experiments,cohorts,resources,errors}.py` -> `core/application/read_models/` -- Split: communication DTOs from `read_models/models.py` -> `core/application/communication/models.py` -- Move: `core/runtime/read_models/communication.py` -> `core/application/communication/service.py` -- Move: remaining read model DTOs -> `core/application/read_models/models.py` -- Move: `core/runtime/context_events.py` -> `core/application/context/events.py` -- Move: `core/runtime/output_extraction.py` -> `core/application/context/output_extraction.py` -- Split: `core/runtime/resources.py` -> `core/application/resources/models.py` and `core/application/resources/repository.py` -- Test: dashboard/read-model/context/resource tests - -- [ ] **Step 1: Move read models** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/read_models -mv ergon_core/ergon_core/core/runtime/read_models/runs.py ergon_core/ergon_core/core/application/read_models/runs.py -mv ergon_core/ergon_core/core/runtime/read_models/run_snapshot.py ergon_core/ergon_core/core/application/read_models/run_snapshot.py -mv ergon_core/ergon_core/core/runtime/read_models/experiments.py ergon_core/ergon_core/core/application/read_models/experiments.py -mv ergon_core/ergon_core/core/runtime/read_models/cohorts.py ergon_core/ergon_core/core/application/read_models/cohorts.py -mv ergon_core/ergon_core/core/runtime/read_models/resources.py ergon_core/ergon_core/core/application/read_models/resources.py -mv ergon_core/ergon_core/core/runtime/read_models/errors.py ergon_core/ergon_core/core/application/read_models/errors.py -mv ergon_core/ergon_core/core/runtime/read_models/models.py ergon_core/ergon_core/core/application/read_models/models.py -touch ergon_core/ergon_core/core/application/read_models/__init__.py -``` - -- [ ] **Step 2: Move communication domain** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/communication -mv ergon_core/ergon_core/core/runtime/read_models/communication.py ergon_core/ergon_core/core/application/communication/service.py -touch ergon_core/ergon_core/core/application/communication/__init__.py -touch ergon_core/ergon_core/core/application/communication/errors.py -touch ergon_core/ergon_core/core/application/communication/models.py -rm ergon_core/ergon_core/core/runtime/read_models/__init__.py -rmdir ergon_core/ergon_core/core/runtime/read_models -``` - -Move `RunCommunicationMessageDto` and `RunCommunicationThreadDto` from `application/read_models/models.py` into `application/communication/models.py`, then update imports to read from `ergon_core.core.application.communication.models`. - -- [ ] **Step 3: Move context domain** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/context -mv ergon_core/ergon_core/core/runtime/context_events.py ergon_core/ergon_core/core/application/context/events.py -mv ergon_core/ergon_core/core/runtime/output_extraction.py ergon_core/ergon_core/core/application/context/output_extraction.py -touch ergon_core/ergon_core/core/application/context/__init__.py -``` - -- [ ] **Step 4: Split resources module** - -Create `ergon_core/ergon_core/core/application/resources/models.py` with `RunResourceView`. - -Create `ergon_core/ergon_core/core/application/resources/repository.py` with `RunResourceRepository`. - -Delete `ergon_core/ergon_core/core/runtime/resources.py`. - -Use this package initializer: - -```python -from ergon_core.core.application.resources.models import RunResourceView -from ergon_core.core.application.resources.repository import RunResourceRepository - -__all__ = ["RunResourceRepository", "RunResourceView"] -``` - -- [ ] **Step 5: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.runtime.read_models.communication": "ergon_core.core.application.communication.service", - "ergon_core.core.runtime.read_models": "ergon_core.core.application.read_models", - "ergon_core.core.runtime.context_events": "ergon_core.core.application.context.events", - "ergon_core.core.runtime.output_extraction": "ergon_core.core.application.context.output_extraction", - "ergon_core.core.runtime.resources": "ergon_core.core.application.resources", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/dashboard/test_communication_threads.py tests/unit/runtime/test_communication_service.py tests/unit/persistence/test_context_event_repository.py tests/unit/runtime/test_persist_outputs_resources.py tests/unit/runtime/test_experiment_read_service.py tests/unit/runtime/test_cohort_service.py -q -``` - -Expected: PASS. - -## Task 7: Split Inngest Handlers Into Application Jobs And Infrastructure Adapters - -**Files:** -- Move semantic logic: `core/runtime/inngest/{handler files}.py` -> `core/application/jobs/{handler files}.py` -- Create: `core/application/jobs/models.py` -- Create thin adapters: `core/infrastructure/inngest/handlers/{handler files}.py` -- Move: `runtime/inngest/client.py` -> `infrastructure/inngest/client.py` -- Move: `runtime/inngest/registry.py` -> `infrastructure/inngest/registry.py` -- Move: `runtime/inngest/contracts.py` -> `infrastructure/inngest/contracts.py` -- Move: `runtime/inngest/errors.py` -> `infrastructure/inngest/errors.py` -- Test: Inngest/runtime unit tests and import registry tests - -- [ ] **Step 1: Move infrastructure primitives** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/infrastructure/inngest/handlers -mv ergon_core/ergon_core/core/runtime/inngest/client.py ergon_core/ergon_core/core/infrastructure/inngest/client.py -mv ergon_core/ergon_core/core/runtime/inngest/registry.py ergon_core/ergon_core/core/infrastructure/inngest/registry.py -mv ergon_core/ergon_core/core/runtime/inngest/contracts.py ergon_core/ergon_core/core/infrastructure/inngest/contracts.py -mv ergon_core/ergon_core/core/runtime/inngest/errors.py ergon_core/ergon_core/core/infrastructure/inngest/errors.py -touch ergon_core/ergon_core/core/infrastructure/__init__.py -touch ergon_core/ergon_core/core/infrastructure/inngest/__init__.py -touch ergon_core/ergon_core/core/infrastructure/inngest/handlers/__init__.py -``` - -- [ ] **Step 2: Move handler semantics into jobs** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/application/jobs -for name in cancel_orphan_subtasks check_evaluators cleanup_cancelled_task complete_workflow evaluate_task_run execute_task fail_workflow persist_outputs propagate_execution run_cleanup sandbox_setup start_workflow worker_execute; do - mv "ergon_core/ergon_core/core/runtime/inngest/${name}.py" "ergon_core/ergon_core/core/application/jobs/${name}.py" -done -touch ergon_core/ergon_core/core/application/jobs/__init__.py -rm ergon_core/ergon_core/core/runtime/inngest/__init__.py 2>/dev/null || true -rmdir ergon_core/ergon_core/core/runtime/inngest -``` - -- [ ] **Step 3: Add thin adapters** - -For each moved job, remove the Inngest decorator from the application job file and expose an async `run_<name>_job(...)` function that contains the semantic behavior. The infrastructure handler owns the `@inngest_client.create_function(...)` decorator and delegates to the application job. - -For `worker_execute`, transform `core/application/jobs/worker_execute.py` so it starts like this: - -```python -"""Application job for worker execution.""" - -import logging -import traceback -from datetime import UTC, datetime - -from ergon_core.api.benchmark import EmptyTaskPayload, Task -from ergon_core.api.worker import WorkerContext -from ergon_core.core.application.context.events import ContextEventService -from ergon_core.core.application.experiments.repository import DefinitionRepository -from ergon_core.core.application.jobs.models import WorkerExecuteJobRequest -from ergon_core.core.application.jobs.models import WorkerExecuteJobResult -from ergon_core.core.domain.generation.context_parts import ContextPartChunk -from ergon_core.core.infrastructure.dashboard.provider import get_dashboard_emitter -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.infrastructure.inngest.errors import RegistryLookupError -from ergon_core.core.infrastructure.tracing import ( - CompletedSpan, - get_trace_sink, - worker_execute_context, -) -from pydantic import BaseModel - -logger = logging.getLogger(__name__) - - -async def run_worker_execute_job(payload: WorkerExecuteJobRequest) -> WorkerExecuteJobResult: - from ergon_builtins.registry import BENCHMARKS, WORKERS - - # Move the current body of worker_execute_fn here, replacing ctx.event.data - # with the typed payload argument. -``` - -Create `core/application/jobs/models.py` for job request/result aliases imported from Inngest contracts during the first migration: - -```python -"""Application job contracts. - -These mirror external Inngest event contracts during the migration so job logic -can be called independently of Inngest decorators. -""" - -from ergon_core.core.infrastructure.inngest.contracts import ( - CleanupCancelledTaskRequest, - CleanupCancelledTaskResult, - CompleteWorkflowRequest, - CompleteWorkflowResult, - EvaluateTaskRequest, - EvaluateTaskResult, - ExecuteTaskRequest, - ExecuteTaskResult, - PropagateExecutionRequest, - PropagateExecutionResult, - SandboxSetupRequest, - SandboxSetupResult, - StartWorkflowRequest, - StartWorkflowResult, - WorkerExecuteRequest as WorkerExecuteJobRequest, - WorkerExecuteResult as WorkerExecuteJobResult, -) - -__all__ = [ - "CleanupCancelledTaskRequest", - "CleanupCancelledTaskResult", - "CompleteWorkflowRequest", - "CompleteWorkflowResult", - "EvaluateTaskRequest", - "EvaluateTaskResult", - "ExecuteTaskRequest", - "ExecuteTaskResult", - "PropagateExecutionRequest", - "PropagateExecutionResult", - "SandboxSetupRequest", - "SandboxSetupResult", - "StartWorkflowRequest", - "StartWorkflowResult", - "WorkerExecuteJobRequest", - "WorkerExecuteJobResult", -] -``` - -Create `core/infrastructure/inngest/handlers/worker_execute.py` as the thin adapter: - -```python -"""Inngest adapter for worker execution.""" - -import inngest - -from ergon_core.core.application.jobs.worker_execute import run_worker_execute_job -from ergon_core.core.infrastructure.inngest.client import inngest_client -from ergon_core.core.infrastructure.inngest.contracts import ( - WorkerExecuteRequest, - WorkerExecuteResult, -) - - -@inngest_client.create_function( - fn_id="worker-execute", - trigger=inngest.TriggerEvent(event="task/worker-execute"), - retries=0, - output_type=WorkerExecuteResult, -) -async def worker_execute_fn(ctx: inngest.Context) -> WorkerExecuteResult: - return await run_worker_execute_job(WorkerExecuteRequest.model_validate(ctx.event.data)) - -__all__ = ["worker_execute_fn"] -``` - -Use the same pattern for every handler: `application/jobs/<name>.py` exports `run_<name>_job`, and `infrastructure/inngest/handlers/<name>.py` owns the decorator and event parsing. Preserve the existing `fn_id`, trigger event, retry policy, and output type from the original handler. - -- [ ] **Step 4: Update registry imports** - -In `core/infrastructure/inngest/registry.py`, import handler modules from `ergon_core.core.infrastructure.inngest.handlers`. - -If the registry currently imports function objects from handler modules, keep the same object names and only change module paths. - -- [ ] **Step 5: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.runtime.inngest.client": "ergon_core.core.infrastructure.inngest.client", - "ergon_core.core.runtime.inngest.registry": "ergon_core.core.infrastructure.inngest.registry", - "ergon_core.core.runtime.inngest.contracts": "ergon_core.core.infrastructure.inngest.contracts", - "ergon_core.core.runtime.inngest.errors": "ergon_core.core.infrastructure.inngest.errors", - "ergon_core.core.runtime.inngest.": "ergon_core.core.application.jobs.", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -After the script, inspect `core/infrastructure/inngest/registry.py` and adapter files. Registry imports should point to `infrastructure.inngest.handlers`, not `application.jobs`. - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/runtime/test_child_function_payloads.py tests/unit/runtime/test_inngest_criterion_executor.py tests/unit/runtime/test_import_boundaries.py tests/unit/registry/test_react_factories.py -q -``` - -Expected: PASS. - -## Task 8: Move Infrastructure Packages - -**Files:** -- Move: `core/sandbox/*` -> `core/infrastructure/sandbox/*` -- Move: `core/dashboard/*` -> `core/infrastructure/dashboard/*` -- Move: `core/runtime/tracing/*` -> `core/infrastructure/tracing/*` -- Move: `core/runtime/dependencies.py` -> `core/infrastructure/dependencies.py` -- Modify: imports across source and tests -- Test: dashboard, sandbox, tracing, dependency tests - -- [ ] **Step 1: Move sandbox** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/infrastructure/sandbox -mv ergon_core/ergon_core/core/sandbox/*.py ergon_core/ergon_core/core/infrastructure/sandbox/ -rmdir ergon_core/ergon_core/core/sandbox -``` - -- [ ] **Step 2: Move dashboard** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/infrastructure/dashboard -mv ergon_core/ergon_core/core/dashboard/*.py ergon_core/ergon_core/core/infrastructure/dashboard/ -rmdir ergon_core/ergon_core/core/dashboard -``` - -- [ ] **Step 3: Move tracing and dependency probe** - -Run: - -```bash -mkdir -p ergon_core/ergon_core/core/infrastructure/tracing -mv ergon_core/ergon_core/core/runtime/tracing/*.py ergon_core/ergon_core/core/infrastructure/tracing/ -rmdir ergon_core/ergon_core/core/runtime/tracing -mv ergon_core/ergon_core/core/runtime/dependencies.py ergon_core/ergon_core/core/infrastructure/dependencies.py -``` - -- [ ] **Step 4: Bulk update imports** - -Run: - -```bash -python - <<'PY' -from pathlib import Path - -replacements = { - "ergon_core.core.sandbox": "ergon_core.core.infrastructure.sandbox", - "ergon_core.core.dashboard": "ergon_core.core.infrastructure.dashboard", - "ergon_core.core.runtime.tracing": "ergon_core.core.infrastructure.tracing", - "ergon_core.core.runtime.dependencies": "ergon_core.core.infrastructure.dependencies", -} - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text - for old, replacement in replacements.items(): - new = new.replace(old, replacement) - if new != text: - path.write_text(new) -PY -``` - -- [ ] **Step 5: Run focused tests** - -Run: - -```bash -uv run pytest tests/unit/dashboard/test_event_contract_types.py tests/unit/runtime/test_sandbox_setup_explicit_slug.py tests/unit/benchmarks/test_swebench_sandbox_manager.py tests/unit/state/test_benchmark_contract.py -q -``` - -Expected: PASS. - -## Task 9: Move Application Events, Remove Runtime Root, And Add Durable Import Direction Guards - -**Files:** -- Move: `ergon_core/ergon_core/core/runtime/events/*` -> `ergon_core/ergon_core/core/application/events/*` -- Delete: `ergon_core/ergon_core/core/runtime/` -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Test: architecture suite - -- [ ] **Step 1: Delete empty runtime root** - -First move the remaining semantic event contracts out of runtime: - -```bash -mkdir -p ergon_core/ergon_core/core/application/events -mv ergon_core/ergon_core/core/runtime/events/*.py ergon_core/ergon_core/core/application/events/ -rmdir ergon_core/ergon_core/core/runtime/events -``` - -Then update imports: - -```bash -python - <<'PY' -from pathlib import Path - -for root in [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests")]: - for path in root.rglob("*.py"): - text = path.read_text() - new = text.replace( - "ergon_core.core.runtime.events", - "ergon_core.core.application.events", - ) - if new != text: - path.write_text(new) -PY -``` - -Now delete the empty runtime root: - -Run: - -```bash -rmdir ergon_core/ergon_core/core/runtime -``` - -Expected: command succeeds because all runtime subpackages and files have moved. - -- [ ] **Step 2: Add durable root guard** - -Append to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_core_uses_hybrid_domain_layout_roots() -> None: - core = ROOT / "ergon_core/ergon_core/core" - - expected_dirs = { - "application", - "domain", - "infrastructure", - "persistence", - "rest_api", - "rl", - "shared", - } - actual_dirs = {path.name for path in core.iterdir() if path.is_dir() and path.name != "__pycache__"} - - assert expected_dirs <= actual_dirs - assert "runtime" not in actual_dirs - assert "api" not in actual_dirs - assert "definitions" not in actual_dirs - assert "composition" not in actual_dirs - assert "sandbox" not in actual_dirs - assert "dashboard" not in actual_dirs -``` - -- [ ] **Step 3: Add import direction guard** - -Append to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_core_hybrid_layout_import_directions() -> None: - forbidden_imports = { - "domain": ( - "ergon_core.core.application", - "ergon_core.core.persistence", - "ergon_core.core.infrastructure", - "ergon_core.core.rest_api", - ), - "persistence": ( - "ergon_core.core.application", - "ergon_core.core.infrastructure", - "ergon_core.core.rest_api", - ), - "application": ( - "ergon_core.core.rest_api", - "ergon_core.core.infrastructure.inngest.handlers", - ), - } - - offenders: list[str] = [] - for root_name, snippets in forbidden_imports.items(): - root = ROOT / "ergon_core/ergon_core/core" / root_name - for path in root.rglob("*.py"): - text = path.read_text() - for snippet in snippets: - if snippet in text: - offenders.append(f"{path.relative_to(ROOT)} imports {snippet}") - - assert offenders == [] -``` - -- [ ] **Step 4: Add job adapter split guard** - -Append to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_application_jobs_do_not_own_inngest_decorators() -> None: - jobs_root = ROOT / "ergon_core/ergon_core/core/application/jobs" - offenders: list[str] = [] - - for path in jobs_root.rglob("*.py"): - text = path.read_text() - if "@inngest_client.create_function" in text or "import inngest" in text: - offenders.append(str(path.relative_to(ROOT))) - if "ergon_core.core.infrastructure.inngest.handlers" in text: - offenders.append(str(path.relative_to(ROOT))) - - assert offenders == [] -``` - -- [ ] **Step 5: Run architecture tests** - -Run: - -```bash -uv run pytest tests/unit/architecture -q -``` - -Expected: PASS except the temporary exact-layout test may still fail if additional unexpected files exist. If it fails, inspect the exact `unexpected` list and decide whether the target doc should include those files or the files should move/delete. - -## Task 10: Finalize Exact Layout, Delete Temporary Test - -**Files:** -- Delete: `tests/unit/architecture/test_core_hybrid_layout_temporary.py` -- Modify: `docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md` if any final file names changed during implementation -- Test: architecture suite and focused regression suite - -- [ ] **Step 1: Run temporary exact-layout test one last time** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_core_hybrid_layout_temporary.py -q -``` - -Expected: PASS. This proves the temporary exact target was achieved before deleting the brittle guard. - -- [ ] **Step 2: Delete the temporary test** - -Run: - -```bash -rm tests/unit/architecture/test_core_hybrid_layout_temporary.py -``` - -- [ ] **Step 3: Run architecture and focused regression tests** - -Run: - -```bash -uv run pytest tests/unit/architecture tests/unit/runtime/test_workflow_service.py tests/unit/runtime/test_task_execution_repository.py tests/unit/runtime/test_inngest_criterion_executor.py tests/unit/dashboard/test_communication_threads.py tests/unit/cli/test_experiment_cli.py tests/unit/benchmarks/test_swebench_sandbox_manager.py -q -``` - -Expected: PASS. - -- [ ] **Step 4: Run ruff on moved source and tests** - -Run: - -```bash -uv run ruff check ergon_core ergon_cli ergon_builtins tests/unit/architecture -``` - -Expected: PASS. - -## Task 11: Broad Verification - -**Files:** -- Modify: none unless tests reveal missed imports -- Test: broad unit/integration suite as time permits - -- [ ] **Step 1: Search for stale paths** - -Run: - -```bash -rg "ergon_core\\.core\\.(runtime|api|definitions|composition|sandbox|dashboard)|core/runtime|core/api|core/definitions|core/composition|core/sandbox|core/dashboard" ergon_core ergon_cli ergon_builtins tests docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md -``` - -Expected: no stale code imports. Documentation may mention old paths only in current-to-target move maps. - -- [ ] **Step 2: Run broad unit tests** - -Run: - -```bash -uv run pytest tests/unit -q -``` - -Expected: PASS, or failures only from known environment import-resolution issues. Fix any migration-related import failures. - -- [ ] **Step 3: Run targeted integration tests** - -Run: - -```bash -uv run pytest tests/integration/propagation tests/integration/restart tests/integration/smokes -q -``` - -Expected: PASS, or failures clearly unrelated to package movement. - -## Self-Review Checklist - -- Every moved package has a target path in the plan. -- The temporary exact folder test is added first and deleted in the final cleanup. -- `core/rl` remains top-level. -- `core/rest_api` is distinct from public `ergon_core.api`. -- Inngest semantic jobs land in `application/jobs`; adapters land in `infrastructure/inngest/handlers`. -- No compatibility aliases are required by the plan. -- No git commits are required by the plan. diff --git a/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md b/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md deleted file mode 100644 index 685b2316f..000000000 --- a/docs/superpowers/plans/2026-04-28-core-hybrid-domain-layout.md +++ /dev/null @@ -1,584 +0,0 @@ -# Core Hybrid Domain Layout - -This documents the implemented hybrid layout for `ergon_core.core`: hard -technical layers stay visible (`rest_api`, `persistence`, `infrastructure`), -while product/application concepts live in explicit clusters under -`core/application`. - -The goal is not "everything is domain-first". The goal is that a new contributor -can answer three questions quickly: - -1. Where do use cases live? -2. Where do SQL/storage rows live? -3. Where do transport/infrastructure adapters live? - -## Implemented Top-Level Shape - -```text -ergon_core/ergon_core/core/ - __init__.py - - rest_api/ - # FastAPI / HTTP transport only. - # Named rest_api to avoid confusion with the public authoring API - # under ergon_core.api. - # Should import application services and read models, not own domain logic. - __init__.py - app.py - cohorts.py - experiments.py - rollouts.py - runs.py - test_harness.py - - application/ - # Product use cases and domain-aware repositories. - # This replaces the current "runtime as second root" feeling. - - experiments/ - # Define experiments, persist authored definitions, launch experiment runs. - # Implemented from: - # - core/definitions/service.py - # - core/definitions/persistence.py - # - core/definitions/repository.py - # - core/definitions/schemas.py - # - runtime/workflows/launch.py - __init__.py - service.py - models.py - repository.py - definition_writer.py - launch.py - - workflows/ - # Run/workflow lifecycle after a definition exists. - # Implemented from: - # - runtime/workflows/service.py - # - runtime/workflows/orchestration.py - # - runtime/workflows/runs.py - # - runtime/workflows/models.py - # - runtime/workflows/errors.py - service.py - orchestration.py - runs.py - models.py - errors.py - - graph/ - # Runtime graph mutations, traversal, lookup, and propagation. - # Implemented from: - # - runtime/graph/* - repository.py - propagation.py - traversal.py - lookup.py - models.py - errors.py - - tasks/ - # Task execution lifecycle and task execution repository. - # Implemented from: - # - runtime/tasks/* - __init__.py - service.py - execution.py - management.py - inspection.py - cleanup.py - repository.py - models.py - errors.py - - evaluation/ - # Evaluation dispatch, criterion runtime, scoring, persistence use cases. - # Implemented from: - # - runtime/evaluation/* - service.py - executors.py - inngest_executor.py - criterion_runtime.py - scoring.py - protocols.py - models.py - errors.py - - read_models/ - # Query-side DTO assembly for UI/API surfaces. - # Implemented from: - # - runtime/read_models/runs.py - # - runtime/read_models/run_snapshot.py - # - runtime/read_models/experiments.py - # - runtime/read_models/cohorts.py - # - runtime/read_models/resources.py - # - runtime/read_models/models.py - # - runtime/read_models/errors.py - __init__.py - runs.py - run_snapshot.py - experiments.py - cohorts.py - resources.py - models.py - errors.py - - communication/ - # Agent-to-agent communication is its own product domain. - # Do not fold this into run read models. - # Implemented from: - # - runtime/read_models/communication.py - # - relevant communication DTOs currently in runtime/read_models/models.py - __init__.py - service.py - models.py - errors.py - - context/ - # Worker context event stream and output extraction. - # Implemented from: - # - runtime/context_events.py - # - runtime/output_extraction.py - __init__.py - events.py - output_extraction.py - - jobs/ - # Core semantic workflows currently implemented inside Inngest handlers. - # These are background job use cases. Inngest should call them, not own - # their branching, persistence, and orchestration rules. - # Implemented from: - # - runtime/inngest/{handler files}.py, after extracting adapter details. - cancel_orphan_subtasks.py - check_evaluators.py - cleanup_cancelled_task.py - complete_workflow.py - evaluate_task_run.py - execute_task.py - fail_workflow.py - persist_outputs.py - propagate_execution.py - run_cleanup.py - sandbox_setup.py - start_workflow.py - worker_execute.py - models.py - - resources/ - # Run resource append/query use cases that are not just API presentation. - # Implemented from: - # - runtime/resources.py - # - sandbox/resource_publisher.py may depend on repository here - __init__.py - repository.py - models.py - - events/ - # Product/application event contracts used by jobs, adapters, and - # dashboard emission. The adapter layer may send these through Inngest, - # but it should not own their semantic schemas. - # Implemented from: - # - runtime/events/* - __init__.py - base.py - task_events.py - infrastructure_events.py - - domain/ - # Pure-ish domain objects that should not know about DB sessions, - # Inngest, FastAPI, or dashboard emission. - - experiments/ - # Authoring/composition objects. - # Implemented from: - # - core/composition/* - __init__.py - experiment.py - handles.py - worker_spec.py - validation.py - - generation/ - # Context stream and generation transcript primitives. - # Implemented from: - # - core/generation.py - context_parts.py - - persistence/ - # SQLModel rows, DB/session helpers, and storage-only repositories. - # Should not own product workflows or read-model assembly. - - shared/ - db.py - enums.py - ids.py - types.py - - definitions/ - models.py - - telemetry/ - models.py - repositories.py - evaluation_summary.py - - graph/ - models.py - status_conventions.py - - context/ - models.py - event_payloads.py - - saved_specs/ - models.py - - infrastructure/ - # External adapters and operational plumbing. - # Infrastructure calls application services; application should not import - # concrete infrastructure except through deliberate adapter seams. - - inngest/ - # Inngest client, contracts, registry, and thin function adapters. - # Implemented from: - # - runtime/inngest/client.py - # - runtime/inngest/registry.py - # - runtime/inngest/contracts.py - # - runtime/inngest/errors.py - # - runtime/inngest/{handler files}.py after semantic logic moves to - # application/jobs. - client.py - registry.py - contracts.py - errors.py - - handlers/ - cancel_orphan_subtasks.py - check_evaluators.py - cleanup_cancelled_task.py - complete_workflow.py - evaluate_task_run.py - execute_task.py - fail_workflow.py - persist_outputs.py - propagate_execution.py - run_cleanup.py - sandbox_setup.py - start_workflow.py - worker_execute.py - - sandbox/ - # E2B/local sandbox managers and sandbox instrumentation. - # Implemented from: - # - core/sandbox/* - __init__.py - manager.py - lifecycle.py - resource_publisher.py - instrumentation.py - event_sink.py - errors.py - utils.py - - dashboard/ - # Dashboard event emission/integration. - # Implemented from: - # - core/dashboard/* - __init__.py - emitter.py - provider.py - event_contracts.py - - tracing/ - # Tracing/OpenTelemetry adapters and sinks. - # Implemented from: - # - runtime/tracing/* - __init__.py - attributes.py - contexts.py - ids.py - noop.py - otel.py - sinks.py - types.py - - dependencies.py - - rl/ - # Keep as a separate bounded context for now. - # Rollouts, rewards, extraction, checkpointing, and vLLM management cut - # across product use cases and are closer to training/research machinery - # than ordinary application services. - __init__.py - rollout_service.py - eval_runner.py - extraction.py - rewards.py - checkpoint.py - rollout_types.py - vllm_manager.py - - shared/ - # Small cross-cutting primitives. Keep this boring and sparse. - json_types.py - settings.py - utils.py -``` - -## Clusters And Ownership Rules - -### `core/application` - -Application packages own use cases. They can import: - -- `core/domain` -- `core/persistence` -- `core/shared` - -They should not import: - -- `core/rest_api` -- Inngest function modules -- FastAPI router modules - -`application` is where the former `runtime` domains landed. The important rename -is conceptual: the old `runtime` package mixed use cases, adapters, and -operational helpers, while `application` now means "use cases over the persisted -product model." - -### `core/domain` - -Domain packages own objects that should be understandable without infrastructure: - -- experiment composition -- worker specs -- definition handles -- context/generation primitives - -These modules should not create DB sessions, emit dashboard events, or know about -Inngest. They may validate invariants and expose plain objects. - -### `core/persistence` - -Persistence owns rows and storage helpers. It should not own product decisions. - -Examples that should stay here: - -- SQLModel row classes -- session helpers -- enum/storage types -- storage-only repositories - -Examples that should not live here: - -- query-bag application workflows -- evaluation summary refresh orchestration -- context event sequencing logic -- run snapshot assembly - -### `core/infrastructure` - -Infrastructure owns adapters: - -- Inngest client, registry, contracts, and thin function adapters -- sandbox manager/resource publisher -- dashboard emitter -- tracing adapters -- package dependency probes - -Infrastructure modules can call application services. They should not become -the canonical home for business rules. Inngest handlers are split so core -semantic logic lives in `application/jobs`, while the Inngest-decorated shell -remains under `infrastructure/inngest/handlers`. - -### `core/rest_api` - -`core/rest_api` is the HTTP layer. The explicit name keeps it visually separate -from `ergon_core.api`, which is the public authoring/types API for builtins, -CLI, and students. It should be thin: - -- validate/deserialize transport requests -- call application services/read models -- map missing resources to HTTP errors - -It should not define reusable domain DTOs just because the frontend consumes -them. Those belong in `application/read_models` or the relevant application -domain. - -## Implemented Move Map - -```text -core/definitions/service.py - -> core/application/experiments/service.py - -core/definitions/schemas.py - -> core/application/experiments/models.py - -core/definitions/repository.py - -> core/application/experiments/repository.py - -core/definitions/persistence.py - -> core/application/experiments/definition_writer.py - -core/composition/* - -> core/domain/experiments/* - -core/runtime/workflows/* - -> core/application/workflows/* - except runtime/workflows/launch.py - -> core/application/experiments/launch.py - -core/runtime/graph/* - -> core/application/graph/* - -core/runtime/tasks/* - -> core/application/tasks/* - -core/runtime/evaluation/* - -> core/application/evaluation/* - -core/runtime/read_models/runs.py -core/runtime/read_models/run_snapshot.py -core/runtime/read_models/experiments.py -core/runtime/read_models/cohorts.py -core/runtime/read_models/resources.py -core/runtime/read_models/errors.py -core/runtime/read_models/models.py - -> core/application/read_models/* - -core/runtime/read_models/communication.py - -> core/application/communication/service.py - -communication DTOs from core/runtime/read_models/models.py - -> core/application/communication/models.py - -core/runtime/context_events.py - -> core/application/context/events.py - -core/runtime/output_extraction.py - -> core/application/context/output_extraction.py - -core/runtime/resources.py - -> core/application/resources/models.py - -> core/application/resources/repository.py - -core/runtime/events/* - -> core/application/events/* - -core/rl/* - -> core/rl/* - # Keep in place for now as a separate bounded context. - -core/runtime/inngest/client.py -core/runtime/inngest/registry.py -core/runtime/inngest/contracts.py -core/runtime/inngest/errors.py - -> core/infrastructure/inngest/* - -core/runtime/inngest/{handler files}.py - -> core/application/jobs/{handler files}.py - -> core/infrastructure/inngest/handlers/{handler files}.py - # Split each handler: semantic background job into application/jobs, - # Inngest decorator/event adapter into infrastructure/inngest/handlers. - -core/sandbox/* - -> core/infrastructure/sandbox/* - -core/dashboard/* - -> core/infrastructure/dashboard/* - -core/runtime/tracing/* - -> core/infrastructure/tracing/* - -core/runtime/dependencies.py - -> core/infrastructure/dependencies.py - -core/generation.py - -> core/domain/generation/context_parts.py - -core/json_types.py -core/settings.py -core/utils.py - -> core/shared/* -``` - -## Deleted Legacy Paths - -```text -core/runtime/ - # Deleted after all subpackages moved. - -core/definitions/ - # Deleted after experiment lifecycle files moved to application/experiments. - -core/composition/ - # Deleted after pure domain objects moved to domain/experiments. - -core/sandbox/ -core/dashboard/ - # Deleted after infrastructure moved. - -core/generation.py -core/json_types.py -core/settings.py -core/utils.py - # Deleted after shared/domain moves. -``` - -## Import Direction Guardrails - -```text -api -> application -> domain -api -> application -> persistence -api -> shared - -infrastructure -> application -infrastructure -> domain -infrastructure -> persistence -infrastructure -> shared - -application -> domain -application -> persistence -application -> shared - -persistence -> shared -persistence -> domain/generation only if row payload parsing requires typed context parts - -domain -> shared -``` - -Forbidden directions: - -```text -domain -> application -domain -> persistence -domain -> infrastructure -domain -> rest_api - -persistence -> application -persistence -> infrastructure -persistence -> rest_api - -application -> rest_api -application -> infrastructure/inngest/handlers -``` - -## Resolved Decisions - -1. This intentionally keeps `communication` separate from run read models. It is - a product domain for agents communicating with each other. -2. `read_models` stays as a query-side application cluster instead of being - split into every domain. That reduces churn while keeping REST - routers thin. -3. `application/jobs` keeps the core semantics of externally-triggered - background workflows visible. `infrastructure/inngest/handlers` should be - thin wrappers around those use cases. -4. `persistence` remains a visible top-level layer because hiding SQL rows - inside product domains would make storage contracts harder to - audit. -5. Old-path compatibility aliases are intentionally avoided. Bulk import renames - keep the finalized package structure explicit. -6. `domain/generation/context_parts.py` remains the name for generation context - primitives. -7. Dashboard emission stays under `infrastructure/dashboard`, while product - event contracts live under `application/events`. -8. `core/rl` remains its own bounded context instead of being renamed to - `core/learning`. diff --git a/docs/superpowers/plans/2026-04-28-core-schema-deduplication.md b/docs/superpowers/plans/2026-04-28-core-schema-deduplication.md deleted file mode 100644 index db086f5de..000000000 --- a/docs/superpowers/plans/2026-04-28-core-schema-deduplication.md +++ /dev/null @@ -1,1178 +0,0 @@ -# Core Schema Deduplication Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make core workflow statuses, evaluation statuses, graph mutation payloads, event causes, and projection schemas have one clear source of truth per domain. - -**Architecture:** Keep persisted table schemas in `core/persistence/*`, graph lifecycle conventions in `core/persistence/graph/status_conventions.py`, typed graph mutation payloads in `core/runtime/services/graph_dto.py`, evaluation summary status in `core/persistence/telemetry/evaluation_summary.py`, and transport-specific projections in `core/api/schemas.py` and `core/dashboard/event_contracts.py`. REST and dashboard layers may project canonical DTOs, but must not redefine domain meaning. - -**Tech Stack:** Python 3.13, Pydantic v2, SQLModel, pytest, ty-compatible type aliases, existing Ergon core runtime/persistence packages. - ---- - -## Source Of Truth Decisions - -| Concept | Source of truth | Consumers should import from | Cleanup rule | -|---|---|---|---| -| Run row lifecycle | `ergon_core.core.persistence.shared.enums.RunStatus` | `core.persistence.shared.enums` | Only use for `RunRecord.status` and run-level orchestration. | -| Task execution row lifecycle | `ergon_core.core.persistence.shared.enums.TaskExecutionStatus` | `core.persistence.shared.enums` | Only use for `RunTaskExecution.status`; do not use it as the graph-node status type. | -| Graph node lifecycle | `ergon_core.core.persistence.graph.status_conventions.NodeStatus` and constants | `core.persistence.graph.status_conventions` | Use for `RunGraphNode.status`, propagation, subtask inspection, dashboard task-node status, and graph DTO status annotations. | -| Graph edge lifecycle | `ergon_core.core.persistence.graph.status_conventions.EdgeStatus` and constants | `core.persistence.graph.status_conventions` | Use for `RunGraphEdge.status` and edge mutation/status changes. | -| Graph target and mutation names | `GraphTargetType`, `MutationType` in `core/persistence/graph/models.py` | `core.persistence.graph.models` | Keep because these are persisted mutation-log contract names. | -| Graph mutation payload body | `GraphMutationValue` union in `core/runtime/services/graph_dto.py` | `core.runtime.services.graph_dto` | REST and dashboard events import this union; no separate payload definitions. | -| Evaluation criterion status | `EvalCriterionStatus` in `core/persistence/telemetry/evaluation_summary.py` | `core.persistence.telemetry.evaluation_summary` | REST evaluation DTOs import this alias. | -| Cancel cause | `CancelCause` in `core/runtime/events/task_events.py` | `core.runtime.events.task_events` | Services that accept cancel causes import the shared alias or narrower named aliases from the same module. | -| Context event payloads | `ContextEventType`, `ContextEventPayload` in `core/persistence/context/event_payloads.py` | `core.persistence.context.event_payloads` | REST/dashboard context event snapshots should use the canonical type where practical. | -| Generation transcript parts | `core/generation.py` | `core.generation` | Keep separate from context event payloads; add adapter tests for the mapping instead of merging naming schemes. | - ---- - -## DTO Collapse Targets - -The cleanup should collapse duplicate DTOs when two classes carry the same domain payload with only superficial transport differences. Keep separate models only when the shape is genuinely different at the boundary. - -| Current duplication | Collapse target | Keep separate? | Why | -|---|---|---|---| -| `GraphMutationDto`, `RunGraphMutationDto`, `DashboardGraphMutationEvent` repeat mutation identity/body fields | Add canonical `GraphMutationRecordDto` in `core/runtime/services/graph_dto.py`; REST returns it, dashboard event embeds it or is a thin envelope around it | Keep dashboard event envelope only | Mutation body and metadata are one concept; REST/dashboard differ only by transport envelope and timestamp naming. | -| `RunContextEventDto` and `DashboardContextEventEvent` repeat context-event fields, but REST is untyped | Add canonical `ContextEventDto` near `core/persistence/context/event_payloads.py` or `core/runtime/services/context_dto.py`; both REST and dashboard use `ContextEventType` + `ContextEventPayload` | Keep event envelope name only | Same persisted event snapshot should not have typed dashboard payload and untyped REST payload. | -| `WorkflowTaskRef` mostly duplicates a subset of `GraphNodeDto` | Prefer `GraphNodeDto` directly where the full node snapshot is acceptable; otherwise create one canonical `GraphTaskRef` in `graph_dto.py` and use it across workflow DTOs | Maybe | CLI/tool responses may intentionally omit fields, but the current separate class adds another status/name surface. | -| `RunTaskDto` and `TaskTreeNode` both represent UI task nodes but one is map-oriented and one is recursive | Extract a shared `TaskNodeSnapshot` payload if frontend compatibility allows; keep `RunSnapshotDto.tasks: dict[str, ...]` and `DashboardWorkflowStartedEvent.task_tree` as containers | Yes, containers differ | Map vs tree is a real transport difference; the task-node payload fields should not drift. | -| `TestGraphNodeDto` and `TestGraphMutationDto` are Playwright-only projections | Leave separate but derive from canonical DTO conversion helpers where possible | Yes | Test harness is intentionally narrow/additive-only, but should not define new domain semantics. | - -Rule: collapse the payload, not necessarily the envelope. For example, `DashboardGraphMutationEvent` can remain an event contract, but it should carry the same canonical mutation record/payload as REST and repository code. - ---- - -## File Structure - -**Modify:** -- `ergon_core/ergon_core/core/persistence/graph/status_conventions.py` — canonical graph status aliases, terminal/settled helpers, and small predicates. -- `ergon_core/ergon_core/core/runtime/execution/propagation.py` — use graph status constants consistently and align failure docs/results with `BLOCKED` behavior. -- `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` — remove stale cancellation wording and stop exposing unused invalidated targets from normal propagation if tests confirm it is dead. -- `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` — remove dead `TaskCancelledEvent` emission from propagation if `invalidated_targets` is removed. -- `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py` — simplify `PropagationResult` around actual ready/block/terminal outcomes. -- `ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py` — use `NodeStatus` directly instead of duplicating or aliasing `SubtaskStatus`. -- `ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py` — keep `EvalCriterionStatus` canonical. -- `ergon_core/ergon_core/core/api/schemas.py` — import `EvalCriterionStatus`, remove duplicate mutation/context payload bodies, and keep REST projection thin. -- `ergon_core/ergon_core/core/runtime/services/graph_dto.py` — make `GraphMutationValue` the only typed mutation payload body and make edge mutation IDs consistent with graph DTO ID types. -- `ergon_core/ergon_core/core/dashboard/event_contracts.py` — keep event envelopes but reuse canonical graph mutation/context event DTO payloads. -- `ergon_core/ergon_core/core/runtime/events/task_events.py` — keep `CancelCause` canonical and add subset aliases if services need narrower inputs. -- `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py` — import shared cancel-cause aliases instead of duplicating string literals. -- `ergon_core/ergon_core/core/runtime/services/subtask_blocking_service.py` — share graph skip predicates from `status_conventions.py`. - -**Add or modify tests:** -- `tests/unit/architecture/test_core_schema_sources.py` — architecture guard for duplicate literals and forbidden imports. -- `tests/unit/runtime/test_propagation_contracts.py` or existing propagation tests — assert failure propagation blocks downstream nodes and does not emit cancellation targets. -- `tests/unit/runtime/test_graph_mutation_contracts.py` or existing graph repository tests — assert REST/dashboard mutation payloads accept the same `GraphMutationValue` body. -- Existing focused tests: `tests/unit/runtime/test_workflow_service.py`, `tests/unit/runtime/test_dynamic_task_evaluation_mapping.py`, `tests/unit/dashboard/test_event_contract_types.py`, `tests/unit/architecture/test_model_field_descriptions.py`. - ---- - -### Task 1: Guard Canonical Status Ownership - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `ergon_core/ergon_core/core/persistence/graph/status_conventions.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py` - -- [ ] **Step 1: Write architecture tests that fail on duplicated graph status literals** - -Create `tests/unit/architecture/test_core_schema_sources.py` with this first test: - -```python -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[3] - - -def test_graph_status_literals_are_defined_only_in_status_conventions() -> None: - offenders: list[str] = [] - duplicate_snippets = ( - 'Literal["pending", "ready", "running", "completed", "failed", "cancelled", "blocked"]', - 'Literal["pending", "ready", "running", "completed", "failed", "blocked", "cancelled"]', - 'Literal["pending", "satisfied", "invalidated"]', - ) - allowed = { - ROOT / "ergon_core/ergon_core/core/persistence/graph/status_conventions.py", - } - - for path in (ROOT / "ergon_core/ergon_core/core").rglob("*.py"): - if path in allowed: - continue - text = path.read_text() - for snippet in duplicate_snippets: - if snippet in text: - offenders.append(f"{path.relative_to(ROOT)} duplicates {snippet}") - - assert offenders == [] -``` - -- [ ] **Step 2: Run the new test and verify it fails** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py::test_graph_status_literals_are_defined_only_in_status_conventions -v` - -Expected: FAIL because `task_inspection_dto.py` duplicates the node status `Literal`. - -- [ ] **Step 3: Add canonical helpers to `status_conventions.py`** - -Update `ergon_core/ergon_core/core/persistence/graph/status_conventions.py`: - -```python -NodeStatus = Literal["pending", "ready", "running", "completed", "failed", "cancelled", "blocked"] - -NON_AUTONOMOUS_STATUSES = TERMINAL_STATUSES | frozenset({BLOCKED}) - - -def is_terminal_node_status(status: str) -> bool: - return status in TERMINAL_STATUSES - - -def is_blockable_node_status(status: str) -> bool: - return status != RUNNING and status not in TERMINAL_STATUSES -``` - -Keep `EdgeStatus` in the same file. Do not move graph statuses to `shared/enums.py`; graph status intentionally remains string-backed because `RunGraphNode.status` is free-form at the database layer. - -- [ ] **Step 4: Replace `SubtaskStatus` with `NodeStatus` at the field boundary** - -Update `ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py`: - -```python -from ergon_core.core.persistence.graph.status_conventions import NodeStatus -from ergon_core.core.persistence.shared.types import NodeId -from pydantic import BaseModel -``` - -Change the model field from: - -```python -status: SubtaskStatus -``` - -to: - -```python -status: NodeStatus -``` - -Delete the `SubtaskStatus` name entirely. If any downstream call site imports `SubtaskStatus`, update that call site to import `NodeStatus` from `status_conventions.py` instead. The goal is one concept name for graph-node lifecycle state. - -- [ ] **Step 5: Run focused tests** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py tests/unit/state/test_subtask_lifecycle_toolkit.py tests/unit/runtime/test_workflow_service.py -v` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add tests/unit/architecture/test_core_schema_sources.py ergon_core/ergon_core/core/persistence/graph/status_conventions.py ergon_core/ergon_core/core/runtime/services/task_inspection_dto.py -git commit -m "Consolidate graph status conventions" -``` - ---- - -### Task 2: Separate Graph Status From Task Execution Status In Propagation - -**Files:** -- Modify: `tests/unit/runtime/test_propagation_contracts.py` -- Modify: `ergon_core/ergon_core/core/runtime/execution/propagation.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_execution_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_initialization_service.py` - -- [ ] **Step 1: Write tests for graph-node status constants at every graph write boundary** - -Add `tests/unit/runtime/test_propagation_contracts.py`: - -```python -from ergon_core.core.persistence.graph import status_conventions as graph_status -from ergon_core.core.runtime.execution import propagation -from ergon_core.core.runtime.services import task_execution_service, task_propagation_service -from ergon_core.core.runtime.services import workflow_initialization_service - - -def _source(module: object) -> str: - loader = getattr(module, "__loader__") - source = loader.get_source(module.__name__) - assert source is not None - return source - - -def test_graph_writers_do_not_use_task_execution_status_for_node_status() -> None: - modules = [ - propagation, - task_execution_service, - task_propagation_service, - workflow_initialization_service, - ] - forbidden_snippets = ( - "new_status=TaskExecutionStatus.", - "initial_node_status=TaskExecutionStatus.", - ) - - offenders = [ - f"{module.__name__}: {snippet}" - for module in modules - for snippet in forbidden_snippets - if snippet in _source(module) - ] - - assert offenders == [] - assert graph_status.READY == "ready" -``` - -This is an architecture test. It is intentionally string-based because the cleanup goal is import-boundary clarity. - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/runtime/test_propagation_contracts.py::test_graph_writers_do_not_use_task_execution_status_for_node_status -v` - -Expected: FAIL because `propagation.py`, `task_propagation_service.py`, `task_execution_service.py`, and `workflow_initialization_service.py` currently use `TaskExecutionStatus` values while writing graph-node status. - -- [ ] **Step 3: Update propagation imports** - -In `ergon_core/ergon_core/core/runtime/execution/propagation.py`, replace direct status imports with a module alias: - -```python -from ergon_core.core.persistence.graph import status_conventions as graph_status -``` - -Remove `TaskExecutionStatus` from `propagation.py` if it becomes unused. This module operates on `RunGraphNode` / `RunGraphEdge`, so all graph-node writes and graph-node comparisons must use `graph_status.*`. - -- [ ] **Step 4: Update graph node writes** - -Change graph-node status writes: - -```python -new_status=graph_status.PENDING -new_status=graph_status.RUNNING -new_status=graph_status.FAILED -new_status=graph_status.BLOCKED -``` - -Change comparisons: - -```python -is_success = terminal_status == graph_status.COMPLETED -if target_node.status == graph_status.RUNNING: -if target_node.status in graph_status.TERMINAL_STATUSES: -is_pending = status == graph_status.PENDING -is_reactivatable_cancelled = status == graph_status.CANCELLED and is_managed_subtask -if all(n is not None and n.status == graph_status.COMPLETED for n in source_nodes): -``` - -- [ ] **Step 5: Update service calls into propagation** - -In `task_propagation_service.py`, call `on_task_completed_or_failed` with graph status constants: - -```python -from ergon_core.core.persistence.graph import status_conventions as graph_status -``` - -Use: - -```python -new_status=graph_status.COMPLETED -terminal_status=graph_status.COMPLETED -new_status=graph_status.FAILED -terminal_status=graph_status.FAILED -new_status=graph_status.PENDING -``` - -- [ ] **Step 6: Update task execution graph writes without changing execution-row writes** - -In `task_execution_service.py`, keep `TaskExecutionStatus` for `RunTaskExecution.status` assignments: - -```python -execution = RunTaskExecution( - ... - status=TaskExecutionStatus.RUNNING, -) -execution.status = TaskExecutionStatus.COMPLETED -execution.status = TaskExecutionStatus.FAILED -``` - -But change graph-node updates and dashboard node-status emissions to graph status constants: - -```python -from ergon_core.core.persistence.graph import status_conventions as graph_status - -await self._graph_repo.update_node_status( - ..., - new_status=graph_status.RUNNING, - ... -) - -await _emit_task_status( - ..., - new_status=graph_status.RUNNING, - ... -) -``` - -For finalization events that are explicitly reporting task-node lifecycle state, use: - -```python -new_status=graph_status.COMPLETED -old_status=graph_status.RUNNING -new_status=graph_status.FAILED -``` - -The rule is: `TaskExecutionStatus` belongs to `RunTaskExecution.status`; `graph_status` belongs to `RunGraphNode.status` and dashboard task-node status payloads. - -- [ ] **Step 7: Update workflow initialization graph seeding** - -In `workflow_initialization_service.py`, keep `RunStatus.EXECUTING` for `RunRecord.status`, but change graph initialization inputs: - -```python -from ergon_core.core.persistence.graph import status_conventions as graph_status - -graph_repo.initialize_from_definition( - ..., - initial_node_status=graph_status.PENDING, - initial_edge_status=graph_status.EDGE_PENDING, - ... -) -``` - -- [ ] **Step 8: Run focused tests** - -Run: `uv run pytest tests/unit/runtime/test_propagation_contracts.py tests/unit/runtime/test_workflow_service.py tests/unit/runtime/test_dynamic_task_evaluation_mapping.py tests/unit/runtime/test_failure_error_json.py tests/unit/runtime/test_worker_execute_factory_call.py tests/unit/runtime/test_smoke_topology_drift.py -v` - -Expected: PASS. - -- [ ] **Step 9: Commit** - -```bash -git add tests/unit/runtime/test_propagation_contracts.py ergon_core/ergon_core/core/runtime/execution/propagation.py ergon_core/ergon_core/core/runtime/services/task_propagation_service.py ergon_core/ergon_core/core/runtime/services/task_execution_service.py ergon_core/ergon_core/core/runtime/services/workflow_initialization_service.py -git commit -m "Use graph status conventions in propagation" -``` - ---- - -### Task 3: Align Failure Propagation Contract With BLOCKED Behavior - -**Files:** -- Modify: `tests/unit/runtime/test_propagation_contracts.py` -- Modify: `ergon_core/ergon_core/core/runtime/execution/propagation.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/orchestration_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/task_propagation_service.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py` - -- [ ] **Step 1: Add a contract test for no cancellation targets from propagation** - -Extend `tests/unit/runtime/test_propagation_contracts.py`: - -```python -from ergon_core.core.runtime.services.orchestration_dto import PropagationResult - - -def test_propagation_result_does_not_expose_invalidated_targets() -> None: - assert "invalidated_targets" not in PropagationResult.model_fields -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/runtime/test_propagation_contracts.py::test_propagation_result_does_not_expose_invalidated_targets -v` - -Expected: FAIL because `PropagationResult` currently has `invalidated_targets`. - -- [ ] **Step 3: Simplify `PropagationResult`** - -In `orchestration_dto.py`, remove the field: - -```python -invalidated_targets: list[UUID] = Field(default_factory=list) -``` - -Keep: - -```python -ready_tasks: list[TaskDescriptor] = Field(default_factory=list) -workflow_terminal_state: WorkflowTerminalState = WorkflowTerminalState.NONE -``` - -- [ ] **Step 4: Update `on_task_completed_or_failed` return type and docs** - -In `propagation.py`, change: - -```python -) -> tuple[list[UUID], list[UUID]]: -``` - -to: - -```python -) -> list[UUID]: -``` - -Update the docstring to say: - -```python -"""Handle a node reaching COMPLETED, FAILED, or CANCELLED. - -Returns newly ready node IDs. - -- COMPLETED: outgoing edges become SATISFIED; targets with all dependencies - satisfied transition to PENDING for scheduling. -- FAILED / CANCELLED: outgoing edges become INVALIDATED; reachable successors - transition to BLOCKED unless they are RUNNING or terminal. -""" -``` - -Remove the local `invalidated: list[UUID] = []` and return only `newly_ready`. - -- [ ] **Step 5: Update `TaskPropagationService`** - -Change: - -```python -newly_ready_node_ids, invalidated_node_ids = await on_task_completed_or_failed(...) -``` - -to: - -```python -newly_ready_node_ids = await on_task_completed_or_failed(...) -``` - -Remove `invalidated_targets=invalidated_node_ids` from returned `PropagationResult`. - -For failure propagation, change: - -```python -_ready, invalidated_node_ids = await on_task_completed_or_failed(...) -``` - -to: - -```python -await on_task_completed_or_failed(...) -``` - -Update docstrings to say failure blocks downstream graph nodes, not cancels them. - -- [ ] **Step 6: Remove dead cancellation emission from `propagate_execution.py`** - -Remove the import: - -```python -TaskCancelledEvent, -``` - -Remove the loop: - -```python -for inv_node_id in propagation.invalidated_targets: - events.append(...) -``` - -Keep `TaskCancelledEvent` in `task_events.py`; it is still used by manager/operator cancellation flows. - -- [ ] **Step 7: Run focused tests** - -Run: `uv run pytest tests/unit/runtime/test_propagation_contracts.py tests/unit/runtime/test_smoke_topology_drift.py tests/unit/runtime/test_dynamic_task_evaluation_mapping.py tests/unit/runtime/test_failed_task_sandbox_cleanup.py -v` - -Expected: PASS. - -- [ ] **Step 8: Commit** - -```bash -git add tests/unit/runtime/test_propagation_contracts.py ergon_core/ergon_core/core/runtime/execution/propagation.py ergon_core/ergon_core/core/runtime/services/orchestration_dto.py ergon_core/ergon_core/core/runtime/services/task_propagation_service.py ergon_core/ergon_core/core/runtime/inngest/propagate_execution.py -git commit -m "Align propagation contract with blocked successors" -``` - ---- - -### Task 4: Consolidate Evaluation Criterion Status - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Confirm: `ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py` - -- [ ] **Step 1: Add architecture test for duplicate evaluation status literals** - -Add to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_eval_criterion_status_literal_is_defined_only_in_evaluation_summary() -> None: - offenders: list[str] = [] - snippet = 'EvalCriterionStatus = Literal["passed", "failed", "errored", "skipped"]' - allowed = { - ROOT / "ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py", - } - - for path in (ROOT / "ergon_core/ergon_core/core").rglob("*.py"): - if path in allowed: - continue - if snippet in path.read_text(): - offenders.append(str(path.relative_to(ROOT))) - - assert offenders == [] -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py::test_eval_criterion_status_literal_is_defined_only_in_evaluation_summary -v` - -Expected: FAIL because `core/api/schemas.py` currently defines the same alias. - -- [ ] **Step 3: Import canonical alias in REST schemas** - -In `core/api/schemas.py`, replace: - -```python -from typing import Any, Literal -EvalCriterionStatus = Literal["passed", "failed", "errored", "skipped"] -``` - -with: - -```python -from typing import Any -from ergon_core.core.persistence.telemetry.evaluation_summary import EvalCriterionStatus -``` - -- [ ] **Step 4: Run focused tests** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py tests/unit/runtime/test_evaluation_summary_contracts.py tests/unit/runtime/test_dynamic_task_evaluation_mapping.py -v` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/architecture/test_core_schema_sources.py ergon_core/ergon_core/core/api/schemas.py -git commit -m "Use canonical evaluation criterion status" -``` - ---- - -### Task 5: Collapse Graph Mutation DTOs Onto One Canonical Record - -**Files:** -- Modify: `tests/unit/runtime/test_graph_mutation_contracts.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/graph_dto.py` -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/dashboard/event_contracts.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/graph_repository.py` -- Modify: `ergon_core/ergon_core/core/dashboard/emitter.py` - -- [ ] **Step 1: Write mutation contract tests** - -Create `tests/unit/runtime/test_graph_mutation_contracts.py`: - -```python -from uuid import uuid4 - -from ergon_core.core.dashboard.event_contracts import DashboardGraphMutationEvent -from ergon_core.core.runtime.services.graph_dto import ( - EdgeAddedMutation, - GraphMutationRecordDto, - GraphMutationValue, -) -from pydantic import TypeAdapter - - -def test_rest_and_dashboard_mutations_share_graph_mutation_record_payloads() -> None: - run_id = uuid4() - mutation_id = uuid4() - edge_id = uuid4() - source_id = uuid4() - target_id = uuid4() - - payload = EdgeAddedMutation( - source_node_id=source_id, - target_node_id=target_id, - status="pending", - ) - - TypeAdapter(GraphMutationValue).validate_python(payload.model_dump(mode="json")) - - record = GraphMutationRecordDto( - id=mutation_id, - run_id=run_id, - sequence=1, - mutation_type="edge.added", - target_type="edge", - target_id=edge_id, - actor="test", - old_value=None, - new_value=payload, - reason=None, - created_at="2026-04-28T00:00:00Z", - ) - dashboard = DashboardGraphMutationEvent( - mutation=record, - ) - - assert dashboard.mutation == record - assert record.new_value == payload -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/runtime/test_graph_mutation_contracts.py::test_rest_and_dashboard_mutations_share_graph_mutation_record_payloads -v` - -Expected: FAIL because `GraphMutationRecordDto` does not exist yet and `DashboardGraphMutationEvent` currently duplicates mutation fields instead of wrapping one canonical record. - -- [ ] **Step 3: Make edge mutation IDs consistent with graph DTO IDs** - -In `graph_dto.py`, change: - -```python -source_node_id: str -target_node_id: str -``` - -to: - -```python -source_node_id: NodeId -target_node_id: NodeId -``` - -for both `EdgeAddedMutation` and `EdgeRemovedMutation`. - -If JSON serialization needs strings, keep conversion at the API/dashboard serialization boundary with `model_dump(mode="json")`; do not weaken the canonical payload type. - -- [ ] **Step 4: Add canonical mutation record DTO** - -In `graph_dto.py`, add: - -```python -from datetime import datetime - - -class GraphMutationRecordDto(BaseModel): - """Append-only graph mutation record with a typed mutation payload.""" - - model_config = {"frozen": True} - - id: UUID - run_id: RunId - sequence: int - mutation_type: MutationType - target_type: GraphTargetType - target_id: UUID - actor: str - old_value: GraphMutationValue | None - new_value: GraphMutationValue - reason: str | None - created_at: datetime -``` - -- [ ] **Step 5: Replace REST mutation DTO with canonical record** - -In `core/api/schemas.py`, remove `RunGraphMutationDto` and import: - -```python -from ergon_core.core.runtime.services.graph_dto import GraphMutationRecordDto -``` - -Update `core/api/runs.py` and `run_read_service.py` so `/runs/{run_id}/mutations` returns `list[GraphMutationRecordDto]`. Keep JSON stringification at FastAPI/Pydantic serialization, not in a second REST DTO. - -- [ ] **Step 6: Collapse dashboard event to a thin envelope** - -In `event_contracts.py`, replace duplicated mutation fields with: - -```python -from ergon_core.core.runtime.services.graph_dto import GraphMutationRecordDto - - -class DashboardGraphMutationEvent(InngestEventContract): - name: ClassVar[str] = "dashboard/graph.mutation" - - mutation: GraphMutationRecordDto -``` - -If frontend contract compatibility requires top-level fields for one release, stop and ask before adding a compatibility shim; the requested direction is to reduce duplicate DTOs. - -- [ ] **Step 7: Update repository/emitter conversion code** - -Search for mutation construction: - -```bash -rg "EdgeAddedMutation|EdgeRemovedMutation|GraphMutationValue|DashboardGraphMutationEvent|RunGraphMutationDto|GraphMutationRecordDto" ergon_core/ergon_core/core tests -n -``` - -Update `_to_mutation_dto` / mutation read paths to produce `GraphMutationRecordDto`. Update `dashboard/emitter.py` to construct `DashboardGraphMutationEvent(mutation=record)` instead of copying fields. Update call sites to pass UUID/`NodeId` values into `EdgeAddedMutation` / `EdgeRemovedMutation`. Use `model_dump(mode="json")` only when writing JSON columns or sending wire payloads. - -- [ ] **Step 8: Run focused mutation/dashboard tests** - -Run: `uv run pytest tests/unit/runtime/test_graph_mutation_contracts.py tests/unit/dashboard/test_event_contract_types.py tests/unit/architecture/test_model_field_descriptions.py -v` - -Expected: PASS. - -- [ ] **Step 9: Commit** - -```bash -git add tests/unit/runtime/test_graph_mutation_contracts.py ergon_core/ergon_core/core/runtime/services/graph_dto.py ergon_core/ergon_core/core/api/schemas.py ergon_core/ergon_core/core/dashboard/event_contracts.py ergon_core/ergon_core/core/runtime/services/graph_repository.py ergon_core/ergon_core/core/dashboard/emitter.py -git commit -m "Unify graph mutation payload contracts" -``` - ---- - -### Task 6: Collapse Task Node Projections Where Shapes Are Accidental - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/graph_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_dto.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/workflow_service.py` -- Modify: `ergon_core/ergon_core/core/dashboard/event_contracts.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/start_workflow.py` - -- [ ] **Step 1: Add tests for task-node DTO collapse** - -Add to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_run_task_dto_does_not_label_worker_slug_as_name() -> None: - path = ROOT / "ergon_core/ergon_core/core/api/schemas.py" - text = path.read_text() - assert "assigned_worker_name" not in text - assert "assigned_worker_slug" in text - - -def test_workflow_task_ref_does_not_duplicate_graph_task_ref() -> None: - path = ROOT / "ergon_core/ergon_core/core/runtime/services/workflow_dto.py" - assert "class WorkflowTaskRef" not in path.read_text() -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py::test_run_task_dto_does_not_label_worker_slug_as_name tests/unit/architecture/test_core_schema_sources.py::test_workflow_task_ref_does_not_duplicate_graph_task_ref -v` - -Expected: FAIL because `RunTaskDto` currently has `assigned_worker_name` and `workflow_dto.py` currently defines `WorkflowTaskRef`. - -- [ ] **Step 3: Rename REST task field to match its actual value** - -In `core/api/schemas.py`, change: - -```python -assigned_worker_name: str | None = None -``` - -to: - -```python -assigned_worker_slug: str | None = None -``` - -In `core/api/runs.py`, change the `_build_task_map` assignment from `assigned_worker_name=...` to `assigned_worker_slug=...`. - -- [ ] **Step 4: Introduce one canonical lightweight graph task ref** - -In `graph_dto.py`, add: - -```python -class GraphTaskRef(BaseModel): - """Lightweight task-node reference for workflow/tool projections.""" - - model_config = {"frozen": True} - - node_id: NodeId - task_slug: str - status: NodeStatus - level: int - parent_node_id: NodeId | None = None - assigned_worker_slug: str | None = None -``` - -Import `NodeStatus` from `status_conventions.py`. - -- [ ] **Step 5: Replace `WorkflowTaskRef` with `GraphTaskRef`** - -In `workflow_dto.py`, remove `WorkflowTaskRef` and import: - -```python -from ergon_core.core.runtime.services.graph_dto import GraphTaskRef -``` - -Update fields: - -```python -source: GraphTaskRef -target: GraphTaskRef -task: GraphTaskRef -task: GraphTaskRef | None = None -``` - -In `workflow_service.py`, update `_task_ref` to return `GraphTaskRef`. - -- [ ] **Step 6: Keep map-vs-tree containers, but share task-node semantics** - -Add or update comments near `RunTaskDto`: - -```python -class RunTaskDto(CamelModel): - """REST projection of RunGraphNode for run detail pages. - - This is not the canonical graph schema; graph semantics live in - runtime/services/graph_dto.py and persistence/graph/status_conventions.py. - """ -``` - -Keep `RunSnapshotDto.tasks: dict[str, RunTaskDto]` and `DashboardWorkflowStartedEvent.task_tree: TaskTreeNode` because map and tree containers are genuinely different. But align their field names and statuses with `GraphTaskRef`: `assigned_worker_slug` means slug, `status` is `NodeStatus`, and dependency/child fields are container-specific additions rather than new task-node semantics. - -- [ ] **Step 7: Run focused API/dashboard/workflow tests** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py tests/unit/cli/test_workflow_cli.py tests/unit/dashboard/test_event_contract_types.py tests/unit/state/test_workflow_cli_tool.py -v` - -Expected: PASS. If frontend TypeScript expects `assignedWorkerName`, update that in a separate frontend-compatible task rather than sneaking it into this backend cleanup. - -- [ ] **Step 8: Commit** - -```bash -git add tests/unit/architecture/test_core_schema_sources.py ergon_core/ergon_core/core/api/schemas.py ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/core/runtime/services/graph_dto.py ergon_core/ergon_core/core/runtime/services/workflow_dto.py ergon_core/ergon_core/core/runtime/services/workflow_service.py ergon_core/ergon_core/core/dashboard/event_contracts.py ergon_core/ergon_core/core/runtime/inngest/start_workflow.py -git commit -m "Collapse duplicate task node projections" -``` - ---- - -### Task 7: Reuse CancelCause Instead Of Local Literal Subsets - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `ergon_core/ergon_core/core/runtime/events/task_events.py` -- Modify: `ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py` -- Modify: any caller that accepts the same literal subset. - -- [ ] **Step 1: Add architecture test for local cancel-cause literals** - -Add to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_cancel_cause_literals_live_in_task_events() -> None: - offenders: list[str] = [] - snippets = ( - 'Literal["parent_terminal", "dep_invalidated"]', - 'Literal["dep_invalidated", "parent_terminal"]', - ) - allowed = { - ROOT / "ergon_core/ergon_core/core/runtime/events/task_events.py", - } - - for path in (ROOT / "ergon_core/ergon_core/core").rglob("*.py"): - if path in allowed: - continue - text = path.read_text() - for snippet in snippets: - if snippet in text: - offenders.append(f"{path.relative_to(ROOT)} duplicates cancel cause subset") - - assert offenders == [] -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/architecture/test_core_schema_sources.py::test_cancel_cause_literals_live_in_task_events -v` - -Expected: FAIL if `subtask_cancellation_service.py` still defines a local subset literal. - -- [ ] **Step 3: Add named subset aliases in `task_events.py`** - -In `task_events.py`, below `CancelCause`, add: - -```python -PropagationCancelCause = Literal["parent_terminal", "dep_invalidated"] -``` - -This keeps narrower service typing but centralizes the strings. - -- [ ] **Step 4: Import the subset alias in services** - -In `subtask_cancellation_service.py`, replace the local `Literal[...]` import/annotation with: - -```python -from ergon_core.core.runtime.events.task_events import PropagationCancelCause -``` - -Use: - -```python -cause: PropagationCancelCause -``` - -- [ ] **Step 5: Run focused cancellation tests** - -Run: `uv run pytest tests/unit/runtime/test_failed_task_sandbox_cleanup.py tests/unit/runtime/test_dynamic_task_evaluation_mapping.py tests/unit/state/test_subtask_lifecycle_toolkit.py tests/unit/architecture/test_core_schema_sources.py -v` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add tests/unit/architecture/test_core_schema_sources.py ergon_core/ergon_core/core/runtime/events/task_events.py ergon_core/ergon_core/core/runtime/services/subtask_cancellation_service.py -git commit -m "Centralize task cancellation causes" -``` - ---- - -### Task 8: Collapse Context Event Snapshot DTOs Onto Typed Payloads - -**Files:** -- Modify: `tests/unit/runtime/test_context_event_contracts.py` -- Modify: `ergon_core/ergon_core/core/api/schemas.py` -- Modify: `ergon_core/ergon_core/core/api/runs.py` -- Modify: `ergon_core/ergon_core/core/dashboard/event_contracts.py` -- Modify: `ergon_core/ergon_core/core/dashboard/emitter.py` - -- [ ] **Step 1: Write a context event DTO sharing test** - -Create `tests/unit/runtime/test_context_event_contracts.py`: - -```python -from uuid import uuid4 - -from ergon_core.core.api.schemas import RunContextEventDto -from ergon_core.core.dashboard.event_contracts import DashboardContextEventEvent -from ergon_core.core.persistence.context.event_payloads import AssistantTextPayload - - -def test_rest_and_dashboard_context_events_share_typed_payload_shape() -> None: - payload = AssistantTextPayload(text="hello") - common = { - "id": uuid4(), - "run_id": uuid4(), - "task_execution_id": uuid4(), - "task_node_id": uuid4(), - "worker_binding_key": "worker", - "sequence": 1, - "event_type": "assistant_text", - "payload": payload, - "created_at": "2026-04-28T00:00:00Z", - "started_at": None, - "completed_at": None, - } - - rest = RunContextEventDto.model_validate(common) - dashboard = DashboardContextEventEvent.model_validate(common) - - assert rest.payload == dashboard.payload - assert rest.event_type == dashboard.event_type -``` - -- [ ] **Step 2: Run the test and verify it fails** - -Run: `uv run pytest tests/unit/runtime/test_context_event_contracts.py::test_rest_and_dashboard_context_events_share_typed_payload_shape -v` - -Expected: FAIL because `RunContextEventDto` currently uses `event_type: str` and `payload: dict[str, Any]`, while dashboard uses `ContextEventType` and `ContextEventPayload`. - -- [ ] **Step 3: Type REST context event DTO with canonical event payloads** - -In `core/api/schemas.py`, import: - -```python -from ergon_core.core.persistence.context.event_payloads import ( - ContextEventPayload, - ContextEventType, -) -``` - -Update: - -```python -event_type: ContextEventType -payload: ContextEventPayload -``` - -- [ ] **Step 4: Update REST context event construction** - -In `core/api/runs.py`, when building `RunContextEventDto`, validate payload with the canonical discriminated payload type. If rows already store dict payloads, use the same validation path as dashboard emitter uses rather than passing raw dicts through REST. - -- [ ] **Step 5: Decide whether to fully collapse class names** - -If `RunContextEventDto` and `DashboardContextEventEvent` now have the same fields except event `name`, move the common fields into a shared model: - -```python -class ContextEventDto(CamelModel or BaseModel): - ... -``` - -Use that model directly in REST and embed it in the dashboard event envelope. If camelCase REST output makes a shared class awkward, keep the two envelope classes but require both to use `ContextEventType` and `ContextEventPayload`. - -- [ ] **Step 6: Run focused tests** - -Run: `uv run pytest tests/unit/runtime/test_context_event_contracts.py tests/unit/dashboard/test_event_contract_types.py tests/unit/architecture/test_model_field_descriptions.py -v` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```bash -git add tests/unit/runtime/test_context_event_contracts.py ergon_core/ergon_core/core/api/schemas.py ergon_core/ergon_core/core/api/runs.py ergon_core/ergon_core/core/dashboard/event_contracts.py ergon_core/ergon_core/core/dashboard/emitter.py -git commit -m "Share typed context event payload schemas" -``` - ---- - -### Task 9: Add Mapping Guard Between Generation Parts And Context Events - -**Files:** -- Modify: `tests/unit/builtins/common/test_transcript_adapters.py` -- Modify: `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` only if the test reveals unmapped kinds. - -- [ ] **Step 1: Add explicit adapter coverage for vocabulary mapping** - -In `tests/unit/builtins/common/test_transcript_adapters.py`, add a test that documents the intended split between `core.generation` kebab-case `part_kind` and context event snake-case `event_type`: - -```python -from ergon_core.core.generation import TextPart, ThinkingPart, ToolCallPart, ToolReturnPart -from ergon_core.core.persistence.context.event_payloads import ContextEventType - - -def test_generation_part_kinds_have_context_event_counterparts() -> None: - assert TextPart(content="x").part_kind == "text" - assert ThinkingPart(content="x").part_kind == "thinking" - assert ToolCallPart(tool_name="t", tool_call_id="1", args={}).part_kind == "tool-call" - assert ToolReturnPart(tool_call_id="1", tool_name="t", content="ok").part_kind == "tool-return" - - assert "assistant_text" in ContextEventType.__args__ - assert "thinking" in ContextEventType.__args__ - assert "tool_call" in ContextEventType.__args__ - assert "tool_result" in ContextEventType.__args__ -``` - -- [ ] **Step 2: Run the test** - -Run: `uv run pytest tests/unit/builtins/common/test_transcript_adapters.py::test_generation_part_kinds_have_context_event_counterparts -v` - -Expected: PASS if the current split is intentional and covered; FAIL if any expected context event value has drifted. - -- [ ] **Step 3: Fix adapter mapping only if the test fails** - -If the test fails because context event values changed, update `ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py` to map the actual canonical context event types. Do not merge generation parts and context events into one model family. - -- [ ] **Step 4: Run focused adapter tests** - -Run: `uv run pytest tests/unit/builtins/common/test_transcript_adapters.py tests/unit/persistence/test_context_event_repository.py -v` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add tests/unit/builtins/common/test_transcript_adapters.py ergon_builtins/ergon_builtins/common/llm_context/adapters/pydantic_ai.py -git commit -m "Guard generation to context event mapping" -``` - ---- - -### Task 10: Final Architecture Sweep - -**Files:** -- Modify: `tests/unit/architecture/test_core_schema_sources.py` -- Modify: `docs/superpowers/plans/2026-04-28-core-schema-deduplication.md` only if implementation reveals a necessary correction. - -- [ ] **Step 1: Add a broad forbidden-duplication guard** - -Add to `tests/unit/architecture/test_core_schema_sources.py`: - -```python -def test_core_schema_source_imports_are_directional() -> None: - forbidden_pairs = { - "ergon_core.core.api.schemas": ( - "EvalCriterionStatus = Literal", - "GraphMutationValue =", - ), - "ergon_core.core.dashboard.event_contracts": ( - "GraphMutationValue =", - "CancelCause = Literal", - ), - } - - offenders: list[str] = [] - for module_path, snippets in forbidden_pairs.items(): - path = ROOT / (module_path.replace(".", "/") + ".py") - text = path.read_text() - for snippet in snippets: - if snippet in text: - offenders.append(f"{path.relative_to(ROOT)} contains local source {snippet!r}") - - assert offenders == [] -``` - -- [ ] **Step 2: Run the full architecture test set** - -Run: `uv run pytest tests/unit/architecture -v` - -Expected: PASS. - -- [ ] **Step 3: Run focused runtime/schema tests** - -Run: - -```bash -uv run pytest \ - tests/unit/runtime/test_workflow_service.py \ - tests/unit/runtime/test_dynamic_task_evaluation_mapping.py \ - tests/unit/runtime/test_evaluation_summary_contracts.py \ - tests/unit/dashboard/test_event_contract_types.py \ - tests/unit/builtins/common/test_transcript_adapters.py \ - tests/unit/architecture/test_model_field_descriptions.py \ - -v -``` - -Expected: PASS. - -- [ ] **Step 4: Search for remaining duplicate literals** - -Run: - -```bash -rg 'Literal\["pending", "ready", "running", "completed", "failed", "cancelled", "blocked"\]|EvalCriterionStatus = Literal|invalidated_targets|assigned_worker_name|Literal\["parent_terminal", "dep_invalidated"\]' ergon_core tests -``` - -Expected output may include only: - -```text -ergon_core/ergon_core/core/persistence/graph/status_conventions.py -ergon_core/ergon_core/core/persistence/telemetry/evaluation_summary.py -tests/unit/architecture/test_core_schema_sources.py -``` - -If other production files appear, either import the canonical alias or explain in a code comment why the duplicate-looking concept is distinct. - -- [ ] **Step 5: Run lints for touched files** - -Use Cursor lints for: - -```text -ergon_core/ergon_core/core/persistence/graph/status_conventions.py -ergon_core/ergon_core/core/runtime/execution/propagation.py -ergon_core/ergon_core/core/runtime/services -ergon_core/ergon_core/core/api/schemas.py -ergon_core/ergon_core/core/dashboard/event_contracts.py -tests/unit/architecture/test_core_schema_sources.py -``` - -Expected: no new diagnostics in touched files. - -- [ ] **Step 6: Commit final guard changes** - -```bash -git add tests/unit/architecture/test_core_schema_sources.py -git commit -m "Guard core schema source ownership" -``` - ---- - -## Execution Notes - -- Do not collapse legitimate transport envelopes into one giant schema. Do collapse duplicated payload bodies: `WorkflowTaskRef` should disappear in favor of `GraphTaskRef`; REST/dashboard task containers can remain map/tree envelopes only if their field semantics align with the canonical graph task ref. -- Do remove duplicate domain definitions. If two modules need the same literal values, one imports from the source-of-truth module. -- Keep table models free-form where the database intentionally allows extension, but make runtime conventions explicit through aliases and constants. -- Keep REST/dashboard serialization at the boundary. Canonical Python DTOs can use UUID/NewType fields; wire models can stringify with `model_dump(mode="json")`. -- Avoid compatibility facades. If a module owns a concept, import it directly from that module. - -## Self-Review - -- Spec coverage: high-priority graph status duplication, evaluation status duplication, stale propagation contract, graph mutation DTO collapse, task-node DTO collapse, context-event DTO typing, cancel-cause duplication, and generation/context event vocabulary mapping are each covered by a task. -- Placeholder scan: no task contains unresolved placeholder markers or an unspecified "add tests" instruction; every task names files and commands. -- Type consistency: graph status aliases live in `status_conventions.py`, evaluation status in `evaluation_summary.py`, mutation payload body in `graph_dto.py`, and cancel-cause aliases in `task_events.py` throughout the plan. diff --git a/docs/superpowers/plans/2026-04-28-ergon-builtins-rebuild-structure.md b/docs/superpowers/plans/2026-04-28-ergon-builtins-rebuild-structure.md deleted file mode 100644 index 167b0b206..000000000 --- a/docs/superpowers/plans/2026-04-28-ergon-builtins-rebuild-structure.md +++ /dev/null @@ -1,709 +0,0 @@ -# Ergon Built-ins Rebuild Structure - -This document lays out the target shape for `ergon_builtins` after the Ergon core public API cleanup. It assumes the core authoring API from `2026-04-28-public-api-target-structure.md`: - -- `Benchmark`, `Task`, `BenchmarkRequirements` -- `Worker`, `WorkerContext`, `WorkerOutput` -- `Criterion`, `CriterionContext`, `CriterionOutcome`, `ScoreScale` -- `Rubric`, `TaskEvaluationResult` -- advanced `Evaluator` only when a fixed `Rubric` is not expressive enough - -The key design rule is that built-ins should be normal public API consumers. The CLI and runtime should discover built-ins through typed registries and service facades, not by importing benchmark internals or rebuilding object graphs by hand. - -## Goals - -- Keep benchmark authoring code small, public-API-first, and easy to copy for external benchmark authors. -- Keep sandbox, dataset loading, and optional dependency code inside benchmark-owned packages. -- Keep the registry as the stable integration boundary for CLI discovery, experiment definition, run launch, and Inngest execution. -- Keep benchmark slugs separate from runtime choices: the CLI must pass worker, evaluator, sandbox, model, and extras/dependency intent explicitly for now. -- Avoid compatibility aliases for renamed public concepts during the coordinated rebuild. - -## Runtime Integration Model - -```mermaid -flowchart TD - accTitle: Builtins Runtime Flow - accDescr: Built-in benchmark, worker, and evaluator slugs flow from the registry through CLI services, persisted definitions, run records, and Inngest execution. - - registry["ergon_builtins.registry<br/>slugs and factories"] - cli["CLI commands<br/>define, run, list"] - facades["core runtime services<br/>experiment, cohort, run"] - experiment["ExperimentRecord<br/>selected samples and explicit choices"] - definition["Workflow definition<br/>task graph and type slugs"] - run["RunRecord<br/>instance key, worker team, evaluator slug"] - inngest["Inngest runtime<br/>worker and evaluator execution"] - - registry --> cli - cli --> facades - facades --> experiment - facades --> definition - facades --> run - definition --> inngest - run --> inngest - registry --> inngest -``` - -The CLI path should be slug-driven: - -1. Validate the explicit `benchmark_slug`, `worker_slug`, `evaluator_slug`, and `sandbox_slug` against `ergon_builtins.registry`. -2. Ask a core service facade to define or launch the experiment. -3. Persist only durable identifiers and slugs in `ExperimentRecord`, workflow definitions, and `RunRecord`. -4. Rehydrate live workers, criteria, rubrics, and sandbox managers from registries at runtime. - -## Proposed Package Tree - -```text -ergon_builtins/ - ergon_builtins/ - __init__.py - - registry.py - # merged public discovery surface - # imports registry_core and optional registries - - registry_core.py - # always-importable built-ins with no [data] dependency - # exports BENCHMARKS, WORKERS, EVALUATORS, SANDBOX_MANAGERS, - # SANDBOX_TEMPLATES, MODEL_BACKENDS - - registry_data.py - # HuggingFace/pandas/datasets-dependent built-ins - # same export names as registry_core - - registry_local_models.py - # optional local model backends - - shared/ - __init__.py - criteria/ - code_check.py - file_check.py - llm_judge.py - sandbox_file_check.py - workers/ - react_worker.py - training_stub_worker.py - react_prompts.py - models/ - cloud_passthrough.py - openrouter_backend.py - openrouter_responses_backend.py - resolution.py - vllm_backend.py - tools/ - # reusable public worker tools only - observability/ - # event/transcript adapters used by shared workers - - benchmarks/ - minif2f/ - __init__.py - benchmark.py - task_schemas.py - worker_factory.py - prompts.py - toolkit.py - criteria.py - rubric.py - sandbox_manager.py - sandbox/ - - swebench_verified/ - __init__.py - benchmark.py - task_schemas.py - worker_factory.py - prompts.py - toolkit.py - criterion.py - rubric.py - sandbox_manager.py - sandbox_manager_support.py - sandbox/ - - gdpeval/ - __init__.py - benchmark.py - task_schemas.py - loader.py - worker_factory.py - criteria.py - rubric.py - sandbox.py - - researchrubrics/ - __init__.py - benchmark.py - vanilla.py - task_schemas.py - worker_factory.py - researcher_worker.py - workflow_cli_react_worker.py - criteria.py - judge_criterion.py - rubric.py - sandbox_manager.py -``` - -### Package Boundary Rules - -- Benchmark packages own their task payload schemas, dataset loaders, sandbox/toolkit wiring, benchmark-specific criteria, and default rubric. -- `shared/` contains reusable primitives that do not know about one benchmark's payload schema. -- Registered worker factories live next to the benchmark when they bind benchmark-specific tools or sandbox setup. -- Generic worker classes live in `shared/workers/`; benchmark packages wrap them with factories. -- Optional data dependencies stay in `registry_data.py` and data-only benchmark packages. Importing `registry_core.py` must not require `datasets`, pandas, `swebench`, or HuggingFace extras. -- CLI code should import only `ergon_builtins.registry` and core service facades. - -## Registry Contract - -The registry should continue to expose dictionaries keyed by stable slugs: - -```python -BENCHMARKS: dict[str, type[Benchmark]] -WORKERS: dict[str, WorkerFactory] -EVALUATORS: dict[str, type[Evaluator]] -SANDBOX_MANAGERS: dict[str, type[BaseSandboxManager]] -SANDBOX_TEMPLATES: dict[str, Path] -MODEL_BACKENDS: dict[str, Callable[..., ResolvedModel]] -``` - -`WorkerFactory` should remain a callable shape that the runtime can use after sandbox setup: - -```python -WorkerFactory = Callable[..., Worker] -``` - -Every registered worker factory must accept: - -```text -name: str -model: str | None -task_id: UUID -sandbox_id: str -``` - -The registry should not provide benchmark-level default profiles in this phase. Explicit beats implicit while the package structure is still moving: callers must specify the worker, evaluator, sandbox, model, and dependency extras they intend to use. - -This gives the CLI enough information to validate explicit requests for: - -- `ergon benchmark list` -- `ergon worker list` -- `ergon evaluator list` -- `ergon experiment define <benchmark>` -- `ergon experiment run <experiment-id>` -- `ergon benchmark run <benchmark>` -- onboarding/setup messages for explicitly requested extras, E2B, HuggingFace, or API keys - -## Public API Usage Rules - -Built-ins should use root imports for ordinary authoring: - -```python -from ergon_core.api import Benchmark, BenchmarkRequirements, Task -from ergon_core.api import Worker, WorkerContext, WorkerOutput -from ergon_core.api import Criterion, CriterionContext, CriterionOutcome -from ergon_core.api import Rubric, TaskEvaluationResult -``` - -Use advanced imports only where the benchmark needs dynamic criteria: - -```python -from ergon_core.api.rubric import Evaluator -``` - -Core composition types stay out of benchmark authoring files: - -- no `Experiment` imports in benchmark packages -- no `WorkerSpec` imports in benchmark packages -- no run/cohort/definition handles in benchmark packages -- no direct DB/session imports in workers, criteria, or rubrics - -## Benchmark Implementation Pattern - -Each benchmark package should follow the same high-level shape: - -```text -benchmark.py - Benchmark subclass - type_slug - task_payload_model - onboarding_deps / BenchmarkRequirements - build_instances() -> Mapping[str, Sequence[Task[Payload]]] - evaluator_requirements() - -task_schemas.py - Pydantic payload models - dataset row conversion helpers when lightweight - -worker_factory.py - factories that bind shared workers to benchmark-specific tools/sandboxes - -criteria.py / criterion.py - benchmark-specific Criterion implementations and builders - -rubric.py - Rubric or Evaluator subclass registered under a stable evaluator slug - -sandbox_manager.py / sandbox.py - benchmark-specific sandbox lifecycle and setup -``` - -`Task` construction should consistently set: - -- `task_slug`: stable dataset sample identifier -- `instance_key`: selected instance key used by experiment/run services -- `description`: worker-facing problem statement -- `evaluator_binding_keys`: usually `("default",)` unless the benchmark has multiple evaluator bindings -- `task_payload`: typed payload model containing all evaluator-only ground truth - -## MiniF2F - -### Folder - -```text -benchmarks/minif2f/ - benchmark.py - task_schemas.py - worker_factory.py - prompts.py - toolkit.py - criteria.py - rubric.py - sandbox_manager.py - sandbox/ -``` - -### Benchmark - -`MiniF2FBenchmark` should remain a public `Benchmark` implementation: - -- `type_slug = "minif2f"` -- `task_payload_model = MiniF2FTaskPayload` -- `onboarding_deps = BenchmarkRequirements(e2b=True)` -- `build_instances()` downloads or reads MiniF2F-v2c and returns one `Task` per theorem. -- `description` should include the informal statement, Lean header, and formal theorem. - -The payload should carry: - -- `name` -- `informal_statement` -- `formal_statement` -- `header` - -Ground truth proof, if available later, belongs in the payload or metadata for evaluation only, not in the worker prompt. - -### Worker - -The recommended first worker pairing is `minif2f-react`, implemented as a benchmark-owned factory around the shared ReAct worker: - -- resolve the live sandbox by `task_id` -- build `MiniF2FToolkit` -- bind Lean tools such as write file, check file, and verify proof -- pass a MiniF2F-specific system prompt -- return a `WorkerOutput` whose final answer includes the proof file path or proof text - -The factory belongs in `benchmarks/minif2f/worker_factory.py` because it knows about Lean, the sandbox manager, and the MiniF2F toolkit. - -### Criteria And Rubric - -`ProofVerificationCriterion` should use `CriterionContext` public capabilities rather than importing a concrete runtime protocol from public files. - -`MiniF2FRubric` should be a fixed `Rubric` with one proof-verification criterion: - -- score `1.0` when Lean verifies the final proof -- score partial credit for syntactically valid but incomplete proof attempts -- score `0.0` for missing or invalid proof artifacts -- return `TaskEvaluationResult` with normalized score and proof metadata - -### Required CLI Pairing - -```text -benchmark_slug: minif2f -worker_slug: minif2f-react -evaluator_slug: minif2f-rubric -sandbox_slug: minif2f -extras: none -model: explicit CLI value, e.g. openai:gpt-4o -``` - -## SWE-Bench Verified - -### Folder - -```text -benchmarks/swebench_verified/ - benchmark.py - task_schemas.py - worker_factory.py - prompts.py - toolkit.py - criterion.py - rubric.py - sandbox_manager.py - sandbox_manager_support.py - sandbox/ -``` - -### Benchmark - -`SweBenchVerifiedBenchmark` should remain the benchmark loader for `princeton-nlp/SWE-bench_Verified`: - -- `type_slug = "swebench-verified"` -- `task_payload_model = SWEBenchTaskPayload` -- `onboarding_deps = BenchmarkRequirements(e2b=True, extras=("ergon-builtins[data]",))` -- `build_instances()` returns one `Task` per SWE-Bench instance. -- the worker-facing `description` should include issue context and repo instructions, not the gold test patch. - -The payload should carry all evaluator-only data: - -- `instance_id` -- repo and base commit identifiers -- problem statement -- test patch -- FAIL_TO_PASS / PASS_TO_PASS metadata needed by the harness - -### Worker - -The recommended first worker pairing is `swebench-react`, implemented as a benchmark-owned factory around the shared ReAct worker: - -- resolve the live sandbox by `task_id` -- build `SWEBenchToolkit` -- expose shell/file/git tools scoped to `/workspace/repo` -- pass a SWE-Bench-specific system prompt -- return patch-oriented output or rely on sandbox diff extraction during evaluation - -The worker should not run the official evaluator. Its job is to modify the repo in the sandbox. - -### Criteria And Rubric - -`SWEBenchTestCriterion` should remain the atomic evaluation unit: - -- extract the agent patch from the sandbox through `CriterionContext` capabilities -- apply the gold test patch -- apply the agent patch -- run the official eval script -- parse the SWE-Bench harness report -- return `CriterionOutcome` with score `1.0` only when the instance is resolved - -`SWEBenchRubric` should live in `benchmarks/swebench_verified/rubric.py`, not in a detached global rubrics folder, because it is benchmark-specific and wraps `SWEBenchTestCriterion`. - -### Required CLI Pairing - -```text -benchmark_slug: swebench-verified -worker_slug: swebench-react -evaluator_slug: swebench-rubric -sandbox_slug: swebench-verified -extras: ergon-builtins[data] -model: explicit CLI value, e.g. openai:gpt-4o -``` - -## GDPEval - -### Folder - -```text -benchmarks/gdpeval/ - benchmark.py - task_schemas.py - loader.py - worker_factory.py - criteria.py - rubric.py - sandbox.py -``` - -### Benchmark - -`GDPEvalBenchmark` should stay in the `[data]` registry: - -- `type_slug = "gdpeval"` -- `task_payload_model = GDPTaskConfig` -- `onboarding_deps = BenchmarkRequirements(e2b=True, extras=("ergon-builtins[data]",))` -- `build_instances()` loads task IDs and reference files from HuggingFace. -- each `Task.description` should be the document-processing instruction extracted from the dataset. - -The payload should carry: - -- `task_id` -- `workflow_type` -- `reference_files` -- any expected output manifest or rubric category references needed by evaluation - -### Worker - -GDPEval should have an explicit recommended worker pairing instead of depending on a generic ReAct slug that has no benchmark tools. The worker can be implemented in either of two ways: - -- `gdpeval-react`: benchmark-owned factory around shared ReAct, with document/file tools and sandbox workspace instructions. -- `gdpeval-workflow-cli-react`: if GDP tasks are meant to exercise the workflow CLI and produce office artifacts through the sandbox. - -The recommended first target is `gdpeval-react` because it keeps the benchmark in the same authoring pattern as MiniF2F and SWE-Bench. - -### Criteria And Rubric - -`StagedRubric` is an advanced evaluator-like rubric because it supports sequential gates and stage-specific failure actions. It should be registered under one stable slug: - -```text -gdpeval-staged-rubric -``` - -If the CLI keeps the shorter compatibility slug during the rebuild, it should be temporary and removed in the coordinated built-ins rename. - -GDPEval criteria should be generated from explicit stage definitions: - -- format/file existence gates -- reference-file consistency checks -- LLM judge criteria for qualitative document quality -- optional code or spreadsheet checks for generated artifacts - -Each criterion should emit structured evidence for auditability: - -- files checked -- sandbox command IDs -- judge prompt messages -- parsed outputs -- failure reason - -### Required CLI Pairing - -```text -benchmark_slug: gdpeval -worker_slug: gdpeval-react -evaluator_slug: gdpeval-staged-rubric -sandbox_slug: gdpeval -extras: ergon-builtins[data] -model: explicit CLI value, e.g. openai:gpt-4o -``` - -## ResearchRubrics - -### Folder - -```text -benchmarks/researchrubrics/ - benchmark.py - vanilla.py - task_schemas.py - worker_factory.py - researcher_worker.py - workflow_cli_react_worker.py - criteria.py - judge_criterion.py - rubric.py - sandbox_manager.py -``` - -### Benchmark - -`ResearchRubricsBenchmark` and `ResearchRubricsVanillaBenchmark` should remain `[data]` benchmarks: - -- `type_slug = "researchrubrics"` and `type_slug = "researchrubrics-vanilla"` -- `task_payload_model = ResearchRubricsTaskPayload` -- `onboarding_deps = BenchmarkRequirements(extras=("ergon-builtins[data]",), optional_keys=("EXA_API_KEY",))` -- `build_instances()` returns one `Task` per dataset sample. -- `description` should be the research prompt. - -The payload should carry: - -- `sample_id` -- `domain` -- `prompt` -- list of weighted rubric criteria - -### Workers - -ResearchRubrics should keep two registered worker choices because they exercise different research-agent paths: - -```text -researchrubrics-researcher -researchrubrics-workflow-cli-react -``` - -`researchrubrics-researcher` should be the recommended first worker pairing: - -- accepts the research prompt -- uses model-backed research behavior -- writes final report artifacts through `WorkerContext` or public resource capabilities -- returns `WorkerOutput` with report summary and final artifact references - -`researchrubrics-workflow-cli-react` should remain an advanced/experimental worker: - -- uses the workflow CLI path inside the sandbox -- is useful for testing tool orchestration and dashboard traces -- should not be the default unless the CLI explicitly requests it - -### Criteria And Rubric - -`ResearchRubricsRubric` should remain an advanced dynamic evaluator or a `Rubric` that overrides `criteria_for(task)`, because its criteria come from each task payload. - -The task-specific path should: - -1. read `ResearchRubricsTaskPayload.rubrics` -2. build one `ResearchRubricsJudgeCriterion` per rubric criterion -3. evaluate the final report against each weighted criterion -4. aggregate positive and negative weights into normalized `TaskEvaluationResult` - -Judge criteria should use `CriterionEvidence` to preserve: - -- judge prompt -- report excerpt or artifact reference -- rubric criterion text -- axis and weight -- model output - -### Required CLI Pairings - -```text -benchmark_slug: researchrubrics -worker_slug: researchrubrics-researcher -evaluator_slug: researchrubrics-rubric -sandbox_slug: researchrubrics -extras: ergon-builtins[data] -model: explicit CLI value, e.g. openai:gpt-4o -``` - -```text -benchmark_slug: researchrubrics-vanilla -worker_slug: researchrubrics-researcher -evaluator_slug: researchrubrics-rubric -sandbox_slug: researchrubrics-vanilla -extras: ergon-builtins[data] -model: explicit CLI value, e.g. openai:gpt-4o -``` - -## CLI Requirements - -The CLI should not know benchmark internals. It should consume registry metadata and call core service facades. - -### Discovery - -`ergon benchmark list` should display: - -- slug -- description -- available registered workers -- available registered evaluators -- sandbox requirement -- data extra requirement - -`ergon worker list` and `ergon evaluator list` should continue to read `WORKERS` and `EVALUATORS`. - -### Experiment Define - -`ergon experiment define <benchmark>` should: - -1. require explicit `--worker`, `--evaluator`, `--sandbox`, `--model`, and `--extras` or equivalent request fields -2. validate those explicit slugs against the registries -3. instantiate the benchmark by slug -4. call `build_instances()` -5. select samples by `--limit`, `--sample`, or future selection flags -6. persist an `ExperimentRecord` with benchmark slug, selected instance keys, explicit worker team JSON, evaluator slug, sandbox slug, model target, extras/dependency intent, and cohort metadata - -It should not instantiate workers or criteria at define time. - -### Experiment Run - -`ergon experiment run <experiment-id>` should: - -1. read the persisted experiment -2. create one run assignment per selected task or instance -3. build a single-sample workflow definition through core composition -4. persist the workflow definition with benchmark, worker, and evaluator slugs -5. create `RunRecord` rows linked to experiment/cohort/definition -6. emit workflow start events - -Workers, criteria, and sandbox managers are instantiated by runtime services from slugs after run creation. - -### Benchmark Run - -`ergon benchmark run <benchmark>` should become a convenience wrapper around define plus run. It should not keep its own separate composition path long term. - -The rebuild should remove drift between: - -- `ergon_cli.commands.benchmark.run_benchmark` -- `ergon_cli.composition.build_experiment` -- `ExperimentDefinitionService` -- `ExperimentLaunchService` - -The preferred end state is: - -```text -benchmark run - -> experiment facade define - -> experiment facade run - -> run facade status/output -``` - -## Migration Order - -### Phase 1: Explicit Registry Contract - -- Keep registries explicit: no benchmark profiles or default pairing layer in this phase. -- Ensure `BENCHMARKS`, `WORKERS`, `EVALUATORS`, `SANDBOX_MANAGERS`, and `SANDBOX_TEMPLATES` are complete and typed. -- Update CLI list commands to display registered components without implying defaults. -- Add tests that every documented CLI pairing references registered benchmark, worker, evaluator, and sandbox slugs. - -### Phase 2: Public API Imports - -- Replace old built-ins imports: - - `BenchmarkTask` -> `Task` - - `BenchmarkDeps` -> `BenchmarkRequirements` - - `EvaluationContext` -> `CriterionContext` - - `CriterionResult` -> `CriterionOutcome` - - `CriterionScoreSpec` -> `ScoreScale` - - `CriterionObservation` -> `CriterionEvidence` - - `CriterionObservationMessage` -> `EvidenceMessage` -- Move SWE-Bench rubric beside the SWE-Bench benchmark. -- Move generic evaluator helpers under `shared/criteria` only if they are truly benchmark-independent. - -### Phase 3: Benchmark-Owned Worker Factories - -- Move `_minif2f_react` into `benchmarks/minif2f/worker_factory.py`. -- Move `_swebench_react` into `benchmarks/swebench_verified/worker_factory.py`. -- Add `gdpeval-react` factory. -- Keep ResearchRubrics workers in the benchmark package or re-export them from benchmark-owned `worker_factory.py`. -- Keep generic `ReActWorker` in `shared/workers/react_worker.py`. - -### Phase 4: CLI Facade Alignment - -- Make `benchmark run` call the same core service facade path as `experiment define` plus `experiment run`. -- Remove direct CLI composition of `Experiment` objects. -- Ensure `create_run` call sites use the current `RunRecord` contract: experiment ID, workflow definition ID, instance key, worker team JSON, evaluator slug, and model target. - -### Phase 5: Runtime And Evaluation Contracts - -- Update Inngest worker execution to construct `Task` from the registered benchmark payload model. -- Update evaluation execution to use `CriterionContext` public capability methods. -- Ensure sandbox setup happens before benchmark-owned worker factories are invoked. -- Ensure criteria never import persistence sessions or concrete runtime protocols through public API modules. - -## Testing Plan - -Core contract tests: - -- every `BENCHMARKS` key has a matching `Benchmark.type_slug` -- every documented required CLI pairing has registered benchmark, worker, evaluator, and sandbox slugs -- every benchmark exposes `task_payload_model` and `BenchmarkRequirements` -- every benchmark's `build_instances(limit=1)` returns at least one `Task` with a valid payload when optional dependencies are available - -Benchmark-specific tests: - -- MiniF2F proof criterion handles verified, syntactically valid incomplete, and invalid proof outputs. -- SWE-Bench criterion handles empty patch, patch extraction failure, git apply failure, unresolved report, and resolved report. -- GDPEval staged rubric handles required gate failure, continue, zero-category, and normalized score bounds. -- ResearchRubrics dynamic criteria build one judge criterion per payload rubric and aggregate negative weights correctly. - -CLI/service tests: - -- `benchmark list` shows registered benchmarks without default worker/evaluator metadata. -- `experiment define` stores slugs and selected sample keys, not live worker/evaluator objects. -- `experiment run` creates one workflow definition and run per selected sample. -- `benchmark run` uses the same facade path as define plus run. -- run records persist worker team JSON, evaluator slug, model target, instance key, experiment ID, and workflow definition ID. - -## Open Decisions - -1. Whether `Evaluator` stays root-public or is imported only from `ergon_core.api.rubric`. -2. Whether `gdpeval-react` should be the recommended GDP worker or GDP should use the workflow CLI worker. -3. Whether `researchrubrics-rubric` is the only final slug, removing `research-rubric`. -4. Whether `benchmark run` should remain as a public CLI command after it becomes a wrapper around experiment services. diff --git a/docs/superpowers/plans/2026-04-28-ergon-cli-refactor-structure.md b/docs/superpowers/plans/2026-04-28-ergon-cli-refactor-structure.md deleted file mode 100644 index 566b72f6a..000000000 --- a/docs/superpowers/plans/2026-04-28-ergon-cli-refactor-structure.md +++ /dev/null @@ -1,772 +0,0 @@ -# Ergon CLI Refactor Structure - -This document specifies the target CLI structure after the Ergon core public API and `ergon_builtins` package refactors. It is a sibling to: - -- `2026-04-28-public-api-target-structure.md` -- `2026-04-28-ergon-builtins-rebuild-structure.md` - -The CLI should become the operator-facing shell over core service facades. It should not assemble low-level graph objects by hand, import benchmark internals, or maintain a second experiment launch path that can drift from the API and runtime. - -## Goals - -- Make `ergon experiment define` and `ergon experiment run` the canonical local lifecycle commands. -- Make API routes, CLI commands, and eval automation call the same core services with the same DTOs. -- Make `benchmark run`, if kept, a thin wrapper over define plus run. -- Use `ergon_builtins.registry` for discovery and validation, but require explicit worker/evaluator/sandbox/model/extras choices in benchmark requests for now. -- Remove stale direct composition paths from the CLI. -- Keep operational commands such as `benchmark setup`, `workflow`, `run list`, `run cancel`, `doctor`, `onboard`, and `train` clearly separated from experiment definition and launch. -- Ensure CLI output remains machine-readable enough for tests, shell scripts, and eval automation. - -## Current Shape - -```text -ergon_cli/ - ergon_cli/ - main.py - # top-level argparse parser and dispatch - - commands/ - benchmark.py - # list, setup, stale run path - experiment.py - # define, run, show, list - run.py - # list, cancel - worker.py - # list - evaluator.py - # list - workflow.py - # sandbox/workflow helper commands - eval.py - # checkpoint eval watcher - train.py - # local RL training - onboard.py - doctor.py - - composition/ - __init__.py - # stale direct Experiment composition helper - - discovery/ - __init__.py - # list BENCHMARKS/WORKERS/EVALUATORS - - rendering/ - __init__.py -``` - -The current parser registers: - -- `benchmark list` -- `benchmark setup` -- `experiment define` -- `experiment run` -- `experiment show` -- `experiment list` -- `run list` -- `run cancel` -- `worker list` -- `evaluator list` -- `workflow ...` -- `eval watch` -- `eval checkpoint` -- `onboard` -- `doctor` -- `train local` - -There is handler code for `benchmark run`, but `main.py` does not register a `benchmark run` subparser. This is intentional in at least one current unit test, but conflicts with dead handler code, old setup messages, and real-LLM tests that still invoke `ergon benchmark run`. - -## Target Command Model - -```mermaid -flowchart TD - accTitle: CLI Command Ownership - accDescr: The CLI command tree routes experiment lifecycle commands through core service facades, while setup, workflow, training, and diagnostics remain separate operational surfaces. - - cli["ergon CLI"] - discovery["discovery commands<br/>benchmark/worker/evaluator list"] - setup["benchmark setup<br/>sandbox template build"] - experiment["experiment lifecycle<br/>define/run/show/list"] - run["run operations<br/>list/cancel/show later"] - workflow["workflow helper<br/>inside sandbox/task context"] - eval["eval watcher<br/>checkpoint scoring"] - train["train local<br/>RL training"] - doctor["doctor/onboard"] - - cli --> discovery - cli --> setup - cli --> experiment - cli --> run - cli --> workflow - cli --> eval - cli --> train - cli --> doctor - - experiment --> services["core runtime service facades"] - eval --> experiment - discovery --> registry["ergon_builtins.registry"] - setup --> sandbox_templates["SANDBOX_TEMPLATES"] -``` - -### Canonical Lifecycle Commands - -These commands define the supported local experiment lifecycle: - -```text -ergon experiment define <benchmark_slug> [selection] --worker ... --evaluator ... --sandbox ... --model ... --extras ... -ergon experiment run <experiment_id> [runtime options] -ergon experiment show <experiment_id> -ergon experiment list -``` - -The HTTP API should remain parallel to this command set: - -```text -POST /api/experiments/define -POST /api/experiments/{id}/run -GET /api/experiments/{id} -GET /api/experiments -``` - -The CLI and HTTP API should use the same service layer: - -- `ExperimentDefinitionService.define_benchmark_experiment` -- `ExperimentLaunchService.run_experiment` -- `ExperimentReadService.get_experiment` -- `ExperimentReadService.list_experiments` -- `ExperimentCohortService.resolve_or_create` -- run read/cancel services - -### Wrapper Commands - -`ergon benchmark run` has two acceptable end states: - -1. Preferred: reintroduce it as a convenience wrapper over `experiment define` plus `experiment run`. -2. Strict: delete the handler and update all docs/tests to use `ergon experiment define` plus `ergon experiment run`. - -The preferred end state is to keep it as a wrapper because it is useful for demos and real-LLM canaries: - -```text -ergon benchmark run minif2f --limit 1 - -equivalent to: - ergon experiment define minif2f --limit 1 --worker minif2f-react --model openai:gpt-4o --evaluator minif2f-rubric --sandbox minif2f --extras none - ergon experiment run <experiment_id> -``` - -The wrapper must not call `ergon_cli.composition.build_experiment` or create `RunRecord` rows itself. - -### Operational Commands - -These commands should stay outside the experiment lifecycle: - -- `ergon benchmark setup <slug>`: build/register E2B sandbox templates. -- `ergon workflow ...`: task-local workflow/resource helper used inside workers and sandboxes. -- `ergon run list`: operator telemetry over recent runs. -- `ergon run cancel <run_id>`: cancellation and cleanup request. -- `ergon eval watch` and `ergon eval checkpoint`: checkpoint evaluation automation. -- `ergon train local`: local training integration. -- `ergon doctor` and `ergon onboard`: environment setup and diagnostics. - -## Target Package Tree - -```text -ergon_cli/ - ergon_cli/ - main.py - # argparse only; no business logic - - commands/ - benchmark.py - # list, setup, wrapper run only - experiment.py - # define, run, show, list through facade helpers - run.py - # list, cancel through run services - worker.py - evaluator.py - workflow.py - eval.py - train.py - onboard.py - doctor.py - - services/ - experiment_cli_facade.py - # CLI-specific orchestration over core service DTOs - # parse args -> requests -> logging/rendering - benchmark_cli_facade.py - # benchmark list/setup/wrapper helpers - run_cli_facade.py - # list/cancel/show helpers - - discovery/ - __init__.py - # registry reads only - - rendering/ - __init__.py - # tables, key=value output, errors - - parsing/ - __init__.py - # optional shared parser helper functions if main.py grows too large -``` - -`ergon_cli.composition` should be removed once `benchmark run` and smoke-only composition paths are replaced by service facade calls or test harness APIs. - -## Service Boundary - -The CLI may import: - -```python -from ergon_builtins.registry import ( - BENCHMARKS, - WORKERS, - EVALUATORS, - SANDBOX_MANAGERS, - SANDBOX_TEMPLATES, -) - -from ergon_core.core.runtime.services.experiment_definition_service import ExperimentDefinitionService -from ergon_core.core.runtime.services.experiment_launch_service import ExperimentLaunchService -from ergon_core.core.runtime.services.experiment_read_service import ExperimentReadService -from ergon_core.core.runtime.services.experiment_schemas import ExperimentDefineRequest, ExperimentRunRequest -from ergon_core.core.runtime.services.cohort_service import experiment_cohort_service -from ergon_core.core.runtime.services.run_service import cancel_run -``` - -The CLI should not import: - -- `ergon_core.core.composition.Experiment` except inside a temporary migration shim. -- `ergon_core.core.composition.WorkerSpec` except inside core services. -- benchmark package internals such as `ergon_builtins.benchmarks.minif2f.*`. -- concrete criterion classes. -- persistence model classes for command logic, except through a temporary run-list shim. -- Inngest event classes for experiment launch, except through core services. - -## Discovery Commands - -### `ergon benchmark list` - -Use `BENCHMARKS` plus the related worker/evaluator/sandbox registries for validation. Do not show or infer benchmark defaults in this phase. - -Target columns: - -```text -Slug -Name -Description -Requires Data Extra -Known Sandboxes -``` - -Rules: - -- Include all registered benchmark slugs. -- Do not display default workers or evaluators. -- Show dependency hints only when they come from `BenchmarkRequirements` or explicit registry metadata. -- A contract test should fail if CLI code starts deriving hidden worker/evaluator/sandbox defaults. - -### `ergon worker list` - -Use `WORKERS`. - -Target columns: - -```text -Slug -Name -Kind -Description -``` - -`Kind` can initially be inferred: - -- `class` -- `factory` - -Long term, worker metadata can move into an explicit descriptor object if the registry grows. - -### `ergon evaluator list` - -Use `EVALUATORS`. - -Target columns: - -```text -Slug -Name -Kind -Description -``` - -`Kind` can be: - -- `rubric` -- `evaluator` - -If `Evaluator` remains advanced public API, list it as an advanced evaluator, not a beginner rubric. - -## Experiment Define - -### Command - -```text -ergon experiment define <benchmark_slug> - (--limit N | --sample-id SAMPLE_ID ...) - [--name NAME] - [--cohort COHORT_NAME] - --worker WORKER_SLUG - --model MODEL_TARGET - --evaluator EVALUATOR_SLUG - --sandbox SANDBOX_SLUG - --extras EXTRAS_SPEC - [--workflow single] - [--max-questions N] -``` - -The CLI should keep these choices compulsory while the package structure is stabilizing. A benchmark slug alone is not enough information to define an experiment. - -### Data Flow - -```mermaid -sequenceDiagram - accTitle: Experiment Define Flow - accDescr: The CLI validates explicit registry slugs, builds a request DTO, and delegates experiment definition to core services. - - participant User - participant CLI - participant Registry - participant Cohorts - participant DefinitionService - participant DB - - User->>CLI: ergon experiment define minif2f --limit 1 - CLI->>Registry: validate explicit benchmark/worker/evaluator/sandbox slugs - CLI->>Cohorts: resolve_or_create when --cohort is set - CLI->>DefinitionService: define_benchmark_experiment(request) - DefinitionService->>Registry: instantiate benchmark by slug - DefinitionService->>DefinitionService: build_instances and select samples - DefinitionService->>DB: persist ExperimentRecord - DefinitionService-->>CLI: ExperimentDefineResult - CLI-->>User: key=value identifiers -``` - -### Request Mapping - -```python -ExperimentDefineRequest( - benchmark_slug=args.benchmark_slug, - name=args.name, - cohort_id=cohort_id, - limit=args.limit, - sample_ids=args.sample_id or None, - default_model_target=args.model, - default_worker_team={"primary": args.worker}, - default_evaluator_slug=args.evaluator, - metadata={ - "workflow": args.workflow, - "max_questions": args.max_questions, - "sandbox_slug": args.sandbox, - "extras": args.extras, - "cli_command": "experiment define", - }, -) -``` - -### Output Contract - -The command should print stable key/value lines: - -```text -EXPERIMENT_ID=<uuid> -COHORT_ID=<uuid> # only when known -BENCHMARK=<slug> -SAMPLES=<comma-separated sample ids> -DEFAULT_WORKER=<slug> -DEFAULT_EVALUATOR=<slug> -DEFAULT_MODEL=<model target> -``` - -Tests and automation should parse these lines rather than human prose. - -## Experiment Run - -### Command - -```text -ergon experiment run <experiment_id> - [--timeout SECONDS] - [--no-wait] -``` - -### Required Core Behavior - -`ExperimentLaunchService.run_experiment` should own: - -1. read `ExperimentRecord` -2. create one `RunAssignment` per selected sample -3. construct a single-sample benchmark wrapper -4. instantiate evaluator binding from `EVALUATORS` -5. call `Experiment.from_single_worker(...)` -6. persist workflow definition through `ExperimentPersistenceService` -7. create `RunRecord` with: - - `experiment_id` - - `workflow_definition_id` - - `instance_key` - - `worker_team_json` - - `evaluator_slug` - - `model_target` - - optional assignment/seed metadata -8. emit `WorkflowStartedEvent` - -The CLI should not implement any of those steps directly. - -### Wait Semantics - -The current schema includes `timeout_seconds` and `wait`, but the launch service does not fully use them. The target semantics: - -- `wait=True`: return after all created runs reach terminal status or timeout. -- `wait=False`: return immediately after workflow start events are emitted. -- `timeout_seconds`: maximum wait time when `wait=True`. -- Timeout should not cancel the run by default; it should return a non-zero CLI code only for the waiting command. - -The result DTO should carry enough status for output: - -```text -EXPERIMENT_ID=<uuid> -RUN_ID=<uuid> -RUN_STATUS=<status> # when wait=True and known -``` - -If multiple runs are launched, print one `RUN_ID=` and `RUN_STATUS=` pair per run, or a tabular block after the stable key/value lines. - -## Experiment Show/List - -`experiment show` should read `ExperimentReadService.get_experiment`. - -Output should include: - -```text -EXPERIMENT_ID=<uuid> -COHORT_ID=<uuid> -NAME=<name> -BENCHMARK=<slug> -STATUS=<status> -SAMPLE_COUNT=<n> -RUN_COUNT=<n> -DEFAULT_WORKER=<slug> -DEFAULT_EVALUATOR=<slug> -DEFAULT_MODEL=<model> -SAMPLE_SELECTION=<json or comma-separated ids> -``` - -If runs exist, print: - -```text -RUNS -<run_id>\t<instance_key>\t<status>\t<model_target> -``` - -`experiment list` should remain a summary table. It should not instantiate benchmarks or workers. - -## Benchmark Setup - -`ergon benchmark setup <slug>` remains separate from experiment lifecycle. - -It should: - -1. read `SANDBOX_TEMPLATES` -2. validate `E2B_API_KEY` -3. load the benchmark template spec -4. build the E2B template -5. write `~/.ergon/sandbox_templates.json` or `ERGON_CONFIG_DIR/sandbox_templates.json` -6. print a follow-up command using the canonical lifecycle - -The success message should not suggest stale `benchmark run` syntax unless `benchmark run` is formally kept. - -Preferred success message: - -```text -Success! Template ID: <template_id> -Next: - ergon experiment define <slug> --limit 1 - ergon experiment run <experiment_id> -``` - -If `benchmark run` is kept: - -```text -Or: - ergon benchmark run <slug> --limit 1 -``` - -## Benchmark Run Wrapper - -If kept, `benchmark run` should be registered in `main.py` and call a wrapper function that does exactly: - -1. require the same explicit worker/evaluator/sandbox/model/extras arguments as `experiment define` -2. validate those explicit choices against registries -3. call the same define facade as `experiment define` -4. call the same run facade as `experiment run` -5. print the same stable key/value output - -Target command: - -```text -ergon benchmark run <benchmark_slug> - [--limit N | --sample-id SAMPLE_ID ...] - [--name NAME] - [--cohort COHORT_NAME] - --worker WORKER_SLUG - --model MODEL_TARGET - --evaluator EVALUATOR_SLUG - --sandbox SANDBOX_SLUG - --extras EXTRAS_SPEC - [--workflow single] - [--timeout SECONDS] - [--no-wait] -``` - -The handler should not call: - -- `build_experiment` -- `Experiment.persist` -- `create_run` directly -- `inngest_client.send` directly - -## Run Commands - -### `ergon run list` - -The current CLI queries `RunRecord` directly. Target state: - -- add a read method in core, either in `RunReadService` or a small `RunListService` -- support `--limit` -- support `--status` -- optionally support `--experiment-id` and `--cohort-id` later - -Output columns: - -```text -RUN_ID -STATUS -EXPERIMENT_ID -WORKFLOW_DEFINITION_ID -INSTANCE_KEY -MODEL -CREATED_AT -UPDATED_AT -``` - -### `ergon run cancel <run_id>` - -Keep routed through `run_service.cancel_run`. - -Target behavior: - -- return `0` if cancellation request is accepted -- return non-zero if run is missing or already terminal and cannot be canceled -- print stable key/value output: - -```text -RUN_ID=<uuid> -STATUS=cancelled -``` - -## Workflow Command - -`ergon workflow` is an internal worker/sandbox helper surface, not an operator experiment lifecycle surface. - -It may continue to call `WorkflowService` directly because it is already scoped by: - -- `--run-id` -- `--node-id` -- `--execution-id` -- `--sandbox-task-key` -- `--benchmark-type` - -Refactor rules: - -- keep it isolated from benchmark definition and launch code -- do not make it import benchmark package internals -- keep `--benchmark-type` as a slug used by sandbox materialization -- add tests that workflow parser changes do not affect experiment parser behavior - -## Eval Commands - -`ergon eval watch` and `ergon eval checkpoint` should use the canonical experiment lifecycle for local evaluation. - -Current target: - -```text -eval checkpoint - -> evaluate_checkpoint - -> local eval path - -> ergon experiment define - -> ergon experiment run - -> read run/evaluation results -``` - -Required cleanup: - -- make `--eval-limit` required for local eval if `_run_local_eval` requires it, or provide a safe default -- ensure subprocess calls use `experiment define/run`, not `benchmark run` -- ensure output parsing relies on stable `EXPERIMENT_ID=` and `RUN_ID=` lines - -## Train Command - -`ergon train local` belongs to training infrastructure and should remain separate from CLI experiment lifecycle. - -It may accept: - -- `--benchmark` -- `--evaluator` -- `--definition-id` -- model/training parameters - -The refactor should not change training semantics unless import paths break. - -## Doctor And Onboard - -`doctor` and `onboard` should use explicit CLI request fields plus benchmark requirements to report missing dependencies. - -Examples: - -- benchmark requires `[data]` -- benchmark requires E2B -- benchmark recommends `EXA_API_KEY` -- benchmark requires sandbox template setup -- model backend requires environment keys - -They should not instantiate benchmark datasets just to list requirements. - -## Migration Plan - -### Phase 1: Parser And Command Contract - -- Decide final `benchmark run` behavior. -- If keeping it, register the parser and implement it as a wrapper. -- If removing it, delete handler code and update tests/docs/real-LLM canaries. -- Update `benchmark setup` success messaging. -- Add parser tests for all command surfaces. - -### Phase 2: Explicit Registry Validation - -- Update `discovery.list_benchmarks()` to display registered benchmarks without implying default pairings. -- Keep `--worker`, `--model`, `--evaluator`, `--sandbox`, and `--extras` required for `experiment define` and `benchmark run`. -- Add validation errors for missing or unknown explicit choices: - - unknown benchmark slug - - unknown worker slug - - unknown evaluator slug - - unknown sandbox slug - - missing model target - - missing extras/dependency intent - -### Phase 3: CLI Facade Extraction - -- Create `ergon_cli/services/experiment_cli_facade.py`. -- Move argument-to-DTO mapping out of command handlers. -- Keep `commands/experiment.py` thin. -- Add `benchmark_cli_facade.py` for list/setup/wrapper run. -- Add `run_cli_facade.py` for list/cancel once run read service exists. - -### Phase 4: Delete Direct Composition Path - -- Remove `ergon_cli.composition.build_experiment` from production CLI flows. -- Move any smoke-only composition behavior into core test harness or test support. -- Ensure no production CLI command imports `Experiment`, `WorkerSpec`, or Inngest events for launch. - -### Phase 5: Wait/Poll Semantics - -- Implement service-level `wait` and `timeout_seconds`, or remove those fields from CLI/schema. -- Prefer implementing them because e2e and demos need blocking behavior. -- Add tests for: - - `--no-wait` returns after dispatch - - timeout returns non-zero without canceling runs - - completed runs return status lines - -### Phase 6: Run Read Service - -- Add a service method for listing recent runs. -- Route `run list` through it. -- Keep `run cancel` through `cancel_run`. -- Add tests for status filtering. - -## Test Plan - -### Unit Tests - -Parser tests: - -- `benchmark list` parses -- `benchmark setup <slug>` parses -- `benchmark run <slug>` parses if kept, fails if removed -- `experiment define` parses only with explicit worker/model/evaluator/sandbox/extras -- `experiment run --no-wait` parses -- `run list --status failed` parses -- `eval checkpoint --eval-limit 1` parses - -Facade tests: - -- define facade builds `ExperimentDefineRequest` with explicit CLI choices -- define facade rejects missing explicit worker/evaluator/sandbox/model/extras -- define facade resolves cohort only when `--cohort` is provided -- run facade builds `ExperimentRunRequest` -- benchmark wrapper calls define then run facades -- benchmark wrapper does not import or call direct composition helpers - -Discovery tests: - -- benchmark list does not imply default worker/evaluator pairings -- worker list includes factory entries -- evaluator list includes rubric/evaluator entries -- discovery does not expose hidden benchmark defaults - -### Integration Tests - -Service/CLI integration tests should cover: - -- `experiment define` persists `ExperimentRecord` with slugs and sample selection -- `experiment run` creates `RunRecord` rows with required foreign keys and assignment JSON -- `benchmark run` wrapper produces the same database shape as define plus run -- `run list` reads persisted runs through service -- `run cancel` emits cancellation and cleanup events - -### E2E Tests - -E2E should keep using: - -```text -ergon experiment define -ergon experiment run -``` - -unless `benchmark run` is explicitly retained as a wrapper, in which case one small canary can prove the wrapper path. - -The full e2e matrix is specified in `2026-04-28-ergon-e2e-refactor-test-plan.md`. - -## Known Drifts To Resolve - -1. `benchmark run` exists in `commands/benchmark.py` but is not registered in `main.py`. -2. `commands/benchmark.py::_create_and_dispatch` calls `create_run` with an old signature. -3. `experiment run --timeout` and `--no-wait` are represented in DTOs but not fully honored by the launch service. -4. `ergon_cli.composition` imports stale public API modules for `Experiment` and `WorkerSpec`. -5. `run list` queries persistence directly instead of using a core read service. -6. `eval checkpoint` can reach local eval without an `eval_limit` even though the local eval helper requires one. -7. `benchmark setup` still prints stale `benchmark run` guidance. - -## Final CLI Contract - -The refactor is complete when: - -- all experiment lifecycle commands go through core service facades -- all discovery commands read registries -- no production CLI command constructs `Experiment` directly -- no production CLI command creates `RunRecord` directly for launch -- `benchmark run` is either a tested wrapper or fully removed -- API, CLI, e2e, and eval automation agree on the same define/run semantics -- stable key/value CLI output is covered by tests diff --git a/docs/superpowers/plans/2026-04-28-ergon-e2e-refactor-test-plan.md b/docs/superpowers/plans/2026-04-28-ergon-e2e-refactor-test-plan.md deleted file mode 100644 index 6052b909a..000000000 --- a/docs/superpowers/plans/2026-04-28-ergon-e2e-refactor-test-plan.md +++ /dev/null @@ -1,831 +0,0 @@ -# Ergon E2E Refactor Test Plan - -This document specifies the test strategy that should accompany the Ergon core, built-ins, and CLI refactor. It is a sibling to: - -- `2026-04-28-public-api-target-structure.md` -- `2026-04-28-ergon-builtins-rebuild-structure.md` -- `2026-04-28-ergon-cli-refactor-structure.md` - -The purpose is to keep the refactor self-consistent: public API objects, built-in registry slugs, CLI commands, runtime rehydration, smoke fixtures, e2e harnesses, and dashboard assertions should all prove the same contract. - -## Goals - -- Preserve the existing four-tier testing model: unit, integration, e2e smoke, real-LLM. -- Keep production built-ins separate from test-only smoke fixtures. -- Use e2e smoke to prove cross-process behavior, not pure benchmark logic. -- Ensure CLI define/run behavior is covered by unit and integration tests before e2e uses it. -- Ensure every production benchmark family has contract tests for registry shape and explicit CLI pairing documentation. -- Ensure runtime execution can rehydrate workers, rubrics, criteria, task payloads, and sandbox managers from persisted slugs. -- Keep dashboard and harness checks aligned with run/cohort semantics. - -## Testing Tier Model - -The source of truth should remain path-based: - -```text -tests/unit/ - pure logic, models, validators, registry shape, parser behavior - -tests/integration/ - real Postgres and real Inngest dev server; service, persistence, API boundaries - -tests/e2e/ - full stack, test harness, real E2B, dashboard, Playwright - -tests/real_llm/ - opt-in or nightly; real model calls and budget-gated canaries -``` - -Markers are developer ergonomics, not the canonical tier definition. If `pyproject.toml` marker descriptions conflict with `docs/architecture/07_testing.md`, update the marker descriptions to match the path-based model. - -## High-Level Coverage Map - -```mermaid -flowchart TD - accTitle: Refactor Coverage Map - accDescr: Each test tier proves a different part of the Ergon refactor, from public API contracts through built-in registry shape and CLI service flow to full-stack dashboard behavior. - - public_api["Public API contracts"] - builtins["Built-ins registries<br/>benchmarks/workers/evaluators"] - cli["CLI facades<br/>define/run/list"] - services["Core runtime services<br/>experiments/runs/cohorts"] - runtime["Inngest runtime<br/>worker/evaluator rehydration"] - smoke["E2E smoke fixtures<br/>happy/sad cohorts"] - dashboard["Dashboard and harness"] - - unit["Unit tests"] - integration["Integration tests"] - e2e["E2E smoke tests"] - real_llm["Real-LLM tests"] - - unit --> public_api - unit --> builtins - unit --> cli - integration --> services - integration --> runtime - e2e --> smoke - e2e --> dashboard - real_llm --> builtins - real_llm --> runtime -``` - -## Fixture Residency Rules - -## Stable E2E Boundary After Core Layout Refactor - -Core behavior is stable, but private repository and persistence modules may move. -E2E code should use only: - -- HTTP endpoints under `/api/test/*` -- `ergon_core.test_support` -- public core API objects from `ergon_core.api` -- application read-model facades, not private repository methods - -The existing smoke behavior assertions remain valid: - -- happy runs complete the 12-node graph -- sad runs fail `l_2` and block `l_3` -- happy runs produce 20 task resources and 26 context events -- happy root produces two score-1.0 evaluations -- sad runs produce one partial artifact and seven completion messages - -### Production Built-ins - -Production benchmark code belongs under: - -```text -ergon_builtins/ergon_builtins/ -``` - -Production built-ins include: - -- benchmark loaders -- production task payload schemas -- production worker factories -- production criteria and rubrics -- production sandbox managers -- production registry entries -- shared production worker/model/tool helpers - -Production built-ins must not import: - -- `ergon_core.test_support` -- `tests` -- smoke fixture workers -- smoke fixture criteria -- smoke benchmark loaders - -### Core Test Support - -Canonical smoke fixtures belong under: - -```text -ergon_core/ergon_core/test_support/smoke_fixtures/ -``` - -This package owns: - -- smoke benchmark replacements for `researchrubrics`, `minif2f`, and `swebench-verified` -- smoke workers -- smoke leaf workers -- recursive smoke workers -- sad-path workers -- smoke criteria and smoke rubrics -- `SmokeSandboxManager` -- registry mutation hook `register_smoke_fixtures()` - -Smoke fixtures register into `ergon_builtins.registry` only when explicitly enabled by: - -- `ERGON_STARTUP_PLUGINS=ergon_core.test_support.smoke_fixtures:register_smoke_fixtures` -- `ENABLE_TEST_HARNESS=1` -- `ENABLE_SMOKE_FIXTURES=1` for any remaining host-side transitional paths - -### Tests - -Test drivers and assertions belong under: - -```text -tests/ -``` - -They own: - -- unit parser tests -- registry and explicit pairing contract tests -- integration service tests -- e2e cohort submission -- e2e harness polling -- dashboard Playwright orchestration -- real-LLM canaries - -Tests can import `ergon_core.test_support` in unit/integration contexts. Black-box e2e client code should not register fixtures in the host process; fixture registration should happen inside the API process through startup plugins. - -## Current Smoke Fixture Shape - -```text -ergon_core/ergon_core/test_support/ - __init__.py - # register_smoke_fixtures public hook - - smoke_fixtures/ - __init__.py - # mutates WORKERS, EVALUATORS, and optionally BENCHMARKS/SANDBOX_MANAGERS - - benchmarks.py - # ResearchRubricsSmokeBenchmark - # MiniF2FSmokeBenchmark - # SweBenchSmokeBenchmark - - sandbox.py - # SmokeSandboxManager - - criteria/ - minif2f_smoke.py - researchrubrics_smoke.py - swebench_smoke.py - smoke_rubrics.py - timing.py - - smoke_base/ - worker_base.py - leaf_base.py - recursive.py - sadpath.py - criterion_base.py - subworker.py - constants.py - - workers/ - minif2f_smoke.py - researchrubrics_smoke.py - researchrubrics_smoke_sadpath.py - swebench_smoke.py -``` - -The smoke benchmarks deliberately reuse production benchmark slugs: - -```text -researchrubrics -minif2f -swebench-verified -``` - -They replace production benchmark loaders only when `ENABLE_TEST_HARNESS=1`, so e2e does not need HuggingFace, production data, or LLM access to materialize root tasks. - -## Canonical Smoke Program - -Every PR should continue to run three e2e legs: - -```text -researchrubrics -minif2f -swebench-verified -``` - -Each leg submits a cohort with: - -- one happy-path run -- one sad-path run - -The topology should stay identical across benchmark slugs: - -```text -Diamond: - d_root - / \ - d_left d_right - \ / - d_join - -Line: - l_1 -> l_2 -> l_3 - -Singletons: - s_a - s_b -``` - -Happy-path `l_2` routes to a recursive worker with nested children: - -```text -l_2 -└─ l_2_a -> l_2_b -``` - -Sad-path `l_2` routes to a failing leaf. `l_3` must remain blocked or cancelled according to the static-sibling failure semantics decision. - -## E2E Submission Flow - -```mermaid -sequenceDiagram - accTitle: Smoke E2E Flow - accDescr: E2E tests submit benchmark cohorts through the HTTP test harness, then assert run state through API and dashboard surfaces. - - participant Pytest - participant Harness as API Test Harness - participant Registry - participant Services - participant Inngest - participant Dashboard - - Pytest->>Harness: POST /api/test/write/cohort - Harness->>Registry: resolve smoke benchmark/worker/evaluator slugs - Harness->>Services: define/persist/dispatch runs - Services->>Inngest: WorkflowStartedEvent - Inngest->>Registry: rehydrate smoke workers/evaluators - Pytest->>Harness: poll /api/test/read/cohort/{key}/runs - Pytest->>Harness: read /api/test/read/run/{id}/state - Pytest->>Dashboard: Playwright assertions by cohort/run -``` - -The black-box e2e tests should not: - -- import production internals -- call `build_experiment` -- call `create_run` -- send Inngest events directly -- register smoke fixtures in the host pytest process - -The API process owns fixture registration through `ERGON_STARTUP_PLUGINS`. - -## CLI Coverage Flow - -CLI tests should be split by tier: - -```text -unit: - parser and facade DTO mapping - -integration: - experiment define/run persistence and dispatch semantics - -e2e: - one small black-box CLI canary only if needed -``` - -The canonical e2e smoke path should use the HTTP test harness, not the CLI, because it is primarily proving cross-process runtime, sandbox, dashboard, and cohort behavior. CLI define/run gets its own unit and integration coverage. - -If `benchmark run` is kept as a wrapper, add exactly one CLI e2e canary proving wrapper wiring. Do not duplicate the full smoke matrix through both HTTP harness and CLI. - -## Unit Test Plan - -### Public API Contract Tests - -Add or update tests under: - -```text -tests/unit/architecture/ -tests/unit/api/ -``` - -Required assertions: - -- `ergon_core.api` exports beginner public symbols: - - `Benchmark` - - `Task` - - `EmptyTaskPayload` - - `BenchmarkRequirements` - - `Worker` - - `WorkerContext` - - `WorkerOutput` - - `Criterion` - - `CriterionContext` - - `CriterionOutcome` - - `ScoreScale` - - `CriterionEvidence` - - `EvidenceMessage` - - `Rubric` - - `TaskEvaluationResult` - - `CriterionCheckError` -- moved core composition types are not root-public authoring concepts: - - `Experiment` - - `WorkerSpec` - - `DefinitionHandle` -- public API modules do not import DB/session modules. -- public worker code does not import context event repositories for default output extraction. - -### Built-ins Registry And Pairing Tests - -Add or update tests under: - -```text -tests/unit/registry/ -tests/unit/builtins/ -tests/unit/benchmarks/ -tests/unit/state/ -``` - -Required assertions: - -- every `BENCHMARKS` key equals the benchmark class `type_slug` -- every benchmark exposes `task_payload_model` -- every benchmark exposes `BenchmarkRequirements` -- every documented CLI pairing references registered benchmark, worker, evaluator, and sandbox slugs -- no production code derives hidden worker/evaluator/sandbox defaults from a benchmark slug -- importing `registry_core.py` does not require `[data]` dependencies -- importing `registry_data.py` is allowed to require optional data extras -- production registries do not include smoke worker slugs -- smoke fixture registration is idempotent -- smoke fixture registration only overrides benchmark loaders when `ENABLE_TEST_HARNESS=1` - -### CLI Unit Tests - -Add or update tests under: - -```text -tests/unit/cli/ -``` - -Required assertions: - -- parser registers all canonical commands -- parser outcome for `benchmark run` matches the decision in the CLI spec -- `experiment define` requires explicit `--worker`, `--model`, `--evaluator`, `--sandbox`, and `--extras` -- missing explicit worker/model/evaluator/sandbox/extras values fail before service calls -- define facade builds `ExperimentDefineRequest` -- run facade builds `ExperimentRunRequest` -- benchmark wrapper calls define plus run facades if kept -- `benchmark setup` success guidance uses canonical commands -- discovery output does not imply hidden benchmark defaults -- `run list` delegates to run read service after that service exists -- `eval checkpoint` handles missing or default `--eval-limit` consistently - -### Smoke Fixture Unit Tests - -Keep and extend tests under: - -```text -tests/unit/smoke_base/ -``` - -Required assertions: - -- topology constants remain the single source of truth -- `SmokeWorkerBase.execute` remains final -- every environment has: - - happy parent worker - - leaf worker - - recursive worker - - sad-path parent - - failing leaf - - smoke rubric -- all smoke workers accept the current public `Worker` constructor contract -- smoke criteria use the public `CriterionContext` capability surface -- smoke benchmark payload schemas match production payload shape enough for runtime serialization -- e2e driver pairs exist for every smoke environment - -### Architecture Boundary Tests - -Keep and extend: - -```text -tests/unit/architecture/test_no_test_logic_in_core.py -tests/unit/architecture/test_smoke_fixture_package_boundary.py -``` - -Target assertions: - -- production core does not import `ergon_core.test_support` except explicit test harness/plugin loading points -- `ergon_builtins` does not import `ergon_core.test_support` -- `ergon_builtins` does not import `tests` -- `ergon_cli` production commands do not import smoke fixture modules -- API startup plugin loader may import configured plugins dynamically -- `/api/test/*` is mounted only when `ENABLE_TEST_HARNESS=1` - -## Integration Test Plan - -Integration tests use real Postgres and real Inngest dev server. They should not require real LLM calls. - -### Experiment Services - -Add or update tests under: - -```text -tests/integration/ -tests/unit/runtime/ -``` - -Required scenarios: - -1. Define experiment from a smoke benchmark slug. -2. Persist selected sample keys and explicit worker/evaluator/sandbox/model/extras choices. -3. Run experiment and create one `RunRecord` per selected sample. -4. Persist workflow definition with benchmark, worker, and evaluator slugs. -5. Emit `WorkflowStartedEvent` for each run. -6. Support `wait=False` path. -7. Support timeout path without deleting or cancelling the run. - -### Runtime Rehydration - -Required scenarios: - -- worker execution rehydrates worker factory from `WORKERS` -- worker execution validates task payload through registered benchmark payload model -- evaluator execution rehydrates evaluator from `EVALUATORS` -- criteria run against `CriterionContext`, not direct concrete runtime imports in public modules -- sandbox manager is resolved from `SANDBOX_MANAGERS` -- sandbox setup completes before benchmark-owned worker factories are invoked -- failed worker path persists partial artifacts and marks downstream dependencies correctly - -### Sandbox Integration - -Keep benchmark-specific sandbox manager tests: - -```text -tests/integration/minif2f/test_sandbox_manager.py -tests/integration/researchrubrics/test_sandbox_manager.py -tests/integration/swebench_verified/test_sandbox_manager.py -tests/integration/sandbox/test_required_env_keys.py -``` - -Refactor expectations: - -- these tests should import benchmark sandbox managers from final package locations -- they should not depend on CLI composition helpers -- they should be skipped or marked clearly when E2B credentials are absent, according to current integration policy - -### Evaluator Integration - -Keep and align: - -```text -tests/integration/minif2f/test_verification_integration.py -tests/integration/swebench_verified/test_criterion.py -tests/integration/swebench_verified/test_rubric.py -``` - -Required updates: - -- import renamed public result/context classes -- assert `CriterionOutcome` evidence fields where appropriate -- avoid old `EvaluationContext` naming -- ensure SWE-Bench criterion patch extraction uses `CriterionContext` capabilities - -## E2E Smoke Test Plan - -### Python E2E Layout - -Target layout: - -```text -tests/e2e/ - conftest.py - # infra preflight, shared DB session, optional CLI helper - - _submit.py - # black-box cohort submission through /api/test/write/cohort - - _asserts.py - # run graph, resources, evaluation, communication, sandbox assertions - - _read_contracts.py - # DTO helpers for /api/test/read endpoints - - test_researchrubrics_smoke.py - test_minif2f_smoke.py - test_swebench_smoke.py -``` - -Each `test_<env>_smoke.py` should: - -1. build a cohort key -2. submit two slots: - - happy smoke worker plus smoke rubric - - sad-path smoke worker plus smoke rubric -3. wait for terminal statuses -4. assert happy run graph/resources/evaluations/messages -5. assert sad run partial artifacts and blocked/cancelled downstream node -6. run the dashboard Playwright smoke spec for that environment - -### Required Per-Run Assertions - -Happy run assertions: - -- root node completed -- expected direct child nodes exist -- nested `l_2_a` and `l_2_b` exist -- dependency edges match canonical topology -- all expected leaf/dynamic nodes completed -- `GenerationTurn` count matches expected topology -- communication thread messages exist in order -- run resources include outputs and probe artifacts -- blob store round-trip works -- root evaluations exist -- evaluation timestamps are after root execution completion -- sandbox health probe succeeded - -Sad run assertions: - -- root node reaches failed or terminal failed-equivalent state -- `l_2` failed -- `l_3` blocked or cancelled until the failure semantics RFC pins final status -- partial artifact from failing leaf exists -- pre-failure sandbox WAL entry exists when WAL persistence exists -- no successful final evaluation score is recorded -- unaffected branches completed as expected - -### Dashboard Assertions - -Dashboard e2e specs under: - -```text -ergon-dashboard/tests/e2e/ -``` - -should assert: - -- cohort page renders both happy and sad runs -- run status is visible -- graph canvas renders -- each expected task node appears by `data-testid` -- environment label appears -- failed/blocked node states are visible on sad path -- evaluation panel shows root evaluation where expected -- resources/artifacts are visible where expected - -Backend harness DTOs should remain the source of truth for data-rich assertions; Playwright should assert that the UI represents the same state. - -## Real-LLM Test Plan - -Real-LLM tests are opt-in and should not block ordinary local development. - -Target directory: - -```text -tests/real_llm/ - benchmarks/ - test_researchrubrics.py - test_minif2f.py # optional future canary - test_swebench.py # optional future canary - test_smoke_stub.py - fixtures/ - stack.py - harness_client.py - playwright_client.py - openrouter_budget.py -``` - -Required canaries: - -- one no-LLM stub model canary proving CLI wrapper behavior if `benchmark run` is kept -- one ResearchRubrics real model run proving report generation and LLM judge path - -Optional canaries: - -- MiniF2F real model proof attempt -- SWE-Bench real model patch attempt - -Real-LLM tests should use strict budgets and explicit environment gates: - -- `ERGON_REAL_LLM=1` -- OpenRouter/OpenAI/Anthropic keys as required -- stack readiness fixtures - -## Test Harness Contract - -The `/api/test/*` harness should remain test-only. - -Mounting rules: - -- enabled only when `ENABLE_TEST_HARNESS=1` -- write endpoints require `X-Test-Secret` or configured secret behavior -- read endpoints are safe for Playwright and pytest polling in test environments - -Required endpoints: - -```text -POST /api/test/write/cohort -GET /api/test/read/cohort/{cohort_key}/runs -GET /api/test/read/run/{run_id}/state -``` - -The write endpoint should use the same core services as production experiment launch. It may use smoke fixture registry entries, but it should not keep a separate run creation path that bypasses service invariants. - -## Coverage Matrix - -| Area | Unit | Integration | E2E Smoke | Real-LLM | -|---|---|---|---|---| -| Public API exports | required | no | no | no | -| Public API import boundaries | required | no | no | no | -| Built-ins registry and explicit pairing shape | required | optional | indirect | optional | -| Benchmark `build_instances` contract | required with stubs | data-dependent paths | smoke replacements | real datasets optional | -| CLI parser/facade mapping | required | optional | one canary only | optional | -| Experiment define/run services | fast mocked unit plus contract tests | required | indirect through harness | indirect | -| Run creation schema | required | required | indirect | indirect | -| Inngest worker rehydration | required | required | required | required for canaries | -| Evaluator/criterion rehydration | required | required | required | required for judge canaries | -| Sandbox manager setup | unit stubs | required per benchmark | required smoke path | optional | -| Dashboard event contracts | required | optional | required | optional | -| Cohort happy/sad behavior | unit topology | service-level partial | required | optional | -| LLM generation quality | no | no | no | required | - -## Migration Plan - -### Phase 1: Freeze Test Boundaries - -- Update this plan and `docs/architecture/07_testing.md` if necessary. -- Align `pyproject.toml` marker descriptions with the path-based tier model. -- Add boundary tests proving production built-ins do not import smoke/test modules. -- Add tests proving smoke fixtures register only through explicit hooks. - -### Phase 2: Public API Rename Tests - -- Update unit tests to use final public names: - - `Task` - - `BenchmarkRequirements` - - `CriterionContext` - - `CriterionOutcome` - - `ScoreScale` - - `CriterionEvidence` - - `EvidenceMessage` -- Keep no compatibility alias tests unless the product decision changes. - -### Phase 3: Built-ins Registry And Pairing Tests - -- Add explicit pairing contract tests for: - - `minif2f` - - `swebench-verified` - - `gdpeval` - - `researchrubrics` - - `researchrubrics-vanilla` -- Add optional dependency import tests for `registry_core.py` versus `registry_data.py`. - -### Phase 4: CLI Contract Tests - -- Update parser tests around the final `benchmark run` decision. -- Add facade tests for define/run DTO mapping. -- Add integration tests for `experiment define` and `experiment run`. -- Update real-LLM tests to use canonical CLI commands or the wrapper if retained. - -### Phase 5: Runtime Rehydration Tests - -- Update Inngest worker execution tests for final `Task` payload paths. -- Update evaluator execution tests for final `CriterionContext` and `CriterionOutcome`. -- Add regression tests for sandbox setup before worker factory invocation. -- Add tests for persisted slugs matching registry keys. - -### Phase 6: E2E Harness Alignment - -- Ensure `/api/test/write/cohort` calls the same core launch service path as CLI/API. -- Ensure e2e host process does not register fixtures. -- Ensure API process registers fixtures by startup plugin. -- Ensure smoke benchmark replacements override production benchmark loaders only when `ENABLE_TEST_HARNESS=1`. -- Keep Playwright specs aligned with expected smoke topology constants. - -### Phase 7: Dashboard And Artifact Assertions - -- Turn soft-skipped sandbox WAL assertions into hard assertions once WAL persistence exists. -- Keep screenshots on failure. -- Verify dashboard `data-testid` attributes remain stable: - - `run-status` - - `task-node-{slug}` - - `graph-canvas` - - `cohort-run-row` - - `cohort-env-label` - -## Required Test Files To Update Or Add - -### Unit - -```text -tests/unit/architecture/test_public_api_shape.py -tests/unit/architecture/test_no_test_logic_in_core.py -tests/unit/architecture/test_smoke_fixture_package_boundary.py -tests/unit/registry/test_builtin_pairings.py -tests/unit/registry/test_react_factories.py -tests/unit/cli/test_experiment_cli.py -tests/unit/cli/test_benchmark_setup.py -tests/unit/cli/test_eval_cli_required_fields.py -tests/unit/smoke_base/test_smoke_fixture_registration.py -tests/unit/smoke_base/test_e2e_smoke_driver_pairs.py -``` - -### Integration - -```text -tests/integration/smokes/test_smoke_harness.py -tests/integration/minif2f/test_verification_integration.py -tests/integration/minif2f/test_sandbox_manager.py -tests/integration/researchrubrics/test_sandbox_manager.py -tests/integration/swebench_verified/test_criterion.py -tests/integration/swebench_verified/test_rubric.py -tests/integration/swebench_verified/test_sandbox_manager.py -tests/integration/sandbox/test_required_env_keys.py -``` - -Add, if missing: - -```text -tests/integration/cli/test_experiment_define_run.py -tests/integration/runtime/test_registry_rehydration.py -tests/integration/runtime/test_experiment_launch_service_wait.py -``` - -### E2E - -```text -tests/e2e/conftest.py -tests/e2e/_submit.py -tests/e2e/_asserts.py -tests/e2e/_read_contracts.py -tests/e2e/test_researchrubrics_smoke.py -tests/e2e/test_minif2f_smoke.py -tests/e2e/test_swebench_smoke.py -``` - -### Dashboard - -```text -ergon-dashboard/tests/e2e/_shared/smoke.ts -ergon-dashboard/tests/e2e/researchrubrics.smoke.spec.ts -ergon-dashboard/tests/e2e/minif2f.smoke.spec.ts -ergon-dashboard/tests/e2e/swebench-verified.smoke.spec.ts -ergon-dashboard/tests/helpers/backendHarnessClient.ts -``` - -### Real-LLM - -```text -tests/real_llm/benchmarks/test_researchrubrics.py -tests/real_llm/benchmarks/test_smoke_stub.py -tests/real_llm/fixtures/stack.py -tests/real_llm/fixtures/harness_client.py -tests/real_llm/fixtures/openrouter_budget.py -``` - -## Acceptance Criteria - -The refactor is test-complete when: - -- unit tests prove public API exports and import boundaries -- unit tests prove built-ins registry and explicit pairing consistency -- unit tests prove CLI parser/facade behavior -- integration tests prove experiment define/run services persist the expected records -- integration tests prove runtime worker/evaluator rehydration from slugs -- e2e tests pass for `researchrubrics`, `minif2f`, and `swebench-verified` -- e2e host process remains a black-box client -- smoke fixtures stay out of production built-ins -- real-LLM tests are updated to the final CLI contract -- dashboard Playwright specs still render and assert cohort/run state - -## 2026-04-29 Finish Plan Update - -The current execution plan for completing the built-ins, CLI, and e2e refactor is: - -```text -docs/superpowers/plans/2026-04-29-finish-builtins-cli-e2e-refactor.md -``` - -That plan supersedes this document's older migration checklist where the two disagree. In particular: - -- `benchmark run` is retained as an explicit `experiment define` plus `experiment run` wrapper. -- E2E smoke submissions must pass explicit `worker`, `evaluator`, `sandbox`, `model`, and `extras` choices through the test harness. -- E2E host-side tests may import `ergon_core.test_support`, public API modules, HTTP `/api/test/*`, and stable application read models, but not private core repository or persistence internals. -- The existing smoke runtime assertions remain hard assertions: happy runs still expect 12 tasks, 10 leaves, 20 resources, 26 context events, 2 root evaluations, and 11 completion messages; sad runs still expect `l_2` failed, `l_3` blocked, one partial artifact, and 7 completion messages. -- Any persistence-level data still needed for e2e assertions should be exposed through `ergon_core.test_support` helpers rather than imported directly by `tests/e2e`. - -## Open Decisions - -1. Whether e2e should include one CLI subprocess canary in addition to HTTP harness submission. -2. Whether sandbox command WAL persistence lands during this refactor or remains a follow-up. -3. Whether `tests/integration/swebench_verified/test_smoke_e2e.py` should be renamed because it is not a full e2e test. diff --git a/docs/superpowers/plans/2026-04-28-evaluation-resource-context-and-scoring.md b/docs/superpowers/plans/2026-04-28-evaluation-resource-context-and-scoring.md deleted file mode 100644 index 593064623..000000000 --- a/docs/superpowers/plans/2026-04-28-evaluation-resource-context-and-scoring.md +++ /dev/null @@ -1,908 +0,0 @@ -# Evaluation Resource Context and Scoring Patch Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make evaluator criteria fetch their own task-scoped resources, judge final artifacts rather than assistant summaries, and preserve evaluator-normalized scores without double-normalizing. - -**Architecture:** Core remains benchmark-agnostic: it exposes task-scoped resource access through `CriterionRuntime`. Benchmark criteria in `ergon_builtins` decide which resources to read, how to sort final outputs vs scratch files, and what to show verifiers or LLM judges. Evaluation persistence assumes all evaluators return normalized scalar task scores. - -**Tech Stack:** Python, Pydantic models, SQLModel, Ergon `CriterionRuntime`, ResearchRubrics LLM judge, real-LLM rollout artifacts. - ---- - -## Code Change Map - -- Modify: `ergon_core/ergon_core/api/criterion_runtime.py` - - Add optional `task_execution_id` to `list_resources`. - - Add `read_resource_by_id` so criteria can read exact SQL rows after listing. - -- Modify: `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` - - Implement optional task-execution scoping for `list_resources`. - - Implement `read_resource_by_id`. - - Keep core generic: no final-vs-scratch classification here. - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py` - - Fetch resources from `context.runtime`. - - Classify ResearchRubrics final outputs vs scratch files locally. - - Build the judge prompt from resource content plus final assistant message. - - Record `evaluated_resource_ids` and `evaluation_input`. - -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py` - - Stop re-normalizing `TaskEvaluationResult.score`. - - Store `summary.normalized_score = result.score`. - -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py` - - Keep existing ResearchRubrics formula, but clarify metadata with normalized score semantics. - -- Modify: `tests/real_llm/artifact_health.py` - - Detect missing final output via task-scoped resource rows and final-output provenance, not durable blob `file_path`. - -- Tests: - - `tests/unit/state/test_criterion_runtime_di.py` - - `tests/unit/state/test_research_rubrics_benchmark.py` - - `tests/unit/runtime/test_evaluation_summary_contracts.py` - - `tests/unit/runtime/test_real_llm_rollout_artifact_health.py` - ---- - -## Task 1: Extend Core Runtime Resource Access - -**Files:** -- Modify: `ergon_core/ergon_core/api/criterion_runtime.py` -- Modify: `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py` -- Test: `tests/unit/state/test_criterion_runtime_di.py` - -### Rationale - -Criteria should own context selection. Core should only provide generic resource primitives: - -- list resources for the evaluated task execution by default; -- optionally list resources for an explicit task execution id; -- read exact resources by id to avoid name collisions. - -Core must not know about ResearchRubrics final reports, scratchpads, or judge prompt layout. - -### Patch: Public Protocol - -In `ergon_core/ergon_core/api/criterion_runtime.py`, add `UUID` under `TYPE_CHECKING` or as a normal import. Since Protocol signatures need the type at runtime under postponed annotations are not enabled in this file, use a normal import: - -```python -from uuid import UUID -``` - -Change the resource methods: - -```python -# ── resource I/O ────────────────────────────────────────────────── -async def read_resource(self, name: str) -> bytes: ... -async def read_resource_by_id(self, resource_id: UUID) -> bytes: ... -async def list_resources( - self, - task_execution_id: UUID | None = None, -) -> "list[RunResourceView]": ... -async def get_all_files_for_task(self) -> "dict[str, bytes]": - """Return ``{name: bytes}`` for every resource produced by this task. - - Scoped to the runtime's evaluator-bound task execution. On duplicate - ``name`` s, the newest ``created_at`` wins. Not size-capped. - """ - ... -``` - -### Patch: Concrete Runtime - -In `ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py`, keep the existing SQLModel imports: - -```python -from sqlmodel import Session, desc, select -``` - -Add exact-id reading after `read_resource`: - -```python -async def read_resource_by_id(self, resource_id: UUID) -> bytes: - """Read one worker-published blob by its RunResource primary key.""" - with get_session() as session: - row = session.get(RunResource, resource_id) - - if row is None or row.run_id != self._run_id: - raise ResourceNotFoundError( - f"No run_resource {resource_id!s} for run {self._run_id}" - ) - - result = Path(row.file_path).read_bytes() - logger.info( - "criterion read_resource_by_id run_id=%s resource_id=%s size_bytes=%d", - self._run_id, - resource_id, - len(result), - ) - return result -``` - -Replace `list_resources` with task-aware behavior: - -```python -async def list_resources( - self, - task_execution_id: UUID | None = None, -) -> list[RunResourceView]: - """Return resource DTOs for this run, newest first. - - Defaults to this runtime's evaluated task execution. Passing - ``task_execution_id`` lets a benchmark criterion inspect a related task - explicitly without core knowing benchmark semantics. - """ - effective_execution_id = ( - task_execution_id if task_execution_id is not None else self._task_id - ) - with get_session() as session: - stmt = select(RunResource).where(RunResource.run_id == self._run_id) - if effective_execution_id is not None: - stmt = stmt.where(RunResource.task_execution_id == effective_execution_id) - stmt = stmt.order_by(desc(RunResource.created_at)) - rows = list(session.exec(stmt).all()) - return [RunResourceView.from_row(r) for r in rows] -``` - -### Tests - -In `tests/unit/state/test_criterion_runtime_di.py`, update the protocol test expected method set: - -```python -expected = { - "ensure_sandbox", - "upload_files", - "write_file", - "run_command", - "execute_code", - "cleanup", - "read_resource", - "read_resource_by_id", - "list_resources", - "get_all_files_for_task", - "db_read_session", - "event_sink", -} -``` - -Add tests: - -```python -@pytest.mark.asyncio -async def test_list_resources_defaults_to_runtime_task_execution() -> None: - task_execution_id = uuid4() - runtime = _make_runtime(task_id=task_execution_id) - - mock_row = MagicMock() - mock_session = MagicMock() - mock_session.__enter__ = MagicMock(return_value=mock_session) - mock_session.__exit__ = MagicMock(return_value=False) - mock_session.exec.return_value.all.return_value = [mock_row] - - with ( - patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session", - return_value=mock_session, - ), - patch.object(RunResourceView, "from_row", return_value=MagicMock()) as mock_from_row, - ): - result = await runtime.list_resources() - - assert len(result) == 1 - mock_from_row.assert_called_once_with(mock_row) - # Keep this assertion broad: SQLModel statements are hard to compare, but - # this ensures a DB query was issued through the runtime path. - mock_session.exec.assert_called_once() -``` - -```python -@pytest.mark.asyncio -async def test_read_resource_by_id_reads_exact_blob(tmp_path: Path) -> None: - blob = tmp_path / "abc" - blob.write_bytes(b"exact-resource") - - run_id = uuid4() - resource_id = uuid4() - row = MagicMock() - row.id = resource_id - row.run_id = run_id - row.file_path = str(blob) - - runtime = _make_runtime(run_id=run_id) - - mock_session = MagicMock() - mock_session.__enter__ = MagicMock(return_value=mock_session) - mock_session.__exit__ = MagicMock(return_value=False) - mock_session.get.return_value = row - - with patch( - "ergon_core.core.runtime.evaluation.criterion_runtime.get_session", - return_value=mock_session, - ): - result = await runtime.read_resource_by_id(resource_id) - - assert result == b"exact-resource" -``` - -Run: - -```bash -uv run pytest tests/unit/state/test_criterion_runtime_di.py -q -``` - -Expected: all tests pass. - ---- - -## Task 2: Make ResearchRubrics Criterion Fetch and Package Its Own Evidence - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py` -- Test: `tests/unit/state/test_research_rubrics_benchmark.py` - -### Rationale - -ResearchRubrics should judge the actual task artifacts, not the final assistant summary. The built-in criterion should use the generic runtime to fetch resources, then apply ResearchRubrics-specific evidence policy: - -- final outputs first; -- scratch/intermediate resources second; -- final assistant message as status/context only. - -### Patch - -Add imports: - -```python -from uuid import UUID - -from ergon_core.api.run_resource import RunResourceView -``` - -Add constants and a small local evidence type: - -```python -_MAX_RESOURCE_CHARS = 30_000 -_FINAL_OUTPUT_PREFIX = "/workspace/final_output/" - - -class _ResourceEvidence(BaseModel): - model_config = {"frozen": True, "arbitrary_types_allowed": True} - - resource: RunResourceView - content: str - - @property - def resource_id(self) -> str: - return str(self.resource.id) -``` - -Change `evaluate`: - -```python -async def evaluate(self, context: EvaluationContext) -> CriterionResult: - final_outputs, scratch_outputs = await _load_researchrubrics_evidence(context) - user_prompt = _build_user_prompt( - context, - final_outputs=final_outputs, - scratch_outputs=scratch_outputs, - ) - verdict = await call_structured_judge( - messages=[ - JudgeMessage(role="system", content=self.system_prompt), - JudgeMessage(role="user", content=user_prompt), - ], - response_type=ResearchRubricsVerdict, - model=self.model, - ) - evaluated_resource_ids = [ - evidence.resource_id for evidence in [*final_outputs, *scratch_outputs] - ] - return CriterionResult( - name=self.name, - score=self.max_score if verdict.passed else 0.0, - passed=verdict.passed, - weight=self.weight, - feedback=verdict.reasoning, - evaluation_input=_summarize_evaluation_input( - final_outputs=final_outputs, - scratch_outputs=scratch_outputs, - final_assistant_message=context.worker_result.output, - ), - evaluated_resource_ids=evaluated_resource_ids, - metadata={ - "primary_evidence_resource_ids": [e.resource_id for e in final_outputs], - "scratch_evidence_resource_ids": [e.resource_id for e in scratch_outputs], - }, - ) -``` - -Add evidence loading helpers: - -```python -async def _load_researchrubrics_evidence( - context: EvaluationContext, -) -> tuple[list[_ResourceEvidence], list[_ResourceEvidence]]: - if context.runtime is None: - return [], [] - - resources = await context.runtime.list_resources() - final_resources = [resource for resource in resources if _is_final_output_resource(resource)] - scratch_resources = [resource for resource in resources if resource not in final_resources] - - final_outputs = await _read_text_resources(context, final_resources) - scratch_outputs = await _read_text_resources(context, scratch_resources) - return final_outputs, scratch_outputs -``` - -```python -async def _read_text_resources( - context: EvaluationContext, - resources: list[RunResourceView], -) -> list[_ResourceEvidence]: - if context.runtime is None: - return [] - - evidence: list[_ResourceEvidence] = [] - for resource in resources: - if not _is_text_like(resource): - continue - content_bytes = await context.runtime.read_resource_by_id(resource.id) - content = content_bytes.decode("utf-8", errors="replace") - if len(content) > _MAX_RESOURCE_CHARS: - content = content[:_MAX_RESOURCE_CHARS] + "\n\n[truncated]" - evidence.append(_ResourceEvidence(resource=resource, content=content)) - return evidence -``` - -```python -def _is_text_like(resource: RunResourceView) -> bool: - return ( - resource.mime_type.startswith("text/") - or resource.mime_type in {"application/json", "application/x-ndjson"} - or resource.name.endswith((".md", ".txt", ".json", ".jsonl", ".csv")) - ) -``` - -```python -def _is_final_output_resource(resource: RunResourceView) -> bool: - origin = resource.metadata.get("sandbox_origin") - return isinstance(origin, str) and origin.startswith(_FINAL_OUTPUT_PREFIX) -``` - -Replace `_build_user_prompt`: - -```python -def _build_user_prompt( - context: EvaluationContext, - *, - final_outputs: list[_ResourceEvidence], - scratch_outputs: list[_ResourceEvidence], -) -> str: - return "\n\n".join( - [ - f"Original research request:\n{context.task.description}", - _format_resource_section( - "Final output resources (primary answer to judge)", - final_outputs, - empty="No final output resources were published.", - ), - _format_resource_section( - "Scratch/intermediate resources (supporting context; do not treat as final answer)", - scratch_outputs, - empty="No scratch resources were published.", - ), - ( - "Final assistant message (execution summary/status, not the primary answer):\n" - f"{context.worker_result.output}" - ), - ] - ) -``` - -Add format helpers: - -```python -def _format_resource_section( - title: str, - resources: list[_ResourceEvidence], - *, - empty: str, -) -> str: - if not resources: - return f"{title}:\n{empty}" - blocks = [f"{title}:"] - for evidence in resources: - resource = evidence.resource - origin = resource.metadata.get("sandbox_origin") - blocks.append( - "\n".join( - [ - f"--- resource_id={resource.id} name={resource.name} kind={resource.kind}", - f"mime_type={resource.mime_type} sandbox_origin={origin}", - evidence.content, - ] - ) - ) - return "\n\n".join(blocks) -``` - -```python -def _summarize_evaluation_input( - *, - final_outputs: list[_ResourceEvidence], - scratch_outputs: list[_ResourceEvidence], - final_assistant_message: str, -) -> str: - return "\n".join( - [ - "Evidence used by ResearchRubrics judge:", - "final_outputs=" - + ", ".join(f"{e.resource.name}:{e.resource.id}" for e in final_outputs), - "scratch_outputs=" - + ", ".join(f"{e.resource.name}:{e.resource.id}" for e in scratch_outputs), - "final_assistant_message=" - + final_assistant_message[:1000], - ] - ) -``` - -### Tests - -In `tests/unit/state/test_research_rubrics_benchmark.py`, add a fake runtime and direct unit test for the criterion. - -```python -class _Runtime: - def __init__(self, resources, blobs): - self._resources = resources - self._blobs = blobs - - async def list_resources(self, task_execution_id=None): - return self._resources - - async def read_resource_by_id(self, resource_id): - return self._blobs[resource_id] -``` - -Patch `call_structured_judge` and assert: - -```python -@pytest.mark.asyncio -async def test_researchrubrics_judge_uses_final_resource_content(monkeypatch): - from uuid import uuid4 - from ergon_core.api.evaluation_context import EvaluationContext - from ergon_core.api.results import WorkerOutput - from ergon_core.api.run_resource import RunResourceKind, RunResourceView - from ergon_builtins.benchmarks.researchrubrics.judge_criterion import ( - ResearchRubricsJudgeCriterion, - ResearchRubricsVerdict, - ) - - report_id = uuid4() - scratch_id = uuid4() - run_id = uuid4() - execution_id = uuid4() - report = RunResourceView( - id=report_id, - run_id=run_id, - task_execution_id=execution_id, - kind=RunResourceKind.REPORT, - name="report.md", - mime_type="text/markdown", - file_path="/tmp/blob/report", - size_bytes=12, - content_hash="abc", - error=None, - metadata={"sandbox_origin": "/workspace/final_output/report.md"}, - ) - scratch = RunResourceView( - id=scratch_id, - run_id=run_id, - task_execution_id=execution_id, - kind=RunResourceKind.NOTE, - name="notes.md", - mime_type="text/markdown", - file_path="/tmp/blob/notes", - size_bytes=5, - content_hash="def", - error=None, - metadata={"sandbox_origin": "/workspace/scratch/notes.md"}, - ) - captured = {} - - async def fake_judge(*, messages, response_type, model): - captured["prompt"] = messages[1].content - return ResearchRubricsVerdict(reasoning="report satisfies criterion", passed=True) - - monkeypatch.setattr( - "ergon_builtins.benchmarks.researchrubrics.judge_criterion.call_structured_judge", - fake_judge, - ) - - criterion = ResearchRubricsJudgeCriterion( - name="criterion_0", - rubric=RubricCriterion(criterion="Includes sources.", axis="Explicit", weight=2.0), - ) - task = BenchmarkTask( - task_slug="sample", - instance_key="default", - description="Write a report.", - ) - context = EvaluationContext( - run_id=run_id, - task_id=uuid4(), - execution_id=execution_id, - task=task, - worker_result=WorkerOutput(output="Wrote report.md"), - runtime=_Runtime( - [report, scratch], - { - report_id: b"# Findings\nFinal report text", - scratch_id: b"draft notes", - }, - ), - ) - - result = await criterion.evaluate(context) - - assert result.passed is True - assert str(report_id) in result.evaluated_resource_ids - assert str(scratch_id) in result.evaluated_resource_ids - assert "Final output resources" in captured["prompt"] - assert "Final report text" in captured["prompt"] - assert "Scratch/intermediate resources" in captured["prompt"] - assert "draft notes" in captured["prompt"] -``` - -Run: - -```bash -uv run pytest tests/unit/state/test_research_rubrics_benchmark.py -q -``` - -Expected: all tests pass. - ---- - -## Task 3: Align Rollout Artifact Health With Task-Scoped Final Outputs - -**Files:** -- Modify: `tests/real_llm/artifact_health.py` -- Test: `tests/unit/runtime/test_real_llm_rollout_artifact_health.py` - -### Rationale - -Health analysis works on dumped JSONL, not live SQL. It should mirror the same policy: - -- group resources by `task_execution_id`; -- a completed task has a final output if at least one resource has `metadata_json.sandbox_origin` under `/workspace/final_output/`; -- do not compare durable blob `file_path` to logical sandbox paths. - -### Patch - -In `tests/real_llm/artifact_health.py`, add helpers near `_tool_budget_signals`: - -```python -_FINAL_OUTPUT_PREFIX = "/workspace/final_output/" - - -def _resource_metadata(resource: dict[str, Any]) -> dict[str, Any]: # slopcop: ignore[no-typing-any] - metadata = resource.get("metadata_json") or resource.get("metadata") or {} - if isinstance(metadata, str): - return json.loads(metadata) - return metadata if isinstance(metadata, dict) else {} - - -def _is_final_output_resource(resource: dict[str, Any]) -> bool: # slopcop: ignore[no-typing-any] - origin = _resource_metadata(resource).get("sandbox_origin") - return isinstance(origin, str) and origin.startswith(_FINAL_OUTPUT_PREFIX) -``` - -Replace current `missing_final_report` calculation: - -```python -completed_execution_ids = { - str(execution.get("id")) - for execution in executions - if execution.get("status") == "completed" and execution.get("id") is not None -} -final_output_execution_ids = { - str(resource.get("task_execution_id")) - for resource in resources - if resource.get("task_execution_id") is not None and _is_final_output_resource(resource) -} -missing_final_report = bool(completed_execution_ids - final_output_execution_ids) -``` - -This field name can stay `missing_final_report` for now to avoid dashboard churn, but the semantics become “completed task is missing a final-output resource.” - -### Tests - -In `tests/unit/runtime/test_real_llm_rollout_artifact_health.py`, update `_write_minimal_rollout` to optionally write final-output metadata: - -```python -def _write_minimal_rollout( - root: Path, - *, - task_count: int = 1, - evaluation_rows: list[dict] | None = None, - resource_rows: list[dict] | None = None, -) -> None: - ... - execution_ids = [str(uuid4()) for _ in range(task_count)] - ... - _write_jsonl( - db / "run_task_executions.jsonl", - [ - { - "id": execution_ids[idx], - "task_slug": f"task-{idx}", - "status": "completed", - } - for idx in range(task_count) - ], - ) - ... - _write_jsonl( - db / "run_resources.jsonl", - resource_rows - if resource_rows is not None - else [ - { - "id": str(uuid4()), - "task_execution_id": execution_ids[0], - "name": "report.md", - "metadata_json": {"sandbox_origin": "/workspace/final_output/report.md"}, - } - ], - ) -``` - -Add: - -```python -def test_artifact_health_detects_final_output_by_task_resource_metadata(tmp_path: Path) -> None: - execution_id = str(uuid4()) - _write_minimal_rollout( - tmp_path, - task_count=1, - evaluation_rows=[ - { - "id": str(uuid4()), - "score": 0.75, - "summary_json": { - "evaluator_name": "research-rubric", - "normalized_score": 0.75, - "criterion_results": [ - { - "criterion_name": "criterion_0", - "criterion_type": "researchrubrics-llm-judge", - "score": 1.0, - "max_score": 1.0, - "passed": True, - "weight": 1.0, - "status": "passed", - "criterion_description": "Includes citations.", - "feedback": "The report cited source material.", - } - ], - }, - } - ], - resource_rows=[ - { - "id": str(uuid4()), - "task_execution_id": execution_id, - "name": "report.md", - "file_path": "/tmp/ergon-blob/abc", - "metadata_json": {"sandbox_origin": "/workspace/final_output/report.md"}, - } - ], - ) -``` - -If `_write_minimal_rollout` generates execution ids internally, return them from the helper or pass explicit ids. Keep the test focused: final-output detection must use `metadata_json.sandbox_origin`, not durable `file_path`. - -Run: - -```bash -uv run pytest tests/unit/runtime/test_real_llm_rollout_artifact_health.py tests/real_llm/test_artifact_health.py -q -``` - -Expected: all tests pass. - ---- - -## Task 4: Preserve Evaluator-Normalized Scores - -**Files:** -- Modify: `ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py` -- Test: `tests/unit/runtime/test_evaluation_summary_contracts.py` -- Test: `tests/unit/state/test_research_rubrics_benchmark.py` - -### Rationale - -New standard: all evaluators return normalized scalar scores in `TaskEvaluationResult.score`. Persistence must record, not reinterpret, that score. - -Current bug: - -```python -total_score = result.score -normalized = total_score / max_score_total if max_score_total > 0 else 0.0 -``` - -For ResearchRubrics, `result.score` is already normalized, so this divides twice. - -### Patch: Persistence - -In `build_evaluation_summary`, replace: - -```python -total_score = result.score -normalized = total_score / max_score_total if max_score_total > 0 else 0.0 -``` - -with: - -```python -normalized = result.score -``` - -Keep `max_score_total` as rubric display metadata: - -```python -return EvaluationSummary( - evaluator_name=result.evaluator_name, - max_score=max_score_total, - normalized_score=normalized, - stages_evaluated=len(stage_names), - stages_passed=stages_passed, - criterion_results=entries, -) -``` - -### Patch: ResearchRubrics Metadata - -In `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py`, keep the formula and add explicit score metadata: - -```python -return TaskEvaluationResult( - task_slug=task.task_slug, - score=normalized_score, - passed=total_score > 0, - evaluator_name=self.name, - criterion_results=results, - metadata={ - "score_scale": "normalized_0_1", - "raw_score": total_score, - "max_possible": max_possible, - "min_possible": min_possible, - }, -) -``` - -### Tests - -In `tests/unit/runtime/test_evaluation_summary_contracts.py`, add: - -```python -def test_build_evaluation_summary_preserves_evaluator_normalized_score() -> None: - summary = build_evaluation_summary( - _service_result( - feedback="criterion ran", - criterion_score=0.5, - criterion_weight=2.0, - passed=True, - ), - evaluation_input=None, - ) - - assert summary.normalized_score == 0.5 - assert summary.max_score == 1.0 -``` - -To make this test prove the no-double-normalization contract, change the helper's `CriterionSpec` for this test case from `max_score=1.0` to `max_score=2.0`. With the old implementation, `summary.normalized_score` would be `0.25`; with the new contract, it remains `0.5`. - -In `tests/unit/state/test_research_rubrics_benchmark.py`, update expected metadata: - -```python -assert result.metadata == { - "score_scale": "normalized_0_1", - "raw_score": 2.0, - "max_possible": 2.0, - "min_possible": -1.0, -} -``` - -Run: - -```bash -uv run pytest tests/unit/runtime/test_evaluation_summary_contracts.py tests/unit/state/test_research_rubrics_benchmark.py -q -``` - -Expected: all tests pass. - ---- - -## Task 5: Verify With One Real Rollout - -**Files:** -- No new code files. - -### Commands - -Run focused checks: - -```bash -uv run pytest \ - tests/unit/state/test_criterion_runtime_di.py \ - tests/unit/state/test_research_rubrics_benchmark.py \ - tests/unit/runtime/test_evaluation_summary_contracts.py \ - tests/unit/runtime/test_real_llm_rollout_artifact_health.py \ - tests/real_llm/test_artifact_health.py \ - -q -``` - -Expected: all tests pass. - -Run lint/compile for touched files: - -```bash -uv run ruff check \ - ergon_core/ergon_core/api/criterion_runtime.py \ - ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py \ - ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py \ - tests/real_llm/artifact_health.py \ - tests/unit/state/test_criterion_runtime_di.py \ - tests/unit/state/test_research_rubrics_benchmark.py \ - tests/unit/runtime/test_evaluation_summary_contracts.py \ - tests/unit/runtime/test_real_llm_rollout_artifact_health.py -``` - -Expected: `All checks passed!` - -Run compile: - -```bash -uv run python -m compileall -q \ - ergon_core/ergon_core/api/criterion_runtime.py \ - ergon_core/ergon_core/core/runtime/evaluation/criterion_runtime.py \ - ergon_core/ergon_core/core/runtime/services/evaluation_persistence_service.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/judge_criterion.py \ - ergon_builtins/ergon_builtins/benchmarks/researchrubrics/rubric.py \ - tests/real_llm/artifact_health.py -``` - -Expected: exit code `0`. - -After rebuild, rerun one real sample: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_MODEL=openrouter:anthropic/claude-opus-4.7 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=1 \ -ERGON_REAL_LLM_BUDGET_USD=25 \ -TEST_HARNESS_SECRET=real-llm-secret \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py --assume-stack-up -vv -s -``` - -Expected rollout properties: - -- terminal status is `completed`; -- artifact health reports `missing_final_report: False`; -- `normalized scores` matches `RunTaskEvaluation.score`; -- criterion `evaluated_resource_ids` contains the report resource id; -- judge feedback references details from the full final report, not just the final assistant summary. - ---- - -## Non-Goals - -- Do not put final-vs-scratch classification in `ergon_core`. -- Do not include full agent conversation in ResearchRubrics judge prompts by default. -- Do not introduce a new persisted table for evidence bundles. -- Do not preserve compatibility with double-normalized summary scores; new runs should use the normalized score invariant. diff --git a/docs/superpowers/plans/2026-04-28-mas-rebase-regression-recovery.md b/docs/superpowers/plans/2026-04-28-mas-rebase-regression-recovery.md deleted file mode 100644 index f5475b3a3..000000000 --- a/docs/superpowers/plans/2026-04-28-mas-rebase-regression-recovery.md +++ /dev/null @@ -1,386 +0,0 @@ -# MAS Rebase Regression Recovery Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Recover changes lost or blurred during the `feature/mas-main-rebase` merge, without undoing intentional main-branch experiment-run work. - -**Architecture:** Treat this as a rebase audit and repair plan. Definite regressions get direct test-first fixes; the older object-first `ExperimentRunHandle` / `Experiment.run()` API is intentionally retired in favor of the newer experiment definition and launch services. - -**Tech Stack:** Python 3.13, Pydantic, SQLModel, pytest, uv, Ergon core/runtime/API packages. - ---- - -## Audit Summary - -The rebase worktree is clean at `feature/mas-main-rebase` with `HEAD` at `ab28db3` (`Merge main into MAS debugger branch`). The broad cleanup survived, but two regressions need action. - -### Preserved Work - -- Public API thinning survived: removed `ergon_core.api.generation`, `json_types`, `run_resource`, `criterion_runtime`, `dependencies`, and `types`. -- Runtime homes survived: `core/runtime/resources.py`, `core/runtime/dependencies.py`, and `core/runtime/evaluation/protocols.py`. -- Context schema consolidation survived: `ContextPart`, `ContextPartChunk`, and `ContextPartChunkLog` are the core stream/log schemas; old `GenerationTurn` and old `*Payload` context-event classes are gone from core. -- File moves survived: Inngest client/registry under `core/runtime/inngest/`, sandbox under `core/sandbox/`, ResearchRubrics sandbox manager under builtins, OpenRouter budget under `tests/real_llm`, and tracing split into `core/runtime/tracing/`. -- `error_payload.py`, `build_error_json`, `RuntimeErrorPayload`, and `_worker_execute_result_from_exception` remain removed. - -### Definite Regression - -`_worker_execute_result_from_output()` has reappeared in `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py`, along with `tests/unit/runtime/test_worker_execute_output_failure.py`. - -Today's intended state was: - -- No private adapter helper for `WorkerOutput -> WorkerExecuteResult`. -- Success result construction inlined at the only callsite. -- No helper-level test importing `_worker_execute_result_from_output`. - -### Intentional Retirement - -`ExperimentRunHandle` and `Experiment.run()` existed on `safety/mas-before-main-rebase`, but are absent in `feature/mas-main-rebase`. - -Current state: - -- `ergon_core/ergon_core/api/handles.py` defines only `PersistedExperimentDefinition`. -- `ergon_core/ergon_core/api/__init__.py` exports only `PersistedExperimentDefinition`, not `ExperimentRunHandle`. -- `ergon_core/ergon_core/api/experiment.py` exposes `persist()` but no `run()`. -- Main added experiment launch/read services under `core/runtime/services/experiment_*`, and that newer model is the one we want to keep. - -Decision: do **not** restore `ExperimentRunHandle` or `Experiment.run()`. Treat the older object-run API as retired. The fix is to remove stale handle/run wording and add tests that prevent the old single-run handle from returning to `ergon_core.api`. - ---- - -## Files To Touch - -### Definite Helper Regression - -- Modify: `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` -- Delete: `tests/unit/runtime/test_worker_execute_output_failure.py` -- Modify or add guard: `tests/unit/runtime/test_import_boundaries.py` or `tests/unit/architecture/test_public_api_boundaries.py` - -### Experiment Handle Retirement - -- Modify: `ergon_core/ergon_core/api/handles.py` docstring -- Modify/add API boundary test confirming no `ExperimentRunHandle` / no `Experiment.run` -- Update docs that still describe `run()` as part of the object-first authoring API. - ---- - -## Task 1: Lock In The Helper Removal Regression - -**Files:** -- Modify: `tests/unit/runtime/test_import_boundaries.py` -- Modify: `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` -- Delete: `tests/unit/runtime/test_worker_execute_output_failure.py` - -- [ ] **Step 1: Add a failing guard for deleted worker helper adapters** - -Add this test to `tests/unit/runtime/test_import_boundaries.py`: - -```python -def test_worker_execute_does_not_expose_result_adapter_helpers() -> None: - import ergon_core.core.runtime.inngest.worker_execute as worker_execute - - assert not hasattr(worker_execute, "_worker_execute_result_from_output") - assert not hasattr(worker_execute, "_worker_execute_result_from_exception") -``` - -- [ ] **Step 2: Run the guard and verify it fails before the fix** - -Run: - -```bash -uv run pytest tests/unit/runtime/test_import_boundaries.py::test_worker_execute_does_not_expose_result_adapter_helpers -q -``` - -Expected before fix: - -```text -FAILED ... assert not hasattr(worker_execute, "_worker_execute_result_from_output") -``` - -- [ ] **Step 3: Inline the success result construction** - -In `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py`, remove: - -```python -def _worker_execute_result_from_output(output: WorkerOutput) -> WorkerExecuteResult: - return WorkerExecuteResult( - success=output.success, - final_assistant_message=output.output, - error=None if output.success else output.output, - ) -``` - -Then replace: - -```python -return _worker_execute_result_from_output(output) -``` - -with: - -```python -return WorkerExecuteResult( - success=output.success, - final_assistant_message=output.output, - error=None if output.success else output.output, -) -``` - -Also remove the now-unused import: - -```python -from ergon_core.api.results import WorkerOutput -``` - -- [ ] **Step 4: Delete helper-specific test** - -Delete: - -```text -tests/unit/runtime/test_worker_execute_output_failure.py -``` - -This test asserts a private helper mapping and should not survive once the helper is gone. The behavior is still covered by `worker_execute_fn` return construction and `WorkerExecuteResult` model validation. - -- [ ] **Step 5: Run focused verification** - -Run: - -```bash -uv run pytest tests/unit/runtime/test_import_boundaries.py tests/unit/runtime/test_failure_error_json.py -q -uv run ruff check ergon_core/ergon_core/core/runtime/inngest/worker_execute.py tests/unit/runtime/test_import_boundaries.py -``` - -Expected: - -```text -passed -All checks passed! -``` - ---- - -## Task 2: Lock In The New Experiment Launch Model - -**Files:** -- Inspect: `ergon_core/ergon_core/api/experiment.py` -- Inspect: `ergon_core/ergon_core/api/handles.py` -- Inspect: `ergon_core/ergon_core/core/runtime/services/run_service.py` -- Inspect: `ergon_core/ergon_core/core/runtime/services/experiment_launch_service.py` -- Inspect: `ergon_cli/ergon_cli/commands/benchmark.py` - -- [ ] **Step 1: Confirm current execution entry points** - -Run: - -```bash -rg "class ExperimentRunHandle|async def run\\(|create_experiment_run|launch" \ - ergon_core/ergon_core/api \ - ergon_core/ergon_core/core/runtime/services \ - ergon_cli/ergon_cli/commands \ - tests -n -``` - -Expected current signal: - -- `ExperimentRunHandle` appears only as a CLI-local class in `ergon_cli/ergon_cli/commands/benchmark.py`. -- `Experiment` has `persist()` but no `run()`. -- Main-branch experiment services own launch/read behavior. - -Step 1 confirms that the newer model is active: - -- `ExperimentRecord` stores the experiment campaign/sample selection. -- `ExperimentLaunchService.run_experiment()` expands one `ExperimentRecord` into many `RunRecord`s. -- `ExperimentRunResult` returns `run_ids: list[UUID]`, not a single `run_id`. -- `ergon_core.api.Experiment` remains a workflow-definition composition object with `persist()` only. - -- [ ] **Step 2: Write a guard for the retired object-run API** - -Add tests to `tests/unit/api/test_public_api_imports.py`: - -```python -def test_object_first_experiment_run_api_is_retired() -> None: - public_api = importlib.import_module("ergon_core.api") - - assert not hasattr(public_api, "ExperimentRunHandle") - assert not hasattr(public_api.Experiment, "run") -``` - -- [ ] **Step 3: Clean stale handle wording** - -Update `ergon_core/ergon_core/api/handles.py` docstring from: - -```python -"""Public lifecycle handle types returned by persist() and run().""" -``` - -to: - -```python -"""Public lifecycle handle types returned by Experiment.persist().""" -``` - -- [ ] **Step 4: Run focused API verification** - -Run: - -```bash -uv run pytest tests/unit/api/test_public_api_imports.py -q -``` - -Expected: - -```text -passed -``` - ---- - -## Task 3: Add A Rebase Recovery Guard For Historical Regressions - -**Files:** -- Modify: `tests/unit/architecture/test_public_api_boundaries.py` -- Modify: `tests/unit/runtime/test_import_boundaries.py` - -- [ ] **Step 1: Guard deleted API facade modules by module spec** - -Add to `tests/unit/architecture/test_public_api_boundaries.py`: - -```python -import importlib.util - - -def test_removed_api_facade_modules_do_not_exist() -> None: - removed_modules = ( - "ergon_core.api.generation", - "ergon_core.api.json_types", - "ergon_core.api.run_resource", - "ergon_core.api.criterion_runtime", - "ergon_core.api.dependencies", - "ergon_core.api.types", - ) - - for module_name in removed_modules: - assert importlib.util.find_spec(module_name) is None -``` - -- [ ] **Step 2: Guard worker private adapter helpers** - -Use the helper guard from Task 1. - -- [ ] **Step 3: Run architecture guards** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_public_api_boundaries.py tests/unit/runtime/test_import_boundaries.py -q -``` - -Expected: - -```text -passed -``` - ---- - -## Task 4: Final Verification - -**Files:** -- All touched files from Tasks 1-3. -- Verify: `tests/integration/smokes/test_smoke_harness.py` -- Verify: `tests/e2e/` - -- [ ] **Step 1: Run focused test group** - -Run: - -```bash -uv run pytest \ - tests/unit/api/test_public_api_imports.py \ - tests/unit/architecture/test_public_api_boundaries.py \ - tests/unit/runtime/test_import_boundaries.py \ - tests/unit/runtime/test_failure_error_json.py \ - -q -``` - -Expected: - -```text -passed -``` - -- [ ] **Step 2: Run targeted lint** - -Run: - -```bash -uv run ruff check \ - ergon_core/ergon_core/core/runtime/inngest/worker_execute.py \ - ergon_core/ergon_core/api/handles.py \ - ergon_core/ergon_core/api/__init__.py \ - ergon_core/ergon_core/api/experiment.py \ - tests/unit/api/test_public_api_imports.py \ - tests/unit/architecture/test_public_api_boundaries.py \ - tests/unit/runtime/test_import_boundaries.py -``` - -Expected: - -```text -All checks passed! -``` - -- [ ] **Step 3: Run local integration/e2e acceptance for the newer cohort -> experiment -> run model** - -Use this as the main system-level confidence metric for the rebase: - -> A local checkout can define an experiment through the newer cohort/experiment model, launch runs for selected samples, drive those runs through the runtime, persist graph/evaluation/resource outputs, and pass the e2e smoke path without relying on retired `Experiment.run()` / `ExperimentRunHandle`. - -Run the local smoke/e2e set used by this branch: - -```bash -uv run pytest tests/integration/smokes/test_smoke_harness.py -q -uv run pytest tests/e2e -q -``` - -Expected: - -```text -passed -``` - -If the e2e suite requires local services, start the normal local stack first, then rerun the same commands. A failure here is a blocker unless it is a documented environment prerequisite rather than a model/API regression. - -- [ ] **Step 4: Check git diff for scope** - -Run: - -```bash -git diff --stat -git diff --name-status -``` - -Expected changed files should be limited to: - -- `docs/superpowers/plans/2026-04-28-mas-rebase-regression-recovery.md` -- `ergon_core/ergon_core/core/runtime/inngest/worker_execute.py` -- `tests/unit/runtime/test_import_boundaries.py` -- `tests/unit/runtime/test_worker_execute_output_failure.py` deleted -- plus the accept-main guard/docstring files from Task 2. - ---- - -## Non-Goals - -- Do not reintroduce `ergon_core.api.generation`, `json_types`, `run_resource`, `criterion_runtime`, `dependencies`, or `types`. -- Do not reintroduce `error_payload.py`, `build_error_json`, or `RuntimeErrorPayload`. -- Do not undo main's experiment-run domain model or revive `ExperimentRunHandle` / `Experiment.run()`. -- Do not edit historical docs/RFCs unless they are actively misleading for the current public API. - -## Completion Criteria - -- `_worker_execute_result_from_output` and `_worker_execute_result_from_exception` are absent. -- `test_worker_execute_output_failure.py` is deleted or rewritten to avoid private helper imports. -- Public API state around `ExperimentRunHandle` is explicit and tested as intentionally absent. -- Local smoke/e2e tests pass through the newer `cohort -> experiment -> run` model without using the retired object-run API. -- Focused pytest and ruff checks pass. diff --git a/docs/superpowers/plans/2026-04-28-public-api-audit-and-ergonomics.md b/docs/superpowers/plans/2026-04-28-public-api-audit-and-ergonomics.md deleted file mode 100644 index 6c7de628b..000000000 --- a/docs/superpowers/plans/2026-04-28-public-api-audit-and-ergonomics.md +++ /dev/null @@ -1,1556 +0,0 @@ -# Public API Audit And Ergonomics Working Doc - -This is a working document for deciding what belongs in `ergon_core.api`, what should move inward to `ergon_core.core`, and what concepts can be merged so the API is easier for students and benchmark authors to use. - -The goal is not to make the public API artificially tiny. The goal is to make it honest. A public symbol should either be: - -- something a benchmark author uses to describe work, -- something a worker author uses to solve work, -- something an evaluator author uses to score work, -- or a deliberately documented advanced extension point. - -Everything else should probably be core, CLI, dashboard, persistence, or runtime plumbing. - -## Current Public API Root - -`ergon_core.api.__all__` currently exports: - -```python -Benchmark -BenchmarkDeps -BenchmarkTask -Criterion -CriterionResult -CriteriaCheckError -DependencyError -EvaluationContext -Evaluator -Experiment -EmptyTaskPayload -PersistedExperimentDefinition -Rubric -TaskEvaluationResult -Worker -WorkerContext -WorkerOutput -WorkerSpec -``` - -Submodule-only public-ish symbols currently used or plausibly imported: - -```python -CriterionScoreSpec -CriterionObservation -CriterionObservationMessage -``` - -Important existing boundary tests: - -- `tests/unit/api/test_public_api_imports.py` already asserts that runtime/tooling concepts like `RunResourceView`, `CriterionRuntime`, `CommandResult`, `SandboxResult`, and `Tool` are not exposed at the root. -- `tests/unit/architecture/test_public_api_boundaries.py` already protects against restoring deleted facade modules like `api.generation`, `api.json_types`, `api.run_resource`, `api.criterion_runtime`, `api.dependencies`, and `api.types`. - -That means the codebase already wants `ergon_core.api` to stay authoring-scoped. The current issue is that some exported authoring-looking objects still pull runtime/persistence concepts through the side door. - -## Current Mental Model - -The current public API effectively asks users to understand this: - -```text -Benchmark -> BenchmarkTask -> Experiment -> WorkerSpec -> persisted definition -> run -Worker -> WorkerContext -> streamed core generation chunks -> WorkerOutput -Criterion -> EvaluationContext -> core CriterionRuntime -> CriterionResult -Evaluator/Rubric -> TaskEvaluationResult -``` - -The student-facing model we probably want is closer to: - -```text -Benchmark -> Task -Worker solves Task -Criterion checks WorkerOutput -Rubric combines Criteria -Core handles experiments, runs, cohorts, persistence, dispatch, and dashboards -``` - -## Usage Map At A Glance - -### CLI - -The built-in CLI imports only a small part of `ergon_core.api` directly: - -- `ergon_cli/ergon_cli/composition/__init__.py` - - imports `Experiment` - - imports `WorkerSpec` -- `ergon_cli/ergon_cli/onboarding/profile.py` - - imports `BenchmarkDeps` - -The CLI otherwise reaches straight into `ergon_core.core` for: - -- DB setup and sessions, -- telemetry models such as `RunRecord`, -- `create_run`, -- cohort resolution, -- Inngest event dispatch, -- experiment define/launch/read services, -- workflow services, -- runtime settings. - -This is a useful signal. `ergon_core.api` is not really the CLI API today. The CLI already operates at the application/runtime layer. - -### Built-ins - -`ergon_builtins` uses the public API heavily as an extension-authoring kit: - -- Benchmarks subclass `Benchmark` and create `BenchmarkTask`. -- Workers subclass `Worker` and receive `WorkerContext`. -- Criteria subclass `Criterion`, receive `EvaluationContext`, and return `CriterionResult`. -- Rubrics subclass `Rubric` and return `TaskEvaluationResult`. -- Registries type their maps as `Benchmark`, `Evaluator`, and `Worker`. -- Onboarding metadata uses `BenchmarkDeps`. - -This is the strongest argument that `Benchmark`, `BenchmarkTask`, `Worker`, `WorkerContext`, `WorkerOutput`, `Criterion`, `CriterionResult`, `CriterionScoreSpec`, `Rubric`, and `TaskEvaluationResult` should remain public or have very deliberate replacements. - -### Core Runtime - -Core runtime imports public API types in several places: - -- `core/runtime/inngest/worker_execute.py` - - uses `BenchmarkTask`, `EmptyTaskPayload`, `WorkerContext` -- `core/runtime/evaluation/inngest_executor.py` - - uses `Criterion`, `EvaluationContext`, `CriterionResult`, `WorkerOutput`, `BenchmarkTask` -- `core/runtime/evaluation/evaluation_schemas.py` - - uses `Criterion` -- `core/runtime/services/rubric_evaluation_service.py` - - uses `Evaluator`, `CriterionResult`, `TaskEvaluationResult`, `BenchmarkTask` -- `core/runtime/services/experiment_persistence_service.py` - - uses `Rubric`, `PersistedExperimentDefinition`, and type-checks `Experiment` -- `core/runtime/services/experiment_launch_service.py` - - uses `Benchmark`, `Evaluator`, `Experiment`, `PersistedExperimentDefinition`, `BenchmarkTask`, `WorkerSpec` -- `core/runtime/services/experiment_definition_service.py` - - uses `Benchmark`, `BenchmarkTask` -- `core/runtime/services/run_service.py` - - uses `PersistedExperimentDefinition` - -Some of that is fine: core runtime naturally consumes public authoring objects. But the reverse direction is more concerning: public API modules also import core runtime/persistence modules. - -### Tests - -Tests use almost every current public type: - -- API contract tests cover imports and public API boundary behavior. -- Runtime tests instantiate criteria, rubrics, contexts, tasks, and result models. -- Built-in benchmark tests instantiate `Benchmark`, `BenchmarkTask`, `BenchmarkDeps`, `EvaluationContext`, `WorkerOutput`, and result models. -- Worker tests use `WorkerContext`, `BenchmarkTask`, and `EmptyTaskPayload`. -- Runtime service tests use `PersistedExperimentDefinition`. - -This means simplification should be staged. Move internal users first, leave compatibility imports where useful, then adjust tests around the intended boundary. - -## Public File Inventory - -```text -ergon_core/ergon_core/api/ - __init__.py - exports the object-first public surface - - benchmark.py - Benchmark base class - currently also validates required packages via core runtime dependencies - - benchmark_deps.py - BenchmarkDeps onboarding metadata - - task_types.py - EmptyTaskPayload - BenchmarkTask - - worker.py - Worker base class - currently imports core generation chunk types - currently reads persisted context events to build default output - - worker_context.py - WorkerContext execution identity model - - worker_spec.py - WorkerSpec config-time registry descriptor - imports ergon_builtins registry during validation - - criterion.py - Criterion base class - currently validates required packages via core runtime dependencies - - evaluation_context.py - EvaluationContext for criteria - currently exposes core CriterionRuntime protocol as a field - - evaluator.py - Evaluator base class - Rubric concrete class - currently validates required packages via core runtime dependencies - - results.py - WorkerOutput - CriterionScoreSpec - CriterionObservationMessage - CriterionObservation - CriterionResult - TaskEvaluationResult - currently imports core JsonObject - - experiment.py - Experiment composition root - validates object graph - persists through core ExperimentPersistenceService - - handles.py - PersistedExperimentDefinition handle returned by Experiment.persist() - imports core utcnow helper - - errors.py - DependencyError - CriteriaCheckError -``` - -## Symbol By Symbol Review - -### `Benchmark` - -Current role: - -- Public base class for benchmark authors. -- Owns `type_slug`, `task_payload_model`, `build_instances()`, `evaluator_requirements()`, `parse_task_payload()`, and dependency validation. - -Where used: - -- Built-in benchmarks: MiniF2F, SWE-Bench Verified, ResearchRubrics, GDPEval. -- Core experiment definition and launch services. -- Registries type benchmark constructors. -- Tests for benchmark contracts and runtime services. - -Keep in public API? - -- Yes. - -Concerns: - -- The name is good for benchmark authors. -- `build_instances()` returning `Mapping[str, Sequence[BenchmarkTask]]` introduces "instance" as an extra concept. That may be necessary for benchmark datasets, but it is one more noun. -- `evaluator_requirements()` exposes evaluator slot binding to benchmark authors. -- `validate()` imports `core.runtime.dependencies.check_packages`. - -Possible cleanup: - -- Keep `Benchmark` public. -- Consider making `evaluator_requirements()` advanced or replacing it with a simpler `default_evaluator_slots = ("default",)` class var. -- Decide whether benchmark authors should declare dependency metadata as: - - `required_packages` plus `install_hint`, - - `onboarding_deps`, - - or one consolidated `requirements` object. -- Move dependency validation implementation inward so `api.benchmark` does not import core runtime. - -Decision question: - -- Should a student writing a benchmark need to know about evaluator binding keys, or should benchmarks just produce tasks and let the experiment/CLI layer attach rubrics? - -### `BenchmarkTask` And `EmptyTaskPayload` - -Current role: - -- `BenchmarkTask` is the public task object passed to workers and criteria. -- `EmptyTaskPayload` is the default Pydantic payload when a benchmark has no structured task data. - -Where used: - -- All built-in benchmarks create `BenchmarkTask`. -- Built-in workers consume `BenchmarkTask`. -- Built-in criteria and rubrics receive task objects. -- Core runtime reconstructs `BenchmarkTask` from persisted task rows. -- Many tests instantiate it directly. - -Keep in public API? - -- Yes. - -Concerns: - -- The name `BenchmarkTask` is precise but slightly more formal than necessary for students. -- It contains `instance_key`, `parent_task_slug`, `dependency_task_slugs`, and `evaluator_binding_keys`, which are runtime/workflow concepts mixed into the authoring task model. - -Possible cleanup: - -- Keep `BenchmarkTask` for compatibility. -- Consider a friendlier alias: - -```python -Task = BenchmarkTask -``` - -- Longer term, split: - - public `Task`: slug, description, payload, - - advanced/internal `WorkflowTaskSpec`: parent/dependencies/evaluator bindings/instance key. - -Decision question: - -- Are task dependencies and evaluator bindings part of the beginner benchmark-authoring story, or are they an advanced workflow story? - -### `Worker` - -Current role: - -- Public base class for workers. -- Authors implement `execute(task, context=...)`. -- `execute()` yields `ContextPartChunk` objects. -- Default `get_output()` reads context events from the database and extracts the last assistant text. - -Where used: - -- Built-in ReAct worker and training stub worker subclass it. -- Smoke fixtures subclass it. -- Registries type worker constructors. -- Core runtime instantiates workers in `worker_execute.py`. -- Tests assert worker contracts. - -Keep in public API? - -- Yes, but slim it down. - -Concerns: - -- `api.worker` imports: - - `core.generation.AssistantTextPart` - - `core.generation.ContextPartChunk` - - `core.persistence.context.repository.ContextEventRepository` - - `core.persistence.shared.db.get_session` - - `core.runtime.dependencies.check_packages` -- That means the public base class knows persistence and generation internals. -- Students writing a worker must understand streaming chunks, not just "return an answer". - -Possible cleanup: - -- Keep `Worker` public. -- Move DB-backed default output extraction to core runtime, probably near `worker_execute.py`. -- Decide whether beginner workers can implement a simpler method: - -```python -async def run(self, task: Task, context: WorkerContext) -> WorkerOutput: - ... -``` - -while advanced workers implement streaming: - -```python -async def execute(self, task: Task, *, context: WorkerContext) -> AsyncGenerator[ContextPartChunk, None]: - ... -``` - -- If streaming remains public, either: - - intentionally export the chunk type as an advanced public type, - - or define a small public event/chunk model that core adapts into context events. - -Decision question: - -- Should the student-facing worker API be "return a WorkerOutput" first, with streaming as advanced, or should all workers remain streaming-first? - -### `WorkerContext` - -Current role: - -- Public model passed to `Worker.execute()`. -- Contains `run_id`, `definition_id`, `task_id`, `execution_id`, `sandbox_id`, `node_id`, and metadata. - -Where used: - -- Built-in workers. -- Built-in tools such as workflow CLI tooling. -- Core runtime worker execution. -- Tests. - -Keep in public API? - -- Yes, but possibly with fewer fields. - -Concerns: - -- `definition_id` and `node_id` are graph/runtime concepts. -- `task_id` is nullable for dynamic subtasks, while `execution_id` is always present. That distinction is important to core but awkward to explain to students. - -Possible cleanup: - -- Public `WorkerContext` could expose: - - `run_id` - - `task_id` or `execution_id` - - `sandbox_id` - - `metadata` -- Internal `CoreWorkerContext` could add: - - `definition_id` - - `node_id` - - static-vs-dynamic task identity. - -Decision question: - -- Which IDs do worker authors actually need in normal code? If most only need `sandbox_id` and maybe `execution_id`, hide the rest. - -### `WorkerOutput` - -Current role: - -- Public result model for worker completion. -- Contains `output`, `success`, and metadata. - -Where used: - -- Built-in workers return it. -- Criteria receive it through `EvaluationContext`. -- Core evaluation executor wraps agent reasoning into it. -- Tests instantiate it. - -Keep in public API? - -- Yes. - -Concerns: - -- Field name `output` is generic but probably fine. -- `success` is useful but can overlap with runtime execution status. - -Possible cleanup: - -- Keep as-is unless we introduce a simpler non-streaming worker API. -- If worker runtime status and worker semantic success diverge, document that `success` means "worker produced a usable answer", not "the process did not crash". - -Decision question: - -- Do we want `WorkerOutput.output` to stay a single string, or should structured outputs become first-class? - -### `Criterion` - -Current role: - -- Public base class for atomic evaluation units. -- Authors implement `evaluate(context) -> CriterionResult`. - -Where used: - -- Built-in criteria for SWE-Bench, MiniF2F, ResearchRubrics, generic code checks, LLM judge, sandbox file check. -- Smoke fixtures. -- Core evaluation executor. -- Core evaluation schemas store `Criterion` in `CriterionSpec`. -- Tests. - -Keep in public API? - -- Yes. - -Concerns: - -- `Criterion.evaluate()` depends on `EvaluationContext`, which currently exposes core runtime capability plumbing. -- `validate()` imports core dependency checking. - -Possible cleanup: - -- Keep `Criterion` public. -- Simplify the context it receives. -- Move dependency checking inward or expose it as a small public helper independent of `core`. - -Decision question: - -- Should criteria own sandbox/resource access directly through context helper methods, or should they receive a separate capability object? - -### `EvaluationContext` - -Current role: - -- Public context passed to `Criterion.evaluate()`. -- Contains run/task/execution IDs, `BenchmarkTask`, `WorkerOutput`, sandbox ID, metadata, and optional runtime capability. - -Where used: - -- Built-in criteria. -- Smoke criteria. -- Core Inngest criterion executor. -- Tests for runtime injection and criterion contracts. - -Keep in public API? - -- Probably yes short-term, but redesign it. - -Concerns: - -- It imports `core.runtime.evaluation.protocols.CriterionRuntime`. -- The public field `runtime` means criterion authors can see an internal protocol rather than a stable student-facing capability. -- It duplicates some identity with `WorkerContext`. - -Possible cleanup: - -- Keep the name `EvaluationContext` if we want stability. -- Change the implementation so it owns public helper methods: - -```python -await context.execute_code("pytest -q") -await context.read_resource("answer.txt") -await context.read_resource_by_id(resource_id) -``` - -- Store the internal runtime in a private field, not as a public typed protocol. -- Or rename to `CriterionContext` if we want "criterion evaluates with criterion context" instead of a broader evaluation context. - -Decision question: - -- Is `EvaluationContext` the right public name, or is `CriterionContext` easier for students? - -### `CriterionScoreSpec` - -Current role: - -- Public-ish score range model for criteria. -- Not exported from `ergon_core.api.__all__`, but imported from `ergon_core.api.results` by tests and built-ins. - -Where used: - -- Criteria constructors. -- MiniF2F proof verification. -- Code check and LLM judge criteria. -- Runtime tests. - -Keep in public API? - -- Yes, if criteria remain configurable with score ranges. - -Concerns: - -- It is public by usage but not top-level exported. -- If top-level exports are the documented API, this mismatch is confusing. - -Possible cleanup: - -- Either export it at the root: - -```python -from ergon_core.api import CriterionScoreSpec -``` - -- Or document `ergon_core.api.results.CriterionScoreSpec` as advanced. - -Decision question: - -- Do we want all common authoring types available from `ergon_core.api`, or do we want submodules for less common result/config types? - -### `CriterionResult` - -Current role: - -- Public result of a single criterion. -- Includes score, pass/fail, weight, feedback, evidence IDs, observations, errors, and metadata. - -Where used: - -- Built-in criteria return it. -- Rubrics aggregate it. -- Core evaluation executor returns it from each criterion step. -- Evaluation persistence converts it into persisted summaries. -- Tests. - -Keep in public API? - -- Yes. - -Concerns: - -- It is fairly large for students. -- It overlaps with internal `CriterionResultEntry` in `core.persistence.telemetry.evaluation_summary`. - -Possible cleanup: - -- Keep public `CriterionResult`. -- Keep persisted `CriterionResultEntry` internal. -- Centralize conversion in a core adapter so authors only learn `CriterionResult`. -- Consider helper constructors: - -```python -CriterionResult.pass_(slug="...", score=1.0, feedback="...") -CriterionResult.fail(slug="...", feedback="...") -``` - -Decision question: - -- Should we add helper constructors to reduce boilerplate in student-written criteria? - -### `CriterionObservation` And `CriterionObservationMessage` - -Current role: - -- Structured observation models nested inside `CriterionResult`. -- Capture prompt messages, evidence resource/action IDs, model details, and output. - -Where used: - -- ResearchRubrics judge criterion and LLM judge criterion. -- Evaluation summary persistence imports `CriterionObservation`. -- Tests likely inspect summary contracts. - -Keep in public API? - -- Keep in `results.py`, but maybe not root export. - -Concerns: - -- This is useful for advanced LLM-as-judge and audit trails. -- It may be too detailed for the beginner path. -- It imports or depends on JSON object typing from core through `results.py`. - -Possible cleanup: - -- Keep as advanced result detail. -- Move JSON type alias local to public API or use `dict[str, object]` style. - -Decision question: - -- Do students need to produce structured observations, or is this mainly for built-in LLM judges and dashboard evidence? - -### `Rubric` - -Current role: - -- Public concrete evaluator with a fixed list of criteria. -- Aggregates criterion scores with weighted average. - -Where used: - -- Built-in rubrics. -- Smoke rubrics. -- Core persistence checks whether an evaluator is a `Rubric` to snapshot criteria names. -- Core runtime service evaluates via `Evaluator` interface. -- Tests. - -Keep in public API? - -- Yes. - -Concerns: - -- It subclasses `Evaluator`, so users see both `Evaluator` and `Rubric`. -- Public `Rubric` is simple, but `RubricEvaluationService` in core has a similar name and is a runtime runner. -- Built-ins like GDPEval subclass `Rubric` but implement staged gating, which stretches the fixed-list weighted-average base concept. - -Possible cleanup: - -- Make `Rubric` the primary student-facing evaluation concept. -- Consider an explicit `WeightedRubric` name if we add multiple rubric types. -- Rename core `RubricEvaluationService` to `TaskEvaluationService` or `EvaluationRunner` to avoid confusing public rubric with internal service. - -Decision question: - -- Is `Rubric` always "a thing with criteria", or should `Evaluator` be the primary abstraction and `Rubric` just one implementation? - -### `Evaluator` - -Current role: - -- Public ABC for objects that select criteria for a task and aggregate criterion results. -- `Rubric` subclasses it. - -Where used: - -- Built-in registry typing. -- Core evaluation service accepts `Evaluator`. -- Core launch service builds evaluator bindings. -- Custom built-in rubrics inherit through `Rubric`. - -Keep in public API? - -- Maybe. - -Concerns: - -- It is a powerful extension point, but it adds another noun for students. -- Most authors probably need `Rubric`, not arbitrary dynamic evaluators. -- ResearchRubrics does need task-specific criteria via `criteria_for(task)`, which is an evaluator behavior. - -Possible cleanup: - -- Keep `Evaluator` for advanced users. -- Do not feature it in beginner docs. -- Potentially move it to `ergon_core.api.advanced` while `Rubric` stays root-exported. -- Or keep it root-exported because registries and dynamic task-specific rubrics already rely on it. - -Decision question: - -- Do we want external users to write custom dynamic evaluators, or only criteria and rubrics? - -### `TaskEvaluationResult` - -Current role: - -- Public aggregated result for one task after criteria run. - -Where used: - -- Rubrics return it. -- Core runtime persists it. -- Tests. - -Keep in public API? - -- Yes if custom rubrics/evaluators remain public. - -Concerns: - -- It overlaps with `EvaluationSummary`, which is internal persisted/dashboard state. - -Possible cleanup: - -- Keep public. -- Make `EvaluationSummary` clearly internal. -- Add adapter for persistence. - -Decision question: - -- Should rubric authors directly construct `TaskEvaluationResult`, or should Rubric have simpler aggregation hooks? - -### `Experiment` - -Current role: - -- Public composition root binding a benchmark, worker specs, evaluator bindings, assignments, and metadata. -- Validates the object graph. -- Persists itself by lazy-importing `ExperimentPersistenceService` from core. - -Where used: - -- CLI composition builds `Experiment`. -- Core launch service builds a temporary single-sample `Experiment`. -- Core persistence service type-checks it. -- Tests cover launch/persistence behavior. - -Keep in public API? - -- Open question. - -Argument to keep: - -- It is a natural word for users: "I want to run an experiment." -- It provides one object that composes benchmark, workers, and evaluators. -- CLI composition already uses it. - -Argument to move or de-emphasize: - -- It is not an authoring primitive like `Benchmark`, `Worker`, or `Criterion`. -- It exposes binding keys, assignments, evaluator maps, and worker specs. -- `persist()` makes public API depend on core persistence. -- There are already core concepts called `ExperimentRecord` and `ExperimentDefinition`, so the word "Experiment" is overloaded. - -Possible cleanup: - -- Short-term: keep exported for compatibility. -- Medium-term: remove `persist()` from the public object. Use a core service: - -```python -definition = experiment_service.persist(experiment) -``` - -- Long-term: decide whether public users should build `Experiment` directly or use a simpler CLI/app facade: - -```python -ergon.define( - benchmark="minif2f", - worker="react", - rubric="minif2f", - model="openai:gpt-4o", -) -``` - -Decision question: - -- Is `Experiment` a public user composition object, or an internal runtime definition draft? - -My current leaning: - -- Keep `Experiment` public short-term, but make it pure composition with no persistence method. -- If the beginner docs do not need it, do not root-feature it. - -### `WorkerSpec` - -Current role: - -- Config-time descriptor for worker binding. -- Contains `worker_slug`, `name`, and `model`. -- Validates worker slug against `ergon_builtins.registry.WORKERS`. - -Where used: - -- CLI composition. -- Core launch service. -- Experiment composition and persistence. -- Tests. - -Keep in public API? - -- Probably not as a beginner concept. - -Concerns: - -- It is registry/config plumbing. -- It imports builtins registry during validation. -- It exists because live `Worker` requires runtime IDs and cannot be used at config time. - -Possible cleanup: - -- Move to core composition. -- Keep compatibility import for now. -- Replace public construction with simpler facade args: - -```python -worker="researchrubrics-workflow-cli-react" -model="openai:gpt-4o" -``` - -Decision question: - -- Do external users need to build multi-worker assignment graphs manually, or can that be an advanced/core composition feature? - -### `PersistedExperimentDefinition` - -Current role: - -- Handle returned by `Experiment.persist()`. -- Contains `definition_id`, benchmark type, worker/evaluator bindings, counts, created timestamp, and metadata. - -Where used: - -- CLI benchmark command renders it and uses it to create a run. -- Core run service takes it. -- Core launch service returns it from workflow definition factory. -- Runtime tests instantiate it. - -Keep in public API? - -- Probably not as student authoring API. - -Concerns: - -- It is a persistence/launch handle, not an authoring concept. -- Its name overlaps with core `ExperimentDefinition` table rows. - -Possible cleanup: - -- Move to core composition or core service DTOs. -- Consider rename: - - `WorkflowDefinitionHandle` - - `DefinitionHandle` - - `PersistedDefinition` -- Keep compatibility import until CLI/core imports are migrated. - -Decision question: - -- Should users ever see persisted definition handles directly, or should they see run IDs/status objects from CLI/app services? - -### `BenchmarkDeps` - -Current role: - -- Onboarding requirements for a benchmark: E2B, extras, optional keys. - -Where used: - -- Built-in benchmark class vars. -- CLI onboarding profile. -- Benchmark contract tests. - -Keep in public API? - -- Maybe, but simplify or rehome. - -Concerns: - -- It duplicates conceptually with `required_packages` and `install_hint`. -- It is not about defining benchmark tasks. It is about onboarding/install/config. -- The `Benchmark` docstring says subclasses must set `onboarding_deps`, but `Benchmark` itself does not define/enforce that class var. - -Possible cleanup: - -- Merge into a single public metadata object: - -```python -requirements = BenchmarkRequirements( - packages=("datasets", "huggingface_hub"), - extras=("ergon-builtins[data]",), - env_keys=("HF_API_KEY",), - e2b=True, -) -``` - -- Or keep `BenchmarkDeps` but move to `ergon_core.api.onboarding`. - -Decision question: - -- Should install/runtime dependencies and onboarding prompts be one concept or two? - -### `DependencyError` - -Current role: - -- Raised when required packages are missing. - -Where used: - -- Public ABC validation methods. -- Tests may catch or assert dependency behavior. - -Keep in public API? - -- Maybe. - -Concerns: - -- If dependency validation moves inward, public users may not need this exception. -- But users might want to catch it around benchmark validation. - -Possible cleanup: - -- Keep if public `.validate()` methods stay. -- Move if validation becomes core launch-time behavior. - -Decision question: - -- Is dependency validation part of authoring, or only part of launching/running? - -### `CriteriaCheckError` - -Current role: - -- Domain-level exception criteria can raise from helpers and catch inside `evaluate()` to return a failed `CriterionResult`. - -Where used: - -- Smoke fixture criteria. -- Built-in criterion tests. - -Keep in public API? - -- Yes. - -Concerns: - -- The name uses plural "Criteria" even though a single criterion raises it. - -Possible cleanup: - -- Keep for compatibility. -- Consider alias: - -```python -CriterionCheckError = CriteriaCheckError -``` - -Decision question: - -- Is the plural name worth correcting with an alias, or not worth the churn? - -## Boundary Problems To Fix - -### Public API Imports Core Persistence - -Worst offender: - -```text -api/worker.py - imports core.persistence.context.repository.ContextEventRepository - imports core.persistence.shared.db.get_session -``` - -Why it matters: - -- A worker author importing `Worker` should not load DB/persistence concerns. -- It creates import-cycle risk. -- It makes the public base class responsible for runtime storage. - -Likely fix: - -- Move default output extraction to core. -- Let worker runtime call a core helper after `execute()` finishes. - -### Public API Imports Core Runtime Protocols - -Offender: - -```text -api/evaluation_context.py - imports core.runtime.evaluation.protocols.CriterionRuntime -``` - -Why it matters: - -- Criteria see an internal runtime protocol as a public field. -- It makes the public context harder to document. - -Likely fix: - -- Make runtime private inside context. -- Expose public methods on context. - -### Public API Imports Builtins Registry - -Offender: - -```text -api/worker_spec.py - validate_spec() imports ergon_builtins.registry.WORKERS -``` - -Why it matters: - -- `ergon_core.api` should not know about built-ins. -- Registry validation is runtime/composition behavior. - -Likely fix: - -- Move `WorkerSpec` to core composition. -- Or inject registry validator from core/CLI. - -### Public API Imports Core Generation Types - -Offender: - -```text -api/worker.py - execute() yields core.generation.ContextPartChunk -``` - -Why it matters: - -- Streaming workers are tightly coupled to Ergon's internal transcript/event model. -- If that is intended, it should be explicitly a public advanced type. - -Likely fix: - -- Decide whether to publicize a stable streaming event type. -- Or add a simpler `run()` API and keep streaming advanced. - -## Consolidation Areas - -### Experiment / Definition / Run / Cohort - -Current nouns: - -```text -Experiment -ExperimentRecord -ExperimentDefinition -PersistedExperimentDefinition -RunRecord -ExperimentCohort -ExperimentCohortStats -``` - -Possible clean story: - -```text -Public: - Benchmark - Worker - Rubric - -Application/CLI: - ExperimentSpec or RunSpec - RunHandle - -Core persistence: - ExperimentRecord - ExperimentDefinition - RunRecord - ExperimentCohort -``` - -Open design choice: - -- If users think in experiments, keep `Experiment` public, but make it a pure spec. -- If students mostly write benchmarks/workers/rubrics, hide experiment composition behind CLI commands or a service facade. - -### Evaluator / Rubric / Evaluation Service - -Current nouns: - -```text -Evaluator -Rubric -RubricEvaluationService -TaskEvaluationResult -EvaluationSummary -CriterionResultEntry -``` - -Possible clean story: - -```text -Public: - Criterion - CriterionResult - Rubric - TaskEvaluationResult - -Advanced public: - Evaluator - -Core: - EvaluationRunner - EvaluationSummary - CriterionResultEntry -``` - -Open design choice: - -- Keep `Evaluator` root-exported if dynamic task-specific evaluators are important. -- Otherwise feature `Rubric` and let custom evaluators live in an advanced namespace. - -### Task / Instance / Workflow Graph - -Current nouns: - -```text -BenchmarkTask -instance_key -parent_task_slug -dependency_task_slugs -evaluator_binding_keys -ExperimentDefinitionTask -RunTaskExecution -RunGraphNode -``` - -Possible clean story: - -```text -Public beginner: - Task(slug, description, payload) - -Public advanced: - WorkflowTask(parent, dependencies, evaluator_slots) - -Core: - ExperimentDefinitionTask - RunTaskExecution - RunGraphNode -``` - -Open design choice: - -- Do benchmark authors commonly need dependency graphs? -- If yes, keep the fields but document them as advanced. -- If no, split simple task authoring from graph authoring. - -## Ergonomic API Options - -### Option A: Minimal Authoring Root - -Root exports: - -```python -from ergon_core.api import ( - Benchmark, - BenchmarkTask, - EmptyTaskPayload, - Worker, - WorkerContext, - WorkerOutput, - Criterion, - CriterionResult, - CriterionScoreSpec, - Rubric, - TaskEvaluationResult, - CriteriaCheckError, -) -``` - -Advanced imports: - -```python -from ergon_core.api.advanced import Evaluator, Experiment, WorkerSpec -``` - -Pros: - -- Cleanest beginner story. -- Easy to document. -- Makes runtime/composition concepts visibly advanced. - -Cons: - -- More migration churn. -- Built-in registry typing and core services need import updates. -- Existing code that imports `Experiment` from public API needs shims. - -### Option B: Keep Object-First API, But Purify It - -Root exports still include: - -```python -Experiment -WorkerSpec -Evaluator -``` - -But: - -- `Experiment.persist()` moves to a service. -- `WorkerSpec.validate_spec()` moves to core composition. -- `Worker.get_output()` no longer reads DB from public base class. -- `EvaluationContext.runtime` becomes private helper-backed capability. - -Pros: - -- Less disruptive. -- Preserves object-first feel. -- Keeps `Experiment` available for users who naturally want to compose runs in Python. - -Cons: - -- Beginner docs still need to explain more nouns. -- The top-level API remains larger. -- Harder to communicate what is "normal" vs "advanced". - -### Option C: Two Layer Public API - -Root beginner API: - -```python -Benchmark -Task -Worker -WorkerOutput -Criterion -CriterionResult -Rubric -``` - -Explicit composition API: - -```python -from ergon_core.composition import Experiment, WorkerSpec, persist_experiment -``` - -or: - -```python -from ergon_core.app import define_experiment, run_benchmark -``` - -Pros: - -- Honest separation without hiding useful power. -- CLI and notebook users get a supported high-level entrypoint. -- Students can start with authoring and only learn composition when needed. - -Cons: - -- Requires new package/module naming decisions. -- Need to avoid having too many "public APIs". - -My current recommendation: - -- Option C, implemented gradually. -- Keep compatibility re-exports during migration. -- Document `ergon_core.api` as authoring. -- Add a separate high-level app/composition facade for running things. - -## Proposed Beginner Docs Shape - -### Writing A Benchmark - -```python -from ergon_core.api import Benchmark, BenchmarkTask - -class MyBenchmark(Benchmark): - type_slug = "my-benchmark" - - def build_instances(self): - return { - "default": [ - BenchmarkTask( - task_slug="task-1", - instance_key="default", - description="Solve this problem.", - ) - ] - } -``` - -Possible future version: - -```python -from ergon_core.api import Benchmark, Task - -class MyBenchmark(Benchmark): - type_slug = "my-benchmark" - - def tasks(self): - yield Task("task-1", "Solve this problem.") -``` - -### Writing A Worker - -Current-ish: - -```python -from ergon_core.api import Worker, WorkerContext, BenchmarkTask - -class MyWorker(Worker): - type_slug = "my-worker" - - async def execute(self, task: BenchmarkTask, *, context: WorkerContext): - ... -``` - -Possible future beginner version: - -```python -from ergon_core.api import Worker, WorkerOutput - -class MyWorker(Worker): - type_slug = "my-worker" - - async def run(self, task, context): - return WorkerOutput(output="answer") -``` - -### Writing A Criterion - -Current-ish: - -```python -from ergon_core.api import Criterion, CriterionResult, EvaluationContext - -class MyCriterion(Criterion): - type_slug = "my-criterion" - - async def evaluate(self, context: EvaluationContext): - return CriterionResult( - slug=self.slug, - name=self.slug, - score=1.0, - passed=True, - ) -``` - -Possible helper version: - -```python -return CriterionResult.pass_(self.slug, score=1.0) -``` - -### Writing A Rubric - -```python -from ergon_core.api import Rubric - -rubric = Rubric( - name="default", - criteria=[MyCriterion(slug="correctness")], -) -``` - -## Decisions To Make Together - -### Public Root Exports - -Suggested categories: - -```text -Definitely root public: - Benchmark - BenchmarkTask or Task - EmptyTaskPayload - Worker - WorkerContext - WorkerOutput - Criterion - CriterionResult - CriterionScoreSpec - Rubric - TaskEvaluationResult - CriteriaCheckError - -Maybe root public: - EvaluationContext - Evaluator - BenchmarkDeps - DependencyError - Experiment - -Probably not root public long-term: - WorkerSpec - PersistedExperimentDefinition -``` - -### Concept Names - -Questions: - -- Keep `BenchmarkTask`, or alias it as `Task`? -- Keep `EvaluationContext`, or rename to `CriterionContext`? -- Keep `Evaluator` visible, or make `Rubric` the main public evaluation abstraction? -- Keep `Experiment`, or move composition to a separate facade? -- Rename `PersistedExperimentDefinition` to `WorkflowDefinitionHandle`? -- Rename `RubricEvaluationService` to `EvaluationRunner` or `TaskEvaluationService`? -- Add `CriterionCheckError` alias for `CriteriaCheckError`? - -### Simplicity Targets - -A clean beginner author should not need to know: - -- Inngest, -- database sessions, -- context event persistence, -- run graph node IDs, -- experiment definition row IDs, -- cohort tables, -- telemetry models, -- evaluator binding keys, -- worker binding keys, -- registry validation internals. - -They may need to know: - -- how to create tasks, -- how a worker receives a task, -- how to return an output, -- how criteria inspect the output, -- how a rubric combines criteria. - -## Recommended Refactor Sequence - -### Phase 1: Document And Test The Boundary - -Add tests that encode: - -- `ergon_core.api.worker` must not import DB/session/persistence modules. -- `ergon_core.api.evaluation_context` must not import core runtime protocols directly. -- root exports are intentionally categorized. -- submodule-only public symbols like `CriterionScoreSpec` are either root-exported or documented. - -### Phase 2: Remove Runtime Leakage From Public Worker - -Move from: - -```text -api/worker.py - ContextEventRepository - get_session - AssistantTextPart -``` - -To: - -```text -core/runtime/output_extraction.py - default_worker_output(context) -``` - -Then `worker_execute.py` owns the runtime behavior. - -### Phase 3: Hide Criterion Runtime Behind Public Context Methods - -Move from: - -```text -EvaluationContext.runtime: CriterionRuntime | None -``` - -To: - -```text -EvaluationContext.execute_code(...) -EvaluationContext.read_resource(...) -EvaluationContext.read_resource_by_id(...) -``` - -Internal runtime remains in `core.runtime.evaluation`. - -### Phase 4: Move Composition Plumbing - -Move: - -```text -api/experiment.py -> core/runtime/composition/experiment.py -api/worker_spec.py -> core/runtime/composition/worker_spec.py -api/handles.py -> core/runtime/composition/handles.py -``` - -Keep compatibility shims temporarily: - -```text -api/experiment.py -api/worker_spec.py -api/handles.py -``` - -But update core and CLI imports to the new home first. - -### Phase 5: Add A CLI/Application Facade - -Create something like: - -```text -core/runtime/services/benchmark_run_facade.py -``` - -It owns: - -- build benchmark from slug, -- attach worker/model/rubric, -- persist definition, -- resolve/create cohort, -- create run, -- emit workflow started event, -- poll run status. - -Then `ergon_cli` becomes mostly command parsing and rendering. - -### Phase 6: Consolidate Evaluation Naming - -Decide: - -- root `Rubric` only, or root `Evaluator` too? -- rename internal `RubricEvaluationService`? -- add public helper constructors for result models? -- centralize `CriterionResult` to `EvaluationSummary` conversion. - -## Proposed End State - -```text -ergon_core.api - The authoring kit. - Used by benchmarks, workers, criteria, rubrics, and students. - -ergon_core.core.runtime.composition - Internal composition layer. - Used by CLI and core services to bind benchmarks, workers, rubrics, assignments. - -ergon_core.core.runtime.services - Application services. - Used by API routers and CLI facade. - -ergon_core.core.persistence - SQLModel rows and repositories. - Not imported by public API. - -ergon_cli - Command parsing and display. - Calls a small core facade, not many low-level services. -``` - -## Working Recommendation - -If we want the cleanest ergonomics for students: - -1. Keep the root public API focused on authoring. -2. Keep `Experiment` available for now, but do not teach it first. -3. Move `WorkerSpec` and `PersistedExperimentDefinition` out of the public root over time. -4. Make `Rubric` the public evaluation concept; keep `Evaluator` advanced. -5. Add helper methods/constructors so basic workers and criteria are short to write. -6. Build a separate run/composition facade for CLI and notebook users. - -The practical next conversation should decide three things: - -1. Is `Experiment` a public composition object or a core definition draft? -2. Is worker authoring streaming-first or output-first? -3. Is `Evaluator` a first-class public concept or an advanced escape hatch behind `Rubric`? diff --git a/docs/superpowers/plans/2026-04-28-public-api-folder-plan.md b/docs/superpowers/plans/2026-04-28-public-api-folder-plan.md deleted file mode 100644 index 1b7bcb475..000000000 --- a/docs/superpowers/plans/2026-04-28-public-api-folder-plan.md +++ /dev/null @@ -1,413 +0,0 @@ -# Public API Folder Refactor Plan - -Goal: make `ergon_core.api` small enough for students to understand while moving runtime, persistence, dashboard, cohort, run, and registry plumbing into `ergon_core.core`. - -The public API should be an authoring kit: define benchmarks, tasks, workers, criteria, rubrics, and simple result objects. It should not expose database sessions, persistence handles, Inngest dispatch, cohort management, run lifecycle, or internal evaluation summaries. - -## Proposed Folder Shape - -```text -ergon_core/ - ergon_core/ - api/ - __init__.py - # keep : only the student-facing authoring exports - # export: Benchmark, BenchmarkTask, EmptyTaskPayload - # export: Worker, WorkerContext, WorkerOutput - # export: Criterion, CriterionResult, CriterionScoreSpec - # export: Rubric, TaskEvaluationResult - # export: CriteriaCheckError - # stop exporting: Experiment, WorkerSpec, PersistedExperimentDefinition - # consider hiding: Evaluator, EvaluationContext, BenchmarkDeps, DependencyError - - benchmark.py - # keep : Benchmark as the public dataset/task generator base class - # keep : type_slug, task_payload_model, build_instances() - # keep : parse_task_payload() - # simplify : evaluator_requirements() should become optional/advanced - # move : dependency package checking to core/runtime/dependencies.py adapter - # merge : onboarding_deps and required_packages into one simpler authoring metadata story - - task_types.py - # keep : BenchmarkTask and EmptyTaskPayload - # consider rename later : BenchmarkTask -> Task or TaskSpec - # keep public because benchmarks, workers, and criteria all share it - # do not expose: ExperimentDefinitionTask persistence model here - - worker.py - # keep : Worker ABC and execute(task, context=...) - # keep : optional from_buffer() only if resumption remains an author-facing extension point - # move : default DB-backed get_output() implementation to core/runtime/output_extraction.py - # move : ContextEventRepository/get_session imports out of public API - # move : AssistantTextPart/ContextPartChunk dependency behind a smaller public streaming type or an advanced namespace - # simplify : base Worker should not know how context events are persisted - - worker_context.py - # keep : WorkerContext as the minimal execution context passed to Worker.execute() - # simplify : expose only run_id, task_id, execution_id, sandbox_id, metadata if possible - # move inward : definition_id and node_id if only runtime/delegation needs them - # consider : a separate internal CoreWorkerContext for graph/runtime identity - - results.py - # keep : WorkerOutput - # keep : CriterionScoreSpec - # keep : CriterionResult - # keep : TaskEvaluationResult - # keep or move advanced : CriterionObservation and CriterionObservationMessage - # move : JsonObject import from core into a public local alias/type - # merge : align CriterionResult fields with core EvaluationSummary conversion in one adapter - - criterion.py - # keep : Criterion ABC - # keep : evaluate(context) -> CriterionResult - # move : dependency package checking to core validation helper - # simplify : criterion authors should not need to import core runtime protocols - - evaluation_context.py - # keep temporarily : EvaluationContext for compatibility - # replace with : CriterionContext or EvaluationContext with public helper methods - # move : CriterionRuntime Protocol import to core/runtime/evaluation/protocols.py only - # hide : sandbox manager/runtime internals behind context.execute_code(), context.read_resource(), etc. - # eventual delete : if Criterion can receive a simpler public CriterionContext - - evaluator.py - # keep : Rubric as the common public evaluation concept - # consider advanced : Evaluator ABC moves to api/advanced/evaluator.py or core/runtime/evaluation - # merge : default weighted aggregation remains Rubric - # move : dynamic evaluator orchestration details to core/runtime/services/rubric_evaluation_service.py - # clarify : Rubric = author-facing grouping of criteria; evaluator service = internal runner - - errors.py - # keep : CriteriaCheckError - # consider move : DependencyError to core/runtime/dependencies.py unless public callers catch it - - benchmark_deps.py - # merge : into Benchmark metadata or move to api/onboarding.py - # keep temporarily : compatibility for ergon_cli/onboarding/profile.py and built-in benchmark declarations - # eventual delete : once onboarding reads a simpler Benchmark.onboarding field - - experiment.py - # move to core/runtime/composition/experiment.py or core/runtime/services/experiment_composition.py - # reason : binds benchmark + worker specs + evaluators + assignments for persistence - # reason : persist() calls core ExperimentPersistenceService - # public replacement : a simple CLI/application facade, not a student authoring primitive - # eventual delete from top-level api - - worker_spec.py - # move to core/runtime/composition/worker_spec.py - # reason : config-time descriptor for registry lookup, not worker authoring - # reason : validate_spec() imports ergon_builtins.registry.WORKERS - # public replacement : CLI accepts worker_slug/model and core builds WorkerSpec internally - # eventual delete from top-level api - - handles.py - # move to core/runtime/services/experiment_handles.py or core/runtime/composition/handles.py - # reason : PersistedExperimentDefinition is a persistence/run launch handle - # public replacement : CLI-facing RunHandle/DefinitionHandle returned by core facade - # eventual delete from top-level api -``` - -```text -ergon_core/ - ergon_core/ - core/ - runtime/ - composition/ - __init__.py - # create : internal composition exports for CLI/core - - experiment.py - # move from api/experiment.py - # keep : Experiment composition root if core still needs object-first persistence - # change : persist() should become service-owned, not a method on Experiment - - worker_spec.py - # move from api/worker_spec.py - # keep : WorkerSpec registry descriptor - # keep : validate_spec() registry lookup here, away from public API - - handles.py - # move from api/handles.py - # keep : PersistedExperimentDefinition or rename to WorkflowDefinitionHandle - - output_extraction.py - # create : default worker output extraction from context events - # move from api/worker.py : ContextEventRepository/get_session/AssistantTextPart logic - # used by : core/runtime/inngest/worker_execute.py - - dependencies.py - # keep : check_packages() - # add : validate_component_dependencies(component_type, slug, packages, install_hint) - # public ABCs call this only through small wrappers, or core validates before launch - - evaluation/ - protocols.py - # keep : CriterionRuntime internal protocol - # no public api imports should depend on this directly - - context.py - # create or rename : internal TaskEvaluationContext/CriterionContext live here - # owns : sandbox/runtime details for criterion execution - - adapters.py - # create : convert public CriterionResult into persisted EvaluationSummary entries - # merge logic currently split between public results and persistence summary models - - evaluation_schemas.py - # keep : internal CriterionSpec, TaskEvaluationContext, CriterionContext - # maybe rename : criterion_specs.py if it remains evaluation-engine only - - services/ - public_api_facade.py - # create : CLI/application facade for common operations - # owns : define benchmark experiment, persist definition, create cohort/run, dispatch, poll - # goal : CLI should import one core facade instead of many core services/models - - experiment_persistence_service.py - # keep : writes Experiment/BenchmarkTask object graph to immutable definition rows - # adjust imports : read Experiment and WorkerSpec from core/runtime/composition - - experiment_definition_service.py - # keep : create ExperimentRecord sample selections - # clarify name : this creates experiment records, not immutable workflow definitions - # possible rename later : benchmark_experiment_service.py - - experiment_launch_service.py - # keep : materializes runs for defined ExperimentRecord rows - # adjust imports : use core composition types, not public api Experiment/WorkerSpec - - rubric_evaluation_service.py - # keep : internal service runner - # clarify : not the same concept as public Rubric - # maybe rename : task_evaluation_service.py - - evaluation_persistence_service.py - # keep : persistence of evaluation summaries - # move conversion from public-ish result shapes into runtime/evaluation/adapters.py - - cohort_service.py - # keep : cohorts are operator/runtime grouping, not student API - # expose via facade only for CLI/dashboard - - run_service.py - # keep : runs are runtime telemetry/lifecycle, not student API - # expose via facade only for CLI/dashboard -``` - -```text -ergon_cli/ - ergon_cli/ - composition/ - __init__.py - # delete or shrink substantially - # current : imports public Experiment + WorkerSpec - # move : build_experiment() logic to core/runtime/composition or services/public_api_facade.py - # replacement : CLI passes slugs/options to core facade - - commands/ - benchmark.py - # keep : command parsing and rendering only - # move inward : create_run, WorkflowStartedEvent, inngest_client, RunRecord polling - # replace with : public_api_facade.run_benchmark(...) - # keep : setup benchmark E2B template logic unless moved to onboarding service - - experiment.py - # keep : command parsing/rendering - # replace multiple core service imports with one facade import - - run.py - # keep : command parsing/rendering - # replace direct RunRecord/run_service access with one run facade - - workflow.py - # keep : command parsing/rendering - # replace direct workflow_service/db access with facade if possible - - onboarding/ - profile.py - # keep : onboarding profile behavior - # change later : read Benchmark.onboarding metadata instead of BenchmarkDeps directly -``` - -```text -ergon_builtins/ - ergon_builtins/ - benchmarks/ - */benchmark.py - # keep public imports : Benchmark, BenchmarkTask, EmptyTaskPayload - # update : BenchmarkDeps if moved/merged - # no direct dependency on core persistence or run concepts - - */rubric.py - # keep public imports : Rubric, CriterionResult, TaskEvaluationResult, BenchmarkTask - # if Evaluator moves advanced/internal, custom rubrics should still subclass Rubric - - */criterion.py - # keep public imports : Criterion, CriterionResult, CriterionScoreSpec - # update : EvaluationContext -> simpler CriterionContext if introduced - - workers/ - */*.py - # keep public imports : Worker, WorkerContext, WorkerOutput, BenchmarkTask - # update : streaming chunk type if ContextPartChunk is hidden or rehomed - - registry.py - # keep : plugin registry for built-ins - # core composition validates WorkerSpec/Benchmark/Evaluator slugs against this - # public API should not import this registry directly -``` - -## Concept Merges And Renames - -### Experiment Concepts - -Current concepts: - -- `api.Experiment`: object graph for benchmark + workers + evaluators + assignments. -- `core.persistence.telemetry.ExperimentRecord`: cohort/sample-selection record. -- `core.persistence.definitions.ExperimentDefinition`: immutable workflow definition rows. - -Plan: - -- Keep `ExperimentDefinition` as a core persistence name. -- Consider renaming `ExperimentRecord` service language to `BenchmarkExperiment` or `ExperimentPlan` later, because it is not the immutable workflow definition. -- Move public `Experiment` into core composition, or rename it `WorkflowDefinitionDraft` if it remains object-first. -- Do not ask students to learn all three names. - -### Worker Concepts - -Current concepts: - -- `Worker`: execution-ready authoring base class. -- `WorkerSpec`: config-time registry descriptor. -- `ExperimentDefinitionWorker`: persisted worker binding row. - -Plan: - -- Keep `Worker` public. -- Move `WorkerSpec` into core composition. -- Keep `ExperimentDefinitionWorker` internal. -- CLI should accept `worker_slug` and `model`; core creates `WorkerSpec`. - -### Evaluation Concepts - -Current concepts: - -- `Criterion`: atomic authoring unit. -- `Rubric`: fixed-list `Evaluator` with aggregation. -- `Evaluator`: abstract dynamic evaluator. -- `RubricEvaluationService`: runtime service that executes criteria and aggregates. -- `CriterionResultEntry` / `EvaluationSummary`: persisted dashboard schema. - -Plan: - -- Keep `Criterion` and `Rubric` public. -- Keep `Evaluator` advanced or internal unless third-party dynamic evaluators are required. -- Rename or document `RubricEvaluationService` as internal task evaluation runner. -- Keep `EvaluationSummary` internal. -- Add one adapter that maps `CriterionResult`/`TaskEvaluationResult` to persisted summary rows. - -### Task Concepts - -Current concepts: - -- `BenchmarkTask`: author-facing task object generated by a benchmark. -- `ExperimentDefinitionTask`: persisted definition row. -- `RunTaskExecution`: runtime execution telemetry row. - -Plan: - -- Keep `BenchmarkTask` public for now. -- Consider future alias `Task = BenchmarkTask` for student docs. -- Keep persistence/runtime task rows internal. -- Core adapters convert public task specs into definition rows. - -### Cohort And Run Concepts - -Current concepts: - -- Cohorts and runs are not in `ergon_core.api`, but CLI imports core services/models directly. -- `ExperimentCohort`, `ExperimentCohortStats`, `RunRecord`, `RunTaskExecution`, `RunTaskEvaluation` are operator/runtime concepts. - -Plan: - -- Keep cohorts and runs out of the student authoring API. -- Add a CLI/application facade so built-in CLI can use cohorts/runs without importing persistence models, Inngest events, or low-level services. -- Dashboard/API routers can still use detailed core services and DTOs. - -## Compatibility Strategy - -1. Add architecture tests for the intended boundary before moving code. -2. Keep compatibility re-exports for one refactor window: - - `ergon_core.api.experiment.Experiment` - - `ergon_core.api.worker_spec.WorkerSpec` - - `ergon_core.api.handles.PersistedExperimentDefinition` - - `ergon_core.api.benchmark_deps.BenchmarkDeps` -3. Update `ergon_cli` and `ergon_core.core` imports first so internal code no longer depends on public API for internal composition. -4. Update `ergon_builtins` imports only after the public authoring surface is stable. -5. Remove compatibility shims once tests and docs no longer reference moved symbols. - -## Suggested Implementation Order - -```text -phase_1_boundary_tests/ - tests/unit/architecture/test_public_api_boundaries.py - # add forbidden import checks for api -> core.persistence, core.runtime.evaluation.protocols, core.generation - # add explicit expected top-level public exports - -phase_2_worker_runtime_split/ - ergon_core/ergon_core/api/worker.py - # keep Worker ABC only - # remove DB/context event imports - - ergon_core/ergon_core/core/runtime/output_extraction.py - # create default output extraction helper - - ergon_core/ergon_core/core/runtime/inngest/worker_execute.py - # use output_extraction helper after worker.execute() - -phase_3_composition_move/ - ergon_core/ergon_core/core/runtime/composition/ - # create experiment.py, worker_spec.py, handles.py - - ergon_core/ergon_core/api/ - # leave temporary import shims for Experiment, WorkerSpec, PersistedExperimentDefinition - - ergon_cli/ergon_cli/composition/__init__.py - # migrate logic or shrink to facade call - -phase_4_cli_facade/ - ergon_core/ergon_core/core/runtime/services/public_api_facade.py - # create stable CLI-facing functions/classes - - ergon_cli/ergon_cli/commands/*.py - # replace direct core service/model/event imports where practical - -phase_5_evaluation_simplification/ - ergon_core/ergon_core/api/evaluation_context.py - # replace raw runtime protocol exposure with public context methods - - ergon_core/ergon_core/core/runtime/evaluation/adapters.py - # centralize result-to-summary conversion - - ergon_core/ergon_core/api/evaluator.py - # make Rubric primary; move Evaluator to advanced/internal if desired - -phase_6_cleanup/ - ergon_core/ergon_core/api/__init__.py - # remove moved concepts from top-level exports - - docs/ - # update student-facing examples to import only the authoring kit -``` - -## Desired Final Student-Facing Mental Model - -```text -I define a Benchmark. -The Benchmark returns Tasks. -A Worker solves each Task. -A Criterion checks the output. -A Rubric combines Criteria into a score. -Ergon core handles experiments, definitions, cohorts, runs, persistence, dispatch, and dashboards. -``` diff --git a/docs/superpowers/plans/2026-04-28-runtime-services-layout-audit.md b/docs/superpowers/plans/2026-04-28-runtime-services-layout-audit.md deleted file mode 100644 index acefd6cdb..000000000 --- a/docs/superpowers/plans/2026-04-28-runtime-services-layout-audit.md +++ /dev/null @@ -1,665 +0,0 @@ -# Runtime Services Layout Audit - -Date: 2026-04-28 - -Scope: `ergon_core/ergon_core/core/runtime/services` in the current core/public API refactor branch. - -This note is an investigation artifact for a later fix/refactor plan. It does not propose a final migration sequence yet. The goal is to identify where `runtime/services` has become a dumping ground, where service shapes are inconsistent, and where logic appears duplicated or split across weak domain boundaries. - -Post-refactor update: this audit has been refreshed after the public API nesting refactor and the first core service moves: - -- `Experiment` and `WorkerSpec` now live under `core/composition`. -- The beginner-facing public API is now nested under `api/benchmark`, `api/worker`, `api/criterion`, and `api/rubric`. -- `experiment_validation_service.py` now owns experiment object-graph validation. -- `workflow_propagation_service.py` now owns the former `runtime/execution/propagation.py` graph propagation helpers. - -Most of the original duplication findings still stand. The new public API shape mainly changes the target boundaries: authoring concepts should stay in `ergon_core.api`, composition/definition concepts should sit near `core/composition` and definition services, and graph/task/workflow lifecycle behavior should stop accumulating in a single flat `runtime/services` package. - -## Executive Summary - -`runtime/services` is doing too many jobs in one flat namespace: - -- Domain orchestration services (`TaskExecutionService`, `WorkflowInitializationService`, `WorkflowFinalizationService`). -- Graph mutation and graph read helpers (`WorkflowGraphRepository`, `GraphNodeLookup`, graph DTOs). -- Agent/tool-facing subtask services (`TaskManagementService`, `TaskInspectionService`). -- API/dashboard read models (`RunReadService`, `WorkflowService`). -- Persistence helpers (`ExperimentPersistenceService`, `EvaluationPersistenceService`). -- Product areas that are not obviously part of runtime orchestration (`CommunicationService`, cohort services). -- Transport contracts for Inngest and API surfaces (`*_dto.py`, `*_schemas.py`, `child_function_payloads.py`, `inngest_function_results.py`). - -The resulting issue is not just file count. The same concepts are implemented with different local conventions: request/response models may be named DTOs, schemas, payloads, or function results; DB access may use explicit sessions, `with get_session()`, or ad hoc repository instances; graph traversal and latest-execution lookup logic are repeated with inconsistent ordering rules. - -## Current File Groups - -### Graph And Graph Mutation - -- `graph_repository.py` -- `graph_lookup.py` -- `graph_dto.py` -- `workflow_propagation_service.py` -- `task_management_service.py` -- `task_inspection_service.py` -- `task_management_dto.py` -- `task_inspection_dto.py` -- `subtask_cancellation_service.py` -- `subtask_cancellation_dto.py` -- `subtask_blocking_service.py` - -This is the densest cluster. It covers graph mutation, graph traversal, task/subtask management, inspection, cancellation, blocking, propagation, and graph DTOs. Moving propagation into services made the domain boundary clearer: the old `runtime/execution` package was not really a separate layer; propagation belongs with graph lifecycle policy. - -### Experiment Definition And Composition - -- `experiment_validation_service.py` -- `experiment_persistence_service.py` -- `experiment_definition_service.py` -- `experiment_launch_service.py` -- `experiment_schemas.py` -- `experiment_read_service.py` - -This group is now more visible because `Experiment` moved out of the public API and into `core/composition`. These files are not all the same kind of service: - -- `experiment_validation_service.py` validates the in-memory composition object graph. -- `experiment_persistence_service.py` materializes immutable definition rows from composition objects. -- `experiment_definition_service.py` defines experiments from registered benchmark/worker/evaluator slugs. -- `experiment_launch_service.py` bridges persisted definitions into runtime orchestration. -- `experiment_read_service.py` and `experiment_schemas.py` are application/API read models. - -The current flat package hides that sequence. A later refactor should make the pipeline explicit: composition -> definition persistence -> launch -> read model. - -### Workflow And Run Lifecycle - -- `run_service.py` -- `workflow_initialization_service.py` -- `workflow_finalization_service.py` -- `workflow_service.py` -- `workflow_dto.py` -- `orchestration_dto.py` -- `run_snapshot_read_model.py` - -This group mixes run lifecycle orchestration with workflow navigation/resource materialization. `workflow_service.py` is read-heavy and tool/API-facing, while `workflow_initialization_service.py` and `workflow_finalization_service.py` are engine lifecycle services. `run_snapshot_read_model.py` is already a move in the right direction because it names read-model shaping separately from orchestration. - -### Task Execution And Propagation - -- `task_execution_service.py` -- `task_propagation_service.py` -- `workflow_propagation_service.py` -- `task_cleanup_service.py` -- `task_cleanup_dto.py` - -This group owns execution row creation/finalization, graph status updates for task execution, propagation after completion/failure, and cleanup of cancelled task executions. `workflow_propagation_service.py` is deliberately listed in both graph and task groups because it is the clearest split point: some functions are graph lifecycle primitives, while `TaskPropagationService` is an orchestration wrapper that turns those transitions into schedulable work. - -### Evaluation - -- `rubric_evaluation_service.py` -- `evaluator_dispatch_service.py` -- `evaluation_persistence_service.py` -- `evaluation_dto.py` - -This group mixes evaluator preparation, rubric execution, persistence, and dashboard DTO shaping. - -### API Read Models And Product Features - -- `run_read_service.py` -- `communication_service.py` -- `communication_schemas.py` -- `cohort_service.py` -- `cohort_stats_service.py` -- `cohort_schemas.py` - -These are valid application services, but they are not the same kind of service as runtime orchestration. Their presence in the same flat package makes ownership harder to read. - -### Transport Contracts - -- `child_function_payloads.py` -- `inngest_function_results.py` -- plus the various `*_dto.py` and `*_schemas.py` files - -These are request/response contracts, not services. They currently sit beside service implementations without a consistent folder or naming convention. - -## Standardization Gaps - -### No Common Service Module Shape - -The desired structure is roughly: - -- request/response models -- DB schema types -- `repository.py` or service implementation -- `errors.py` for custom domain/service exceptions -- optional `utils.py` - -The current structure is flat and inconsistent: - -- Some service request/response models live in `*_dto.py`. -- Some live in `*_schemas.py`. -- Inngest request models live in `child_function_payloads.py`. -- Inngest outputs live in `inngest_function_results.py`. -- Some service-specific helper models live in the same service file. -- Persistence-facing repositories live partly in `core/persistence` and partly in `runtime/services`. -- Custom exceptions live mostly in broad runtime error modules, not beside the service/domain that raises them. - -This makes it difficult to infer whether a file is a domain service, transport contract, read model, or persistence adapter. - -### Public API Boundary Is Cleaner, But Core Still Needs Adapters - -The public API refactor has reduced the authoring surface to nested packages: - -- `api/benchmark`: `Benchmark`, `Task`, `EmptyTaskPayload`, `BenchmarkRequirements` -- `api/worker`: `Worker`, `WorkerContext`, `WorkerOutput` -- `api/criterion`: `Criterion`, `CriterionContext`, `CriterionOutcome`, `ScoreScale`, evidence types -- `api/rubric`: `Rubric`, `TaskEvaluationResult`, and advanced `Evaluator` - -That is a useful constraint for the services refactor. Runtime services should consume public authoring objects at the boundary where user-authored concepts enter core, but they should not treat `ergon_core.api` as the place for operational concepts like runs, cohorts, graph nodes, or persisted definition handles. - -Current service imports are mostly consistent with that direction: - -- `experiment_validation_service.py`, `experiment_definition_service.py`, `experiment_launch_service.py`, and `rubric_evaluation_service.py` legitimately consume authoring concepts such as `Benchmark`, `Task`, `Evaluator`, `Rubric`, and criterion outcomes. -- `run_read_service.py`, `run_snapshot_read_model.py`, `communication_service.py`, and `evaluation_persistence_service.py` still import API-layer DTOs from `core/api/schemas.py`. Those are not beginner-facing authoring API objects, but the import direction is still awkward: runtime read-model code depends upward on API schemas. - -The revised target should be: public authoring API in `ergon_core.api`; internal composition in `core/composition`; runtime read models in a runtime/application read-model package; HTTP/API routers adapt those read models to wire schemas. - -### Error Types Are Not Domain-Local - -Some custom errors already exist under `core/runtime/errors`, for example graph, delegation, and Inngest-specific error modules. That is better than raising generic `ValueError` everywhere, but it still leaves service packages without local ownership of their failure modes. - -The target convention should be: each runtime domain package owns an `errors.py` file for exceptions that are part of that domain contract. For example: - -- `runtime/graph/errors.py` for graph structural and mutation errors. -- `runtime/tasks/errors.py` for task execution, task management, cleanup, cancellation, and inspection failures. -- `runtime/workflows/errors.py` for workflow initialization/finalization/lifecycle failures. -- `runtime/evaluation/errors.py` for evaluator dispatch, rubric evaluation, and evaluation persistence failures. -- `runtime/inngest/errors.py` for Inngest wrapper/contract/non-retryable errors. - -This does not mean every exception class needs to move immediately. The refactor plan should move errors opportunistically with the package they belong to, and should prefer explicit custom exceptions over generic `ValueError`, `RuntimeError`, or assertion-style checks at service boundaries. - -### Repository Naming Is Ambiguous - -`WorkflowGraphRepository` is in `runtime/services/graph_repository.py`, while persistence repositories live in: - -- `core/persistence/context/repository.py` -- `core/persistence/telemetry/repositories.py` - -This is understandable because `WorkflowGraphRepository` owns runtime graph mutation semantics and audit-log writes, not just raw CRUD. Still, the package shape blurs whether repositories are persistence infrastructure or runtime domain services. - -### Session Ownership Varies - -Patterns include: - -- Methods accepting an explicit `Session`. -- Services opening `with get_session() as session`. -- Services using `session = get_session()` with manual `finally: session.close()`. -- Repository classes receiving a session from callers. - -Examples: - -- `TaskManagementService`, `SubtaskCancellationService`, and `WorkflowService` accept caller-owned sessions. -- `RunReadService`, `RunService`, `WorkflowInitializationService`, and `WorkflowFinalizationService` open sessions internally. -- `EvaluationPersistenceService` manually opens and closes sessions instead of using `with get_session()`. - -This makes transaction boundaries harder to reason about and complicates any future service package convention. - -## Concrete Duplication Findings - -### P1: Duplicate Latest Execution Lookup - -Two files define the same helper: - -- `task_management_service.py` -- `subtask_cancellation_service.py` - -Both query `RunTaskExecution.id` by `node_id`, ordered by `RunTaskExecution.started_at.desc()`, and use it to populate `TaskCancelledEvent.execution_id`. - -Related methods in other services define "latest execution" differently: - -- `WorkflowService.get_latest_execution` orders by `attempt_number DESC`, then `started_at DESC`. -- `TaskInspectionService._latest_output` and `_latest_error` order only by `started_at DESC`. - -This is a real semantic duplication. There should be one canonical helper for "latest execution for node", with a clearly documented ordering rule. - -### P1: Duplicate Containment Subtree Traversal - -The same parent-child BFS pattern appears in: - -- `task_management_service.py` via `_count_non_terminal_descendants`. -- `subtask_cancellation_service.py` via `cancel_orphans`. -- `subtask_blocking_service.py` via `block_pending_descendants`. - -All query `RunGraphNode` children by `run_id` and `parent_node_id`, then apply a different policy: - -- Count non-terminal descendants. -- Cancel non-terminal descendants. -- Block non-terminal, non-running descendants. - -This should become a shared graph traversal primitive, with the policy supplied by the caller or by domain-specific cascade services. - -### P1: Scattered Graph Status Transitions - -Graph node and edge status writes appear across: - -- `task_execution_service.py` -- `task_propagation_service.py` -- `workflow_propagation_service.py` -- `task_management_service.py` -- `subtask_cancellation_service.py` -- `subtask_blocking_service.py` -- `workflow_initialization_service.py` -- `graph_repository.py` - -`WorkflowGraphRepository` intentionally does not validate transitions; it only records mutations and enforces structural invariants. That boundary is reasonable, but the transition policy above it is distributed across many services. - -The refactor plan should decide whether there is a single graph lifecycle domain service, or at least a small set of named transition operations such as: - -- start node execution -- complete node execution -- fail node execution -- reset node for restart -- cancel subtree -- block subtree -- satisfy dependency edge - -### P2: Duplicated Graph Mapping / Read Loading - -`GraphNodeLookup` batch-loads mappings from definition task IDs and edges to run graph IDs. - -`RunReadService.build_run_snapshot` builds similar maps inline: - -- `execution_task_map` -- `defn_to_node` -- task maps and context-event maps through API helper functions - -`WorkflowService` also builds node maps through `_nodes_by_id` and tree/resource scopes through local queries. - -These are not identical consumers, but the primitives overlap: load run graph, map definition IDs to node IDs, map executions to nodes, and traverse parent/child relationships. - -### P2: Evaluation Score Semantics Drift - -`WorkflowFinalizationService` computes: - -- `final_score = sum(scores)` -- `normalized_score = final_score / len(scores)` - -`RunReadService.build_run_snapshot` computes: - -- `final_score = sum(scores) / len(scores)` - -`TelemetryRepository.refresh_run_evaluation_summary` also updates summary fields from evaluation rows. - -`cohort_service.py` and `cohort_stats_service.py` then read `normalized_score` and `final_score` from summary JSON. This should be centralized because downstream consumers depend on the meaning of these fields. - -### P2: Read Model Shaping Depends On API Helpers - -`RunReadService` imports DTOs from `ergon_core.core.api.schemas` and imports `ergon_core.core.api.runs` helper functions inside `build_run_snapshot`. - -That means a runtime service depends upward on API helpers. This is likely a layering smell. `run_snapshot_read_model.py` is a partial correction because it moves snapshot shaping into a named runtime read model, but it still imports DTO classes from `core/api/schemas.py`. The pure DTO helper functions and run snapshot DTOs should either move into a runtime/read-model package, or the API should own the service and not call it "runtime". - -The new public API nesting makes this more important. `ergon_core.api` should mean authoring API, not operational wire schemas. Runtime read models should not be coupled to the benchmark/worker/criterion authoring package or to HTTP schema modules. - -### P3: Repeated Graph Repository Construction - -`WorkflowGraphRepository()` is constructed in many places: - -- `task_execution_service.py` -- `task_propagation_service.py` -- `workflow_initialization_service.py` -- `task_management_service.py` -- `subtask_cancellation_service.py` -- `subtask_blocking_service.py` - -The repository is mostly stateless, but it has mutation listeners. `TaskManagementService` registers `dashboard_emitter.graph_mutation`; other construction sites do not. If listeners are meant to be consistently applied, construction should be standardized. If not, the listener behavior should be explicit at call sites or separated from repository construction. - -### P3: DTO Naming And Boundaries Are Mixed - -Current naming patterns include: - -- `graph_dto.py` -- `workflow_dto.py` -- `task_management_dto.py` -- `task_inspection_dto.py` -- `evaluation_dto.py` -- `cohort_schemas.py` -- `communication_schemas.py` -- `child_function_payloads.py` -- `inngest_function_results.py` - -The differences may have history, but they do not communicate ownership. A student/user reading the package cannot easily tell whether "schema", "DTO", "payload", and "result" are meaningful distinctions. - -### P3: Task Reference Shapes Overlap - -The following are related but split: - -- `GraphTaskRef` in `graph_dto.py` -- `TaskDescriptor` in `orchestration_dto.py` -- `SubtaskInfo` in `task_inspection_dto.py` -- `WorkflowDependencyRef.source` / `target` in `workflow_dto.py` -- `AddSubtaskResult`, `CancelTaskResult`, and `RestartTaskResult` in `task_management_dto.py` - -Some separation is legitimate, but the shared task identity payload should be explicit. The current split risks reintroducing separate names/status fields for the same runtime graph node. - -## Boundary Assessment - -### Persistence Layer Boundary - -Keep `core/persistence` as storage infrastructure, not as a home for domain behavior. - -These belong in `core/persistence`: - -- SQLModel table definitions in `core/persistence`. -- Shared DB session creation in `core/persistence/shared/db.py`. -- Shared persisted enums and types in `core/persistence/shared`. -- Thin append/read/write helpers that do not encode runtime policy. - -These should move out of `core/persistence`, or should not be added there: - -- Domain repositories that encode graph/task/workflow/evaluation semantics. -- "Latest execution" selection rules. -- Graph lifecycle transition rules. -- Evaluation score aggregation semantics. -- Experiment-definition materialization from authored composition objects. - -In other words, `core/persistence` answers "what rows exist and how do we store them?" Domain packages answer "what does it mean to add a graph node, complete a task, select an attempt, or persist an authored experiment definition?" - -Candidate to split or dissolve: - -- `core/persistence/queries.py` - -It currently contains domain-shaped query objects (`DefinitionsQueries`, `TaskExecutionsQueries`, child-execution lookup, status lookup). Those should be redistributed over time into definition, task, graph, and read-model packages. - -Candidate to reframe: - -- `experiment_persistence_service.py` - -It writes immutable experiment definition tables, but the important behavior is not raw SQL persistence; it is materializing an authored `Experiment` into a persisted definition graph. That makes it a definition/composition domain operation that imports persistence table models, not a persistence-layer module. - -### Things That Belong Near Composition - -`Experiment` and `WorkerSpec` are now under `core/composition`, which gives the services refactor a better boundary than the original audit had. Composition owns the in-memory definition before it becomes persisted runtime state. - -Candidate to move or reframe: - -- `experiment_validation_service.py` - -It validates `Experiment`, benchmark task graph structure, evaluator bindings, and worker assignments. That is composition/definition validation, not runtime DAG execution. It can live under `runtime/services` temporarily, but the target should probably be `core/composition/validation.py` or `core/composition/services/validation.py` unless we decide all composition use cases belong under a broader `core/application` layer later. - -Related files that should be considered together: - -- `core/composition/experiment.py` -- `core/composition/worker_spec.py` -- `core/composition/handles.py` -- `runtime/services/experiment_validation_service.py` -- `runtime/services/experiment_persistence_service.py` -- `runtime/services/experiment_definition_service.py` - -### Things That Belong In Runtime Domain Packages - -These are runtime domain behavior, not raw persistence: - -- Graph mutation repository and mutation DTOs. -- Task execution lifecycle. -- Propagation and graph lifecycle transitions. -- Agent/tool-facing task management and inspection. -- Inngest command/result contracts. - -Candidate runtime packages: - -- `runtime/graph` -- `runtime/tasks` -- `runtime/workflows` -- `runtime/evaluation` -- `runtime/read_models` -- `runtime/inngest/contracts` - -The exact package names can wait for the refactor plan, but the target should be domain packages rather than one `services` bucket. `workflow_propagation_service.py` should be treated as a graph lifecycle module during that migration, not as a generic workflow service. - -### Things Inngest Should Own - -The Inngest function implementations already live under `core/runtime/inngest`, but two Inngest-owned modules currently sit at the top of `core/runtime`: - -- `inngest_client.py` -- `inngest_registry.py` - -These should move under `runtime/inngest` with the function modules. The Inngest package should own: - -- the client singleton and shared cancellation configuration -- the function registry / function list passed to `serve()` -- function modules -- child-function request contracts and function result contracts, unless those contracts are better colocated with the specific function module -- Inngest-specific errors - -This would make `runtime/inngest` the runtime boundary for event orchestration instead of spreading its setup across `runtime` and `runtime/services`. - -### Things That Are Product/Application Services - -These may belong outside the runtime kernel, or in separate runtime subdomains: - -- `communication_service.py` -- `cohort_service.py` -- `cohort_stats_service.py` -- `run_read_service.py` - -They are valid application concerns, but colocating them with graph mutation and task execution weakens the meaning of `services`. - -## Suggested Target Shape - -This is a sketch, not a final implementation plan. - -```text -core/runtime/ - # imports table/session infrastructure from core/persistence, - # but owns domain-specific persistence operations. - - composition_services/ # optional; may instead live under core/composition - validation.py # ExperimentValidationService or pure validation functions - - graph/ - models.py # runtime DTOs for graph snapshots and mutation records - repository.py # WorkflowGraphRepository; domain-aware graph writes over persistence graph tables - errors.py # graph structural and mutation errors - traversal.py # subtree and dependency traversal primitives - lookup.py # GraphNodeLookup or successor - lifecycle.py # named graph status transitions, if introduced - propagation.py # former workflow_propagation_service graph edge/node propagation helpers - - tasks/ - models.py # task execution commands/results, task refs - errors.py # task execution/management/cancellation errors - repository.py # latest execution / attempt selection over RunTaskExecution rows - execution.py # TaskExecutionService - management.py # agent-initiated subtask operations - inspection.py # read-only subtask snapshots - cleanup.py # per-execution cleanup - cascades.py # cancellation/blocking/downstream invalidation - - workflows/ - models.py # workflow lifecycle commands/results - errors.py - initialization.py - finalization.py - service.py # workflow navigation/resource materialization, if kept here - launch.py # ExperimentLaunchService if launch remains runtime-facing - - evaluation/ - models.py - errors.py - dispatch.py - rubric.py - persistence.py - scoring.py # shared score aggregation semantics - - read_models/ - errors.py - run_snapshot.py # RunReadService and pure DTO shaping helpers - experiments.py # ExperimentReadService - cohorts.py # cohort read/detail/stats DTO shaping - - definitions/ - models.py # define/persist commands/results if kept out of persistence - definition.py # ExperimentDefinitionService - persistence.py # ExperimentPersistenceService; materializes composition objects into definition rows - - inngest/ - client.py # Inngest singleton and cancellation config - registry.py # ALL_FUNCTIONS / serve() function list - contracts.py # child payloads and function results, or per-event modules - errors.py # Inngest/non-retryable/contract wrapper errors - functions/ # optional if we want one subdirectory below package root -``` - -The key convention is that each domain package should make its file roles obvious: - -- `models.py` for request/response/domain DTOs. -- `repository.py` only where the module owns persisted mutation/read-write behavior. -- `errors.py` for exceptions that are part of that service/domain contract. -- `service.py` or named service files for use-case orchestration. -- `utils.py` or more specific helper modules only for reusable pure helpers. - -For Inngest specifically, avoid a separate top-level `runtime/inngest_client.py` or `runtime/inngest_registry.py`; the `runtime/inngest` package should own those pieces directly. - -## High-Value Refactor Candidates - -### 0. Keep The New Public API Boundary Out Of Runtime Read Models - -The public API is now an authoring API. Do not move run/cohort/graph/read-model concepts into `ergon_core.api` to make service imports easier. - -Immediate cleanup direction: - -- Leave `Benchmark`, `Task`, `Worker`, `Criterion`, `Rubric`, and their result/context objects in the nested public API packages. -- Keep `Experiment`, `WorkerSpec`, and definition handles in `core/composition`. -- Move operational DTO shaping out of `core/api/schemas.py` and into runtime/application read models before doing large package moves. - -This is mostly a boundary rule for the plan, but it prevents the services refactor from undoing the public API simplification. - -### 1. Extract Graph Traversal Primitives - -Create a small module for containment traversal by `parent_node_id`. - -Initial consumers: - -- `task_management_service._count_non_terminal_descendants` -- `subtask_cancellation_service.cancel_orphans` -- `subtask_blocking_service.block_pending_descendants` -- `workflow_service._descendant_ids` - -This is the clearest low-risk cleanup because the duplicated query shape is visible and bounded. - -### 2. Centralize Latest Execution Selection - -Create one helper or repository method for "latest execution for node". - -It should define ordering once, probably: - -1. `attempt_number DESC` -2. `started_at DESC` - -Then update: - -- `WorkflowService.get_latest_execution` -- `TaskInspectionService._latest_output` -- `TaskInspectionService._latest_error` -- `task_management_service._latest_execution_id` -- `subtask_cancellation_service._latest_execution_id` - -### 3. Centralize Evaluation Score Aggregation - -Create one score aggregation helper that returns a named object: - -- `final_score` -- `normalized_score` -- `evaluators_count` - -Then update: - -- `WorkflowFinalizationService` -- `TelemetryRepository.refresh_run_evaluation_summary` -- `RunReadService.build_run_snapshot` -- cohort summary readers if their semantics need adjustment - -### 4. Split DTO/Schema Contracts From Service Implementations - -Normalize naming inside any new package: - -- Use `models.py` for request/response DTOs within runtime domain packages. -- Reserve `schemas.py` for API wire schemas only if the codebase keeps that distinction. -- Avoid mixing Inngest contracts with service DTOs unless the package name makes that explicit. - -### 5. Move API Snapshot Helpers Out Of API Layer - -`RunReadService` should not need to import `ergon_core.core.api.runs` helper functions. Move pure task/resource/evaluation snapshot builders to a runtime read-model module, or move `RunReadService` behind the API layer. - -### 6. Decide Whether `WorkflowGraphRepository` Is A Repository Or Domain Service - -Keep it in runtime, but move it to `runtime/graph/repository.py` and make clear that it is a domain repository for graph mutations, not a generic persistence repository. - -The repository writes audit mutations and encodes structural invariants, not just SQL CRUD. It should import `core/persistence/graph/models.py` table classes, but the operation names and invariants belong to the graph domain. - -Use this as the general persistence rule for the refactor: - -- Table definitions and session setup stay under `core/persistence`. -- Domain-specific repositories live with their domain package. -- Generic query bags such as `core/persistence/queries.py` should shrink or dissolve as their methods move to domain packages. - -### 7. Move Experiment Validation Toward Composition - -`experiment_validation_service.py` is useful as a first extraction, but it should not make `runtime/services` the permanent home for composition validation. - -Candidate target: - -- `core/composition/validation.py` - -The target file can expose either `ExperimentValidationService` or pure validation functions. The important boundary is that this logic validates authored/composed definitions before persistence; it does not participate in live runtime execution. - -### 8. Move Inngest Ownership Into The Inngest Package - -Move or plan to move: - -- `runtime/inngest_client.py` to `runtime/inngest/client.py` -- `runtime/inngest_registry.py` to `runtime/inngest/registry.py` -- `services/child_function_payloads.py` to `runtime/inngest/contracts.py` or per-function contract modules -- `services/inngest_function_results.py` to `runtime/inngest/contracts.py` or per-function result modules -- `runtime/errors/inngest_errors.py` to `runtime/inngest/errors.py` - -This should be mostly import churn, but the plan should include architecture tests so Inngest setup does not drift back into `runtime/services`. - -### 9. Add Domain-Local Error Modules - -As packages are split, add `errors.py` to each domain package. The first pass can be mechanical: - -- graph errors follow `WorkflowGraphRepository` -- delegation/task errors follow task management and inspection -- Inngest errors follow the Inngest client and functions -- evaluation-specific contract violations move with evaluation services if they are not broadly runtime-level - -The plan should not require inventing custom errors for every possible branch in one pass. It should require that new service boundary failures use domain-specific exception types, and that moved services do not keep reaching into a shared dumping-ground error module when a local `errors.py` is clearer. - -## Questions For The Refactor Plan - -1. Should `services` disappear entirely in favor of domain packages, or should it remain only for files not yet moved during direct bulk renames? -2. Should request/response models live in `models.py` beside each domain package, or in separate `contracts.py` files when they are consumed by Inngest/API boundaries? -3. Should `WorkflowGraphRepository` emit/listen to dashboard mutations directly, or should dashboard emission sit above the repository? -4. Should read-model services be considered runtime services, API services, or their own `runtime/read_models` layer? -5. Which `core/persistence/queries.py` methods should dissolve into definition/task/graph/read-model domain repositories first? -6. Should each package expose its domain errors from `__init__.py`, or should callers import directly from `package.errors` to avoid new barrel behavior? -7. Should Inngest contracts be centralized in one `runtime/inngest/contracts.py`, or colocated with each function module? -8. Should `experiment_validation_service.py` move into `core/composition`, or should all experiment definition use cases live under a new definition/application package? -9. Should `workflow_propagation_service.py` become `runtime/graph/propagation.py`, or should propagation be split between graph lifecycle primitives and task orchestration? -10. Should operational DTOs currently in `core/api/schemas.py` move before or after the services package split? -11. Should the first domain repository extraction be `runtime/tasks/repository.py` for latest execution/attempt selection, since that duplication is already concrete? - -## Recommended Next Step - -Write a refactor plan that starts with mechanical, low-risk extractions before package moves. Revised order after the public API and service moves: - -1. Lock the boundary rule in tests: public `ergon_core.api` remains authoring-only; runtime/read-model services do not import beginner-facing API modules except at authoring/evaluation adapter boundaries. -2. Lock the persistence rule in tests or architecture notes: `core/persistence` owns tables/session/storage infrastructure; domain repositories live with runtime/composition/definition packages. -3. Extract shared latest-execution and attempt-selection logic into a task-domain repository/helper. -4. Extract graph containment traversal helper. -5. Move `workflow_propagation_service.py` behind a graph lifecycle module or package, preserving the current import behavior through direct bulk updates rather than aliasing. -6. Extract evaluation score aggregation helper. -7. Move pure run snapshot helper functions and operational DTO shaping out of `core.api.runs` / `core.api.schemas`. -8. Move `experiment_validation_service.py` toward `core/composition` and keep `experiment_persistence_service.py` in a definition/composition domain package rather than under raw persistence. -9. Move Inngest client, registry, contracts, results, and errors under `runtime/inngest`. -10. Introduce domain package structure with one package at a time, starting with `runtime/graph`. -11. Dissolve `core/persistence/queries.py` incrementally as each domain repository takes over its methods. -12. Add `errors.py` to each package as services move, and replace generic service-boundary exceptions where the domain already has a clear failure type. -13. Move/rename services only after tests prove the helpers preserve behavior. - -This order reduces risk because it fixes semantic duplication before large import churn. diff --git a/docs/superpowers/plans/2026-04-29-core-component-registry-refactor.md b/docs/superpowers/plans/2026-04-29-core-component-registry-refactor.md deleted file mode 100644 index de57361b8..000000000 --- a/docs/superpowers/plans/2026-04-29-core-component-registry-refactor.md +++ /dev/null @@ -1,1229 +0,0 @@ -# Core Component Registry Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Move component registration ownership into `ergon_core` public API so core never imports `ergon_builtins`, builtins/tests explicitly register components, and experiment definition/runtime have a clear slug-to-component mental model. - -**Architecture:** Add a Pydantic-based `ComponentRegistry` and process-global `registry` under `ergon_core.api.registry`. Builtins, optional builtins capabilities, and tests contribute components through explicit registration functions. Core application/runtime code resolves persisted slugs through the core registry only. - -**Tech Stack:** Python, Pydantic models, pytest, Inngest job handlers, FastAPI startup, existing Ergon public APIs. - ---- - -## Mental Model To Preserve - -The final model should be easy to explain to students: - -1. Components are Python classes/functions: `Benchmark`, `Worker`, `Evaluator`/`Rubric`, `BaseSandboxManager`. -2. Registration says which component slugs are available in this process. -3. Experiment authoring passes concrete objects/specs into `Experiment`. -4. Persistence stores only stable identities: benchmark slug, worker slug, evaluator slug, sandbox slug, model target. -5. Runtime jobs turn those stored slugs back into Python classes/functions via `ergon_core.api.registry.registry`. - -The registry is not the main experiment authoring API. It is the catalog that validates slugs and rehydrates persisted definitions across process boundaries. - -## File Structure - -- Create `ergon_core/ergon_core/api/registry.py` - - Defines `WorkerFactory`, `ComponentRegistry`, `registry`, duplicate handling, `require_*` lookup helpers, and reset/snapshot helpers for tests. -- Modify `ergon_core/ergon_core/api/__init__.py` - - Re-export `ComponentRegistry`, `WorkerFactory`, and `registry`. -- Modify `ergon_builtins/ergon_builtins/registry_core.py` - - Replace exported dict ownership with `register_core_builtins(target=registry)`. -- Modify `ergon_builtins/ergon_builtins/registry_data.py` - - Replace exported dict ownership with `register_data_builtins(target=registry)`. -- Modify `ergon_builtins/ergon_builtins/registry_local_models.py` - - Replace exported dict ownership with `register_local_model_builtins(target=registry)` or a returned model backend mapping, depending on model backend constraints. -- Modify `ergon_builtins/ergon_builtins/registry.py` - - Becomes explicit composition function `register_builtins(target=registry)`. - - Optional: keep backwards-compatible module attributes temporarily only if necessary for existing tests, but core must not use them. -- Modify core runtime imports in: - - `ergon_core/ergon_core/core/application/jobs/worker_execute.py` - - `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` - - `ergon_core/ergon_core/core/application/jobs/persist_outputs.py` - - `ergon_core/ergon_core/core/application/jobs/sandbox_setup.py` - - `ergon_core/ergon_core/core/application/experiments/launch.py` - - `ergon_core/ergon_core/core/application/experiments/service.py` - - `ergon_core/ergon_core/core/application/workflows/service.py` - - `ergon_core/ergon_core/core/application/tasks/management.py` - - `ergon_core/ergon_core/core/domain/experiments/worker_spec.py` - - `ergon_core/ergon_core/core/rest_api/app.py` -- Move test-only smoke fixture component definitions from: - - `ergon_core/ergon_core/test_support/smoke_fixtures/**` - - into `tests/e2e/fixtures/smoke_components/**` or `tests/fixtures/smoke_components/**`. -- Modify E2E/test startup: - - `tests/e2e/conftest.py` - - current startup plugin module(s) referenced by `ERGON_STARTUP_PLUGINS` - - tests currently importing `ergon_core.test_support.smoke_fixtures` -- Modify unit tests: - - `tests/unit/registry/test_builtin_pairings.py` - - add `tests/unit/registry/test_component_registry.py` - - add/adjust core tests that assert no `ergon_core` file imports `ergon_builtins.registry`. - ---- - -### Task 1: Add Core Public Component Registry - -**Files:** -- Create: `ergon_core/ergon_core/api/registry.py` -- Modify: `ergon_core/ergon_core/api/__init__.py` -- Test: `tests/unit/registry/test_component_registry.py` - -- [ ] **Step 1: Write failing registry unit tests** - -Create `tests/unit/registry/test_component_registry.py`: - -```python -import pytest - -from ergon_core.api import Benchmark, Rubric, Worker -from ergon_core.api.registry import ComponentRegistry -from ergon_core.core.infrastructure.sandbox.manager import BaseSandboxManager - - -class ExampleWorker(Worker): - type_slug = "example-worker" - - -class ReplacementWorker(Worker): - type_slug = "example-worker" - - -class ExampleBenchmark(Benchmark): - type_slug = "example-benchmark" - - -class ExampleRubric(Rubric): - type_slug = "example-rubric" - - -class ExampleSandboxManager(BaseSandboxManager): - pass - - -def test_registers_components_by_explicit_or_type_slug() -> None: - registry = ComponentRegistry() - - registry.register_worker(ExampleWorker.type_slug, ExampleWorker) - registry.register_benchmark(ExampleBenchmark) - registry.register_evaluator(ExampleRubric) - registry.register_sandbox_manager("example-benchmark", ExampleSandboxManager) - - assert registry.require_worker("example-worker") is ExampleWorker - assert registry.require_benchmark("example-benchmark") is ExampleBenchmark - assert registry.require_evaluator("example-rubric") is ExampleRubric - assert registry.sandbox_managers["example-benchmark"] is ExampleSandboxManager - - -def test_duplicate_slug_rejects_different_object() -> None: - registry = ComponentRegistry() - registry.register_worker("example-worker", ExampleWorker) - - with pytest.raises(ValueError, match="Duplicate worker slug 'example-worker'"): - registry.register_worker("example-worker", ReplacementWorker) - - -def test_duplicate_slug_allows_idempotent_registration() -> None: - registry = ComponentRegistry() - registry.register_worker("example-worker", ExampleWorker) - registry.register_worker("example-worker", ExampleWorker) - - assert registry.require_worker("example-worker") is ExampleWorker - - -def test_unknown_slug_error_lists_registered_values() -> None: - registry = ComponentRegistry() - registry.register_worker("example-worker", ExampleWorker) - - with pytest.raises( - ValueError, - match="Unknown worker slug 'missing-worker'; registered workers: example-worker", - ): - registry.require_worker("missing-worker") -``` - -- [ ] **Step 2: Run failing registry tests** - -Run: - -```bash -pytest tests/unit/registry/test_component_registry.py -q -``` - -Expected: FAIL because `ergon_core.api.registry` does not exist. - -- [ ] **Step 3: Implement `ergon_core.api.registry`** - -Create `ergon_core/ergon_core/api/registry.py`: - -```python -"""Public process-level component registry. - -The registry maps stable slugs stored in experiment definitions back to the -Python classes/factories needed by runtime jobs. Packages such as -``ergon_builtins`` and test fixtures contribute components explicitly during -startup; ``ergon_core`` never imports those packages to discover components. -""" - -from collections.abc import Callable, Mapping -from typing import TypeVar - -from ergon_core.api.benchmark import Benchmark -from ergon_core.api.rubric import Evaluator -from ergon_core.api.worker import Worker -from ergon_core.core.infrastructure.sandbox.manager import BaseSandboxManager -from pydantic import BaseModel, ConfigDict, Field - -WorkerFactory = Callable[..., Worker] -T = TypeVar("T") - - -class ComponentRegistry(BaseModel): - """Catalog of component types available in the current Python process.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - workers: dict[str, WorkerFactory] = Field(default_factory=dict) - benchmarks: dict[str, type[Benchmark]] = Field(default_factory=dict) - evaluators: dict[str, type[Evaluator]] = Field(default_factory=dict) - sandbox_managers: dict[str, type[BaseSandboxManager]] = Field(default_factory=dict) - - def register_worker(self, slug: str, factory: WorkerFactory) -> None: - self._register(self.workers, "worker", slug, factory) - - def register_benchmark(self, benchmark_cls: type[Benchmark], slug: str | None = None) -> None: - self._register(self.benchmarks, "benchmark", slug or benchmark_cls.type_slug, benchmark_cls) - - def register_evaluator(self, evaluator_cls: type[Evaluator], slug: str | None = None) -> None: - self._register(self.evaluators, "evaluator", slug or evaluator_cls.type_slug, evaluator_cls) - - def register_sandbox_manager( - self, - slug: str, - manager_cls: type[BaseSandboxManager], - ) -> None: - self._register(self.sandbox_managers, "sandbox manager", slug, manager_cls) - - def require_worker(self, slug: str) -> WorkerFactory: - return self._require(self.workers, "worker", slug) - - def require_benchmark(self, slug: str) -> type[Benchmark]: - return self._require(self.benchmarks, "benchmark", slug) - - def require_evaluator(self, slug: str) -> type[Evaluator]: - return self._require(self.evaluators, "evaluator", slug) - - def _register(self, target: dict[str, T], kind: str, slug: str, value: T) -> None: - existing = target.get(slug) - if existing is not None and existing is not value: - raise ValueError(f"Duplicate {kind} slug {slug!r}") - target[slug] = value - - def _require(self, target: Mapping[str, T], kind: str, slug: str) -> T: - try: - return target[slug] - except KeyError: - known = ", ".join(sorted(target)) or "<none>" - raise ValueError( - f"Unknown {kind} slug {slug!r}; registered {kind}s: {known}" - ) from None - - -registry = ComponentRegistry() -``` - -- [ ] **Step 4: Re-export the registry from public API** - -Modify `ergon_core/ergon_core/api/__init__.py`: - -```python -"""Beginner-facing Ergon authoring API surface.""" - -from ergon_core.api.benchmark import Benchmark, BenchmarkRequirements, EmptyTaskPayload, Task -from ergon_core.api.criterion import ( - Criterion, - CriterionContext, - CriterionEvidence, - CriterionOutcome, - EvidenceMessage, - ScoreScale, -) -from ergon_core.api.errors import CriterionCheckError -from ergon_core.api.registry import ComponentRegistry, WorkerFactory, registry -from ergon_core.api.rubric import Rubric, TaskEvaluationResult -from ergon_core.api.worker import Worker, WorkerContext, WorkerOutput, WorkerStreamItem - -__all__ = [ - "Benchmark", - "BenchmarkRequirements", - "ComponentRegistry", - "Criterion", - "CriterionCheckError", - "CriterionContext", - "CriterionEvidence", - "CriterionOutcome", - "EmptyTaskPayload", - "EvidenceMessage", - "Rubric", - "ScoreScale", - "Task", - "TaskEvaluationResult", - "Worker", - "WorkerContext", - "WorkerFactory", - "WorkerOutput", - "WorkerStreamItem", - "registry", -] -``` - -- [ ] **Step 5: Run registry tests** - -Run: - -```bash -pytest tests/unit/registry/test_component_registry.py -q -``` - -Expected: PASS. - ---- - -### Task 2: Convert Builtins Registry To Explicit Registration - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/registry_core.py` -- Modify: `ergon_builtins/ergon_builtins/registry_data.py` -- Modify: `ergon_builtins/ergon_builtins/registry_local_models.py` -- Modify: `ergon_builtins/ergon_builtins/registry.py` -- Test: `tests/unit/registry/test_builtin_pairings.py` - -- [ ] **Step 1: Update builtin pairing tests to register into a fresh registry** - -Modify `tests/unit/registry/test_builtin_pairings.py` so tests no longer import dicts from `ergon_builtins.registry_core` or `ergon_builtins.registry`. Use a fresh `ComponentRegistry`: - -```python -"""Documented built-in benchmark pairings are explicit and registered.""" - -import pytest - -from ergon_core.api.registry import ComponentRegistry - - -CORE_PAIRINGS = [ - { - "benchmark": "minif2f", - "worker": "minif2f-react", - "evaluator": "minif2f-rubric", - "sandbox": "minif2f", - "extras": ("none",), - }, - { - "benchmark": "swebench-verified", - "worker": "swebench-react", - "evaluator": "swebench-rubric", - "sandbox": "swebench-verified", - "extras": ("ergon-builtins[data]",), - }, -] - -DATA_PAIRINGS = [ - { - "benchmark": "gdpeval", - "worker": "gdpeval-react", - "evaluator": "gdpeval-staged-rubric", - "sandbox": "gdpeval", - "extras": ("ergon-builtins[data]",), - }, - { - "benchmark": "researchrubrics", - "worker": "researchrubrics-researcher", - "evaluator": "researchrubrics-rubric", - "sandbox": "researchrubrics", - "extras": ("ergon-builtins[data]",), - }, - { - "benchmark": "researchrubrics-vanilla", - "worker": "researchrubrics-researcher", - "evaluator": "researchrubrics-rubric", - "sandbox": "researchrubrics-vanilla", - "extras": ("ergon-builtins[data]",), - }, -] - - -@pytest.mark.parametrize("pairing", CORE_PAIRINGS) -def test_core_pairings_reference_registered_slugs(pairing: dict[str, object]) -> None: - from ergon_builtins.registry_core import register_core_builtins - - registry = ComponentRegistry() - register_core_builtins(registry) - - _assert_pairing(pairing, registry) - - -@pytest.mark.parametrize("pairing", DATA_PAIRINGS) -def test_data_pairings_reference_registered_slugs(pairing: dict[str, object]) -> None: - pytest.importorskip("datasets", reason="ergon-builtins[data] not installed") - from ergon_builtins.registry import register_builtins - - registry = ComponentRegistry() - register_builtins(registry) - - _assert_pairing(pairing, registry) - - -def _assert_pairing(pairing: dict[str, object], registry: ComponentRegistry) -> None: - benchmark = pairing["benchmark"] - worker = pairing["worker"] - evaluator = pairing["evaluator"] - sandbox = pairing["sandbox"] - extras = pairing["extras"] - - assert benchmark in registry.benchmarks - assert worker in registry.workers - assert evaluator in registry.evaluators - assert sandbox in registry.sandbox_managers - assert isinstance(extras, tuple) - assert extras -``` - -- [ ] **Step 2: Run updated builtin pairing tests** - -Run: - -```bash -pytest tests/unit/registry/test_builtin_pairings.py -q -``` - -Expected: FAIL because the `register_*` functions do not exist. - -- [ ] **Step 3: Replace `registry_core.py` dicts with `register_core_builtins`** - -Modify `ergon_builtins/ergon_builtins/registry_core.py` to keep imports but replace exported dicts with: - -```python -from ergon_core.api.registry import ComponentRegistry, registry - - -def register_core_builtins(target: ComponentRegistry = registry) -> None: - """Register builtins that have no optional dependency extras.""" - - target.register_worker("training-stub", TrainingStubWorker) - target.register_worker("minif2f-react", minif2f_react) - target.register_worker("swebench-react", swebench_react) - - target.register_benchmark(MiniF2FBenchmark) - target.register_benchmark(SweBenchVerifiedBenchmark) - - target.register_evaluator(StagedRubric) - target.register_evaluator(StagedRubric, slug="gdpeval-staged-rubric") - target.register_evaluator(MiniF2FRubric) - target.register_evaluator(SWEBenchRubric) - - target.register_sandbox_manager("gdpeval", GDPEvalSandboxManager) - target.register_sandbox_manager("minif2f", MiniF2FSandboxManager) - target.register_sandbox_manager("swebench-verified", SWEBenchSandboxManager) -``` - -Do not remove `SANDBOX_TEMPLATES` yet unless all uses are known. Leave it as a plain exported mapping: - -```python -SANDBOX_TEMPLATES: dict[str, Path] = { - "minif2f": Path(__file__).parent / "benchmarks/minif2f/sandbox", - "swebench-verified": Path(__file__).parent / "benchmarks/swebench_verified/sandbox", -} -``` - -- [ ] **Step 4: Replace `registry_data.py` dicts with `register_data_builtins`** - -Modify `ergon_builtins/ergon_builtins/registry_data.py`: - -```python -from ergon_core.api.registry import ComponentRegistry, registry - - -def register_data_builtins(target: ComponentRegistry = registry) -> None: - """Register builtins that require the [data] optional dependency group.""" - - target.register_benchmark(GDPEvalBenchmark) - target.register_benchmark(ResearchRubricsBenchmark) - target.register_benchmark(ResearchRubricsVanillaBenchmark) - - target.register_evaluator(ResearchRubricsRubric, slug="research-rubric") - target.register_evaluator(ResearchRubricsRubric) - - target.register_worker("gdpeval-react", gdpeval_react) - target.register_worker(ResearchRubricsResearcherWorker.type_slug, ResearchRubricsResearcherWorker) - target.register_worker( - ResearchRubricsWorkflowCliReActWorker.type_slug, - ResearchRubricsWorkflowCliReActWorker, - ) - - target.register_sandbox_manager("researchrubrics", ResearchRubricsSandboxManager) - target.register_sandbox_manager("researchrubrics-vanilla", ResearchRubricsSandboxManager) -``` - -If `GDPEvalBenchmark` requires a sandbox manager but the current data registry does not register one, decide during implementation whether to add: - -```python -target.register_sandbox_manager("gdpeval", GDPEvalSandboxManager) -``` - -only if `GDPEvalSandboxManager` can be imported from the data module without creating an optional dependency problem. Otherwise keep the current core registration for `"gdpeval"`. - -- [ ] **Step 5: Convert top-level `ergon_builtins.registry` to an explicit registration function** - -Modify `ergon_builtins/ergon_builtins/registry.py`: - -```python -"""Register built-in Ergon components into the core public registry.""" - -import structlog - -from ergon_core.api.registry import ComponentRegistry, registry -from ergon_builtins.models.resolution import register_model_backend -from ergon_builtins.registry_core import register_core_builtins - -log = structlog.get_logger() - - -def register_builtins(target: ComponentRegistry = registry) -> None: - """Register builtins available in the current environment. - - This is intentionally explicit: importing ``ergon_core`` does not import - builtins, and importing builtins does not mutate core unless startup calls - this function. - """ - - register_core_builtins(target) - _register_local_model_builtins() - _register_data_builtins(target) - - -def _register_local_model_builtins() -> None: - try: - from ergon_builtins.registry_local_models import register_local_model_builtins - except ImportError: - log.info("ergon-builtins[local-models] not installed; local transformers inference unavailable") - return - - register_local_model_builtins() - - -def _register_data_builtins(target: ComponentRegistry) -> None: - try: - from ergon_builtins.registry_data import register_data_builtins - except ImportError: - log.info( - "ergon-builtins[data] not installed; gdpeval and researchrubrics benchmarks unavailable" - ) - return - - register_data_builtins(target) - - -INSTALL_HINTS: dict[str, str] = { - "transformers": "pip install 'ergon-builtins[local-models]'", - "gdpeval": "pip install 'ergon-builtins[data]'", - "researchrubrics": "pip install 'ergon-builtins[data]'", - "research-rubric": "pip install 'ergon-builtins[data]'", -} -``` - -- [ ] **Step 6: Convert local model registry** - -Modify `ergon_builtins/ergon_builtins/registry_local_models.py`: - -```python -"""Components that require the [local-models] capability.""" - -from ergon_builtins.models.resolution import register_model_backend -from ergon_builtins.models.transformers_backend import resolve_transformers - - -def register_local_model_builtins() -> None: - register_model_backend("transformers", resolve_transformers) -``` - -Keep core model backends registered wherever they are currently registered. If `registry_core.py` currently owns `"vllm"`, `"openai"`, `"anthropic"`, `"google"`, `"openrouter"`, and `"openai-responses"`, move that into a helper in `ergon_builtins.registry_core` called by `register_core_builtins()`: - -```python -def _register_core_model_backends() -> None: - register_model_backend("vllm", resolve_vllm) - register_model_backend("openai", resolve_cloud) - register_model_backend("anthropic", resolve_cloud) - register_model_backend("google", resolve_cloud) - register_model_backend("openrouter", resolve_openrouter) - register_model_backend("openai-responses", resolve_openrouter_responses) -``` - -- [ ] **Step 7: Run builtin registry tests** - -Run: - -```bash -pytest tests/unit/registry/test_builtin_pairings.py tests/unit/registry/test_component_registry.py -q -``` - -Expected: PASS. - ---- - -### Task 3: Add Startup Registration For Runtime Processes - -**Files:** -- Modify: runtime startup location that is imported by CLI/API before defining/running experiments. -- Likely modify: `ergon_core/ergon_core/core/rest_api/app.py` -- Search and modify: CLI entrypoints under `ergon_cli/**` -- Test: existing CLI/API tests that define experiments. - -- [ ] **Step 1: Locate CLI and startup entrypoints** - -Run: - -```bash -rg "experiment define|ERGON_STARTUP_PLUGINS|startup_plugins|register_builtins|def main|typer|click" ergon_cli ergon_core tests -n -``` - -Expected: identify the CLI initialization path and FastAPI lifespan path. - -- [ ] **Step 2: Add explicit builtin registration during API startup** - -In `ergon_core/ergon_core/core/rest_api/app.py`, import only the core registry at module or function scope. In the lifespan before sandbox event sink wiring, call builtins registration as a startup plugin decision: - -```python -from ergon_core.api.registry import registry - - -def _register_default_components() -> None: - from ergon_builtins.registry import register_builtins - - register_builtins(registry) -``` - -Then call `_register_default_components()` early in `lifespan`, before runtime services need sandbox managers. - -Important: this is acceptable at app startup because the application chooses to install builtins. Core library modules still must not import `ergon_builtins.registry`. - -- [ ] **Step 3: Update sandbox event sink wiring to use core registry** - -Replace: - -```python -from ergon_builtins.registry import SANDBOX_MANAGERS -... -for manager_cls in SANDBOX_MANAGERS.values(): - manager_cls.set_event_sink(sink) -logger.info("sandbox event sink wired on %d manager subclass(es)", 1 + len(SANDBOX_MANAGERS)) -``` - -with: - -```python -from ergon_core.api.registry import registry -... -for manager_cls in registry.sandbox_managers.values(): - manager_cls.set_event_sink(sink) -logger.info( - "sandbox event sink wired on %d manager subclass(es)", - 1 + len(registry.sandbox_managers), -) -``` - -- [ ] **Step 4: Add explicit builtin registration during CLI startup** - -In the CLI root entrypoint, add a small registration helper and call it before commands that define or run experiments: - -```python -from ergon_core.api.registry import registry - - -def register_default_components() -> None: - from ergon_builtins.registry import register_builtins - - register_builtins(registry) -``` - -Do not scatter this call through individual commands if there is a central CLI startup hook. If no central hook exists, call it at the top of experiment define/run command handlers and note the duplication for later cleanup. - -- [ ] **Step 5: Run fast CLI/API tests affected by startup** - -Run the narrowest available tests after locating them: - -```bash -pytest tests/unit tests/integration -q -k "experiment or registry or cli" -``` - -Expected: PASS or unrelated pre-existing failures documented before continuing. - ---- - -### Task 4: Replace Core Imports Of Builtins Registry - -**Files:** -- Modify listed core files containing `from ergon_builtins.registry import ...` -- Test: add import-boundary test under `tests/unit/registry/test_core_registry_boundary.py` - -- [ ] **Step 1: Add boundary test that core does not import builtins registry** - -Create `tests/unit/registry/test_core_registry_boundary.py`: - -```python -from pathlib import Path - - -def test_ergon_core_does_not_import_builtins_registry() -> None: - root = Path("ergon_core/ergon_core") - offenders: list[str] = [] - - for path in root.rglob("*.py"): - text = path.read_text() - if "ergon_builtins.registry" in text: - offenders.append(str(path)) - - assert offenders == [] -``` - -- [ ] **Step 2: Run boundary test and verify it fails** - -Run: - -```bash -pytest tests/unit/registry/test_core_registry_boundary.py -q -``` - -Expected: FAIL listing the current core files that import `ergon_builtins.registry`. - -- [ ] **Step 3: Update worker execution lookup** - -Modify `ergon_core/ergon_core/core/application/jobs/worker_execute.py`: - -```python -from ergon_core.api.registry import registry -``` - -Inside `run_worker_execute_job`, remove: - -```python -from ergon_builtins.registry import BENCHMARKS, WORKERS -``` - -Replace worker lookup: - -```python -worker_cls = registry.workers.get(payload.worker_type) -``` - -Replace benchmark lookup: - -```python -benchmark_cls = registry.benchmarks.get(payload.benchmark_type) -``` - -Keep existing `RegistryLookupError` behavior for workers by checking `None` as today. - -- [ ] **Step 4: Update evaluation job lookup** - -Modify `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py`: - -```python -from ergon_core.api.registry import registry -``` - -Remove the builtins import inside `run_evaluate_task_run_job`. Replace: - -```python -evaluator_cls = EVALUATORS.get(evaluator_type) -manager_cls = SANDBOX_MANAGERS.get(benchmark_type, DefaultSandboxManager) -benchmark_cls = BENCHMARKS.get(benchmark_type) if benchmark_type is not None else None -``` - -with: - -```python -evaluator_cls = registry.evaluators.get(evaluator_type) -manager_cls = ( - registry.sandbox_managers.get(benchmark_type, DefaultSandboxManager) - if benchmark_type is not None - else DefaultSandboxManager -) -benchmark_cls = registry.benchmarks.get(benchmark_type) if benchmark_type is not None else None -``` - -- [ ] **Step 5: Update sandbox and output jobs** - -Modify `ergon_core/ergon_core/core/application/jobs/persist_outputs.py` and `ergon_core/ergon_core/core/application/jobs/sandbox_setup.py`: - -```python -from ergon_core.api.registry import registry -``` - -Replace: - -```python -manager_cls = SANDBOX_MANAGERS.get(..., DefaultSandboxManager) -``` - -with: - -```python -manager_cls = registry.sandbox_managers.get(..., DefaultSandboxManager) -``` - -- [ ] **Step 6: Update experiment launch and define services** - -Modify `ergon_core/ergon_core/core/application/experiments/launch.py`: - -```python -from ergon_core.api.registry import registry -``` - -Replace evaluator and benchmark lookups with: - -```python -evaluator_cls = registry.require_evaluator(evaluator_slug) -source = registry.require_benchmark(benchmark_slug)() -``` - -Modify `ergon_core/ergon_core/core/application/experiments/service.py` so `_benchmark_cls` caches `registry.benchmarks`, not builtins dicts: - -```python -from ergon_core.api.registry import registry -... -if self._benchmarks is None: - self._benchmarks = registry.benchmarks -return self._benchmarks[benchmark_slug] -``` - -- [ ] **Step 7: Update workflow/task mutation validation** - -Modify `ergon_core/ergon_core/core/application/workflows/service.py`, `ergon_core/ergon_core/core/application/tasks/management.py`, and `ergon_core/ergon_core/core/domain/experiments/worker_spec.py`: - -```python -from ergon_core.api.registry import registry -``` - -Replace membership checks: - -```python -if slug not in WORKERS: -``` - -with: - -```python -if slug not in registry.workers: -``` - -For error messages listing known workers, use: - -```python -known = ", ".join(sorted(registry.workers)) -``` - -- [ ] **Step 8: Run boundary and affected unit tests** - -Run: - -```bash -pytest tests/unit/registry/test_core_registry_boundary.py tests/unit/registry/test_component_registry.py tests/unit/registry/test_builtin_pairings.py -q -``` - -Expected: PASS. - ---- - -### Task 5: Move Smoke Test Helpers Out Of Core - -**Files:** -- Move from: `ergon_core/ergon_core/test_support/smoke_fixtures/**` -- Move to: `tests/fixtures/smoke_components/**` -- Modify: `tests/e2e/conftest.py` -- Modify: startup plugin referenced by E2E environment -- Test: E2E smoke tests and import-boundary tests. - -- [ ] **Step 1: Add a test proving smoke fixtures do not live under core** - -Create or extend `tests/unit/registry/test_core_registry_boundary.py`: - -```python -def test_core_package_has_no_smoke_fixture_registration_package() -> None: - assert not Path("ergon_core/ergon_core/test_support/smoke_fixtures").exists() -``` - -Expected initially: FAIL. - -- [ ] **Step 2: Create tests fixture package** - -Create: - -```text -tests/fixtures/smoke_components/ -tests/fixtures/smoke_components/__init__.py -tests/fixtures/smoke_components/benchmarks.py -tests/fixtures/smoke_components/sandbox.py -tests/fixtures/smoke_components/criteria/ -tests/fixtures/smoke_components/workers/ -``` - -Move files from `ergon_core/ergon_core/test_support/smoke_fixtures/**` into the new package, preserving internal folder shape where possible. - -- [ ] **Step 3: Update imports in moved files** - -Search: - -```bash -rg "ergon_core\\.test_support\\.smoke_fixtures|test_support\\.smoke_fixtures" tests/fixtures/smoke_components tests ergon_core -n -``` - -Replace imports such as: - -```python -from ergon_core.test_support.smoke_fixtures.workers.swebench_smoke import SweBenchSmokeWorker -``` - -with: - -```python -from tests.fixtures.smoke_components.workers.swebench_smoke import SweBenchSmokeWorker -``` - -- [ ] **Step 4: Replace smoke registration function** - -In `tests/fixtures/smoke_components/__init__.py`, define: - -```python -"""Test-only smoke component registration.""" - -import os - -from ergon_core.api.registry import ComponentRegistry, registry -from tests.fixtures.smoke_components.benchmarks import ( - MiniF2FSmokeBenchmark, - ResearchRubricsSmokeBenchmark, - SweBenchSmokeBenchmark, -) -from tests.fixtures.smoke_components.criteria.smoke_rubrics import ( - MiniF2FSmokeRubric, - ResearchRubricsSmokeRubric, - SweBenchSmokeRubric, -) -from tests.fixtures.smoke_components.criteria.timing import SmokePostRootTimingRubric -from tests.fixtures.smoke_components.sandbox import SmokeSandboxManager -from tests.fixtures.smoke_components.workers.minif2f_smoke import ( - MiniF2FFailingLeafWorker, - MiniF2FRecursiveSmokeWorker, - MiniF2FSadPathSmokeWorker, - MiniF2FSmokeLeafWorker, - MiniF2FSmokeWorker, -) -from tests.fixtures.smoke_components.workers.researchrubrics_smoke import ( - ResearchRubricsFailingLeafWorker, - ResearchRubricsRecursiveSmokeWorker, - ResearchRubricsSadPathSmokeWorker, - ResearchRubricsSmokeLeafWorker, - ResearchRubricsSmokeWorker, -) -from tests.fixtures.smoke_components.workers.swebench_smoke import ( - SweBenchFailingLeafWorker, - SweBenchRecursiveSmokeWorker, - SweBenchSadPathSmokeWorker, - SweBenchSmokeLeafWorker, - SweBenchSmokeWorker, -) - - -def register_smoke_components(target: ComponentRegistry = registry) -> None: - """Register test-only smoke components into the supplied registry.""" - - if os.environ.get("ENABLE_TEST_HARNESS") == "1": - target.register_benchmark(ResearchRubricsSmokeBenchmark) - target.register_benchmark(MiniF2FSmokeBenchmark) - target.register_benchmark(SweBenchSmokeBenchmark) - target.register_sandbox_manager(ResearchRubricsSmokeBenchmark.type_slug, SmokeSandboxManager) - target.register_sandbox_manager(MiniF2FSmokeBenchmark.type_slug, SmokeSandboxManager) - target.register_sandbox_manager(SweBenchSmokeBenchmark.type_slug, SmokeSandboxManager) - - target.register_worker(ResearchRubricsSmokeWorker.type_slug, ResearchRubricsSmokeWorker) - target.register_worker(ResearchRubricsSmokeLeafWorker.type_slug, ResearchRubricsSmokeLeafWorker) - target.register_worker( - ResearchRubricsRecursiveSmokeWorker.type_slug, - ResearchRubricsRecursiveSmokeWorker, - ) - target.register_evaluator(ResearchRubricsSmokeRubric) - target.register_evaluator(SmokePostRootTimingRubric) - target.register_worker(ResearchRubricsSadPathSmokeWorker.type_slug, ResearchRubricsSadPathSmokeWorker) - target.register_worker(ResearchRubricsFailingLeafWorker.type_slug, ResearchRubricsFailingLeafWorker) - - target.register_worker(MiniF2FSmokeWorker.type_slug, MiniF2FSmokeWorker) - target.register_worker(MiniF2FSmokeLeafWorker.type_slug, MiniF2FSmokeLeafWorker) - target.register_worker(MiniF2FRecursiveSmokeWorker.type_slug, MiniF2FRecursiveSmokeWorker) - target.register_worker(MiniF2FSadPathSmokeWorker.type_slug, MiniF2FSadPathSmokeWorker) - target.register_worker(MiniF2FFailingLeafWorker.type_slug, MiniF2FFailingLeafWorker) - target.register_evaluator(MiniF2FSmokeRubric) - - target.register_worker(SweBenchSmokeWorker.type_slug, SweBenchSmokeWorker) - target.register_worker(SweBenchSmokeLeafWorker.type_slug, SweBenchSmokeLeafWorker) - target.register_worker(SweBenchRecursiveSmokeWorker.type_slug, SweBenchRecursiveSmokeWorker) - target.register_worker(SweBenchSadPathSmokeWorker.type_slug, SweBenchSadPathSmokeWorker) - target.register_worker(SweBenchFailingLeafWorker.type_slug, SweBenchFailingLeafWorker) - target.register_evaluator(SweBenchSmokeRubric) -``` - -- [ ] **Step 5: Update E2E startup plugin** - -Locate the startup plugin currently importing `ergon_core.test_support.smoke_fixtures`. Replace it with: - -```python -from tests.fixtures.smoke_components import register_smoke_components - - -def register() -> None: - register_smoke_components() -``` - -If the startup plugin loader expects a different function name, preserve that function name and call `register_smoke_components()` inside it. - -- [ ] **Step 6: Remove old core smoke fixture package** - -Delete `ergon_core/ergon_core/test_support/smoke_fixtures/**` only after all imports have been updated. - -- [ ] **Step 7: Run smoke fixture import and boundary tests** - -Run: - -```bash -pytest tests/unit/registry/test_core_registry_boundary.py -q -pytest tests/e2e/test_swebench_smoke.py --collect-only -q -``` - -Expected: PASS. - ---- - -### Task 6: Update E2E And Integration Tests To Use Explicit Registry Setup - -**Files:** -- Modify: `tests/e2e/conftest.py` -- Modify: E2E startup plugin module(s) -- Modify: tests currently using `ergon_builtins.registry` dict mutation -- Test: E2E smoke suite. - -- [ ] **Step 1: Search for remaining dict mutation against old registries** - -Run: - -```bash -rg "BENCHMARKS|WORKERS|EVALUATORS|SANDBOX_MANAGERS|ergon_builtins\\.registry|register_smoke_fixtures|smoke_fixtures" tests ergon_core ergon_builtins -n -``` - -Expected: remaining references are either in `ergon_builtins` registration implementation, tests asserting pairings via `ComponentRegistry`, or places to update. - -- [ ] **Step 2: Update tests that temporarily patch registries** - -Replace code like: - -```python -from ergon_builtins.registry import BENCHMARKS, SANDBOX_MANAGERS - -original_benchmarks = {slug: BENCHMARKS[slug] for slug in slugs} -BENCHMARKS[slug] = SmokeBenchmark -``` - -with fresh registry injection if the code under test accepts a registry, or explicit registration into global `registry` if the code under test is runtime-like: - -```python -from ergon_core.api.registry import registry - -registry.register_benchmark(SmokeBenchmark) -registry.register_sandbox_manager(SmokeBenchmark.type_slug, SmokeSandboxManager) -``` - -If a test mutates global `registry`, restore state in `finally`: - -```python -original_benchmarks = dict(registry.benchmarks) -original_sandbox_managers = dict(registry.sandbox_managers) -try: - registry.register_benchmark(SmokeBenchmark) - registry.register_sandbox_manager(SmokeBenchmark.type_slug, SmokeSandboxManager) - ... -finally: - registry.benchmarks.clear() - registry.benchmarks.update(original_benchmarks) - registry.sandbox_managers.clear() - registry.sandbox_managers.update(original_sandbox_managers) -``` - -- [ ] **Step 3: Keep host-side E2E black-box behavior** - -`tests/e2e/conftest.py` currently documents that smoke fixture registration lives in the API container via `ERGON_STARTUP_PLUGINS`. Keep that mental model. Update the note to reference `tests.fixtures.smoke_components.register_smoke_components`, not `ergon_core.test_support`. - -- [ ] **Step 4: Run E2E smoke collect and selected tests** - -Run: - -```bash -pytest tests/e2e/test_swebench_smoke.py --collect-only -q -``` - -Then, if the E2E stack is running: - -```bash -pytest tests/e2e/test_swebench_smoke.py -q -``` - -Expected: collect passes. Runtime E2E passes when required infrastructure is available. - ---- - -### Task 7: Improve Experiment Validation Error Messages - -**Files:** -- Modify: `ergon_core/ergon_core/core/domain/experiments/worker_spec.py` -- Modify: `ergon_core/ergon_core/core/domain/experiments/validation.py` -- Test: existing or new experiment validation unit tests. - -- [ ] **Step 1: Add tests for clear missing component errors** - -Create or update `tests/unit/experiments/test_experiment_validation.py` with tests covering: - -```python -import pytest - -from ergon_core.core.domain.experiments import WorkerSpec - - -def test_worker_spec_unknown_worker_lists_registered_workers() -> None: - spec = WorkerSpec(worker_slug="missing-worker", name="primary", model="stub:constant") - - with pytest.raises(ValueError, match="Unknown worker slug 'missing-worker'"): - spec.validate_spec() -``` - -If the registry is process-global and other tests register workers, isolate this test by snapshotting/restoring `registry.workers`. - -- [ ] **Step 2: Update `WorkerSpec.validate_spec`** - -Use `ergon_core.api.registry.registry`: - -```python -from ergon_core.api.registry import registry - - -def validate_spec(self) -> None: - """Check that ``worker_slug`` refers to a known registry entry.""" - if self.worker_slug not in registry.workers: - known = ", ".join(sorted(registry.workers)) or "<none>" - raise ValueError( - f"Unknown worker slug {self.worker_slug!r}; registered workers: {known}" - ) - if not self.name: - raise ValueError("WorkerSpec.name must be a non-empty string") - if not self.model: - raise ValueError("WorkerSpec.model must be a non-empty string") -``` - -- [ ] **Step 3: Add benchmark pairing metadata only if needed** - -Do not add a large new abstraction in this refactor unless tests show a concrete gap. If student-facing validation needs “benchmark X expects worker Y,” add a small optional method to benchmark classes later: - -```python -def recommended_worker_slugs(self) -> tuple[str, ...]: - return () -``` - -For this plan, keep pairing validation in tests and docs unless an existing runtime path requires it. - -- [ ] **Step 4: Run experiment validation tests** - -Run: - -```bash -pytest tests/unit -q -k "validation or WorkerSpec or registry" -``` - -Expected: PASS. - ---- - -### Task 8: Final Search, Lint, And Regression Verification - -**Files:** -- No planned source files beyond cleanup. - -- [ ] **Step 1: Verify no core imports of builtins registry remain** - -Run: - -```bash -rg "ergon_builtins\\.registry" ergon_core/ergon_core -n -``` - -Expected: no matches. - -- [ ] **Step 2: Verify old smoke fixture location is gone** - -Run: - -```bash -test ! -d ergon_core/ergon_core/test_support/smoke_fixtures -``` - -Expected: exit code 0. - -- [ ] **Step 3: Verify remaining registry references are intentional** - -Run: - -```bash -rg "BENCHMARKS|WORKERS|EVALUATORS|SANDBOX_MANAGERS" ergon_core ergon_builtins tests -n -``` - -Expected: no core runtime imports from `ergon_builtins.registry`; remaining uppercase dict names should either be deleted or constrained to docs/backwards compatibility tests. - -- [ ] **Step 4: Run focused tests** - -Run: - -```bash -pytest tests/unit/registry -q -pytest tests/unit -q -k "experiment or workflow or task or sandbox or registry" -``` - -Expected: PASS. - -- [ ] **Step 5: Run E2E collect** - -Run: - -```bash -pytest tests/e2e --collect-only -q -``` - -Expected: PASS. - -- [ ] **Step 6: Run full available test suite** - -Run: - -```bash -pytest tests/unit -q -``` - -Expected: PASS. If E2E infrastructure is available, also run: - -```bash -pytest tests/e2e -q -``` - -Expected: PASS or documented infrastructure failures unrelated to this refactor. - ---- - -## Self-Review - -- Spec coverage: The plan covers core registry creation, builtins update, removal of `BENCHMARKS`/`WORKERS`/`EVALUATORS`/`SANDBOX_MANAGERS` imports from core, moving smoke test helpers out of core, and updating integration/E2E registration flow. -- Placeholder scan: No unfinished placeholder markers remain. The only conditional areas are explicitly bounded implementation checks where the current codebase must be searched first, such as CLI entrypoint location and optional data dependency import constraints. -- Type consistency: `ComponentRegistry`, `WorkerFactory`, `registry`, and `register_*` function names are used consistently across tasks. diff --git a/docs/superpowers/plans/2026-04-29-dashboard-boundary-refactor.md b/docs/superpowers/plans/2026-04-29-dashboard-boundary-refactor.md deleted file mode 100644 index 0fbaad83c..000000000 --- a/docs/superpowers/plans/2026-04-29-dashboard-boundary-refactor.md +++ /dev/null @@ -1,1224 +0,0 @@ -# Dashboard Boundary Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Pay down dashboard frontend debt by making wire contracts, server harness/backend reads, dashboard store state, and UI state explicit and testable. - -**Architecture:** Keep visual components mostly intact. Refactor the dashboard at its boundaries: REST/socket payloads enter through parsers, convert once into dashboard domain state, update through shared reducers, and serialize back through a single wire serializer. Server components and API route handlers should use one server-side data adapter so harness fallback behavior is consistent. - -**Tech Stack:** Next.js 14 App Router, TypeScript, Zod, Socket.io, Playwright, Node test runner via `tsx --test`, existing generated OpenAPI/event contracts. - ---- - -## Non-Goals - -- Do not redesign the dashboard UI. -- Do not replace Socket.io. -- Do not remove the dashboard test harness. -- Do not reintroduce deleted backend env-var gates. -- Do not preserve compatibility with stale dashboard fixture payloads; update fixtures to the current generated contract. - -## Clean Result Folder Layout - -The final dashboard boundary code should be organized as follows: - -```text -ergon-dashboard/ - src/ - app/ - api/ - cohorts/ - route.ts # thin route handler, delegates to server-data adapter - [cohortId]/route.ts # thin route handler, delegates to server-data adapter - danger/ - test-harness/dashboard/ # dashboard-only harness routes; Next cannot route __danger__ folders - experiments/[experimentId]/page.tsx # uses server-data adapter, no bespoke harness fallback - run/[runId]/page.tsx # uses server-data adapter - cohorts/[cohortId]/page.tsx # uses server-data adapter - cohorts/[cohortId]/runs/[runId]/page.tsx # uses server-data adapter - lib/ - contracts/ - rest.ts # generated REST parser wrappers only - events.ts # generated/socket parser wrappers only - contextEvents.ts # UI/domain context event payload types - run-state/ - domain.ts # dashboard domain/store types, if moved out of lib/types.ts - hydrate.ts # wire RunSnapshot -> WorkflowRunState - serialize.ts # WorkflowRunState -> wire RunSnapshot - contextEvents.ts # context part/event_type conversions - reducers.ts # pure run-state reducers shared by store + client hook - metrics.ts # run/task metric recalculation - index.ts # public run-state boundary exports - server-data/ - fetchBackend.ts # fetchErgonApi wrapper + status/timeout policy - harness.ts # read-only harness access helpers - runs.ts # loadRunSnapshot/loadRunPageData - cohorts.ts # loadCohortDetail/loadCohortList/updateCohortStatus - experiments.ts # loadExperimentDetail/loadExperimentList - responses.ts # NextResponse helpers for API routes - testing/ - dashboardHarness.ts # in-memory harness implementation; server-only - state/ - store.ts # thin singleton store using run-state reducers - socket/ - server.ts # socket rooms/broadcasts only - hooks/ - useRunState.ts # client socket subscription + reducer dispatch - components/ - run/ - RunWorkspacePage.tsx # orchestration shell only - useRunDisplayState.ts # extracted after boundary work - useRunKeyboardShortcuts.ts # extracted after boundary work - useRunPanelLayout.ts # extracted after boundary work - tests/ - contracts/ - rest.contract.test.ts - run-state-roundtrip.contract.test.ts - context-events.contract.test.ts - server-data.contract.test.ts - e2e/ - ... -``` - -Do not create this whole tree in one mechanical move. Create files only when a task needs them. The important ownership rule is: - -```text -generated/rest/events -> contract wrappers -> run-state/server-data boundary -> UI/store/hooks -``` - -No component should parse backend DTOs directly. No route/page should hand-roll harness fallback. - -## Mental Model - -The dashboard should have three explicit shapes: - -```text -WireRunSnapshot # parsed backend/API payload, generated-contract shaped -WorkflowRunState # dashboard runtime/store state with Maps, history, edges, annotations -RunDisplayState # UI-selected state, maybe live or replay snapshot -``` - -Data flow: - -```text -FastAPI / dashboard harness / socket sync - ↓ -contract parser - ↓ -hydrateRunSnapshot() - ↓ -WorkflowRunState - ↓ -reducers for live updates - ↓ -RunWorkspacePage / TaskWorkspace / graph/activity UI -``` - -When state leaves the dashboard store over REST or socket sync: - -```text -WorkflowRunState - ↓ -serializeRunSnapshot() - ↓ -WireRunSnapshot - ↓ -parseRunSnapshot() round-trip test must pass -``` - -## Route Naming Constraint - -Backend danger harness routes intentionally use: - -```text -/api/__danger__/test-harness/... -``` - -The Next.js dashboard must not use an app directory named `__danger__`, because underscore-prefixed App Router segments are treated as private and do not route. Dashboard-only harness routes should stay at: - -```text -/api/danger/test-harness/dashboard/... -``` - -Document this in route comments and tests so a future cleanup does not move them back to `__danger__`. - -## Task 1: Wire Existing Frontend Tests Into One Reliable Fast Path - -**Files:** -- Modify: `ergon-dashboard/package.json` -- Modify: `.github/workflows/ci-fast.yml` -- Modify: `ergon-dashboard/tests/e2e/health.spec.ts` -- Test: all existing dashboard `*.test.ts` files - -- [ ] **Step 1: Add a unit test script that runs all non-Playwright Node tests** - -In `ergon-dashboard/package.json`, replace the narrow contract script with explicit fast scripts: - -```json -{ - "scripts": { - "test:unit": "tsx --test \"src/**/*.test.ts\" \"tests/**/*.test.ts\"", - "test:contracts": "tsx --test tests/contracts/contracts.test.ts src/features/graph/contracts/graphMutations.test.ts", - "test": "pnpm run test:unit" - } -} -``` - -If `test:unit` accidentally includes Playwright specs, narrow it to: - -```json -"test:unit": "tsx --test \"src/**/*.test.ts\" \"tests/contracts/**/*.test.ts\" \"tests/graph/**/*.test.ts\"" -``` - -- [ ] **Step 2: Verify the unit script catches currently orphaned tests** - -Run: - -```bash -pnpm -C ergon-dashboard run test:unit -``` - -Expected: all Node test files run, including: - -```text -src/app/api/health/health.test.ts -src/features/activity/*.test.ts -src/features/evaluation/selectors.test.ts -src/hooks/useRunState.socketHydration.test.ts -src/lib/timeFormat.test.ts -src/components/workspace/filterTaskEvidenceForTime.test.ts -tests/contracts/contracts.test.ts -tests/graph/taskTiming.test.ts -``` - -- [ ] **Step 3: Align health e2e with Playwright baseURL** - -In `ergon-dashboard/tests/e2e/health.spec.ts`, remove the hardcoded `BASE` constant and use relative URLs: - -```ts -test("returns 200 with healthy status when build is fresh", async ({ request }) => { - const res = await request.get("/api/health"); - expect(res.status()).toBe(200); -}); -``` - -For page navigation: - -```ts -await page.goto("/"); -await expect(page.locator('[data-testid="build-health-toast"]')).not.toBeVisible(); -``` - -Avoid `networkidle`; wait for the specific DOM state instead. - -- [ ] **Step 4: Add frontend tests to CI fast** - -In `.github/workflows/ci-fast.yml`, after TypeScript check: - -```yaml - - name: Frontend unit tests - run: pnpm -C ergon-dashboard run test:unit - - - name: Install Playwright browsers - run: pnpm -C ergon-dashboard exec playwright install --with-deps chromium - - - name: Dashboard harness e2e - env: - ERGON_API_BASE_URL: http://127.0.0.1:9000 - run: pnpm -C ergon-dashboard run e2e -``` - -If the e2e job is too slow for the lint/typecheck job, split it into a separate `frontend-e2e` job that `needs: frontend-checks`. - -- [ ] **Step 5: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run e2e -pnpm -C ergon-dashboard run typecheck -pnpm -C ergon-dashboard run lint -``` - -Expected: - -```text -unit tests pass -20 local dashboard e2e pass, live smoke specs skip without live env -typecheck passes -lint passes -``` - -## Task 2: Split Wire Run Snapshot From Dashboard Runtime State - -**Files:** -- Create: `ergon-dashboard/src/lib/run-state/domain.ts` -- Create: `ergon-dashboard/src/lib/run-state/hydrate.ts` -- Create: `ergon-dashboard/src/lib/run-state/serialize.ts` -- Create: `ergon-dashboard/src/lib/run-state/contextEvents.ts` -- Create: `ergon-dashboard/src/lib/run-state/index.ts` -- Modify: `ergon-dashboard/src/lib/runState.ts` (temporary compatibility re-export) -- Modify: `ergon-dashboard/src/lib/types.ts` -- Test: `ergon-dashboard/tests/contracts/run-state-roundtrip.contract.test.ts` - -- [ ] **Step 1: Write the failing round-trip test** - -Create `ergon-dashboard/tests/contracts/run-state-roundtrip.contract.test.ts`: - -```ts -import assert from "node:assert/strict"; -import test from "node:test"; - -import { parseRunSnapshot } from "../../src/lib/contracts/rest"; -import { hydrateRunSnapshot, serializeRunSnapshot } from "../../src/lib/run-state"; -import { createDashboardSeed, FIXTURE_IDS } from "../helpers/dashboardFixtures"; - -test("run state serializes back into a valid wire run snapshot", () => { - const run = createDashboardSeed().runs?.[0]; - assert.ok(run); - - const state = hydrateRunSnapshot(run); - const wire = serializeRunSnapshot(state); - const reparsed = parseRunSnapshot(wire); - - assert.equal(reparsed.id, FIXTURE_IDS.runId); - assert.equal(reparsed.tasks[FIXTURE_IDS.solveTaskId]?.id, FIXTURE_IDS.solveTaskId); -}); -``` - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test tests/contracts/run-state-roundtrip.contract.test.ts -``` - -Expected: FAIL because `src/lib/run-state` does not exist yet. - -- [ ] **Step 2: Create domain exports** - -Create `ergon-dashboard/src/lib/run-state/domain.ts`: - -```ts -import type { RunSnapshot } from "@/lib/contracts/rest"; -import type { WorkflowRunState } from "@/lib/types"; - -export type WireRunSnapshot = RunSnapshot; -export type DashboardRunState = WorkflowRunState; -``` - -- [ ] **Step 3: Move hydrate logic** - -Create `ergon-dashboard/src/lib/run-state/hydrate.ts` by moving `deserializeRunState()` and helper functions from `src/lib/runState.ts`. - -Export: - -```ts -export function hydrateRunSnapshot(input: unknown): WorkflowRunState { - const data = parseRunSnapshot(input); - // existing deserializeRunState body -} -``` - -Keep a compatibility alias: - -```ts -export const deserializeRunState = hydrateRunSnapshot; -``` - -- [ ] **Step 4: Move serialize logic** - -Create `ergon-dashboard/src/lib/run-state/serialize.ts` by moving `serializeRunState()` and context-event serialization from `src/lib/runState.ts`. - -Export: - -```ts -export function serializeRunSnapshot(run: WorkflowRunState): WireRunSnapshot { - // existing serializeRunState body without broad unknown casts where possible -} - -export const serializeRunState = serializeRunSnapshot; -``` - -- [ ] **Step 5: Add index exports** - -Create `ergon-dashboard/src/lib/run-state/index.ts`: - -```ts -export type { DashboardRunState, WireRunSnapshot } from "./domain"; -export { hydrateRunSnapshot, deserializeRunState } from "./hydrate"; -export { serializeRunSnapshot, serializeRunState } from "./serialize"; -export { compareContextEvents } from "./contextEvents"; -``` - -- [ ] **Step 6: Convert old file to compatibility wrapper** - -Replace `ergon-dashboard/src/lib/runState.ts` with: - -```ts -export { - compareContextEvents, - deserializeRunState, - hydrateRunSnapshot, - serializeRunSnapshot, - serializeRunState, -} from "@/lib/run-state"; -``` - -This keeps imports working while future tasks migrate call sites. - -- [ ] **Step 7: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test tests/contracts/run-state-roundtrip.contract.test.ts -pnpm -C ergon-dashboard run test:contracts -pnpm -C ergon-dashboard run typecheck -``` - -Expected: all pass. - -## Task 3: Make Context Event Conversion A Named Boundary - -**Files:** -- Create/modify: `ergon-dashboard/src/lib/run-state/contextEvents.ts` -- Modify: `ergon-dashboard/src/lib/run-state/hydrate.ts` -- Modify: `ergon-dashboard/src/lib/run-state/serialize.ts` -- Test: `ergon-dashboard/tests/contracts/context-events.contract.test.ts` - -- [ ] **Step 1: Write conversion tests** - -Create `ergon-dashboard/tests/contracts/context-events.contract.test.ts`: - -```ts -import assert from "node:assert/strict"; -import test from "node:test"; - -import { - contextPartToUiPayload, - uiPayloadToContextPart, -} from "../../src/lib/run-state/contextEvents"; - -test("tool_call context part converts to UI payload", () => { - const payload = contextPartToUiPayload({ - part: { - part_kind: "tool_call", - tool_call_id: "call-1", - tool_name: "lean_check", - args: { file: "proof.lean" }, - }, - token_ids: [1, 2], - logprobs: null, - sequence: 0, - worker_binding_key: "react-worker", - turn_id: "turn-1", - started_at: "2026-03-18T12:00:00.000Z", - completed_at: "2026-03-18T12:00:01.000Z", - policy_version: null, - }); - - assert.deepEqual(payload, { - event_type: "tool_call", - tool_call_id: "call-1", - tool_name: "lean_check", - args: { file: "proof.lean" }, - turn_id: "turn-1", - turn_token_ids: [1, 2], - turn_logprobs: null, - }); -}); - -test("UI tool_result payload serializes to context part", () => { - const payload = uiPayloadToContextPart( - { - event_type: "tool_result", - tool_call_id: "call-1", - tool_name: "lean_check", - result: "ok", - is_error: false, - }, - { - sequence: 3, - workerBindingKey: "react-worker", - startedAt: null, - completedAt: null, - }, - ); - - assert.equal(payload.part.part_kind, "tool_result"); - assert.equal(payload.part.tool_name, "lean_check"); - assert.equal(payload.sequence, 3); -}); -``` - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test tests/contracts/context-events.contract.test.ts -``` - -Expected: FAIL because functions do not exist. - -- [ ] **Step 2: Implement explicit conversion functions** - -Create or update `ergon-dashboard/src/lib/run-state/contextEvents.ts`: - -```ts -import type { ContextEventPayload, TokenLogprob } from "@/lib/contracts/contextEvents"; - -type ContextPartChunk = { - part: Record<string, unknown>; - token_ids?: number[] | null; - logprobs?: TokenLogprob[] | null; - sequence: number; - worker_binding_key: string; - turn_id?: string | null; - started_at?: string | null; - completed_at?: string | null; - policy_version?: string | null; -}; - -function asRecord(value: unknown): Record<string, unknown> { - return typeof value === "object" && value !== null ? (value as Record<string, unknown>) : {}; -} - -export function contextPartToUiPayload(payload: unknown): ContextEventPayload { - const record = asRecord(payload); - if (typeof record.event_type === "string") { - return payload as ContextEventPayload; - } - const part = asRecord(record.part); - const tokenIds = (record.token_ids as number[] | null | undefined) ?? null; - const logprobs = (record.logprobs as TokenLogprob[] | null | undefined) ?? null; - const turnId = String(record.turn_id ?? ""); - - switch (part.part_kind) { - case "system_prompt": - return { event_type: "system_prompt", text: String(part.content ?? "") }; - case "user_message": - return { event_type: "user_message", text: String(part.content ?? ""), from_worker_key: null }; - case "assistant_text": - return { event_type: "assistant_text", text: String(part.content ?? ""), turn_id: turnId, turn_token_ids: tokenIds, turn_logprobs: logprobs }; - case "tool_call": - return { event_type: "tool_call", tool_call_id: String(part.tool_call_id ?? ""), tool_name: String(part.tool_name ?? ""), args: asRecord(part.args), turn_id: turnId, turn_token_ids: tokenIds, turn_logprobs: logprobs }; - case "tool_result": - return { event_type: "tool_result", tool_call_id: String(part.tool_call_id ?? ""), tool_name: String(part.tool_name ?? ""), result: part.content ?? null, is_error: Boolean(part.is_error ?? false) }; - case "thinking": - return { event_type: "thinking", text: String(part.content ?? ""), turn_id: turnId, turn_token_ids: tokenIds, turn_logprobs: logprobs }; - default: - throw new Error(`Unsupported context part kind: ${String(part.part_kind)}`); - } -} - -export function uiPayloadToContextPart( - payload: ContextEventPayload, - meta: { - sequence: number; - workerBindingKey: string; - startedAt: string | null; - completedAt: string | null; - }, -): ContextPartChunk { - let part: Record<string, unknown>; - switch (payload.event_type) { - case "system_prompt": - part = { part_kind: "system_prompt", content: payload.text }; - break; - case "user_message": - part = { part_kind: "user_message", content: payload.text }; - break; - case "assistant_text": - part = { part_kind: "assistant_text", content: payload.text }; - break; - case "tool_call": - part = { part_kind: "tool_call", tool_call_id: payload.tool_call_id, tool_name: payload.tool_name, args: payload.args }; - break; - case "tool_result": - part = { part_kind: "tool_result", tool_call_id: payload.tool_call_id, tool_name: payload.tool_name, content: typeof payload.result === "string" ? payload.result : JSON.stringify(payload.result) ?? "", is_error: payload.is_error }; - break; - case "thinking": - part = { part_kind: "thinking", content: payload.text }; - break; - } - const turnPayload = payload as { - turn_id?: string | null; - turn_token_ids?: number[] | null; - turn_logprobs?: TokenLogprob[] | null; - }; - return { - part, - token_ids: turnPayload.turn_token_ids ?? null, - logprobs: turnPayload.turn_logprobs ?? null, - sequence: meta.sequence, - worker_binding_key: meta.workerBindingKey, - turn_id: turnPayload.turn_id ?? null, - started_at: meta.startedAt, - completed_at: meta.completedAt, - policy_version: null, - }; -} -``` - -- [ ] **Step 3: Use conversion functions in hydrate and serialize** - -In `hydrate.ts`, replace inline payload normalization with: - -```ts -payload: contextPartToUiPayload(event.payload), -``` - -In `serialize.ts`, replace inline payload serialization with: - -```ts -payload: uiPayloadToContextPart(event.payload, { - sequence: event.sequence, - workerBindingKey: event.workerBindingKey, - startedAt: event.startedAt, - completedAt: event.completedAt, -}) as unknown as ContextEventState["payload"], -``` - -- [ ] **Step 4: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test tests/contracts/context-events.contract.test.ts -pnpm -C ergon-dashboard run test:contracts -pnpm -C ergon-dashboard run typecheck -``` - -Expected: all pass. - -## Task 4: Centralize Harness/Backend Server Data Access - -**Files:** -- Create: `ergon-dashboard/src/lib/server-data/runs.ts` -- Create: `ergon-dashboard/src/lib/server-data/cohorts.ts` -- Create: `ergon-dashboard/src/lib/server-data/experiments.ts` -- Create: `ergon-dashboard/src/lib/server-data/responses.ts` -- Modify: `ergon-dashboard/src/app/api/runs/[runId]/route.ts` -- Modify: `ergon-dashboard/src/app/api/cohorts/route.ts` -- Modify: `ergon-dashboard/src/app/api/cohorts/[cohortId]/route.ts` -- Modify: `ergon-dashboard/src/app/cohorts/[cohortId]/runs/[runId]/page.tsx` -- Modify: `ergon-dashboard/src/app/run/[runId]/page.tsx` -- Modify: `ergon-dashboard/src/app/cohorts/[cohortId]/page.tsx` -- Modify: `ergon-dashboard/src/app/experiments/[experimentId]/page.tsx` -- Test: `ergon-dashboard/tests/contracts/server-data.contract.test.ts` - -- [ ] **Step 1: Write a focused experiment fallback test** - -Create `ergon-dashboard/tests/contracts/server-data.contract.test.ts`: - -```ts -import assert from "node:assert/strict"; -import test from "node:test"; - -import { getHarnessExperiment, resetDashboardHarness } from "../../src/lib/testing/dashboardHarness"; - -test("harness miss for experiment is represented as null, not notFound policy", () => { - resetDashboardHarness(); - assert.equal(getHarnessExperiment("missing-experiment"), null); -}); -``` - -This test documents the low-level harness behavior. The route/page fallback is verified by Playwright in Task 4 Step 5. - -- [ ] **Step 2: Create run data adapter** - -Create `ergon-dashboard/src/lib/server-data/runs.ts`: - -```ts -import { config } from "@/lib/config"; -import { parseRunSnapshot } from "@/lib/contracts/rest"; -import { fetchErgonApi } from "@/lib/serverApi"; -import { getHarnessRun } from "@/lib/testing/dashboardHarness"; -import type { SerializedWorkflowRunState } from "@/lib/types"; - -export async function loadRunSnapshot(runId: string): Promise<SerializedWorkflowRunState | null> { - if (config.enableTestHarness) { - const harnessRun = getHarnessRun(runId); - if (harnessRun !== null) return harnessRun; - } - const response = await fetchErgonApi(`/runs/${runId}`); - if (response.status === 404) return null; - if (!response.ok) throw new Error(`Run API returned ${response.status}`); - return parseRunSnapshot(await response.json()); -} -``` - -- [ ] **Step 3: Create cohort and experiment adapters** - -Create `ergon-dashboard/src/lib/server-data/experiments.ts`: - -```ts -import { notFound } from "next/navigation"; - -import { config } from "@/lib/config"; -import { fetchErgonApi } from "@/lib/serverApi"; -import { getHarnessExperiment } from "@/lib/testing/dashboardHarness"; -import type { ExperimentDetail } from "@/lib/types"; - -export async function loadExperimentDetail(experimentId: string): Promise<ExperimentDetail | null> { - if (config.enableTestHarness) { - const harnessExperiment = getHarnessExperiment(experimentId); - if (harnessExperiment !== null) return harnessExperiment; - } - const response = await fetchErgonApi(`/experiments/${experimentId}`); - if (response.status === 404) return null; - if (!response.ok) { - throw new Error(`Failed to load experiment ${experimentId}: ${response.status}`); - } - return (await response.json()) as ExperimentDetail; -} - -export async function requireExperimentDetail(experimentId: string): Promise<ExperimentDetail> { - const detail = await loadExperimentDetail(experimentId); - if (detail === null) notFound(); - return detail; -} -``` - -Create `ergon-dashboard/src/lib/server-data/cohorts.ts` with the same pattern: - -```ts -import { config } from "@/lib/config"; -import { parseCohortDetail, parseCohortSummaryList } from "@/lib/contracts/rest"; -import { fetchErgonApi } from "@/lib/serverApi"; -import { getHarnessCohort, listHarnessCohorts } from "@/lib/testing/dashboardHarness"; -import type { CohortDetail, CohortSummary } from "@/lib/types"; - -export async function loadCohortDetail(cohortId: string): Promise<CohortDetail | null> { - if (config.enableTestHarness) { - const detail = getHarnessCohort(cohortId); - if (detail !== null) return detail; - } - const response = await fetchErgonApi(`/cohorts/${cohortId}`); - if (response.status === 404) return null; - if (!response.ok) throw new Error(`Cohort API returned ${response.status}`); - return parseCohortDetail(await response.json()); -} - -export async function loadCohortList(includeArchived: string | null): Promise<CohortSummary[]> { - if (config.enableTestHarness) { - const cohorts = listHarnessCohorts(); - if (cohorts.length > 0) return cohorts; - } - const path = includeArchived === null ? "/cohorts" : `/cohorts?include_archived=${includeArchived}`; - const response = await fetchErgonApi(path); - if (!response.ok) throw new Error(`Cohort list API returned ${response.status}`); - return parseCohortSummaryList(await response.json()); -} -``` - -- [ ] **Step 4: Update pages and routes to call adapters** - -In `src/app/experiments/[experimentId]/page.tsx`, replace the bespoke harness block with: - -```ts -const detail = await requireExperimentDetail(experimentId); -``` - -In run/cohort pages and API routes, call `loadRunSnapshot`, `loadCohortDetail`, and `loadCohortList`. - -- [ ] **Step 5: Add browser coverage for experiment fallback** - -Add to `ergon-dashboard/tests/e2e/cohort.snapshot.spec.ts` or a new `experiment.snapshot.spec.ts`: - -```ts -test("experiment detail falls back to backend when dashboard harness has no fixture", async ({ page }) => { - await page.goto(`/experiments/${FIXTURE_IDS.experimentId}`); - await expect(page.getByRole("heading", { name: "minif2f smoke n=3" })).toBeVisible(); -}); -``` - -If this fixture is harness-owned, add a separate route-level test with mocked backend for the fallback path instead. - -- [ ] **Step 6: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run e2e -pnpm -C ergon-dashboard run typecheck -``` - -Expected: all pass. - -## Task 5: Share Client And Server Run-State Reducers - -**Files:** -- Create: `ergon-dashboard/src/lib/run-state/reducers.ts` -- Create: `ergon-dashboard/src/lib/run-state/metrics.ts` -- Modify: `ergon-dashboard/src/lib/state/store.ts` -- Modify: `ergon-dashboard/src/hooks/useRunState.ts` -- Test: `ergon-dashboard/src/hooks/useRunState.socketHydration.test.ts` -- Test: `ergon-dashboard/tests/contracts/run-state-roundtrip.contract.test.ts` - -- [ ] **Step 1: Write reducer parity tests** - -Extend `src/hooks/useRunState.socketHydration.test.ts`: - -```ts -import assert from "node:assert/strict"; -import test from "node:test"; - -import { applyTaskStatusChanged } from "@/lib/run-state/reducers"; -import { createDashboardSeed, FIXTURE_IDS } from "../../tests/helpers/dashboardFixtures"; -import { hydrateRunSnapshot } from "@/lib/run-state"; -import { TaskStatus } from "@/lib/types"; - -test("task status reducer records transition history", () => { - const run = createDashboardSeed().runs?.[0]; - assert.ok(run); - - const state = hydrateRunSnapshot(run); - const next = applyTaskStatusChanged(state, { - runId: FIXTURE_IDS.runId, - taskId: FIXTURE_IDS.solveTaskId, - status: TaskStatus.COMPLETED, - timestamp: "2026-03-18T12:01:00.000Z", - assignedWorkerId: null, - assignedWorkerSlug: null, - }); - - const task = next.tasks.get(FIXTURE_IDS.solveTaskId); - assert.equal(task?.status, TaskStatus.COMPLETED); - assert.equal(task?.history?.at(-1)?.to, TaskStatus.COMPLETED); -}); -``` - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test src/hooks/useRunState.socketHydration.test.ts -``` - -Expected: FAIL because reducer does not exist. - -- [ ] **Step 2: Implement metrics helper** - -Create `src/lib/run-state/metrics.ts`: - -```ts -import { TaskStatus, type TaskState, type WorkflowRunState } from "@/lib/types"; - -export function recalculateTaskMetrics(tasks: Map<string, TaskState>): Pick< - WorkflowRunState, - "completedTasks" | "runningTasks" | "failedTasks" | "cancelledTasks" -> { - let completedTasks = 0; - let runningTasks = 0; - let failedTasks = 0; - let cancelledTasks = 0; - - for (const task of tasks.values()) { - if (!task.isLeaf) continue; - if (task.status === TaskStatus.COMPLETED) completedTasks += 1; - if (task.status === TaskStatus.RUNNING) runningTasks += 1; - if (task.status === TaskStatus.FAILED) failedTasks += 1; - if (task.status === TaskStatus.CANCELLED) cancelledTasks += 1; - } - - return { completedTasks, runningTasks, failedTasks, cancelledTasks }; -} -``` - -- [ ] **Step 3: Implement task status reducer** - -Create `src/lib/run-state/reducers.ts`: - -```ts -import { inferTrigger } from "@/lib/runEvents"; -import { TaskStatus, type ExecutionAttemptState, type TaskState, type WorkflowRunState } from "@/lib/types"; -import { recalculateTaskMetrics } from "./metrics"; - -export interface TaskStatusChangedInput { - runId: string; - taskId: string; - status: TaskStatus; - timestamp: string; - assignedWorkerId?: string | null; - assignedWorkerSlug?: string | null; -} - -function nextExecutionStatus(status: TaskStatus): TaskStatus { - return status === TaskStatus.READY ? TaskStatus.PENDING : status; -} - -export function applyTaskStatusChanged( - state: WorkflowRunState, - data: TaskStatusChangedInput, -): WorkflowRunState { - const task = state.tasks.get(data.taskId); - if (!task) return state; - - const previousStatus = task.status; - const status = data.status; - const trigger = previousStatus !== status ? inferTrigger(previousStatus, status) : task.lastTrigger; - const updatedTask: TaskState = { - ...task, - status, - assignedWorkerId: data.assignedWorkerId ?? task.assignedWorkerId, - assignedWorkerSlug: data.assignedWorkerSlug ?? task.assignedWorkerSlug, - startedAt: status === TaskStatus.RUNNING && !task.startedAt ? data.timestamp : task.startedAt, - completedAt: - status === TaskStatus.COMPLETED || status === TaskStatus.FAILED || status === TaskStatus.CANCELLED - ? data.timestamp - : task.completedAt, - history: - previousStatus !== status - ? [ - ...(task.history ?? []), - { - from: previousStatus, - to: status, - trigger, - at: data.timestamp, - sequence: null, - actor: data.assignedWorkerSlug ?? task.assignedWorkerSlug ?? null, - reason: null, - }, - ] - : task.history, - lastTrigger: trigger, - }; - - const tasks = new Map(state.tasks); - tasks.set(data.taskId, updatedTask); - - const existingExecutions = state.executionsByTask.get(data.taskId) ?? []; - const latestExecution = existingExecutions[existingExecutions.length - 1]; - let nextExecutions = existingExecutions; - - if (status === TaskStatus.RUNNING) { - if (!latestExecution || latestExecution.status === TaskStatus.COMPLETED || latestExecution.status === TaskStatus.FAILED) { - const createdExecution: ExecutionAttemptState = { - id: `${data.taskId}:attempt:${existingExecutions.length + 1}`, - taskId: data.taskId, - attemptNumber: existingExecutions.length + 1, - status: TaskStatus.RUNNING, - agentId: data.assignedWorkerId ?? task.assignedWorkerId, - agentName: data.assignedWorkerSlug ?? task.assignedWorkerSlug, - startedAt: data.timestamp, - completedAt: null, - finalAssistantMessage: null, - outputResourceIds: [], - errorMessage: null, - score: null, - evaluationDetails: {}, - }; - nextExecutions = [...existingExecutions, createdExecution]; - } - } else if (latestExecution) { - nextExecutions = existingExecutions.map((execution, index) => - index === existingExecutions.length - 1 - ? { - ...execution, - status: nextExecutionStatus(status), - completedAt: - status === TaskStatus.COMPLETED || status === TaskStatus.FAILED || status === TaskStatus.CANCELLED - ? data.timestamp - : execution.completedAt, - errorMessage: - status === TaskStatus.FAILED - ? execution.errorMessage ?? "Task execution failed" - : execution.errorMessage, - } - : execution, - ); - } - - const executionsByTask = new Map(state.executionsByTask); - executionsByTask.set(data.taskId, nextExecutions); - const metrics = recalculateTaskMetrics(tasks); - - return { ...state, tasks, executionsByTask, ...metrics }; -} -``` - -- [ ] **Step 4: Use reducer in `useRunState`** - -In `src/hooks/useRunState.ts`, replace the task status `setRunState` body with: - -```ts -setRunState((prev) => { - if (!prev) return prev; - return applyTaskStatusChanged(prev, { - runId: data.runId, - taskId: data.taskId, - status, - timestamp: data.timestamp, - assignedWorkerId: data.assignedWorkerId, - assignedWorkerSlug: data.assignedWorkerSlug, - }); -}); -``` - -- [ ] **Step 5: Use reducer in `DashboardStore`** - -In `src/lib/state/store.ts`, replace `updateTaskStatus` internals with: - -```ts -const updated = applyTaskStatusChanged(run, { - runId, - taskId, - status: newStatus, - timestamp, - assignedWorkerId, - assignedWorkerSlug, -}); -this.runs.set(runId, updated); -``` - -- [ ] **Step 6: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard exec tsx --test src/hooks/useRunState.socketHydration.test.ts -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run e2e -``` - -Expected: all pass. - -## Task 6: Normalize Sandbox Event Ordering - -**Files:** -- Modify: `ergon-dashboard/src/lib/run-state/reducers.ts` -- Modify: `ergon-dashboard/src/hooks/useRunState.ts` -- Modify: `ergon-dashboard/src/lib/state/store.ts` -- Test: `ergon-dashboard/src/hooks/useRunState.socketHydration.test.ts` - -- [ ] **Step 1: Write out-of-order sandbox test** - -Add: - -```ts -test("sandbox command before sandbox creation is preserved", () => { - // Build a run state without sandbox for solve task. - // Apply command reducer first, then sandbox-created reducer. - // Assert the sandbox contains the pending command. -}); -``` - -Use exact fixture IDs from `tests/helpers/dashboardFixtures.ts`. - -- [ ] **Step 2: Add pending command support to reducers** - -Add a client/server-compatible pending command map if needed, or simplify by making `sandbox:command` trigger a snapshot reload when the sandbox is missing. - -Preferred implementation: - -```ts -export function applySandboxCommand(state: WorkflowRunState, taskId: string, command: SandboxCommandState): WorkflowRunState { - const sandbox = state.sandboxesByTask.get(taskId); - if (!sandbox) { - return { - ...state, - pendingSandboxCommands: addPendingCommand(state.pendingSandboxCommands, taskId, command), - }; - } - // append command -} -``` - -If adding `pendingSandboxCommands` to `WorkflowRunState` creates too much type churn, use the snapshot-reload fallback in `useRunState` and keep server buffering as-is. - -- [ ] **Step 3: Preserve backend timestamps on close** - -In `useRunState.ts`, replace: - -```ts -closedAt: new Date().toISOString(), -``` - -with: - -```ts -closedAt: data.timestamp, -``` - -If `parseSandboxClosedSocketData` does not expose `timestamp`, update `src/lib/contracts/events.ts` and its tests. - -- [ ] **Step 4: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run e2e -``` - -Expected: all pass. - -## Task 7: Make Route/API Parsing And Error Semantics Consistent - -**Files:** -- Modify: `ergon-dashboard/src/app/api/cohorts/[cohortId]/route.ts` -- Modify: `ergon-dashboard/src/app/api/runs/[runId]/mutations/route.ts` -- Modify: `ergon-dashboard/src/app/api/training/sessions/route.ts` -- Modify: `ergon-dashboard/src/lib/serverApi.ts` -- Test: `ergon-dashboard/src/app/api/health/health.test.ts` or new API route tests - -- [ ] **Step 1: Parse successful cohort detail responses** - -In `src/app/api/cohorts/[cohortId]/route.ts`, successful GET should return: - -```ts -return NextResponse.json(parseCohortDetail(payload), { status: response.status }); -``` - -not raw `payload`. - -- [ ] **Step 2: Choose a dependency-failure status** - -Use `503` for unavailable upstream dependency across API proxy routes unless the upstream returned a concrete HTTP response. - -Example: - -```ts -return NextResponse.json( - { - detail: "Ergon API is unavailable.", - error: error instanceof Error ? error.message : "Unknown backend fetch failure", - }, - { status: 503 }, -); -``` - -- [ ] **Step 3: Allow per-route backend timeout** - -In `src/lib/serverApi.ts`, add: - -```ts -export async function fetchErgonApi(path: string, init: RequestInit & { timeoutMs?: number } = {}) { - const { timeoutMs = 5_000, ...requestInit } = init; - // existing AbortController logic uses timeoutMs -} -``` - -Use a longer timeout in resource content proxy: - -```ts -await fetchErgonApi(path, { timeoutMs: 30_000 }); -``` - -- [ ] **Step 4: Verify** - -Run: - -```bash -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run typecheck -``` - -Expected: all pass. - -## Task 8: Slim `RunWorkspacePage` After Boundaries Are Stable - -**Files:** -- Create: `ergon-dashboard/src/components/run/useRunDisplayState.ts` -- Create: `ergon-dashboard/src/components/run/useRunKeyboardShortcuts.ts` -- Create: `ergon-dashboard/src/components/run/useRunPanelLayout.ts` -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` -- Test: existing Playwright `run.snapshot.spec.ts` - -- [ ] **Step 1: Extract panel layout persistence** - -Move localStorage layout code into: - -```ts -export function useRunPanelLayout() { - return { - verticalLayout, - setVerticalLayout, - horizontalLayout, - setHorizontalLayout, - hasLoadedPanelLayouts, - }; -} -``` - -- [ ] **Step 2: Extract live/replay display state** - -Create: - -```ts -export function useRunDisplayState(runState: WorkflowRunState | null, mutations: DashboardGraphMutationData[]) { - return { - displayState, - selectedActivity, - selectedActivityId, - setSelectedActivityId, - snapshotSequence, - }; -} -``` - -Document the invariant: - -```text -Activities and trace rows are built from full live run state. -Inspector and graph may render replay display state. -``` - -- [ ] **Step 3: Extract keyboard shortcuts** - -Create: - -```ts -export function useRunKeyboardShortcuts(options: { clearSnapshot: () => void }) { - // Escape clears snapshot selection. -} -``` - -- [ ] **Step 4: Verify no UI behavior changed** - -Run: - -```bash -pnpm -C ergon-dashboard exec playwright test tests/e2e/run.snapshot.spec.ts -pnpm -C ergon-dashboard run typecheck -``` - -Expected: run workspace behavior remains identical. - -## Final Verification - -Run the full local verification matrix: - -```bash -pnpm -C ergon-dashboard run test:unit -pnpm -C ergon-dashboard run test:contracts -pnpm -C ergon-dashboard run typecheck -pnpm -C ergon-dashboard run lint -ERGON_API_BASE_URL=http://127.0.0.1:9000 pnpm -C ergon-dashboard run e2e -ERGON_DATABASE_URL=postgresql://ergon:ergon_dev@localhost:5433/ergon \ -INNGEST_API_BASE_URL=http://localhost:8289 \ -INNGEST_DEV=1 \ -INNGEST_EVENT_KEY=dev \ -ERGON_API_BASE_URL=http://127.0.0.1:9000 \ -PLAYWRIGHT_BASE_URL=http://127.0.0.1:3001 \ -SCREENSHOT_DIR=/tmp/playwright \ -PYTHONUNBUFFERED=1 \ -uv run pytest tests/e2e -v --timeout=300 -``` - -Expected: - -```text -frontend unit/contract/type/lint checks pass -dashboard local e2e passes with live smoke specs skipped if env is absent -Python full-stack e2e passes -``` - -## Self-Review - -- Spec coverage: covers test wiring, type split, context event conversion, harness/backend fallback, shared reducers, sandbox ordering, route semantics, and later component slimming. -- Placeholder scan: no `TBD` or open-ended “add tests” instructions remain; each task has target files and verification commands. -- Type consistency: canonical names are `WireRunSnapshot`, `DashboardRunState`, `hydrateRunSnapshot`, `serializeRunSnapshot`, `contextPartToUiPayload`, `uiPayloadToContextPart`, and `applyTaskStatusChanged`. - diff --git a/docs/superpowers/plans/2026-04-29-finish-builtins-cli-e2e-refactor.md b/docs/superpowers/plans/2026-04-29-finish-builtins-cli-e2e-refactor.md deleted file mode 100644 index 78a069329..000000000 --- a/docs/superpowers/plans/2026-04-29-finish-builtins-cli-e2e-refactor.md +++ /dev/null @@ -1,841 +0,0 @@ -# Finish Built-ins, CLI, And E2E Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the Ergon built-ins, CLI, and e2e refactor after the core public API and test-support facade have stabilized, while avoiding private core internals that may continue moving. - -**Architecture:** Treat `ergon_core.api`, core service/facade DTOs, `ergon_core.test_support`, HTTP `/api/test/*`, and application read models as the stable boundary. Production built-ins own benchmark-specific workers/rubrics/sandboxes; CLI commands validate explicit slugs and call core facades; e2e tests assert black-box runtime behavior and use test-support constants rather than private repository methods. - -**Tech Stack:** Python, pytest, FastAPI test harness endpoints, Playwright, Inngest, E2B, `ergon_core.test_support`, `ergon_builtins.registry`, `ergon_cli`. - ---- - -## Current Working Assumptions - -- Core runtime behavior is stable: the canonical smoke topology, resource counts, task states, communication threads, and evaluation outcomes are still expected to match existing e2e assertions. -- Core internal layout has changed substantially. Tests should not import private repository modules or persistence models unless there is no stable public/test-support read helper yet. -- `ergon_core.test_support` is stable and may be imported by unit/integration/e2e host-side test code. -- The API process, not the host e2e process, should register smoke fixtures via startup plugin/environment. -- Built-ins and CLI work may proceed as long as it stays on public API/service boundaries and avoids core repository implementation files. - -## E2E Behavior That Should Remain True - -These expected values are derived from stable smoke fixture constants and should remain hard assertions unless `ergon_core.test_support.smoke_fixtures` changes intentionally. - -```text -Happy path: -- 12 total tasks: 1 root + 9 direct subtasks + 2 nested subtasks -- 10 leaf tasks -- direct level-1 slugs match EXPECTED_SUBTASK_SLUGS -- nested level-2 slugs match NESTED_LINE_SLUGS -- l_2 is non-leaf; l_2_a and l_2_b are children of l_2 -- all nodes complete -- 20 task artifact resources: 10 benchmark artifacts + 10 probe_*.json -- no worker_output resources; final assistant messages stay on executions -- 26 context events: parent 3 + recursive 3 + 10 leaves x 2 -- 2 root evaluations, both score 1.0, created after root execution completion -- final score is 1.0 -- one smoke-completion thread with 11 ordered messages - -Sad path: -- l_2 fails -- l_3 is blocked, never starts, and has no execution attempts -- root does not complete -- independent leaves complete -- exactly one partial_*.md artifact persists from l_2 -- at least one pre-failure partial wc WAL/probe entry exists -- smoke-completion thread has 7 messages -- l_2 and l_3 do not send completion messages -- final score is None or 0.0 -``` - -Benchmark-specific artifact assertions should also remain: - -```text -MiniF2F: -- 10 proof_*.lean resources -- each proof contains "theorem smoke_trivial" and ":=" - -SWE-Bench: -- 10 patch_*.py resources -- each patch parses as Python and defines add() - -ResearchRubrics: -- report/probe artifacts and dashboard-visible resource panels match the shared smoke assertions -``` - -## File Responsibility Map - -Built-ins: - -- `ergon_builtins/ergon_builtins/registry.py`: merged public registry surface. -- `ergon_builtins/ergon_builtins/registry_core.py`: always-importable benchmarks/workers/evaluators/sandboxes/model backends. -- `ergon_builtins/ergon_builtins/registry_data.py`: `[data]` benchmark registrations. -- `ergon_builtins/ergon_builtins/benchmarks/*/worker_factory.py`: benchmark-owned worker factories or benchmark-owned re-export surfaces. -- `ergon_builtins/ergon_builtins/shared/`: generic worker, criteria, model, prompt import surfaces. - -CLI: - -- `ergon_cli/ergon_cli/main.py`: parser contract only. -- `ergon_cli/ergon_cli/commands/experiment.py`: thin command handler for `experiment define/run/show/list`. -- `ergon_cli/ergon_cli/commands/benchmark.py`: `list`, `setup`, and `run` wrapper behavior. -- `ergon_cli/ergon_cli/discovery/__init__.py`: registry list helpers. -- Future target: `ergon_cli/ergon_cli/services/*_facade.py` if command handlers remain too stateful. - -E2E: - -- `tests/e2e/_submit.py`: black-box cohort submission client for `/api/test/write/cohort`. -- `tests/e2e/_read_contracts.py`: stable read-model wrapper for run snapshots. -- `tests/e2e/_asserts.py`: behavior assertions; should import test-support constants and stable read helpers. -- `tests/e2e/test_{researchrubrics,minif2f,swebench}_smoke.py`: per-benchmark e2e drivers. -- `ergon-dashboard/tests/e2e/*.smoke.spec.ts`: dashboard assertions. - -Stable core/test-support surfaces: - -- `ergon_core.api` -- `ergon_core.test_support` -- `ergon_core.core.application.read_models.*`, if accepted as the application-level read facade -- `/api/test/*` HTTP endpoints - -Private core surfaces to avoid in new e2e code: - -- `ergon_core.core.persistence.*` models and queries -- `ergon_core.core.runtime.tasks.repository` -- `ergon_core.core.runtime.evaluation.persistence` -- Inngest child payload modules -- repository method names or table-specific access patterns - -## Task 1: Freeze And Document The Stable E2E Boundary - -**Files:** -- Modify: `docs/superpowers/plans/2026-04-28-ergon-e2e-refactor-test-plan.md` -- Test: `tests/unit/architecture/test_public_api_boundaries.py` - -- [ ] **Step 1: Add a “stable e2e boundary” section to the e2e plan** - -Add this section near the existing `Fixture Residency Rules` section: - -```markdown -## Stable E2E Boundary After Core Layout Refactor - -Core behavior is stable, but private repository and persistence modules may move. -E2E code should use only: - -- HTTP endpoints under `/api/test/*` -- `ergon_core.test_support` -- public core API objects from `ergon_core.api` -- application read-model facades, not private repository methods - -The existing smoke behavior assertions remain valid: - -- happy runs complete the 12-node graph -- sad runs fail `l_2` and block `l_3` -- happy runs produce 20 task resources and 26 context events -- happy root produces two score-1.0 evaluations -- sad runs produce one partial artifact and seven completion messages -``` - -- [ ] **Step 2: Add or update a boundary test** - -Add/extend a test in `tests/unit/architecture/test_public_api_boundaries.py`: - -```python -from pathlib import Path - - -def test_e2e_tests_do_not_import_private_core_repositories() -> None: - e2e_dir = Path("tests/e2e") - forbidden = ( - "ergon_core.core.persistence.", - "ergon_core.core.runtime.tasks.repository", - "ergon_core.core.runtime.evaluation.persistence", - "ergon_core.core.runtime.inngest.", - ) - offenders: list[tuple[str, str]] = [] - for path in e2e_dir.rglob("*.py"): - text = path.read_text() - for needle in forbidden: - if needle in text: - offenders.append((str(path), needle)) - assert not offenders -``` - -- [ ] **Step 3: Run the boundary test and confirm failure before cleanup** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_public_api_boundaries.py::test_e2e_tests_do_not_import_private_core_repositories -q -``` - -Expected before cleanup: fail with current `tests/e2e/_asserts.py` private persistence imports. - -## Task 2: Update E2E Submission To Explicit Runtime Choices - -**Files:** -- Modify: `tests/e2e/_submit.py` -- Modify: `tests/e2e/test_researchrubrics_smoke.py` -- Modify: `tests/e2e/test_minif2f_smoke.py` -- Modify: `tests/e2e/test_swebench_smoke.py` -- Test: `tests/unit/smoke_base/test_e2e_smoke_driver_pairs.py` - -- [ ] **Step 1: Add a unit test for explicit e2e submission payloads** - -Create or update `tests/unit/smoke_base/test_e2e_smoke_driver_pairs.py`: - -```python -from tests.e2e._submit import build_cohort_payload - - -def test_build_cohort_payload_includes_explicit_runtime_choices() -> None: - payload = build_cohort_payload( - benchmark_slug="minif2f", - slots=[("minif2f-smoke-worker", "minif2f-smoke-criterion")], - cohort_key="ci-smoke-minif2f", - sandbox_slug="minif2f", - dependency_extras=("none",), - model="openai:gpt-4o", - ) - - assert payload["benchmark_slug"] == "minif2f" - assert payload["sandbox_slug"] == "minif2f" - assert payload["dependency_extras"] == ["none"] - assert payload["model"] == "openai:gpt-4o" - assert payload["slots"] == [ - { - "worker_slug": "minif2f-smoke-worker", - "evaluator_slug": "minif2f-smoke-criterion", - } - ] -``` - -- [ ] **Step 2: Implement `build_cohort_payload()`** - -In `tests/e2e/_submit.py`, add: - -```python -def build_cohort_payload( - *, - benchmark_slug: str, - slots: list[tuple[str, str]], - cohort_key: str, - sandbox_slug: str, - dependency_extras: tuple[str, ...], - model: str = "openai:gpt-4o", -) -> dict: - return { - "benchmark_slug": benchmark_slug, - "slots": [ - {"worker_slug": worker, "evaluator_slug": evaluator} - for worker, evaluator in slots - ], - "cohort_key": cohort_key, - "sandbox_slug": sandbox_slug, - "dependency_extras": list(dependency_extras), - "model": model, - } -``` - -- [ ] **Step 3: Route `submit_cohort()` through the payload builder** - -Change `submit_cohort()` signature to accept explicit fields: - -```python -async def submit_cohort( - *, - benchmark_slug: str, - slots: list[tuple[str, str]], - cohort_key: str, - sandbox_slug: str, - dependency_extras: tuple[str, ...], - model: str = "openai:gpt-4o", - timeout: int = 300, -) -> list[UUID]: - payload = build_cohort_payload( - benchmark_slug=benchmark_slug, - slots=slots, - cohort_key=cohort_key, - sandbox_slug=sandbox_slug, - dependency_extras=dependency_extras, - model=model, - ) - async with httpx.AsyncClient(base_url=_api_base(), timeout=30.0) as client: - response = await client.post("/api/test/write/cohort", json=payload) - ... -``` - -- [ ] **Step 4: Update each e2e driver call** - -For `tests/e2e/test_minif2f_smoke.py`: - -```python -run_ids = await submit_cohort( - benchmark_slug=ENV, - slots=[(worker, criterion) for _, worker, criterion in smoke_slots], - cohort_key=cohort_key, - sandbox_slug=ENV, - dependency_extras=("none",), - timeout=PER_RUN_TIMEOUT, -) -``` - -For `tests/e2e/test_swebench_smoke.py`: - -```python -run_ids = await submit_cohort( - benchmark_slug=ENV, - slots=[(worker, criterion) for _, worker, criterion in smoke_slots], - cohort_key=cohort_key, - sandbox_slug=ENV, - dependency_extras=("none",), - timeout=PER_RUN_TIMEOUT, -) -``` - -For `tests/e2e/test_researchrubrics_smoke.py`: - -```python -run_ids = await submit_cohort( - benchmark_slug=ENV, - slots=[(worker, criterion) for _, worker, criterion in smoke_slots], - cohort_key=cohort_key, - sandbox_slug=ENV, - dependency_extras=("none",), - timeout=PER_RUN_TIMEOUT, -) -``` - -Smoke fixtures replace production benchmark loaders, so e2e smoke should use `("none",)` unless the API harness explicitly requires package extras to test onboarding messaging. - -- [ ] **Step 5: Run unit payload test** - -Run: - -```bash -uv run pytest tests/unit/smoke_base/test_e2e_smoke_driver_pairs.py -q -``` - -Expected: pass. - -## Task 3: Replace Private E2E Reads With Test-Support Or Application Read Models - -**Files:** -- Modify: `tests/e2e/_asserts.py` -- Modify: `tests/e2e/_read_contracts.py` -- Optional create: `ergon_core/ergon_core/test_support/e2e_read_helpers.py` -- Test: `tests/unit/smoke_base/test_e2e_read_helpers.py` - -- [ ] **Step 1: Inventory direct private imports in `_asserts.py`** - -Search: - -```bash -rg "ergon_core.core.persistence|sqlmodel|select\\(" tests/e2e/_asserts.py -``` - -Expected current private access areas: - -- graph node rows for temporal ordering -- `RunResource` rows for blob/artifact assertions -- `RunTaskEvaluation` rows for evaluation timestamp assertions -- sandbox WAL/event rows - -- [ ] **Step 2: Keep `require_run_snapshot()` as the primary read path** - -`tests/e2e/_read_contracts.py` may keep: - -```python -from ergon_core.core.application.read_models.models import RunSnapshotDto -from ergon_core.core.application.read_models.runs import RunReadService -``` - -Do not import private repository classes in e2e drivers. If `RunReadService` moves, fix this wrapper only. - -- [ ] **Step 3: Add test-support helpers only for data not exposed in snapshots** - -If WAL/resource byte paths/evaluation timestamps are not exposed through `RunSnapshotDto`, create `ergon_core/ergon_core/test_support/e2e_read_helpers.py`: - -```python -"""Stable test-support reads for e2e assertions.""" - -from pathlib import Path -from uuid import UUID - -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.telemetry.models import ( - RunResource, - RunTaskEvaluation, - RunTaskExecution, - SandboxCommandWalEntry, - SandboxEvent, -) -from sqlmodel import select - - -def list_run_resources(run_id: UUID) -> list[RunResource]: - with get_session() as session: - return list(session.exec(select(RunResource).where(RunResource.run_id == run_id)).all()) - - -def read_resource_bytes(resource: RunResource) -> bytes: - return Path(resource.file_path).read_bytes() - - -def list_sandbox_command_wal(run_id: UUID) -> list[SandboxCommandWalEntry]: - with get_session() as session: - return list( - session.exec( - select(SandboxCommandWalEntry).where(SandboxCommandWalEntry.run_id == run_id), - ).all() - ) - - -def list_sandbox_events(run_id: UUID) -> list[SandboxEvent]: - with get_session() as session: - return list(session.exec(select(SandboxEvent).where(SandboxEvent.run_id == run_id)).all()) - - -def list_root_evaluation_rows(run_id: UUID) -> tuple[RunTaskExecution | None, list[RunTaskEvaluation]]: - # Implementation may use the current core layout internally. - # E2E tests should import this function, not the private models directly. - ... -``` - -If the core agent has already created stable equivalents under `ergon_core.test_support`, use those instead of adding this file. - -- [ ] **Step 4: Move `_asserts.py` imports to stable helper functions** - -Change `tests/e2e/_asserts.py` so private persistence imports are replaced by: - -```python -from ergon_core.test_support.e2e_read_helpers import ( - list_root_evaluation_rows, - list_run_resources, - list_sandbox_command_wal, - list_sandbox_events, - read_resource_bytes, -) -``` - -Keep these direct test-support imports: - -```python -from ergon_core.test_support.smoke_fixtures.smoke_base.constants import EXPECTED_SUBTASK_SLUGS -from ergon_core.test_support.smoke_fixtures.smoke_base.leaf_base import BaseSmokeLeafWorker -from ergon_core.test_support.smoke_fixtures.smoke_base.recursive import ( - NESTED_LINE_SLUGS, - RecursiveSmokeWorkerBase, -) -from ergon_core.test_support.smoke_fixtures.smoke_base.worker_base import SmokeWorkerBase -``` - -- [ ] **Step 5: Re-run the boundary test** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_public_api_boundaries.py::test_e2e_tests_do_not_import_private_core_repositories -q -``` - -Expected after cleanup: pass. - -## Task 4: Finish Built-ins Registry And Factory Contracts - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/registry_core.py` -- Modify: `ergon_builtins/ergon_builtins/registry_data.py` -- Modify/create: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/worker_factory.py` -- Modify/create: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/worker_factory.py` -- Modify: `tests/unit/registry/test_builtin_pairings.py` -- Modify: `tests/unit/registry/test_react_factories.py` - -- [ ] **Step 1: Verify explicit pairing table** - -`tests/unit/registry/test_builtin_pairings.py` must contain registered pairings: - -```python -PAIRINGS = [ - ("minif2f", "minif2f-react", "minif2f-rubric", "minif2f", ("none",)), - ("swebench-verified", "swebench-react", "swebench-rubric", "swebench-verified", ("none",)), - ("gdpeval", "gdpeval-react", "gdpeval-staged-rubric", "gdpeval", ("ergon-builtins[data]",)), - ("researchrubrics", "researchrubrics-researcher", "researchrubrics-rubric", "researchrubrics", ("ergon-builtins[data]",)), - ("researchrubrics-vanilla", "researchrubrics-researcher", "researchrubrics-rubric", "researchrubrics-vanilla", ("ergon-builtins[data]",)), -] -``` - -Use `("none",)` for e2e smoke replacement submissions, but keep production pairing documentation accurate for production data benchmarks. - -- [ ] **Step 2: Register final evaluator slugs** - -`registry_core.py` should expose both during migration: - -```python -EVALUATORS = { - "staged-rubric": StagedRubric, - "gdpeval-staged-rubric": StagedRubric, - ... -} -``` - -`registry_data.py` should expose: - -```python -EVALUATORS = { - "research-rubric": ResearchRubricsRubric, - "researchrubrics-rubric": ResearchRubricsRubric, -} -``` - -- [ ] **Step 3: Keep benchmark-owned worker factory surfaces** - -Required files: - -```text -ergon_builtins/ergon_builtins/benchmarks/minif2f/worker_factory.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/worker_factory.py -ergon_builtins/ergon_builtins/benchmarks/gdpeval/worker_factory.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/worker_factory.py -``` - -`researchrubrics/worker_factory.py` may re-export existing worker classes until a later physical move. - -- [ ] **Step 4: Run registry tests** - -Run: - -```bash -uv run pytest tests/unit/registry/test_builtin_pairings.py tests/unit/registry/test_react_factories.py -q -``` - -Expected: pass. - -## Task 5: Finish CLI Contract And Wrapper Behavior - -**Files:** -- Modify: `ergon_cli/ergon_cli/main.py` -- Modify: `ergon_cli/ergon_cli/commands/experiment.py` -- Modify: `ergon_cli/ergon_cli/commands/benchmark.py` -- Modify: `tests/unit/cli/test_experiment_cli.py` -- Modify: `tests/unit/cli/test_benchmark_setup.py` - -- [ ] **Step 1: Keep explicit define args required** - -Parser requirements: - -```text -ergon experiment define <benchmark> - --worker <worker> - --model <backend:model> - --evaluator <evaluator> - --sandbox <sandbox> - --extras <extra-or-none> -``` - -Test with: - -```bash -uv run pytest tests/unit/cli/test_experiment_cli.py::test_experiment_define_requires_explicit_runtime_choices -q -``` - -- [ ] **Step 2: Keep `benchmark run` as define-plus-run wrapper** - -`benchmark run` should parse the same explicit fields: - -```text -ergon benchmark run <benchmark> - --limit 1 - --worker <worker> - --model <backend:model> - --evaluator <evaluator> - --sandbox <sandbox> - --extras <extra-or-none> -``` - -If `ExperimentLaunchService.wait/timeout_seconds` is not implemented, do not expose `--timeout` or `--no-wait` on `benchmark run`. The wrapper should submit and print run IDs, not pretend to block. - -- [ ] **Step 3: Keep `benchmark setup` success hint explicit** - -Expected hint shape: - -```text -ergon benchmark run <slug> --limit 1 --worker <worker> --model <model> --evaluator <evaluator> --sandbox <slug> --extras none -``` - -Regression test: - -```python -def test_setup_success_hint_uses_explicit_runtime_choices(...): - rc = setup_benchmark(_make_args()) - out = capsys.readouterr().out - assert "--worker" in out - assert "--evaluator" in out - assert "--sandbox" in out - assert "--extras" in out -``` - -- [ ] **Step 4: Run CLI tests** - -Run: - -```bash -uv run pytest tests/unit/cli/test_experiment_cli.py tests/unit/cli/test_benchmark_setup.py -q -``` - -Expected: pass. - -## Task 6: Align `/api/test/write/cohort` With Explicit Test Harness Contract - -**Files:** -- Modify: `ergon_core/ergon_core/core/api/test_harness.py` or the current stable test harness module if moved -- Modify: `tests/integration/smokes/test_smoke_harness.py` -- Modify: `tests/e2e/_submit.py` - -- [ ] **Step 1: Ensure request DTO accepts explicit sandbox/extras** - -The stable test harness write request should accept: - -```python -class SubmitCohortRequest(BaseModel): - benchmark_slug: str - slots: list[CohortSlotRequest] - cohort_key: str - sandbox_slug: str | None = None - dependency_extras: tuple[str, ...] = ("none",) - model: str = "openai:gpt-4o" - limit: int = 1 -``` - -- [ ] **Step 2: Ensure the harness uses the same define/run service path** - -The handler should pass: - -```python -ExperimentDefineRequest( - benchmark_slug=body.benchmark_slug, - cohort_id=cohort.id, - limit=body.limit, - default_model_target=body.model, - default_worker_team={"primary": slot.worker_slug}, - default_evaluator_slug=slot.evaluator_slug, - sandbox_slug=body.sandbox_slug or body.benchmark_slug, - dependency_extras=body.dependency_extras, - metadata={"source": "test-harness"}, -) -``` - -If the core facade DTO names differ after the core refactor, adapt to the stable facade shape rather than private repositories. - -- [ ] **Step 3: Add integration assertion** - -In `tests/integration/smokes/test_smoke_harness.py`, assert the write endpoint accepts a payload with `sandbox_slug` and `dependency_extras` and returns run IDs. - -- [ ] **Step 4: Run smoke harness integration test** - -Run: - -```bash -uv run pytest tests/integration/smokes/test_smoke_harness.py -q -``` - -Expected: pass if stack dependencies for integration are available; otherwise skip should be environment-gated. - -## Task 7: Preserve E2E Runtime Assertions While Updating Access Paths - -**Files:** -- Modify: `tests/e2e/_asserts.py` -- Modify: `tests/e2e/test_researchrubrics_smoke.py` -- Modify: `tests/e2e/test_minif2f_smoke.py` -- Modify: `tests/e2e/test_swebench_smoke.py` -- Modify: `ergon-dashboard/tests/e2e/*.smoke.spec.ts` - -- [ ] **Step 1: Keep the behavioral assertions hard** - -Do not weaken these assertions: - -```python -assert snapshot.total_tasks == 12 -assert snapshot.total_leaf_tasks == 10 -assert len(probes) == 10 -assert len(resources) == 20 -assert event_count == 26 -assert len(evaluations) == 2 -assert scores == [1.0, 1.0] -assert len(msgs) == 11 -``` - -Sad path: - -```python -assert by_slug["l_2"].status == FAILED -assert by_slug["l_3"].status == BLOCKED -assert by_slug["l_3"].started_at is None -assert len(msgs) == 7 -``` - -- [ ] **Step 2: Update imports only** - -Replace any private core imports with: - -```python -from tests.e2e._read_contracts import require_run_snapshot -from ergon_core.test_support.smoke_fixtures.smoke_base.constants import EXPECTED_SUBTASK_SLUGS -``` - -And, where direct DB access is still needed: - -```python -from ergon_core.test_support.e2e_read_helpers import ... -``` - -- [ ] **Step 3: Keep dashboard assertions aligned** - -Playwright specs should assert visible behavior: - -```text -- run status is completed/failed as appropriate -- all expected task nodes appear -- failed l_2 and blocked l_3 are visible on sad path -- resource/evaluation panels render when expected -``` - -Do not assert private API response shapes unless the dashboard API marks them public/stable. - -## Task 8: Run The Non-E2E Verification Gate - -**Files:** -- No code changes unless tests fail. - -- [ ] **Step 1: Run focused unit/integration tests** - -Run: - -```bash -uv run pytest \ - tests/unit/registry/test_react_factories.py \ - tests/unit/registry/test_builtin_pairings.py \ - tests/unit/cli/test_experiment_cli.py \ - tests/unit/cli/test_benchmark_setup.py \ - tests/unit/smoke_base/test_e2e_smoke_driver_pairs.py \ - tests/unit/architecture/test_public_api_boundaries.py \ - tests/integration/smokes/test_smoke_harness.py \ - -q -``` - -Expected: pass or environment-gated integration skip. Any import failure from `tests/e2e` is a blocker. - -- [ ] **Step 2: Run e2e collection without executing live stack** - -Run: - -```bash -uv run pytest tests/e2e --collect-only -q -``` - -Expected: collection succeeds. This catches stale import paths without needing the stack. - -- [ ] **Step 3: Run lint diagnostics on touched test/docs paths** - -Use IDE lints for: - -```text -tests/e2e/ -tests/unit/registry/ -tests/unit/cli/ -tests/unit/smoke_base/ -docs/superpowers/plans/ -``` - -Expected: no new code-specific diagnostics. Environment import-resolution warnings are non-blocking only if pytest confirms imports. - -## Task 9: Full E2E Execution Gate - -**Files:** -- No code changes unless runtime evidence fails. - -- [ ] **Step 1: Verify stack env** - -Required environment: - -```text -ENABLE_TEST_HARNESS=1 -ENABLE_SMOKE_FIXTURES=1 -ERGON_STARTUP_PLUGINS=ergon_core.test_support.smoke_fixtures:register_smoke_fixtures -ERGON_API_BASE_URL=http://127.0.0.1:9000 -TEST_HARNESS_SECRET=<configured secret if required> -E2B_API_KEY=<available for real sandbox e2e> -``` - -- [ ] **Step 2: Run one smoke leg first** - -Run: - -```bash -uv run pytest tests/e2e/test_minif2f_smoke.py -q -s -``` - -Expected: - -- one happy run reaches `completed` -- one sad run reaches `failed` -- all hard assertions pass -- Playwright spec completes or captures failure screenshots - -- [ ] **Step 3: Run all smoke legs** - -Run: - -```bash -uv run pytest tests/e2e -q -s -``` - -Expected: - -- ResearchRubrics, MiniF2F, and SWE-Bench each submit happy/sad cohorts -- happy runs pass graph/resource/turn/evaluation/dashboard assertions -- sad runs pass blocked/failure/partial-artifact assertions - -## Task 10: Review And Handoff To Real-LLM Canaries - -**Files:** -- Modify only if review finds issues. - -- [ ] **Step 1: Request code review** - -Send reviewer scope: - -```text -Review built-ins, CLI, and e2e refactor completion. -Check that: -- no benchmark profiles/default pairings remain -- CLI requires explicit worker/model/evaluator/sandbox/extras -- e2e uses HTTP/test-support/read-model boundaries -- runtime behavior assertions remain hard -- no private core repository imports remain in e2e tests -``` - -- [ ] **Step 2: Fix Critical and Important review findings** - -Follow review feedback with tests for each fix. - -- [ ] **Step 3: Decide real-LLM canary timing** - -Only after e2e smoke is green, run or schedule: - -```bash -ERGON_REAL_LLM=1 uv run pytest tests/real_llm -q -s -``` - -If real-LLM tests still use stale CLI paths, update them to the same explicit runtime choice contract before running. - -## Completion Criteria - -- `tests/e2e --collect-only` succeeds without private core import failures. -- `tests/unit/architecture/test_public_api_boundaries.py` confirms e2e tests do not import private core repository/runtime internals. -- `tests/unit/registry/test_builtin_pairings.py` covers all documented production benchmark pairings. -- CLI parser tests prove explicit arguments are required. -- `/api/test/write/cohort` accepts explicit sandbox/extras and uses the same define/run facade path. -- Full e2e smoke suite preserves existing behavior assertions: - - 12 tasks, 10 leaves, 20 resources, 26 turns, 2 root evaluations on happy path - - `l_2` failed, `l_3` blocked, 7 completion messages on sad path -- Code review has no unresolved Critical or Important findings. - diff --git a/docs/superpowers/plans/2026-04-29-persistent-component-catalog-and-test-layout.md b/docs/superpowers/plans/2026-04-29-persistent-component-catalog-and-test-layout.md deleted file mode 100644 index 10b02a873..000000000 --- a/docs/superpowers/plans/2026-04-29-persistent-component-catalog-and-test-layout.md +++ /dev/null @@ -1,1944 +0,0 @@ -# Persistent Component Catalog And Test Layout Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make component registration understandable across processes by splitting tests by package ownership, persisting component slug-to-import references in Postgres, and deleting test/fixture env-var switches. - -**Architecture:** First reorganize tests so package boundaries are visible and cross-process E2E stays black-box. Then add a trusted `component_catalog` table in `ergon_core` that stores component kind, slug, module, qualname, and metadata. Finally, update the Pydantic registry to publish/load catalog rows, make runtime jobs resolve components through the catalog-backed registry, and remove `ENABLE_TEST_HARNESS`, `TEST_HARNESS_SECRET`, `ERGON_STARTUP_PLUGINS`, `ENABLE_SMOKE_FIXTURES`, and `ERGON_SKIP_INFRA_CHECK`. - -**Tech Stack:** Python 3.13, SQLModel, Alembic, Pydantic v2, pytest, FastAPI, argparse CLI, existing uv/pnpm scripts. - ---- - -## Service Design Constraint - -Use one catalog boundary: `ComponentCatalogService`. Do not implement both a service and repository for the catalog. The service owns the contract for publishing refs, requiring refs, and loading import refs; keep the API small so it does not become a second registry. - -## Mental Model - -The final system should be explainable as: - -1. Packages define components in Python code. -2. Packages publish component references into Postgres as trusted catalog rows. -3. Experiment definitions store stable slugs. -4. API/Inngest/CLI resolve slugs through the shared catalog, import the Python reference, and instantiate the component. -5. Tests are package-owned; only black-box E2E crosses process boundaries. - -The Pydantic registry remains useful as an authoring and publishing helper, but runtime resolution should read from Postgres every time. These lookups are not hot enough to justify an in-memory process-local cache, and always reading the catalog keeps cross-process behavior easier to reason about. - -## ID Model - -Use two task shapes, because authoring and runtime happen at different times: - -```python -TaskSpec # definition/authoring shape; no runtime id -Task.task_id # worker-facing runtime id -Task.task_id == RunGraphNode.id -``` - -Benchmark code defines static task specs before any run exists. Those specs are persisted as `ExperimentDefinitionTask` rows. When a run starts, core materializes/copies those definition tasks into `RunGraphNode` rows. Worker execution only receives the runtime `Task` built from a `RunGraphNode`. - -The faithful lifecycle is: - -```text -Benchmark.build_instances() returns TaskSpec objects - ↓ -ExperimentDefinitionTask rows store the static template - ↓ -Run launch creates RunGraphNode rows for that run - ↓ -Worker receives Task(task_id=RunGraphNode.id, ...) -``` - -Use one worker-facing task identity: - -```python -Task.task_id == RunGraphNode.id -``` - -`RunGraphNode.id` is the runtime task id. It exists for every executable task in a run, including dynamically spawned subtasks. This is the only task id worker authors should see. - -Use explicit names for internal/template identity: - -```python -definition_id # ExperimentDefinition.id, the static experiment template -definition_task_id # ExperimentDefinitionTask.id, persisted template link only -node_id # RunGraphNode.id, the runtime task identity -execution_id # RunTaskExecution.id, one attempt to execute a node -``` - -Do not pass `definition_task_id` through public worker-facing `Task` or runtime event/job payloads. Keep it only as an optional persisted relationship on rows such as `RunGraphNode` / `RunTaskExecution` when the application layer needs static-template joins. If runtime needs definition data, resolve it from `node_id` by loading `RunGraphNode` and following `RunGraphNode.definition_task_id`, or by using the already available run/definition context in the application layer. - -## File Structure - -- Create package-owned test roots: - - `ergon_core/tests/` - - `ergon_builtins/tests/` - - `ergon_cli/tests/` - - optionally `ergon_infra/tests/` -- Keep cross-package black-box tests at: - - `tests/e2e/` - - `tests/real_llm/` - - `tests/fixtures/` only for fixtures intentionally shared by black-box tests. -- Create component catalog files: - - `ergon_core/ergon_core/core/persistence/components/models.py` - - `ergon_core/ergon_core/core/application/components/catalog.py` - - `ergon_core/migrations/versions/<new>_add_component_catalog.py` -- Modify registry/bootstrap files: - - `ergon_core/ergon_core/api/benchmark/task.py` - - `ergon_core/ergon_core/api/worker/context.py` - - `ergon_core/ergon_core/api/worker/worker.py` - - `ergon_core/ergon_core/api/worker/__init__.py` - - `ergon_core/ergon_core/api/registry.py` - - `ergon_builtins/ergon_builtins/registry.py` - - `ergon_builtins/ergon_builtins/registry_core.py` - - `ergon_builtins/ergon_builtins/registry_data.py` - - `tests/fixtures/smoke_components/__init__.py` -- Modify runtime resolution files: - - `ergon_core/ergon_core/core/application/events/task_events.py` - - `ergon_core/ergon_core/core/application/jobs/models.py` - - `ergon_core/ergon_core/core/application/jobs/worker_execute.py` - - `ergon_core/ergon_core/core/application/jobs/execute_task.py` - - `ergon_core/ergon_core/core/application/workflows/orchestration.py` - - `ergon_core/ergon_core/core/application/jobs/evaluate_task_run.py` - - `ergon_core/ergon_core/core/application/jobs/sandbox_setup.py` - - `ergon_core/ergon_core/core/application/jobs/persist_outputs.py` - - `ergon_core/ergon_core/core/application/experiments/service.py` - - `ergon_core/ergon_core/core/application/experiments/launch.py` - - `ergon_core/ergon_core/core/application/workflows/service.py` - - `ergon_core/ergon_core/core/application/tasks/management.py` - - `ergon_core/ergon_core/core/domain/experiments/worker_spec.py` -- Modify harness/env-var files: - - `ergon_core/ergon_core/core/shared/settings.py` - - `ergon_core/ergon_core/core/rest_api/app.py` - - `ergon_core/ergon_core/core/rest_api/test_harness.py` - - `docker-compose.yml` - - `.github/workflows/e2e-benchmarks.yml` - - `.github/workflows/ci-fast.yml` - - `package.json` - - `scripts/smoke_local_up.sh` - - `scripts/smoke_local_run.sh` - - `tests/e2e/conftest.py` - - `tests/integration/conftest.py` - - dashboard test harness clients/routes that reference `TEST_HARNESS_SECRET`. - ---- - -### Task 1: Create Package-Owned Test Layout Guardrails - -**Files:** -- Create: `tests/unit/architecture/test_package_test_layout.py` -- Modify later: `package.json` - -- [ ] **Step 1: Write architecture test for target test layout** - -Create `tests/unit/architecture/test_package_test_layout.py`: - -```python -from pathlib import Path - - -def test_package_owned_test_roots_exist() -> None: - assert Path("ergon_core/tests").is_dir() - assert Path("ergon_builtins/tests").is_dir() - assert Path("ergon_cli/tests").is_dir() - - -def test_root_tests_are_black_box_or_shared_only() -> None: - allowed = { - "__init__.py", - "__pycache__", - "conftest.py", - "e2e", - "fixtures", - "integration", - "real_llm", - } - root_entries = {path.name for path in Path("tests").iterdir()} - assert root_entries <= allowed -``` - -- [ ] **Step 2: Run the architecture test and verify it fails** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_package_test_layout.py -q -``` - -Expected: FAIL because package-owned test roots do not exist and `tests/unit` still contains package-owned tests. - -- [ ] **Step 3: Create package-owned test directories** - -Create: - -```text -ergon_core/tests/unit/ -ergon_core/tests/integration/ -ergon_builtins/tests/unit/ -ergon_builtins/tests/integration/ -ergon_cli/tests/unit/ -ergon_cli/tests/integration/ -``` - -Add empty `__init__.py` files only if import/package semantics require them. Prefer no `__init__.py` for pytest discovery unless an existing pattern depends on package imports. - -- [ ] **Step 4: Update `package.json` scripts to include both old and new roots** - -Modify backend test scripts temporarily so moved tests can be discovered while migration is incremental: - -```json -"test:be:unit": "uv run pytest ergon_core/tests/unit ergon_builtins/tests/unit ergon_cli/tests/unit tests/unit -q -n auto --durations=20", -"test:be:coverage": "uv run pytest ergon_core/tests/unit ergon_builtins/tests/unit ergon_cli/tests/unit tests/unit tests/integration --cov=ergon_core --cov=ergon_builtins --cov-report=term-missing --cov-report=xml:coverage.xml" -``` - -- [ ] **Step 5: Run package layout test** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_package_test_layout.py -q -``` - -Expected: still FAIL until tests are moved in Tasks 2-4. - ---- - -### Task 2: Move Core-Owned Unit Tests To `ergon_core/tests` - -**Files:** -- Move tests from `tests/unit/api`, `tests/unit/runtime`, `tests/unit/sandbox`, selected `tests/unit/architecture`, selected `tests/unit/state`, and core app tests into `ergon_core/tests/unit`. -- Modify imports only where they reference moved fixture paths. - -- [ ] **Step 1: Move clearly core-owned directories** - -Move: - -```text -tests/unit/api/ -> ergon_core/tests/unit/api/ -tests/unit/runtime/ -> ergon_core/tests/unit/runtime/ -tests/unit/sandbox/ -> ergon_core/tests/unit/sandbox/ -tests/unit/persistence/ -> ergon_core/tests/unit/persistence/ -tests/unit/dashboard/ -> ergon_core/tests/unit/dashboard/ -``` - -Move standalone core app tests: - -```text -tests/unit/test_app_mounts_harness_conditionally.py -> ergon_core/tests/unit/test_app_mounts_harness_conditionally.py -tests/unit/test_dashboard_emitter_wiring.py -> ergon_core/tests/unit/test_dashboard_emitter_wiring.py -tests/unit/test_rollouts_di.py -> ergon_core/tests/unit/test_rollouts_di.py -tests/unit/test_test_harness.py -> ergon_core/tests/unit/test_test_harness.py -tests/unit/test_swebench_criterion_no_sandbox.py -> ergon_core/tests/unit/test_swebench_criterion_no_sandbox.py -``` - -- [ ] **Step 2: Move registry/core architecture tests** - -Move: - -```text -tests/unit/registry/ -> ergon_core/tests/unit/registry/ -tests/unit/architecture/test_api_runs_boundary.py -> ergon_core/tests/unit/architecture/test_api_runs_boundary.py -tests/unit/architecture/test_core_schema_sources.py -> ergon_core/tests/unit/architecture/test_core_schema_sources.py -tests/unit/architecture/test_model_field_descriptions.py -> ergon_core/tests/unit/architecture/test_model_field_descriptions.py -tests/unit/architecture/test_no_test_logic_in_core.py -> ergon_core/tests/unit/architecture/test_no_test_logic_in_core.py -tests/unit/architecture/test_persistence_boundaries.py -> ergon_core/tests/unit/architecture/test_persistence_boundaries.py -tests/unit/architecture/test_public_api_boundaries.py -> ergon_core/tests/unit/architecture/test_public_api_boundaries.py -tests/unit/architecture/test_public_api_target_structure.py -> ergon_core/tests/unit/architecture/test_public_api_target_structure.py -tests/unit/architecture/test_smoke_fixture_package_boundary.py -> ergon_core/tests/unit/architecture/test_smoke_fixture_package_boundary.py -``` - -Leave `tests/unit/architecture/test_package_test_layout.py` at root until the migration is complete because it governs the whole repo. - -- [ ] **Step 3: Run moved core tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit -q -``` - -Expected: PASS or failures that reveal imports still pointing at old `tests/unit/...` paths. - -- [ ] **Step 4: Fix import paths revealed by failures** - -For each failure, update imports to either: - -```python -from tests.fixtures... -``` - -for intentionally shared black-box fixtures, or local package test helpers under: - -```python -from ergon_core.tests... -``` - -Do not import `ergon_builtins` in core unit tests unless the test is explicitly an integration/boundary test that names that dependency. - -- [ ] **Step 5: Run old and new unit suites** - -Run: - -```bash -uv run pytest ergon_core/tests/unit tests/unit -q -``` - -Expected: PASS, with fewer tests left under `tests/unit`. - ---- - -### Task 3: Move Builtins-Owned Tests To `ergon_builtins/tests` - -**Files:** -- Move benchmark, worker, builtins state, smoke component tests that assert builtins behavior. - -- [ ] **Step 1: Move builtins benchmark/worker tests** - -Move: - -```text -tests/unit/benchmarks/ -> ergon_builtins/tests/unit/benchmarks/ -tests/unit/builtins/ -> ergon_builtins/tests/unit/builtins/ -tests/unit/workers/ -> ergon_builtins/tests/unit/workers/ -tests/unit/state/test_benchmark_contract.py -> ergon_builtins/tests/unit/state/test_benchmark_contract.py -tests/unit/state/test_gdpeval_benchmark.py -> ergon_builtins/tests/unit/state/test_gdpeval_benchmark.py -tests/unit/state/test_research_rubrics_benchmark.py -> ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py -tests/unit/state/test_research_rubrics_workers.py -> ergon_builtins/tests/unit/state/test_research_rubrics_workers.py -tests/unit/state/test_llm_judge_runtime_injection.py -> ergon_builtins/tests/unit/state/test_llm_judge_runtime_injection.py -tests/unit/state/test_criteria_do_not_spawn_sandboxes.py -> ergon_builtins/tests/unit/state/test_criteria_do_not_spawn_sandboxes.py -``` - -- [ ] **Step 2: Move smoke component unit tests** - -Move: - -```text -tests/unit/smoke_base/ -> ergon_builtins/tests/unit/smoke_base/ -``` - -Rationale: the fixture source remains at `tests/fixtures/smoke_components` because E2E consumes it as shared black-box fixture code, but unit tests for that fixture behavior should not live in root `tests/unit`. - -- [ ] **Step 3: Run builtins tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit -q -``` - -Expected: PASS or import failures from moved helper paths. - -- [ ] **Step 4: Fix moved builtins imports** - -Update any relative references from old root locations. Keep production imports from `ergon_builtins.*` unchanged. - -- [ ] **Step 5: Run package test subset** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit ergon_core/tests/unit tests/unit -q -``` - -Expected: PASS. - ---- - -### Task 4: Move CLI-Owned Tests To `ergon_cli/tests` - -**Files:** -- Move CLI unit tests and CLI-specific state tests. - -- [ ] **Step 1: Move CLI tests** - -Move: - -```text -tests/unit/cli/ -> ergon_cli/tests/unit/cli/ -tests/unit/state/test_onboard_profile.py -> ergon_cli/tests/unit/state/test_onboard_profile.py -tests/unit/state/test_env_writer.py -> ergon_cli/tests/unit/state/test_env_writer.py -tests/unit/state/test_openrouter_model_resolution.py -> ergon_cli/tests/unit/state/test_openrouter_model_resolution.py -tests/unit/state/test_subtask_lifecycle_toolkit.py -> ergon_cli/tests/unit/state/test_subtask_lifecycle_toolkit.py -tests/unit/state/test_workflow_cli_tool.py -> ergon_cli/tests/unit/state/test_workflow_cli_tool.py -``` - -- [ ] **Step 2: Run CLI tests** - -Run: - -```bash -uv run pytest ergon_cli/tests/unit -q -``` - -Expected: PASS or import failures that identify old paths. - -- [ ] **Step 3: Update `package.json` to remove old unit root once empty** - -After Tasks 2-4, if `tests/unit` contains only architecture migration tests or is empty, update scripts: - -```json -"test:be:unit": "uv run pytest ergon_core/tests/unit ergon_builtins/tests/unit ergon_cli/tests/unit -q -n auto --durations=20" -``` - -If a small root `tests/unit` remains for repo-wide architecture tests, include it explicitly: - -```json -"test:be:unit": "uv run pytest ergon_core/tests/unit ergon_builtins/tests/unit ergon_cli/tests/unit tests/unit -q -n auto --durations=20" -``` - -- [ ] **Step 4: Run package layout guardrail** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_package_test_layout.py -q -``` - -Expected: PASS. - ---- - -### Task 5: Add Component Catalog Persistence Model And Migration - -**Files:** -- Create: `ergon_core/ergon_core/core/persistence/components/models.py` -- Modify: `ergon_core/migrations/env.py` -- Create: `ergon_core/migrations/versions/<revision>_add_component_catalog.py` -- Test: `ergon_core/tests/unit/registry/test_component_catalog_model.py` - -- [ ] **Step 1: Write catalog model tests** - -Create `ergon_core/tests/unit/registry/test_component_catalog_model.py`: - -```python -import pytest - -from ergon_core.core.persistence.components.models import ComponentCatalogEntry - - -def test_component_catalog_entry_round_trips_metadata() -> None: - entry = ComponentCatalogEntry( - kind="worker", - slug="training-stub", - module="ergon_builtins.shared.workers.training_stub_worker", - qualname="TrainingStubWorker", - package="ergon-builtins", - metadata_json={"description": "offline worker"}, - ) - - assert entry.parsed_metadata() == {"description": "offline worker"} - - -def test_component_catalog_entry_rejects_invalid_kind() -> None: - with pytest.raises(ValueError, match="kind must be one of"): - ComponentCatalogEntry( - kind="not-a-kind", - slug="bad", - module="pkg.mod", - qualname="Thing", - ) -``` - -- [ ] **Step 2: Run catalog model tests and verify they fail** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_catalog_model.py -q -``` - -Expected: FAIL because the model module does not exist. - -- [ ] **Step 3: Implement SQLModel catalog entry** - -Create `ergon_core/ergon_core/core/persistence/components/models.py`: - -```python -"""Persistent component catalog shared across CLI/API/Inngest processes.""" - -from datetime import datetime -from uuid import UUID, uuid4 - -from ergon_core.core.shared.json_types import JsonObject -from ergon_core.core.shared.utils import utcnow as _utcnow -from pydantic import model_validator -from sqlalchemy import JSON, Column, DateTime, UniqueConstraint -from sqlmodel import Field, SQLModel - -TZDateTime = DateTime(timezone=True) -COMPONENT_KINDS = {"worker", "benchmark", "evaluator", "sandbox_manager"} - - -class ComponentCatalogEntry(SQLModel, table=True): - __tablename__ = "component_catalog" - __table_args__ = (UniqueConstraint("kind", "slug", name="uq_component_catalog_kind_slug"),) - - id: UUID = Field(default_factory=uuid4, primary_key=True) - kind: str = Field(index=True) - slug: str = Field(index=True) - module: str - qualname: str - package: str | None = Field(default=None, index=True) - version: str | None = None - metadata_json: dict = Field(default_factory=dict, sa_column=Column(JSON)) - created_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - updated_at: datetime = Field(default_factory=_utcnow, sa_type=TZDateTime) - - def parsed_metadata(self) -> JsonObject: - return self.__class__._parse_metadata(self.metadata_json) - - @classmethod - def _parse_metadata(cls, data: dict) -> JsonObject: - if not isinstance(data, dict): - raise ValueError(f"metadata_json must be a dict, got {type(data).__name__}") - return data - - @model_validator(mode="after") - def _validate_entry(self) -> "ComponentCatalogEntry": - if self.kind not in COMPONENT_KINDS: - allowed = ", ".join(sorted(COMPONENT_KINDS)) - raise ValueError(f"kind must be one of: {allowed}") - if not self.slug: - raise ValueError("slug must be non-empty") - if not self.module: - raise ValueError("module must be non-empty") - if not self.qualname: - raise ValueError("qualname must be non-empty") - self.__class__._parse_metadata(self.metadata_json) - return self -``` - -- [ ] **Step 4: Import component models in Alembic env** - -Modify `ergon_core/migrations/env.py`: - -```python -import ergon_core.core.persistence.components.models -``` - -Add it beside the other persistence model imports. - -- [ ] **Step 5: Add Alembic migration** - -Create a migration file under `ergon_core/migrations/versions/` with a new revision id: - -```python -"""add component catalog - -Revision ID: d1e2f3a4b5c6 -Revises: c2d3e4f5a6b7 -Create Date: 2026-04-29 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -revision: str = "d1e2f3a4b5c6" -down_revision: str | None = "c2d3e4f5a6b7" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.create_table( - "component_catalog", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("kind", sa.String(), nullable=False), - sa.Column("slug", sa.String(), nullable=False), - sa.Column("module", sa.String(), nullable=False), - sa.Column("qualname", sa.String(), nullable=False), - sa.Column("package", sa.String(), nullable=True), - sa.Column("version", sa.String(), nullable=True), - sa.Column("metadata_json", sa.JSON(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("kind", "slug", name="uq_component_catalog_kind_slug"), - ) - op.create_index("ix_component_catalog_kind", "component_catalog", ["kind"], unique=False) - op.create_index("ix_component_catalog_slug", "component_catalog", ["slug"], unique=False) - op.create_index("ix_component_catalog_package", "component_catalog", ["package"], unique=False) - - -def downgrade() -> None: - op.drop_index("ix_component_catalog_package", table_name="component_catalog") - op.drop_index("ix_component_catalog_slug", table_name="component_catalog") - op.drop_index("ix_component_catalog_kind", table_name="component_catalog") - op.drop_table("component_catalog") -``` - -Before choosing `down_revision`, inspect the current migration head with: - -```bash -uv run alembic -c ergon_core/alembic.ini heads -``` - -Use the actual head instead of the placeholder if different. - -- [ ] **Step 6: Run catalog model tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_catalog_model.py -q -``` - -Expected: PASS. - ---- - -### Task 6: Add Component Catalog Service And Import Reference Loader - -**Files:** -- Create: `ergon_core/ergon_core/core/application/components/__init__.py` -- Create: `ergon_core/ergon_core/core/application/components/catalog.py` -- Test: `ergon_core/tests/unit/registry/test_component_catalog_service.py` - -- [ ] **Step 1: Write catalog service tests** - -Create `ergon_core/tests/unit/registry/test_component_catalog_service.py`: - -```python -import pytest -from sqlalchemy.pool import StaticPool -from sqlmodel import Session, SQLModel, create_engine - -from ergon_core.core.application.components.catalog import ( - ComponentCatalogService, - ComponentRef, - import_component_ref, -) -from ergon_core.core.persistence.components.models import ComponentCatalogEntry - - -def _session() -> Session: - engine = create_engine( - "sqlite://", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - SQLModel.metadata.create_all(engine) - return Session(engine) - - -def test_upsert_and_require_component_ref() -> None: - session = _session() - service = ComponentCatalogService() - - service.upsert( - session, - ComponentRef( - kind="worker", - slug="training-stub", - module="ergon_builtins.shared.workers.training_stub_worker", - qualname="TrainingStubWorker", - package="ergon-builtins", - metadata={"install_hint": "none"}, - ), - ) - session.commit() - - ref = service.require(session, kind="worker", slug="training-stub") - assert ref.module == "ergon_builtins.shared.workers.training_stub_worker" - assert ref.qualname == "TrainingStubWorker" - assert ref.metadata == {"install_hint": "none"} - - -def test_upsert_updates_existing_ref() -> None: - session = _session() - service = ComponentCatalogService() - - service.upsert(session, ComponentRef(kind="worker", slug="x", module="old", qualname="Thing")) - service.upsert(session, ComponentRef(kind="worker", slug="x", module="new", qualname="Other")) - session.commit() - - rows = session.query(ComponentCatalogEntry).all() - assert len(rows) == 1 - assert service.require(session, kind="worker", slug="x").module == "new" - - -def test_import_component_ref_imports_module_qualname() -> None: - ref = ComponentRef( - kind="worker", - slug="component-ref", - module="ergon_core.core.application.components.catalog", - qualname="ComponentRef", - ) - - assert import_component_ref(ref) is ComponentRef - - -def test_require_unknown_component_lists_kind_and_slug() -> None: - session = _session() - - with pytest.raises(ValueError, match="Unknown worker component slug 'missing'"): - ComponentCatalogService().require(session, kind="worker", slug="missing") -``` - -- [ ] **Step 2: Run catalog service tests and verify they fail** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_catalog_service.py -q -``` - -Expected: FAIL because `ComponentCatalogService` does not exist. - -- [ ] **Step 3: Implement component catalog service** - -Create the package marker: - -```python -"""Component catalog application services.""" -``` - -Create `ergon_core/ergon_core/core/application/components/catalog.py`: - -```python -"""Application service for trusted component catalog references.""" - -from importlib import import_module -from typing import Any - -from ergon_core.core.persistence.components.models import ComponentCatalogEntry -from ergon_core.core.shared.json_types import JsonObject -from ergon_core.core.shared.utils import utcnow -from pydantic import BaseModel, ConfigDict, Field -from sqlmodel import Session, select - - -class ComponentRef(BaseModel): - model_config = ConfigDict(frozen=True) - - kind: str - slug: str - module: str - qualname: str - package: str | None = None - version: str | None = None - metadata: JsonObject = Field(default_factory=dict) - - -class ComponentCatalogService: - def upsert(self, session: Session, ref: ComponentRef) -> ComponentCatalogEntry: - existing = session.exec( - select(ComponentCatalogEntry).where( - ComponentCatalogEntry.kind == ref.kind, - ComponentCatalogEntry.slug == ref.slug, - ) - ).one_or_none() - - row = existing or ComponentCatalogEntry( - kind=ref.kind, - slug=ref.slug, - module=ref.module, - qualname=ref.qualname, - ) - row.module = ref.module - row.qualname = ref.qualname - row.package = ref.package - row.version = ref.version - row.metadata_json = dict(ref.metadata) - row.updated_at = utcnow() - session.add(row) - return row - - def require(self, session: Session, *, kind: str, slug: str) -> ComponentRef: - row = session.exec( - select(ComponentCatalogEntry).where( - ComponentCatalogEntry.kind == kind, - ComponentCatalogEntry.slug == slug, - ) - ).one_or_none() - if row is None: - raise ValueError(f"Unknown {kind} component slug {slug!r}") - return _row_to_ref(row) - - def load_ref(self, ref: ComponentRef) -> Any: # slopcop: ignore[no-typing-any] - return import_component_ref(ref) - - -def import_component_ref(ref: ComponentRef) -> Any: # slopcop: ignore[no-typing-any] - target: Any = import_module(ref.module) # slopcop: ignore[no-typing-any] - for part in ref.qualname.split("."): - target = getattr(target, part) - return target - - -def _row_to_ref(row: ComponentCatalogEntry) -> ComponentRef: - return ComponentRef( - kind=row.kind, - slug=row.slug, - module=row.module, - qualname=row.qualname, - package=row.package, - version=row.version, - metadata=row.parsed_metadata(), - ) -``` - -- [ ] **Step 4: Run catalog service tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_catalog_service.py -q -``` - -Expected: PASS. - ---- - -### Task 7: Split Definition Task Specs From Runtime Worker Tasks - -**Files:** -- Modify: `ergon_core/ergon_core/api/benchmark/task.py` -- Modify: `ergon_core/ergon_core/api/benchmark/__init__.py` -- Modify: `ergon_core/ergon_core/api/worker/context.py` -- Modify: `ergon_core/ergon_core/api/worker/worker.py` -- Modify: `ergon_core/ergon_core/core/application/events/task_events.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/models.py` -- Modify: `ergon_core/ergon_core/core/application/workflows/orchestration.py` -- Modify: `ergon_core/ergon_core/core/application/experiments/definition_writer.py` -- Modify: `ergon_core/ergon_core/core/application/jobs/execute_task.py` -- Modify worker subclasses/factories that still require `task_id` or `sandbox_id` -- Test: `ergon_core/tests/unit/api/test_task_spec_contract.py` -- Test: `ergon_core/tests/unit/api/test_worker_contract.py` - -- [ ] **Step 1: Write task spec and worker task contract tests** - -Create `ergon_core/tests/unit/api/test_task_spec_contract.py`: - -```python -from uuid import uuid4 - -import pytest -from pydantic import ValidationError - -from ergon_core.api.benchmark import EmptyTaskPayload, Task, TaskSpec - - -def test_task_spec_is_definition_time_and_has_no_runtime_id() -> None: - spec = TaskSpec( - task_slug="root", - instance_key="default", - description="Definition-time task", - task_payload=EmptyTaskPayload(), - ) - - assert spec.task_slug == "root" - assert not hasattr(spec, "task_id") - - -def test_worker_task_requires_runtime_graph_node_identity() -> None: - node_id = uuid4() - - task = Task( - task_id=node_id, - task_slug="root", - instance_key="default", - description="Runtime task", - ) - - assert task.task_id == node_id - - -def test_worker_task_rejects_missing_runtime_identity() -> None: - with pytest.raises(ValidationError, match="task_id"): - Task( - task_slug="root", - instance_key="default", - description="Runtime task", - ) -``` - -Create `ergon_core/tests/unit/api/test_worker_contract.py`: - -```python -from collections.abc import AsyncGenerator - -from ergon_core.api.benchmark import Task -from ergon_core.api.worker import Worker, WorkerContext, WorkerOutput -from ergon_core.api.worker.worker import WorkerStreamItem - - -class ContractSmokeWorker(Worker): - type_slug = "contract-smoke-worker" - - async def execute( - self, - task: Task, - *, - context: WorkerContext, - ) -> AsyncGenerator[WorkerStreamItem, None]: - yield WorkerOutput(output="ok", success=True) - - -def test_worker_constructor_has_only_authoring_configuration() -> None: - worker = ContractSmokeWorker(name="primary", model="stub:constant") - - assert isinstance(worker, ContractSmokeWorker) - assert worker.name == "primary" - assert worker.model == "stub:constant" -``` - -- [ ] **Step 2: Run task and worker contract tests and verify they fail** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/api/test_task_spec_contract.py ergon_core/tests/unit/api/test_worker_contract.py -q -``` - -Expected: FAIL because `TaskSpec` does not exist yet, worker-facing `Task.task_id` is not required yet, and `Worker.__init__` still requires `task_id` and `sandbox_id`. - -- [ ] **Step 3: Add `TaskSpec` for definition-time authoring and make `Task` runtime-only** - -Modify `ergon_core/ergon_core/api/benchmark/task.py`: - -```python -from typing import Generic, TypeVar -from uuid import UUID - -from pydantic import BaseModel, Field - - -class EmptyTaskPayload(BaseModel): - """Default payload for benchmarks that do not need task-specific data.""" - - model_config = {"extra": "forbid", "frozen": True} - - -PayloadT = TypeVar( - "PayloadT", - bound=BaseModel, - default=EmptyTaskPayload, - covariant=True, -) - - -class TaskSpec(BaseModel, Generic[PayloadT]): - """Definition-time task template produced by benchmark authoring code.""" - - model_config = {"frozen": True} - - task_slug: str - instance_key: str - description: str - parent_task_slug: str | None = None - dependency_task_slugs: tuple[str, ...] = () - evaluator_binding_keys: tuple[str, ...] = () - task_payload: PayloadT = Field(default_factory=EmptyTaskPayload) # ty: ignore[invalid-assignment] - - -class Task(BaseModel, Generic[PayloadT]): - """Runtime task passed to Worker.execute().""" - - model_config = {"frozen": True} - - task_id: UUID - task_slug: str - instance_key: str - description: str - parent_task_slug: str | None = None - dependency_task_slugs: tuple[str, ...] = () - evaluator_binding_keys: tuple[str, ...] = () - task_payload: PayloadT = Field(default_factory=EmptyTaskPayload) # ty: ignore[invalid-assignment] -``` - -Modify `ergon_core/ergon_core/api/benchmark/__init__.py` to export both names: - -```python -from ergon_core.api.benchmark.task import EmptyTaskPayload, Task, TaskSpec -``` - -Benchmark authoring code should construct `TaskSpec`, because no run graph node exists yet. Worker execution code should construct `Task`, because it has a concrete `RunGraphNode.id`. - -`Task.task_id` is the worker-facing runtime task identity. It must always be `RunGraphNode.id`, not `ExperimentDefinitionTask.id`. Static definition tasks and dynamic subtasks both have a `RunGraphNode`, so worker authors get one non-null task id for every execution. - -- [ ] **Step 3a: Update benchmark authoring APIs to return `TaskSpec`** - -Update benchmark protocol/type hints and tests so `Benchmark.build_instances()` returns definition-time specs: - -```python -from ergon_core.api.benchmark import TaskSpec - -def build_instances(self) -> Mapping[str, Sequence[TaskSpec[BaseModel]]]: - return { - "sample-1": [ - TaskSpec( - task_slug="root", - instance_key="sample-1", - description="Solve sample 1", - ) - ] - } -``` - -Update `ergon_core/ergon_core/core/application/experiments/definition_writer.py` to persist `TaskSpec` objects into `ExperimentDefinitionTask` rows. Do not invent a `task_id` during this phase. - -The persistence mapping should remain: - -```python -ExperimentDefinitionTask( - id=uuid4(), - experiment_definition_id=definition_id, - instance_id=instance_id, - task_slug=spec.task_slug, - parent_task_id=parent_task_id, - description=spec.description, - task_payload_json=spec.task_payload.model_dump(mode="json"), -) -``` - -- [ ] **Step 3b: Keep launch/materialization as the definition-to-run copy boundary** - -When a run starts, core must materialize static definition tasks into `RunGraphNode` rows. Preserve the persisted template link but do not pass it through worker-facing payloads: - -```python -RunGraphNode( - run_id=run_id, - definition_task_id=definition_task.id, - instance_key=instance.instance_key, - task_slug=definition_task.task_slug, - description=definition_task.description, - status=TaskExecutionStatus.PENDING, - assigned_worker_slug=assigned_worker_slug, - parent_node_id=parent_node_id, - level=level, -) -``` - -Dynamic subtasks skip the definition layer and insert directly into `RunGraphNode` with `definition_task_id=None`. - -- [ ] **Step 3c: Remove nullable task identity from runtime payloads** - -Remove the old nullable event/request `task_id` from runtime payloads. Runtime events/jobs should carry `node_id` as the task identity: - -```python -node_id: UUID # RunGraphNode.id; runtime task identity -``` - -Then remove the nullable worker-facing `task_id` from `WorkerContext`. The worker-facing contract should be: - -```python -task.task_id # non-null RunGraphNode.id -context.sandbox_id # non-null sandbox identity -``` - -If helper tools need a sandbox/task key, pass `task.task_id` to those helpers explicitly when building them. Do not use `WorkerContext.task_id` as a second, nullable source of truth. - -Remove internal event and job fields that currently use nullable `task_id` for `ExperimentDefinitionTask.id`: - -```python -class TaskReadyEvent(InngestEventContract): - run_id: UUID - definition_id: UUID - node_id: UUID -``` - -Apply the same shape to: - -- `TaskStartedEvent` -- `TaskCompletedEvent` -- `TaskFailedEvent` -- `PrepareTaskExecutionCommand` -- `WorkerExecuteRequest` -- `EvaluateTaskRunRequest` - -Keep `PreparedTaskExecution.node_id` as the canonical runtime task identity. Keep `RunGraphNode.definition_task_id` and `RunTaskExecution.definition_task_id` only as persisted relationships for static-template joins. If a service needs the static definition task row, it should load `RunGraphNode` by `node_id` and follow `RunGraphNode.definition_task_id`; do not carry that id through event payloads or public `Task`. - -- [ ] **Step 4: Simplify `Worker.__init__`** - -Modify `ergon_core/ergon_core/api/worker/worker.py`: - -```python -def __init__( - self, - *, - name: str, - model: str | None, - metadata: Mapping[str, Any] | None = None, # slopcop: ignore[no-typing-any] -) -> None: - self.name = name - self.model = model - self.metadata: dict[str, Any] = dict(metadata or {}) # slopcop: ignore[no-typing-any] -``` - -Do not keep `self.task_id` or `self.sandbox_id` on `Worker`. Workers should use `task.task_id` and `context.sandbox_id` inside `execute(...)`. - -- [ ] **Step 5: Refactor builtin worker factories into Worker subclasses** - -Replace factory functions such as `minif2f_react(...)` and `swebench_react(...)` with importable `Worker` subclasses. Those classes should build sandbox-bound tools inside `execute(...)`, using the runtime objects they already receive: - -```python -async def execute(self, task: Task, *, context: WorkerContext) -> AsyncGenerator[WorkerStreamItem, None]: - sandbox = MiniF2FSandboxManager().reconnect(context.sandbox_id) - toolkit = MiniF2FToolkit(...) - delegate = ReActWorker( - name=self.name, - model=self.model, - tools=list(toolkit.get_tools()), - system_prompt=MINIF2F_SYSTEM_PROMPT, - max_iterations=30, - ) - async for item in delegate.execute(task, context=context): - yield item -``` - -If a sandbox manager currently only looks up sandboxes by definition task id, add a public lookup/reconnect path by `sandbox_id`. Do not force worker construction to know about sandbox registry keys. - -- [ ] **Step 6: Run task spec and worker contract tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/api/test_task_spec_contract.py ergon_core/tests/unit/api/test_worker_contract.py -q -``` - -Expected: PASS. - ---- - -### Task 8: Update Pydantic Registry To Produce And Publish Component Refs - -**Files:** -- Modify: `ergon_core/ergon_core/api/registry.py` -- Test: `ergon_core/tests/unit/registry/test_component_registry.py` - -- [ ] **Step 1: Add tests for ref generation and deregistration** - -Extend `ergon_core/tests/unit/registry/test_component_registry.py`: - -```python -def test_registry_records_import_refs_for_registered_components() -> None: - registry = ComponentRegistry(catalog_service=ComponentCatalogService()) - - registry.register_worker(ExampleWorker.type_slug, ExampleWorker) - ref = registry.component_refs[("worker", "example-worker")] - - assert ref.kind == "worker" - assert ref.slug == "example-worker" - assert ref.module == __name__ - assert ref.qualname == "ExampleWorker" - - -def test_registry_deregister_removes_component_and_ref() -> None: - registry = ComponentRegistry(catalog_service=ComponentCatalogService()) - registry.register_worker("example-worker", ExampleWorker) - - registry.deregister("worker", "example-worker") - - assert "example-worker" not in registry.workers - assert ("worker", "example-worker") not in registry.component_refs -``` - -- [ ] **Step 2: Run registry tests and verify they fail** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_registry.py -q -``` - -Expected: FAIL because `component_refs` and `deregister` do not exist. - -- [ ] **Step 3: Add `ComponentRef` tracking to `ComponentRegistry`** - -Modify `ergon_core/ergon_core/api/registry.py`: - -```python -from ergon_core.core.application.components.catalog import ComponentCatalogService, ComponentRef -from sqlmodel import Session -``` - -Add field: - -```python -catalog_service: ComponentCatalogService -component_refs: dict[tuple[str, str], ComponentRef] = Field(default_factory=dict) -``` - -Update register methods to call a private helper after `_register`: - -```python -self._remember_ref("worker", slug, worker_cls) -``` - -Implement: - -```python -def deregister(self, kind: str, slug: str) -> None: - mapping = self._mapping_for(kind) - mapping.pop(slug, None) - self.component_refs.pop((kind, slug), None) - -def publish(self, session: Session) -> None: - for ref in self.component_refs.values(): - self.catalog_service.upsert(session, ref) - -def _remember_ref(self, kind: str, slug: str, value: object) -> None: - self.component_refs[(kind, slug)] = ComponentRef( - kind=kind, - slug=slug, - module=value.__module__, - qualname=value.__qualname__, - ) -``` - -For worker classes, `__qualname__` is sufficient if the class is module-level. If a value lacks `__module__` or `__qualname__`, raise `ValueError` with a clear message. Do not preserve the old `WorkerFactory` public alias; workers should be registered as importable `Worker` subclasses and constructed by the catalog with only authoring configuration (`name`, `model`, metadata). - -Construct the global authoring registry with an explicit service dependency: - -```python -registry = ComponentRegistry(catalog_service=ComponentCatalogService()) -``` - -Do not use nullable service parameters or ad hoc fallback construction such as `service or ComponentCatalogService()`. Tests that need isolation should pass their own `ComponentCatalogService()` when constructing a fresh `ComponentRegistry`. - -- [ ] **Step 4: Run registry tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_component_registry.py -q -``` - -Expected: PASS. - ---- - -### Task 9: Register Builtins And Smoke Components Into The Catalog - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/registry.py` -- Modify: `tests/fixtures/smoke_components/__init__.py` -- Test: `ergon_builtins/tests/unit/registry/test_builtin_pairings.py` or moved equivalent. - -- [ ] **Step 1: Add tests that builtins can publish refs into a DB session** - -Create or extend builtins registry tests: - -```python -from sqlalchemy.pool import StaticPool -from sqlmodel import Session, SQLModel, create_engine - -from ergon_core.api.registry import ComponentRegistry -from ergon_core.core.application.components.catalog import ComponentCatalogService - - -def _session() -> Session: - engine = create_engine( - "sqlite://", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - SQLModel.metadata.create_all(engine) - return Session(engine) - - -def test_register_builtins_can_publish_component_refs() -> None: - from ergon_builtins.registry import register_builtins - - service = ComponentCatalogService() - registry = ComponentRegistry(catalog_service=service) - register_builtins(registry) - session = _session() - - registry.publish(session) - session.commit() - - ref = service.require(session, kind="worker", slug="training-stub") - assert ref.module.endswith("training_stub_worker") - assert ref.qualname == "TrainingStubWorker" -``` - -- [ ] **Step 2: Run publishing test and verify it fails if refs are incomplete** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/registry -q -``` - -Expected: PASS if Task 8 is complete; otherwise FAIL on missing refs. - -- [ ] **Step 3: Keep publishing explicit and outside registration functions** - -Keep registration functions focused on filling the in-process authoring registry: - -```python -def register_builtins(target: ComponentRegistry = registry) -> None: - register_core_builtins(target) - _register_local_model_builtins() - _register_data_builtins(target) -``` - -Do not make builtins import DB/session code. Keep publishing as an explicit caller responsibility: - -```python -register_builtins(registry) -with get_session() as session: - registry.publish(session) - session.commit() -``` - -This keeps builtins package independent of persistence. - -- [ ] **Step 4: Run builtins registry tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/registry -q -``` - -Expected: PASS. - -- [ ] **Step 5: Remove legacy builtins registry dict snapshots** - -After publishing tests pass, delete legacy dict snapshot exports from `ergon_builtins/ergon_builtins/registry.py`. The top-level builtins registry module should expose registration functions and install hints only, not old process-local maps. - -Remove exports named: - -```python -BENCHMARKS -WORKERS -EVALUATORS -SANDBOX_MANAGERS -MODEL_BACKENDS -``` - -Keep sub-registry implementation details in `registry_core.py` and `registry_data.py` only as inputs to `register_core_builtins()` and `register_data_builtins()`. Update tests/callers that imported top-level dict snapshots to use either `ComponentRegistry` in authoring tests or `ComponentCatalogService` in runtime/catalog tests. - -- [ ] **Step 6: Convert worker factory functions to Worker subclasses** - -Before publishing worker refs into the catalog, ensure every registered worker slug points at an importable `Worker` subclass. If any existing builtins are module-level factory functions that return workers, replace them with small `Worker` subclasses or move their construction logic into the subclass initializer. - -This keeps the public mental model simple: - -```python -register_worker("training-stub", TrainingStubWorker) -worker = catalog.build_worker(session, slug="training-stub", name="primary", model="stub:constant") -``` - -There should be no public `Callable[..., Worker]` / `WorkerFactory` API after this migration. - ---- - -### Task 10: Add Catalog-Only Runtime Loading - -**Files:** -- Modify: `ergon_core/ergon_core/core/application/components/catalog.py` -- Modify runtime files listed in file structure. -- Test: core runtime registry tests. - -- [ ] **Step 1: Add test for catalog-backed runtime loading** - -Create `ergon_core/tests/unit/registry/test_catalog_backed_registry_resolution.py`: - -```python -from collections.abc import AsyncGenerator -from sqlalchemy.pool import StaticPool -from sqlmodel import Session, SQLModel, create_engine - -from ergon_core.api.benchmark import Task -from ergon_core.api.worker import Worker, WorkerContext, WorkerOutput -from ergon_core.api.worker.worker import WorkerStreamItem -from ergon_core.core.application.components.catalog import ComponentCatalogService, ComponentRef - - -class CatalogSmokeWorker(Worker): - type_slug = "catalog-smoke-worker" - - async def execute( - self, - task: Task, - *, - context: WorkerContext, - ) -> AsyncGenerator[WorkerStreamItem, None]: - yield WorkerOutput(output="ok", success=True) - - -def _session() -> Session: - engine = create_engine( - "sqlite://", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - SQLModel.metadata.create_all(engine) - return Session(engine) - - -def test_build_worker_imports_worker_class_without_local_registration() -> None: - session = _session() - service = ComponentCatalogService() - service.upsert( - session, - ComponentRef( - kind="worker", - slug=CatalogSmokeWorker.type_slug, - module=__name__, - qualname="CatalogSmokeWorker", - ), - ) - session.commit() - - loaded = service.build_worker( - session, - slug=CatalogSmokeWorker.type_slug, - name="primary", - model="stub:constant", - ) - - assert isinstance(loaded, CatalogSmokeWorker) - assert loaded.name == "primary" -``` - -This test proves the catalog imports the persisted worker class and returns a real `Worker` without requiring process-local registry state or execution-only constructor arguments. - -- [ ] **Step 2: Run test and verify it fails** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry/test_catalog_backed_registry_resolution.py -q -``` - -Expected: FAIL because `build_worker` does not exist yet. - -- [ ] **Step 3: Add catalog loading without registry caching** - -Do not extend `ComponentRegistry.require_*` into a cache-loading runtime API. Keep `ComponentRegistry` focused on in-process authoring, validation of explicitly registered objects, and publishing refs into the catalog. - -Add one generic loading helper to `ComponentCatalogService` for non-worker component types: - -```python -def load_ref(self, ref: ComponentRef) -> object: - return import_component_ref(ref) -``` - -Runtime code should call catalog resolution directly and not populate `registry.workers`, `registry.benchmarks`, `registry.evaluators`, or `registry.sandbox_managers`. - -- [ ] **Step 4: Add typed catalog loading helpers** - -Add typed helpers on `ComponentCatalogService` because they make runtime call sites easier to read. Workers should produce a real `Worker`, not a factory/constructor object. - -```python -def build_worker( - self, - session: Session, - *, - slug: str, - name: str, - model: str | None, -) -> Worker: - ref = self.require(session, kind="worker", slug=slug) - worker_cls = self.load_ref(ref) - if not isinstance(worker_cls, type) or not issubclass(worker_cls, Worker): - raise TypeError( - f"Worker component {slug!r} resolved to {worker_cls!r}, expected a Worker subclass" - ) - return worker_cls( - name=name, - model=model, - metadata=ref.metadata, - ) - -def resolve_benchmark(self, session: Session, slug: str) -> type[Benchmark]: - return self.load_ref(self.require(session, kind="benchmark", slug=slug)) - -def resolve_evaluator(self, session: Session, slug: str) -> type[Evaluator]: - return self.load_ref(self.require(session, kind="evaluator", slug=slug)) - -def resolve_sandbox_manager(self, session: Session, slug: str) -> type[BaseSandboxManager]: - return self.load_ref(self.require(session, kind="sandbox_manager", slug=slug)) -``` - -These helpers must still read from Postgres and import the component on each call; do not populate `registry.workers`, `registry.benchmarks`, `registry.evaluators`, or `registry.sandbox_managers`. - -- [ ] **Step 5: Run catalog-backed registry tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/registry -q -``` - -Expected: PASS. - ---- - -### Task 11: Publish Catalog Rows During CLI/API/Test Bootstrap - -**Files:** -- Modify: `ergon_cli/ergon_cli/main.py` -- Modify: `ergon_core/ergon_core/core/rest_api/app.py` -- Modify: test setup files. - -- [ ] **Step 1: Replace env-var plugin startup with explicit bootstrap helper** - -Create a function in a non-core module, for example `ergon_cli/ergon_cli/bootstrap.py`: - -```python -"""Process bootstrap for local CLI/API components.""" - -from ergon_builtins.registry import register_builtins -from ergon_core.api.registry import registry -from ergon_core.core.persistence.shared.db import get_session - - -def register_and_publish_builtins() -> None: - register_builtins(registry) - with get_session() as session: - registry.publish(session) - session.commit() -``` - -- [ ] **Step 2: Call bootstrap from CLI startup** - -Modify `ergon_cli/ergon_cli/main.py`: - -```python -from ergon_cli.bootstrap import register_and_publish_builtins -``` - -Call it before command handlers run. If commands like `doctor` should not require DB, skip publishing for those commands by calling it only in experiment/benchmark/eval/workflow handlers. - -- [ ] **Step 3: Add API startup bootstrap without env plugins** - -Do not import tests from core app. For local Docker, choose one explicit bootstrap: - -Option A, if `app.py` is local/dev-only: - -```python -from ergon_builtins.registry import register_builtins -from ergon_core.api.registry import registry - -register_builtins(registry) -with get_session() as session: - registry.publish(session) - session.commit() -``` - -Option B, if strict core independence is still desired: - -Create `ergon_cli/ergon_cli/api_app.py` or a top-level `ergon_app/local_api.py` that imports core `app`, registers/publishes builtins, registers/publishes smoke fixtures, and is the uvicorn target used by docker compose. - -Recommendation: use Option B to avoid recreating core-to-builtins coupling. - -- [ ] **Step 4: Add smoke publishing in test bootstrap** - -For E2E/local Docker, explicit Python bootstrap should call: - -```python -from tests.fixtures.smoke_components import register_smoke_components - -register_smoke_components(registry) -with get_session() as session: - registry.publish(session) - session.commit() -``` - -Host-side pytest can still call this for in-process tests, but E2E must publish inside the API/Inngest process or before the stack starts against the shared DB. - -- [ ] **Step 5: Run CLI/API bootstrap tests** - -Run: - -```bash -uv run pytest ergon_cli/tests/unit ergon_core/tests/unit/test_app_mounts_harness_conditionally.py -q -``` - -Expected: PASS after tests are updated for no `ENABLE_TEST_HARNESS`. - ---- - -### Task 12: Update Runtime Jobs To Resolve Through Catalog When Needed - -**Files:** -- Modify runtime files listed in file structure. -- Test: existing runtime job tests plus new catalog-backed tests. - -- [ ] **Step 1: Update worker execute job** - -In `worker_execute.py`, when resolving worker and benchmark: - -```python -with get_session() as session: - worker = catalog.build_worker( - session, - slug=payload.worker_type, - name=payload.assigned_worker_slug, - model=payload.model_target, - ) -``` - -Build the `Task` with the runtime graph node identity. Do not derive this from the nullable static definition task id: - -```python -if payload.node_id is None: - raise ContractViolationError("worker-execute requires node_id") - -task = Task( - task_id=payload.node_id, - task_slug=payload.task_slug, - instance_key=instance_key, - description=payload.task_description, - task_payload=task_payload or EmptyTaskPayload(), -) -``` - -Build `WorkerContext` without duplicating task identity: - -```python -worker_context = WorkerContext( - run_id=payload.run_id, - definition_id=payload.definition_id, - execution_id=payload.execution_id, - sandbox_id=payload.sandbox_id, -) -``` - -`WorkerExecuteRequest` should carry only the runtime task id: - -```python -node_id: UUID # runtime task id, always present -``` - -If worker execution needs static task payload or instance data, resolve it from the persisted graph node: - -```python -node = session.get(RunGraphNode, payload.node_id) -if node is None: - raise ContractViolationError(f"RunGraphNode {payload.node_id} not found") - -if node.definition_task_id is not None: - task_row, instance_row = DefinitionRepository().task_with_instance( - session, - node.definition_task_id, - ) - task_payload = task_row.task_payload_as(benchmark_cls.task_payload_model) - instance_key = instance_row.instance_key -else: - task_payload = None - instance_key = str(payload.node_id) -``` - -Avoid opening duplicate sessions if the function already opens a session for task rows. Reuse the existing session where practical. - -- [ ] **Step 2: Update evaluate task job** - -Use: - -```python -evaluator_cls = catalog.resolve_evaluator(session, evaluator_type) -benchmark_cls = catalog.resolve_benchmark(session, benchmark_type) -manager_cls = catalog.resolve_sandbox_manager(session, benchmark_type) -``` - -Do not keep the previous `DefaultSandboxManager` fallback for known benchmark/sandbox slugs. If a persisted benchmark or sandbox slug has no catalog entry, raise immediately; that means definition-time validation or catalog publishing failed. - -- [ ] **Step 3: Update sandbox setup and persist outputs** - -Use catalog resolution where a sandbox slug is explicit. Do not fall back to `DefaultSandboxManager` for unknown explicit slugs. The purpose of definition-time validation is to prevent unknown slugs from being persisted; if one still reaches runtime, fail loudly with the missing slug and registry/catalog context. - -```python -manager_cls = catalog.resolve_sandbox_manager(session, slug) -``` - -- [ ] **Step 4: Update experiment service and launch** - -Resolve benchmark/evaluator via catalog-backed `require_*` using the DB session already used in the service. - -- [ ] **Step 5: Update workflow/task validation** - -Replace `slug in registry.workers` checks with catalog-backed existence checks: - -```python -catalog.require(session, kind="worker", slug=slug) -``` - -This is the point where cross-process correctness improves: validation no longer depends on the current process having imported builtins first. - -- [ ] **Step 6: Run runtime tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit/runtime ergon_core/tests/unit/registry -q -``` - -Expected: PASS. - ---- - -### Task 13: Delete `ERGON_STARTUP_PLUGINS` And `ENABLE_SMOKE_FIXTURES` - -**Files:** -- Modify: `ergon_core/ergon_core/core/shared/settings.py` -- Modify: `ergon_core/ergon_core/core/rest_api/app.py` -- Modify: `ergon_cli/ergon_cli/composition/__init__.py` -- Modify: `docker-compose.yml`, `.github/workflows/e2e-benchmarks.yml`, scripts/docs/tests. - -- [ ] **Step 1: Add grep-based env-var deletion test** - -Create `tests/unit/architecture/test_retired_env_vars.py`: - -```python -from pathlib import Path - - -RETIRED = { - "ERGON_STARTUP_PLUGINS", - "ENABLE_SMOKE_FIXTURES", -} - - -def test_retired_plugin_and_smoke_env_vars_are_not_used_in_code() -> None: - offenders: list[str] = [] - roots = [Path("ergon_core"), Path("ergon_cli"), Path("ergon_builtins"), Path("tests"), Path("scripts")] - for root in roots: - for path in root.rglob("*"): - if path.is_file() and path.suffix in {".py", ".sh", ".ts", ".tsx", ".yml", ".yaml", ".json"}: - text = path.read_text(errors="ignore") - if any(name in text for name in RETIRED): - offenders.append(str(path)) - assert offenders == [] -``` - -- [ ] **Step 2: Run env-var deletion test and verify it fails** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_retired_env_vars.py -q -``` - -Expected: FAIL listing current usage. - -- [ ] **Step 3: Remove startup plugin settings and loader** - -Delete from `Settings`: - -```python -startup_plugin_specs -startup_plugins -``` - -Delete `_run_startup_plugins` from `app.py`. - -- [ ] **Step 4: Remove `ENABLE_SMOKE_FIXTURES` fallback** - -In `ergon_cli/ergon_cli/composition/__init__.py`, delete: - -```python -os.environ.get("ENABLE_SMOKE_FIXTURES", ...) -``` - -Smoke registration should happen through explicit test/bootstrap code, not inside generic CLI composition. - -- [ ] **Step 5: Remove env vars from compose/workflows/scripts** - -Delete `ERGON_STARTUP_PLUGINS` and `ENABLE_SMOKE_FIXTURES` from: - -```text -docker-compose.yml -.github/workflows/e2e-benchmarks.yml -scripts/smoke_local_up.sh -tests/real_llm/benchmarks/test_smoke_stub.py -``` - -- [ ] **Step 6: Run deletion test** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_retired_env_vars.py -q -``` - -Expected: PASS. - ---- - -### Task 14: Delete `ENABLE_TEST_HARNESS` And `TEST_HARNESS_SECRET` - -**Files:** -- Modify: `ergon_core/ergon_core/core/shared/settings.py` -- Modify: `ergon_core/ergon_core/core/rest_api/app.py` -- Modify: `ergon_core/ergon_core/core/rest_api/test_harness.py` -- Modify dashboard test clients/routes referencing `TEST_HARNESS_SECRET`. -- Modify compose/workflows/package scripts/docs. - -- [ ] **Step 1: Extend retired env-var test** - -Add to `RETIRED`: - -```python -"ENABLE_TEST_HARNESS", -"TEST_HARNESS_SECRET", -``` - -Run: - -```bash -uv run pytest tests/unit/architecture/test_retired_env_vars.py -q -``` - -Expected: FAIL listing all remaining uses. - -- [ ] **Step 2: Always mount test harness under a danger-prefixed route** - -Change test harness router: - -```python -router = APIRouter(prefix="/api/__danger__/test-harness", tags=["danger-test-harness"]) -``` - -Update all clients from `/api/test/...` to `/api/__danger__/test-harness/...`. - -- [ ] **Step 3: Remove secret requirement from write endpoints** - -Delete `_require_secret` from `test_harness.py`. - -Remove `x_test_secret` parameters and `_require_secret(x_test_secret)` calls from: - -```python -seed_run -reset_test_rows -``` - -Decide whether `submit_cohort` should remain write-but-unguarded; with the danger-prefixed route, it should also be under the same unauthenticated local harness policy. - -- [ ] **Step 4: Remove conditional mount** - -In `app.py`, replace: - -```python -if settings.enable_test_harness: - app.include_router(_test_harness_router) -``` - -with: - -```python -app.include_router(_test_harness_router) -``` - -Delete `enable_test_harness` from `Settings`. - -- [ ] **Step 5: Update dashboard and Python clients** - -Update: - -```text -ergon-dashboard/tests/helpers/backendHarnessClient.ts -ergon-dashboard/src/app/api/test/dashboard/seed/route.ts -ergon-dashboard/src/lib/config.ts -tests/e2e/_asserts.py -tests/e2e/test_*_smoke.py -tests/integration/smokes/test_smoke_harness.py -package.json -scripts/smoke_local_run.sh -``` - -Remove `X-Test-Secret` headers and env lookups. Update URL paths to danger-prefixed harness routes. - -- [ ] **Step 6: Update tests for always-mounted harness** - -Replace `test_app_mounts_harness_conditionally.py` with a test named: - -```python -def test_app_mounts_danger_test_harness_routes() -> None: - routes = {route.path for route in app.routes} - assert "/api/__danger__/test-harness/read/run/{run_id}/state" in routes -``` - -- [ ] **Step 7: Run retired env-var test** - -Run: - -```bash -uv run pytest tests/unit/architecture/test_retired_env_vars.py -q -``` - -Expected: PASS. - ---- - -### Task 15: Verification - -**Files:** -- No planned source files beyond fixes revealed by tests. - -- [ ] **Step 1: Verify retired env vars are gone** - -Run: - -```bash -rg "ENABLE_TEST_HARNESS|TEST_HARNESS_SECRET|ERGON_STARTUP_PLUGINS|ENABLE_SMOKE_FIXTURES|ERGON_SKIP_INFRA_CHECK" ergon_core ergon_builtins ergon_cli tests scripts docker-compose.yml .github package.json ergon-dashboard -n -``` - -Expected: no matches, except historical docs if the team chooses not to update old planning documents. The architecture test should search code/config, not historical plans. - -- [ ] **Step 2: Verify component catalog migration imports** - -Run: - -```bash -uv run alembic -c ergon_core/alembic.ini upgrade head -``` - -Expected: migration succeeds on a local/dev DB. - -- [ ] **Step 3: Run package-owned unit tests** - -Run: - -```bash -uv run pytest ergon_core/tests/unit ergon_builtins/tests/unit ergon_cli/tests/unit tests/unit -q -``` - -Expected: PASS. - -- [ ] **Step 4: Run backend unit script** - -Run: - -```bash -pnpm run test:be:unit -``` - -Expected: PASS. - -- [ ] **Step 5: Run E2E collection** - -Run: - -```bash -uv run pytest tests/e2e --collect-only -q -``` - -Expected: PASS. - -- [ ] **Step 6: Run lint on changed Python paths** - -Run: - -```bash -uv run ruff check ergon_core ergon_builtins ergon_cli tests scripts -``` - -Expected: PASS. - ---- - -## Self-Review - -- Spec coverage: The plan covers package-owned test layout, PG component catalog schema, catalog service, registry publishing/loading, runtime refactor, and deletion of all five env vars named in the discussion. -- Placeholder scan: The plan contains no placeholder instructions. The migration revision id must be chosen from the actual Alembic head during execution, and the plan explicitly instructs how to do that. -- Type consistency: The same names are used throughout: `ComponentCatalogEntry`, `ComponentCatalogService`, `ComponentRef`, `component_catalog`, `registry.publish`, and catalog-backed `require_*` methods. diff --git a/docs/superpowers/plans/2026-05-05-sharded-export-and-deadline-dataset-release.md b/docs/superpowers/plans/2026-05-05-sharded-export-and-deadline-dataset-release.md deleted file mode 100644 index b28ca9a38..000000000 --- a/docs/superpowers/plans/2026-05-05-sharded-export-and-deadline-dataset-release.md +++ /dev/null @@ -1,420 +0,0 @@ -# Sharded Export And Deadline Dataset Release Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add Ergon-native incremental sharded export so the rollout-card artifact can publish verified sub-1GB datasets first, retar Ergon, and leave large-dataset ETL/export jobs running overnight. - -**Architecture:** Ergon owns export semantics: pagination, Parquet/JSONL shard writing, resource descriptors, resource copy/linking, manifests, checksums, and resume state. The rollout-card artifact remains a thin orchestrator that fetches sources, invokes Ergon CLI commands, packages generated shards for Hugging Face, and documents NeurIPS submission status. - -**Tech Stack:** Python 3.13, SQLModel/Postgres, Ergon ingestion models, `pyarrow` for Parquet, content-addressed resource files, Hugging Face dataset layout. - ---- - -## Deadline Strategy - -The submission path is intentionally staged: - -1. Implement Ergon-native sharded export and retar Ergon. -2. Run and publish all sub-1GB datasets first within the next one to two hours. -3. Verify shard format, manifests, resource hashes, reducer rows, and drops rows. -4. Push verified sub-1GB exports to the Hugging Face org and include that org/link in the NeurIPS upload. -5. Start large ETL/export jobs overnight with the same resumable exporter. -6. Update the hosted HF collection as large datasets complete. - -The OpenReview upload should not contain full 50-100GB derived data. It should contain code, Ergon tarball, Croissant/manifests where available, smoke fixtures, and links to hosted HF datasets. - -## Dataset Triage - -Use the latest local size audit from `rollout-card-artifact/outputs/verification/dataset_size_audit.json`. - -### First-wave datasets - -Run these first because configured input/prepared size is below 1GB: - -- `agentharm` -- `atbench` -- `bfcl` -- `debate_mallm` -- `gpqa` -- `maestro` -- `miniwob` -- `stabletoolbench` -- `tau_bench` -- `tot_crosswords` -- `tot_game24` -- `weblinx` - -`debate_mallm`, `gpqa`, and `stabletoolbench` are still meaningful enough to watch, but they are under 1GB and should finish quickly relative to the large sets. - -### Overnight datasets - -Run after the first wave is published: - -- `agent_reward_bench` -- `openhands_swe_rebench` -- `swe_smith` - -These dominate local storage and runtime. They must use the same sharded/resumable exporter and must not require one giant in-memory or single-file export. - -## File Structure - -### Add to Ergon - -- `ergon_ingestion/ergon_ingestion/exports/__init__.py` - - Package marker for export helpers. -- `ergon_ingestion/ergon_ingestion/exports/models.py` - - Pydantic/dataclass models for export config, shard manifests, resource descriptors, and dataset manifests. -- `ergon_ingestion/ergon_ingestion/exports/sharded.py` - - Core sharded exporter. Owns DB pagination, Parquet/JSONL writing, resource copy/linking, checksums, and resume state. -- `ergon_ingestion/ergon_ingestion/exports/verify.py` - - Verifier for generated export directories: checks counts, resources, hashes, non-empty reducers/drops, and no truncation markers. -- `ergon_ingestion/ergon_ingestion/cli.py` - - Add `ergon ingest export` or wire an export subcommand under the existing ingestion CLI surface if that is the least invasive deadline path. -- `ergon_cli/ergon_cli/main.py` - - Register the CLI route if needed. -- `ergon_ingestion/tests/unit/test_sharded_export.py` - - Unit tests for shard rollover, resume behavior, manifest shape, and resource hash validation. -- `docs/architecture/cross_cutting/artifacts.md` - - Update architecture docs to state Ergon owns sharded dataset exports and artifact repos only orchestrate them. - -### Keep in rollout-card-artifact as wrapper/orchestration - -- `artifact_tools/export_reanalysis.py` - - Convert into a compatibility shim that imports Ergon exporter functions and passes artifact-specific arguments through to them. - - It must not implement pagination, sharding, resource copying, checksum generation, manifest writing, or verification itself. -- `artifact_tools/package_hf.py` - - Keep HF-specific packaging/readme/upload behavior here, but make it consume Ergon export directories rather than inventing export semantics. -- `scripts/05_export_reanalysis_inputs.sh` - - Point at the new Ergon export command. -- `scripts/run_all_cards.sh` - - Run datasets in first-wave/overnight order, with resume enabled. - -## Export Format - -Each dataset export directory should look like: - -```text -outputs/ergon-export/<batch>/<dataset>/ - manifest.json - checksums.json - state.json - runs/ - runs-00000.parquet - runs-00001.parquet - reducers/ - reducers-00000.parquet - drops/ - drops-00000.parquet - resources/ - ab/ - <sha256>.json -``` - -The Hugging Face package can mirror this directory or copy it under `outputs/huggingface/<dataset>/`. - -### Manifest requirements - -`manifest.json` must include: - -- `dataset` -- `batch` -- `created_at` -- `exporter_version` -- `source_url` -- `source_version_ref` -- `run_count` -- `reducer_count` -- `drop_count` -- `resource_count` -- `resource_total_bytes` -- `shards` -- `malformed_source_records` -- `verification` - -### Shard rules - -- Default `shard_size_mb`: 256. -- Default `page_size`: 1000 runs. -- Runs, reducers, and drops are separate shard families. -- Shards are append-only within a run. On resume, completed shards listed in `state.json` are not rewritten. -- Temporary shard path is `<name>.tmp`; rename atomically to final name after writing and checksum. - -### Resource rules - -- Postgres stores only descriptors: path, size, hash, kind, MIME type, metadata. -- Full payloads live as blob files. -- Export should copy or hardlink blob files into `resources/<sha256[:2]>/<sha256><suffix>`. -- If a destination resource already exists with matching hash and size, skip copy. -- If a destination resource exists with mismatched hash or size, fail loudly. - -## CLI Design - -Deadline-friendly command: - -```bash -ergon ingest export \ - --batch hf-trace-publication-v1 \ - --dataset weblinx \ - --output outputs/ergon-export/hf-trace-publication-v1/weblinx \ - --format parquet \ - --page-size 1000 \ - --shard-size-mb 256 \ - --resume -``` - -Batch wrapper command, if time permits: - -```bash -ergon ingest export-batch \ - --batch hf-trace-publication-v1 \ - --output outputs/ergon-export/hf-trace-publication-v1 \ - --dataset agentharm \ - --dataset atbench \ - --resume -``` - -If adding a top-level `ergon export` command is straightforward, prefer that. If routing is risky before the deadline, use `ergon ingest export` now and rename later. - -## Implementation Tasks - -### Task 1: Define Export Models - -**Files:** - -- Create: `ergon_ingestion/ergon_ingestion/exports/__init__.py` -- Create: `ergon_ingestion/ergon_ingestion/exports/models.py` -- Test: `ergon_ingestion/tests/unit/test_sharded_export.py` - -- [ ] Add models for `ShardedExportConfig`, `ShardRecord`, `DatasetExportManifest`, and `ExportState`. -- [ ] Include fields for dataset, batch, output path, page size, shard size, resume flag, and resource policy. -- [ ] Test JSON serialization produces stable sorted keys. -- [ ] Test `ExportState` can record completed shard names and resume cursor fields. - -### Task 2: Implement Paginated DB Reader - -**Files:** - -- Create/modify: `ergon_ingestion/ergon_ingestion/exports/sharded.py` -- Test: `ergon_ingestion/tests/unit/test_sharded_export.py` - -- [ ] Move the query logic currently embedded in `rollout-card-artifact/artifact_tools/export_reanalysis.py` into Ergon. -- [ ] Page `RunRecord` rows by stable order: `sample_id`, `instance_key`, `id`. -- [ ] For each run, fetch annotations, resources, reducers, and drops. -- [ ] Convert rows into export dictionaries equivalent to the current reanalysis payload, but without inlining resource payloads. -- [ ] Print progress every 100 runs: `exported_runs: N`. - -### Task 3: Add Parquet Shard Writer - -**Files:** - -- Modify: `ergon_ingestion/ergon_ingestion/exports/sharded.py` -- Test: `ergon_ingestion/tests/unit/test_sharded_export.py` - -- [ ] Use `pyarrow` to write `runs`, `reducers`, and `drops` shard families. -- [ ] Use JSON string columns for nested structures: - - `observed_fields_json` - - `missing_fields_json` - - `annotations_json` - - `resources_json` - - `output_json` - - `evidence_json` -- [ ] Roll over to a new shard when estimated uncompressed row payload exceeds `shard_size_mb`. -- [ ] Write to `.tmp` and atomically rename after checksum calculation. -- [ ] Record each completed shard in `state.json`. - -### Task 4: Copy Or Link Resources Incrementally - -**Files:** - -- Modify: `ergon_ingestion/ergon_ingestion/exports/sharded.py` -- Test: `ergon_ingestion/tests/unit/test_sharded_export.py` - -- [ ] For every `RunResource`, resolve its `file_path`. -- [ ] Copy or hardlink to `resources/<sha256[:2]>/<sha256><suffix>`. -- [ ] Skip resources already present with matching hash and size. -- [ ] Fail on missing source files. -- [ ] Fail on hash mismatch. -- [ ] Add resource copy counts and bytes to manifest/state. - -### Task 5: Add Export Verification - -**Files:** - -- Create: `ergon_ingestion/ergon_ingestion/exports/verify.py` -- Test: `ergon_ingestion/tests/unit/test_sharded_export.py` - -- [ ] Verify `manifest.json`, `checksums.json`, and `state.json` exist. -- [ ] Verify every listed shard exists and matches checksum. -- [ ] Verify resource files match hash and size. -- [ ] Verify `run_count > 0`, `reducer_count > 0`, and `drop_count > 0` unless `--allow-empty-drops` is explicitly set. -- [ ] Scan exported JSON fields and resources for truncation markers: - - `[truncated]` - - `<truncated>` - - `...<truncated>` - - `__TRUNCATED__` - - `truncated_payload` - -### Task 6: Wire CLI - -**Files:** - -- Modify: `ergon_ingestion/ergon_ingestion/cli.py` -- Modify: `ergon_cli/ergon_cli/main.py` if required by command routing. -- Test: CLI unit or smoke command. - -- [ ] Add command parser support for `ergon ingest export`. -- [ ] Add arguments: - - `--dataset` - - `--batch` - - `--output` - - `--format parquet` - - `--page-size` - - `--shard-size-mb` - - `--resume` - - `--allow-empty-drops` -- [ ] Print final summary: - - runs - - reducers - - drops - - resources - - total bytes - - output path - -### Task 7: Convert Artifact Export Wrapper - -**Files:** - -- Modify: `rollout-card-artifact/artifact_tools/export_reanalysis.py` -- Modify: `rollout-card-artifact/scripts/05_export_reanalysis_inputs.sh` - -- [ ] Keep the old JSONL exporter available only as a legacy mode if needed. -- [ ] Default to invoking the new Ergon sharded export command. -- [ ] Write outputs to `outputs/ergon-export/<batch>/<dataset>/`. -- [ ] Keep current summary JSON output for compatibility if scripts expect it. - -### Task 8: Package Hugging Face Layout - -**Files:** - -- Modify: `rollout-card-artifact/artifact_tools/package_hf.py` -- Modify: `rollout-card-artifact/scripts/10_package_hf_datasets.sh` - -- [ ] Package sharded Ergon export directories without rewriting shards. -- [ ] Copy README and `dataset_info.json`. -- [ ] Include `manifest.json`, `checksums.json`, and resource files. -- [ ] Avoid copying resource blobs if package directory already contains matching hash/size. -- [ ] Emit `package_manifest.json` with HF repo names and local paths. - -### Task 9: Update Architecture Docs - -**Files:** - -- Modify: `docs/architecture/cross_cutting/artifacts.md` - -- [ ] Add a section: “Sharded Dataset Export”. -- [ ] State that Ergon owns export format, resource integrity, resume state, and manifest generation. -- [ ] State that paper/artifact repos own orchestration and paper-specific packaging only. - -### Task 10: Retar Ergon - -**Files:** - -- Output in rollout artifact: `rollout-card-artifact/third_party/ergon-anonymous-src.tar.gz` -- Verify with: `rollout-card-artifact/scripts/00_checkout_ergon.sh` - -- [ ] Copy or synchronize the modified local Ergon checkout into `rollout-card-artifact/third_party/ergon`. -- [ ] Rebuild `third_party/ergon-anonymous-src.tar.gz`. -- [ ] Extract into a temporary directory and verify the new export modules exist. -- [ ] Run anonymity audit. - -### Task 11: First-Wave Run And Publish - -**Files/Commands:** - -- `rollout-card-artifact/scripts/run_all_cards.sh` -- `rollout-card-artifact/scripts/10_package_hf_datasets.sh` -- `rollout-card-artifact/scripts/11_upload_hf_datasets.sh` - -- [ ] Run ingest/export/package for first-wave datasets only: - -```bash -FIRST_WAVE=( - agentharm - atbench - bfcl - debate_mallm - gpqa - maestro - miniwob - stabletoolbench - tau_bench - tot_crosswords - tot_game24 - weblinx -) -``` - -- [ ] Verify sharded output for each dataset. -- [ ] Package for HF. -- [ ] Upload to HF org. -- [ ] Record HF URLs in upload notes. - -### Task 12: Overnight Large Runs - -**Datasets:** - -- `agent_reward_bench` -- `openhands_swe_rebench` -- `swe_smith` - -- [ ] Start each large dataset as an independent resumable job. -- [ ] Log output to `outputs/logs/<dataset>-overnight.log`. -- [ ] Verify progress every 30-60 minutes if awake, otherwise rely on `state.json`. -- [ ] On failure, rerun with `--resume`. -- [ ] Upload completed large datasets to the same HF org/collection. - -## Verification Commands - -Run these before claiming success: - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon -PYTHONPATH="ergon_ingestion:ergon_core" uv run pytest -q ergon_ingestion/tests/unit -``` - -```bash -cd /Users/charliemasters/Desktop/synced_vm_002/ergon_submission_bundle/rollout-card-artifact -scripts/00_checkout_ergon.sh "$(mktemp -d)/ergon" -scripts/audit_anonymity.sh -``` - -For each exported dataset: - -```bash -ergon ingest export \ - --dataset DATASET \ - --batch hf-trace-publication-v1 \ - --output outputs/ergon-export/hf-trace-publication-v1/DATASET \ - --format parquet \ - --resume -``` - -Then verify: - -```bash -ergon ingest verify-export \ - --output outputs/ergon-export/hf-trace-publication-v1/DATASET -``` - -If `verify-export` is not implemented as a CLI command, run the verifier module directly from the Ergon project. - -## Acceptance Criteria - -- Ergon contains the sharded export implementation. -- Artifact export logic is a wrapper, not the owner of export semantics. -- Ergon tarball is refreshed and checkout-verified. -- At least all first-wave sub-1GB datasets export to sharded Parquet with manifests and checksums. -- First-wave datasets are uploaded to HF and linked in the NeurIPS upload material. -- Large datasets can run overnight independently and resume from `state.json`. -- No generated full dataset needs to be included in the OpenReview zip. - diff --git a/docs/superpowers/plans/2026-05-18-pr11-local-smoke-debugging-and-coverage.md b/docs/superpowers/plans/2026-05-18-pr11-local-smoke-debugging-and-coverage.md deleted file mode 100644 index 846a89026..000000000 --- a/docs/superpowers/plans/2026-05-18-pr11-local-smoke-debugging-and-coverage.md +++ /dev/null @@ -1,313 +0,0 @@ -# PR11 Local Smoke Debugging And Coverage Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:systematic-debugging before changing runtime behavior, then use superpowers:test-driven-development for each code fix. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Stop using GitHub CI as the debugger, reproduce the PR11 smoke failure locally, add integration coverage for dynamic task propagation, and only push once all three local smokes are green end to end. - -**Architecture:** Treat the smoke failure as a multi-component runtime bug across `Worker.execute`, `TaskManagementService.plan_subtasks`, Inngest event dispatch, graph dependency propagation, and evaluator timing. The current evidence says unit/integration tests cover dispatch pieces but not the full parent-plans-children-children-execute-parent/evaluator-observes-completion path. The plan first adds a focused integration reproduction, then uses the full local stack to confirm the real Inngest behavior before making the smallest root-cause fix. - -**Tech Stack:** Python, pytest, SQLModel/Postgres, Inngest dev server, Docker Compose, existing `scripts/smoke_local_up.sh` / `scripts/smoke_local_run.sh`, Ergon smoke fixtures. - ---- - -## Working Hypothesis To Test, Not Assume - -The current smoke parent and recursive workers plan subtasks and then wait inside the same `worker-execute` Inngest function for those subtasks to complete. Child `task/ready` events are emitted from that function, but the children remain pending while the parent function is still running. That looks like an Inngest orchestration deadlock or event-delivery timing problem, not a simple serialization bug. - -The fix must be driven by local evidence. If local traces disprove this hypothesis, update the plan before implementing a different fix. - -## Files - -- Modify: `ergon_core/ergon_core/core/application/jobs/worker_execute.py` -- Modify: `ergon_core/ergon_core/core/application/tasks/management.py` -- Modify: `tests/fixtures/smoke_components/smoke_base/worker_base.py` -- Modify: `tests/fixtures/smoke_components/smoke_base/recursive.py` -- Modify: `tests/fixtures/smoke_components/smoke_base/criterion_base.py` -- Modify or create: `ergon_core/tests/integration/runtime/test_dynamic_task_propagation.py` -- Possibly modify: `ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py` -- Possibly modify: `tests/e2e/_asserts.py` -- Run: `scripts/smoke_local_up.sh` -- Run: `scripts/smoke_local_run.sh` - -## Task 1: Establish The Local Full-Stack Reproduction - -- [ ] **Step 1: Start the local stack** - -Run from repo root: - -```bash -scripts/smoke_local_up.sh -``` - -Expected: Postgres, API, dashboard, and Inngest dev are healthy. If Docker is already running stale containers, run `docker compose ps` and inspect before restarting. - -- [ ] **Step 2: Export the smoke environment** - -Use the values printed by `scripts/smoke_local_up.sh`, correcting the duplicated `export` typo in the printed `SCREENSHOT_DIR` line: - -```bash -export ERGON_DATABASE_URL=postgresql://ergon:ergon_dev@localhost:5433/ergon -export INNGEST_API_BASE_URL=http://localhost:8289 -export INNGEST_DEV=1 -export INNGEST_EVENT_KEY=dev -export ERGON_API_BASE_URL=http://127.0.0.1:9000 -export PLAYWRIGHT_BASE_URL=http://127.0.0.1:3001 -export SCREENSHOT_DIR=/tmp/playwright -``` - -- [ ] **Step 3: Run the smallest failing smoke** - -Run one cohort member first: - -```bash -scripts/smoke_local_run.sh swebench-verified 1 -``` - -Expected current failure: timeout with root still running and child graph nodes pending. Capture the run id from the pytest failure. - -- [ ] **Step 4: Capture runtime evidence at component boundaries** - -For the failed run, collect: - -```bash -docker compose logs --since=15m api > /tmp/ergon-api-smoke.log -docker compose logs --since=15m inngest > /tmp/ergon-inngest-smoke.log -rg -n "task/ready|worker-execute|plan_subtasks|RunGraphNode|pending|completed|failed|error|Exception" /tmp/ergon-api-smoke.log /tmp/ergon-inngest-smoke.log -``` - -Expected: enough evidence to locate whether `task/ready` events are not emitted, emitted but not received by Inngest, received but not invoking `worker-execute`, or invoking but failing before graph state changes. - -## Task 2: Add Missing Integration Coverage For Dynamic Task Propagation - -- [ ] **Step 1: Write a failing integration test around the real propagation contract** - -Create or extend `ergon_core/tests/integration/runtime/test_dynamic_task_propagation.py` with a test shaped like: - -```python -async def test_parent_planned_ready_children_are_dispatched_and_reach_terminal_status(): - """A parent worker planning dynamic subtasks must not leave children pending.""" -``` - -The test should construct a real run graph using existing test-support factories, execute the parent path through the application job/service boundary, and assert: - -```python -assert root_node.status in {"completed", "failed"} -assert sorted(child.task_slug for child in children) == sorted(EXPECTED_SUBTASK_SLUGS) -assert all(child.status != "pending" for child in children) -assert any(execution.node_id == child.id for child in children for execution in executions) -``` - -This test must fail on the current broken behavior before the fix. - -- [ ] **Step 2: Cover replay/idempotency explicitly** - -Add a second test that simulates Inngest step replay around `plan_subtasks`: - -```python -async def test_plan_subtasks_step_replay_does_not_duplicate_children_or_drop_ready_events(): - """Replay must preserve one graph mutation and one ready dispatch per root child.""" -``` - -Assert: - -```python -assert len(children) == len(EXPECTED_SUBTASK_SLUGS) -assert len({child.task_slug for child in children}) == len(EXPECTED_SUBTASK_SLUGS) -assert ready_dispatch_count == expected_root_ready_count -``` - -This locks in the fix from `c03a219`: no duplicate child nodes under step replay. - -- [ ] **Step 3: Run the new tests and confirm they fail for the right reason** - -Run: - -```bash -uv run pytest ergon_core/tests/integration/runtime/test_dynamic_task_propagation.py -v --tb=short -``` - -Expected before fix: at least one test fails because child tasks remain pending or because no child execution row is created. If it fails due to fixtures/imports instead, fix the test harness first. - -## Task 3: Decide The Runtime Shape From Evidence - -- [ ] **Step 1: Compare local evidence to the two plausible fixes** - -Use this decision table: - -```text -Evidence: child task/ready events never reach Inngest -Fix: event dispatch wiring bug in TaskManagementService / Inngest client config. - -Evidence: events reach Inngest but child worker-execute does not start until parent unwinds -Fix: smoke/runtime contract must not wait inside the parent worker for children that are scheduled through sibling events. - -Evidence: child worker-execute starts and fails immediately -Fix: inspect child payload/context construction; repair the failing service boundary. - -Evidence: children complete but parent/evaluator reads stale state -Fix: criterion/read-model polling or transaction visibility issue. -``` - -- [ ] **Step 2: Write down the chosen root cause in the PR notes** - -Update the PR description or a short local note with: - -```text -Root cause: -Evidence: -Fix selected: -Why not the other fixes: -``` - -Do this before implementation. - -## Task 4: Implement The Smallest Root-Cause Fix - -- [ ] **Step 1: If the deadlock hypothesis is confirmed, remove child waiting from smoke workers** - -Modify `tests/fixtures/smoke_components/smoke_base/worker_base.py` so the parent worker plans children, emits its three expected chunks, and returns a successful `WorkerOutput` without polling direct children. - -Expected final product: - -```python -yield WorkerOutput( - output=waiting_message, - success=True, - metadata={ - "planned_children": sorted(result.nodes.keys()), - "child_wait_mode": "criterion", - }, -) -``` - -Remove unused direct polling imports if this path is selected. - -- [ ] **Step 2: If the deadlock hypothesis is confirmed, remove nested child waiting from the recursive worker** - -Modify `tests/fixtures/smoke_components/smoke_base/recursive.py` so `l_2` plans `l_2_a -> l_2_b`, emits its fixed three chunks, sends its recursive completion message, and returns without polling nested children. - -Expected final product: - -```python -yield WorkerOutput( - output="nested smoke recursion planned", - success=True, - metadata={ - "planned_children": sorted(result.nodes.keys()), - "child_wait_mode": "criterion", - }, -) -``` - -Keep `RecursiveSmokeWorkerBase.RECURSIVE_TURN_COUNT == 3` unless local assertions prove the smoke contract should change. - -- [ ] **Step 3: Move child completion waiting to the smoke criterion** - -Modify `tests/fixtures/smoke_components/smoke_base/criterion_base.py` so `evaluate` waits for the graph and artifact state it already asserts. - -Expected helper shape: - -```python -async def _wait_for_artifact_state( - self, - context: CriterionContext, - *, - timeout_s: float = 180.0, - interval_s: float = 2.0, -) -> tuple[list[RunGraphNode], list[RunGraphNode], dict[UUID, ProbeResult]]: - deadline = time.monotonic() + timeout_s - last_error: CriterionCheckError | None = None - while time.monotonic() < deadline: - try: - children = await self._pull_children(context) - self._check_graph_shape(children) - self._check_children_completed(children) - artifact_children = await self._artifact_children(children) - self._check_children_completed(artifact_children) - probes = await self._pull_probe_results(context, artifact_children) - self._check_probes_succeeded(probes, artifact_children) - return children, artifact_children, probes - except CriterionCheckError as err: - last_error = err - await asyncio.sleep(interval_s) - raise CriterionCheckError(f"timed out waiting for smoke child artifacts: {last_error}") -``` - -Then `evaluate` should call this helper before `_verify_env_content`. - -- [ ] **Step 4: If local evidence points elsewhere, do not apply the smoke-worker change** - -If Task 3 identifies dispatch wiring or child payload failure instead, fix the responsible production file and keep the smoke fixture waiting semantics unchanged until the integration test proves otherwise. - -## Task 5: Verify Locally Before Pushing - -- [ ] **Step 1: Run focused Python checks** - -```bash -uv run ruff check ergon_core/ergon_core/core/application/jobs/worker_execute.py ergon_core/ergon_core/core/application/tasks/management.py tests/fixtures/smoke_components/smoke_base/worker_base.py tests/fixtures/smoke_components/smoke_base/recursive.py tests/fixtures/smoke_components/smoke_base/criterion_base.py -uv run ruff format --check ergon_core/ergon_core/core/application/jobs/worker_execute.py ergon_core/ergon_core/core/application/tasks/management.py tests/fixtures/smoke_components/smoke_base/worker_base.py tests/fixtures/smoke_components/smoke_base/recursive.py tests/fixtures/smoke_components/smoke_base/criterion_base.py -uv run ty check ergon_core/ergon_core/core/application/jobs/worker_execute.py ergon_core/ergon_core/core/application/tasks/management.py -``` - -Expected: all pass. - -- [ ] **Step 2: Run focused unit and integration tests** - -```bash -uv run pytest ergon_core/tests/unit/core/application/jobs/test_worker_execute_live_sandbox_attach.py ergon_core/tests/unit/runtime/test_child_function_payloads.py -v -uv run pytest ergon_core/tests/unit/runtime/test_smoke_topology_drift.py ergon_core/tests/unit/architecture/test_smoke_fixture_package_boundary.py -v -uv run pytest ergon_core/tests/integration/runtime/test_dynamic_task_propagation.py -v --tb=short -``` - -Expected: all pass. - -- [ ] **Step 3: Run all three local smokes with one cohort member** - -```bash -scripts/smoke_local_run.sh swebench-verified 1 -scripts/smoke_local_run.sh minif2f 1 -scripts/smoke_local_run.sh researchrubrics 1 -``` - -Expected: all pass end to end. - -- [ ] **Step 4: Run all three local smokes with CI-sized cohorts** - -```bash -scripts/smoke_local_run.sh swebench-verified 3 -scripts/smoke_local_run.sh minif2f 3 -scripts/smoke_local_run.sh researchrubrics 3 -``` - -Expected: all pass end to end. This is the gate before pushing. - -## Task 6: Commit, Push, Then Use CI Only As Confirmation - -- [ ] **Step 1: Commit the tested fix and coverage** - -```bash -git status --short -git add ergon_core tests docs/superpowers/plans/2026-05-18-pr11-local-smoke-debugging-and-coverage.md -git commit -m "Cover dynamic task propagation in PR11 smokes" -``` - -- [ ] **Step 2: Push once local smokes are green** - -```bash -git push -``` - -- [ ] **Step 3: Poll CI only after local green** - -```bash -gh pr checks 65 --watch -``` - -Expected: CI confirms the local result. If CI disagrees, compare CI logs to local logs before changing code. - -## Done Definition - -- Local integration coverage fails on the old propagation behavior and passes after the fix. -- Local `swebench-verified`, `minif2f`, and `researchrubrics` smokes pass with cohort size 3. -- Fast CI remains green. -- Smoke CI is used only as final confirmation, not as the primary debugger. diff --git a/docs/superpowers/plans/2026-05-19-builtins-pr04-todo-cleanup.md b/docs/superpowers/plans/2026-05-19-builtins-pr04-todo-cleanup.md deleted file mode 100644 index b25a24586..000000000 --- a/docs/superpowers/plans/2026-05-19-builtins-pr04-todo-cleanup.md +++ /dev/null @@ -1,1583 +0,0 @@ -# Builtins PR 4 TODO Cleanup Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. - -**Goal:** Remove the TODO-backed cleanup debt in `ergon_builtins` as part of PR 4, preserving the current public benchmark/worker behavior while making the builtins code library-tier clean. - -**Architecture:** Keep PR 4 focused on builtins domain cleanup: move domain models into semantic modules, delete empty/re-export-only modules where they add no boundary, replace constructor shims with Pydantic validators, and rely on existing core runtime containment instead of duplicating it in builtins. Do not change benchmark behavior, task JSON shape, or worker/evaluator slugs. - -**Tech Stack:** Python 3.13, Pydantic v2, pydantic-ai tools, `ergon_core.api` public `Task`/`Sandbox`/`Criterion` contracts, ruff, ty, pytest. - ---- - -## TODO Inventory - -Source TODOs were read from: - -`/Users/charliemasters/.config/superpowers/worktrees/ergon/codex-builtins-pr01-hygiene-guardrails/ergon_builtins` - -Current PR 4 branch: - -`/tmp/ergon-builtins-restack`, branch `codex/builtins-pr04-benchmark-domains-v2` - -Implement these in PR 4: - -- Strongly type GDPEval raw rubric loading instead of returning bare `dict`. -- Move GDPEval criteria factory helpers out of `criteria/__init__.py`. -- Remove empty `operations.py` placeholder modules. -- Move tool response models out of `tool_builder.py` and stop `response_models.py` re-exporting from builders. -- Move GDPEval sandbox operation response DTOs out of `task_schemas.py`. -- Replace `Any` annotations for `sandbox`/`task` in benchmark tool builders/toolkits. -- Remove `max_score` magic constructors from GDPEval criteria and normalize HF rubric data into canonical `score_spec` at the dataset-loading/conversion boundary. -- Replace the ResearchRubrics judge constructor override with Pydantic pre-validation. -- Move the ResearchRubrics settings import to module scope. -- Delete builtins Logfire observability hooks and remove the `ReActWorker` call site. -- Remove stale subtask-containment TODO text because `WorkerContext` already enforces descendant containment. -- Decide `StagedRubric.validate_runtime_deps`: keep dependency validation but remove the redundant stage-shape validation from that method, because Pydantic field constraints already own stage shape. - -Non-goals for PR 4: - -- Do not remove `validate_runtime_deps` from core public APIs. `WorkerExecuteService` and `EvaluationService` call it. -- Do not redesign GDPEval rubric semantics or load real HF data in unit tests. -- Do not change task/evaluator/criterion type slugs. - ---- - -## Worker / Tool / Model Domain Layout - -The current top-level `workers/`, `tools/`, and `models/` packages read like grab bags: - -- `workers/react_worker.py` and `workers/react_output.py` are one coherent ReAct implementation, but sit beside the unrelated `training_stub_worker.py`. -- `workers/toolkit.py` is not really a worker; it is the serializable base class for benchmark toolkits used by ReAct-style agents. -- `workers/tool_budget.py`, `tools/workflow_cli_tool.py`, `tools/subtask_lifecycle_toolkit.py`, `tools/bash_sandbox_tool.py`, `tools/graph_toolkit.py`, and `tools/graph_toolkit_types.py` are separate toolkit surfaces, not one shared tools domain. -- `models/*` are not domain models; they are LLM provider/backend infrastructure. - -Restructure toward ownership-first packages: - -```text -ergon_builtins/ - agents/ - react/ - __init__.py - worker.py # ReActWorker - output.py # worker_output_from_chunks - training/ - __init__.py - synthetic_worker.py # TrainingStubWorker + synthetic chunk helpers - toolkits/ - common/ - __init__.py - base.py # Toolkit base for serializable benchmark toolkits - budgets.py # AgentToolBudgetState / deps / exhausted result - subagents/ - __init__.py - toolkit.py # SubtaskLifecycleToolkit + builder - models.py # subtask lifecycle response DTOs - sandbox_bash.py # manager/subagent bash helper - workflow_cli/ - __init__.py - tool.py # make_workflow_cli_tool - resources/ - __init__.py - toolkit.py # ResearchGraphToolkit - models.py # ResourceRef / TaskExecutionRef - llm/ - __init__.py - resolution.py - capture_settings.py - providers/ - __init__.py - openrouter.py - openrouter_responses.py - transformers.py - vllm.py -``` - -Compatibility strategy: - -- Move implementation modules to the new layout in PR 4. -- Keep thin compatibility modules at the old import paths for this stack only where external tests or likely user code import them: - - `ergon_builtins.workers.react_worker` - - `ergon_builtins.workers.react_output` - - `ergon_builtins.workers.toolkit` - - `ergon_builtins.workers.training_stub_worker` - - `ergon_builtins.workers.tool_budget` - - `ergon_builtins.tools.subtask_lifecycle_toolkit` - - `ergon_builtins.tools.bash_sandbox_tool` - - `ergon_builtins.tools.workflow_cli_tool` - - `ergon_builtins.tools.graph_toolkit` - - `ergon_builtins.tools.graph_toolkit_types` - - `ergon_builtins.models.resolution` -- Compatibility modules must contain imports/`__all__` only, plus a short deprecation note in the docstring. No logic should remain there. -- Update builtins-internal imports to the new paths immediately. -- Add architecture tests that fail if old compatibility modules contain class/function definitions. - -This keeps PR 4 reviewable: each toolkit folder contains the callable factory, response models, and helper logic for one agent-facing surface. Shared abstractions go in `toolkits/common`, not in a generic `tools` package. - ---- - -### Task 1: Move GDPEval Criterion Factories Out Of `__init__.py` - -**Files:** -- Create: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria/factories.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria/__init__.py` -- Test: `ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py` - -- [x] **Step 1: Write the failing tests** - -Create `ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py`: - -```python -from ergon_builtins.benchmarks.gdpeval.criteria import ( - GDPEvalCriterion, - content_quality_judge, - make_code_check, - make_llm_judge, - output_file_exists, -) -from ergon_core.api.criterion import ScoreScale -from ergon_builtins.benchmarks.gdpeval.criteria.code_check import CodeCheckCriterion -from ergon_builtins.benchmarks.gdpeval.criteria.llm_judge import LLMJudgeCriterion - - -def test_gdpeval_criterion_union_accepts_domain_criteria() -> None: - criterion: GDPEvalCriterion = make_code_check( - name="has-output", - code_template="True", - description="checks output", - ) - - assert isinstance(criterion, CodeCheckCriterion) - - -def test_make_code_check_requires_explicit_description() -> None: - criterion = make_code_check( - name="has-output", - code_template="True", - description="checks output", - score_spec=ScoreScale(max_score=2.5), - ) - - assert criterion.slug == "has-output" - assert criterion.description == "checks output" - assert criterion.score_spec.max_score == 2.5 - - -def test_make_llm_judge_requires_explicit_description() -> None: - criterion = make_llm_judge( - name="quality", - prompt_template="Judge this.", - description="checks quality", - model="openai:gpt-4o-mini", - score_spec=ScoreScale(max_score=3.0), - ) - - assert isinstance(criterion, LLMJudgeCriterion) - assert criterion.description == "checks quality" - assert criterion.score_spec.max_score == 3.0 - - -def test_gdpeval_presets_supply_descriptions() -> None: - assert output_file_exists("*.docx").description == ( - "Verify output file matching *.docx was produced" - ) - assert content_quality_judge("accuracy").description == "Judge content quality on: accuracy" -``` - -- [x] **Step 2: Run the failing tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py -q -``` - -Expected: fail because `criteria/factories.py` does not exist yet and descriptions still default to `""`. - -- [x] **Step 3: Implement the factory module** - -Move the factory/preset functions from `criteria/__init__.py` into `criteria/factories.py`: - -```python -"""Factory helpers for GDP-specific criterion configurations.""" - -from ergon_builtins.benchmarks.gdpeval.criteria.code_check import CodeCheckCriterion -from ergon_builtins.benchmarks.gdpeval.criteria.llm_judge import LLMJudgeCriterion -from ergon_core.api.criterion import ScoreScale - - -def make_code_check( - name: str, - code_template: str, - *, - description: str, - weight: float = 1.0, - score_spec: ScoreScale | None = None, -) -> CodeCheckCriterion: - """Create a GDP code-check criterion.""" - return CodeCheckCriterion( - slug=name, - code_template=code_template, - description=description, - weight=weight, - score_spec=score_spec or ScoreScale(), - ) - - -def make_llm_judge( - name: str, - prompt_template: str, - *, - description: str, - weight: float = 1.0, - score_spec: ScoreScale | None = None, - model: str = "openai:gpt-4o", -) -> LLMJudgeCriterion: - """Create a GDP LLM-judge criterion.""" - return LLMJudgeCriterion( - slug=name, - prompt_template=prompt_template, - description=description, - weight=weight, - score_spec=score_spec or ScoreScale(), - model=model, - ) - - -def output_file_exists( - file_pattern: str = "*.docx", - *, - weight: float = 1.0, - score_spec: ScoreScale | None = None, -) -> CodeCheckCriterion: - """Check that at least one output file matching *file_pattern* exists.""" - return make_code_check( - name=f"output-exists-{file_pattern}", - code_template=( - "import glob; " - f"files = glob.glob('/workspace/final_output/{file_pattern}'); " - "len(files) > 0" - ), - description=f"Verify output file matching {file_pattern} was produced", - weight=weight, - score_spec=score_spec, - ) - - -def content_quality_judge( - aspect: str = "completeness", - *, - weight: float = 1.0, - score_spec: ScoreScale | None = None, - model: str = "openai:gpt-4o", -) -> LLMJudgeCriterion: - """LLM judge that evaluates content quality on a specific *aspect*.""" - return make_llm_judge( - name=f"content-quality-{aspect}", - prompt_template=( - f"Evaluate the {aspect} of the worker's output. " - "Score 1.0 if the output fully satisfies expectations, " - "0.5 for partial, 0.0 for absent or incorrect." - ), - description=f"Judge content quality on: {aspect}", - weight=weight, - score_spec=score_spec, - model=model, - ) -``` - -Make `criteria/__init__.py` only expose symbols: - -```python -"""GDP-specific criterion package.""" - -from ergon_builtins.benchmarks.gdpeval.criteria.code_check import CodeCheckCriterion -from ergon_builtins.benchmarks.gdpeval.criteria.factories import ( - content_quality_judge, - make_code_check, - make_llm_judge, - output_file_exists, -) -from ergon_builtins.benchmarks.gdpeval.criteria.llm_judge import LLMJudgeCriterion - -GDPEvalCriterion = CodeCheckCriterion | LLMJudgeCriterion - -__all__ = [ - "CodeCheckCriterion", - "GDPEvalCriterion", - "LLMJudgeCriterion", - "content_quality_judge", - "make_code_check", - "make_llm_judge", - "output_file_exists", -] -``` - -- [x] **Step 4: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py -q -``` - -Expected: pass. - ---- - -### Task 2: Remove GDPEval Criterion Constructor Shims - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria/code_check.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria/llm_judge.py` -- Test: `ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py` -- Test: `ergon_builtins/tests/unit/state/test_llm_judge_runtime_injection.py` - -- [x] **Step 1: Add tests that require canonical `score_spec` input** - -Append to `test_gdpeval_criteria_factories.py`: - -```python -import pytest -from ergon_core.api.criterion import ScoreScale -from pydantic import ValidationError - - -def test_code_check_rejects_legacy_max_score_constructor_input() -> None: - with pytest.raises(ValidationError): - CodeCheckCriterion(slug="code", code_template="True", max_score=4.0) - - -def test_llm_judge_rejects_legacy_max_score_constructor_input() -> None: - with pytest.raises(ValidationError): - LLMJudgeCriterion(slug="judge", prompt_template="Judge.", max_score=5.0) - - -def test_gdpeval_criteria_accept_canonical_score_spec() -> None: - code = CodeCheckCriterion( - slug="code", - code_template="True", - score_spec=ScoreScale(max_score=4.0), - ) - judge = LLMJudgeCriterion( - slug="judge", - prompt_template="Judge.", - score_spec=ScoreScale(max_score=5.0), - ) - - assert code.score_spec.max_score == 4.0 - assert judge.score_spec.max_score == 5.0 -``` - -- [x] **Step 2: Run tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py ergon_builtins/tests/unit/state/test_llm_judge_runtime_injection.py -q -``` - -Expected before implementation: fail because the criteria currently accept `max_score` through constructor shims. - -- [x] **Step 3: Delete constructor overrides and disallow extra fields** - -In both `CodeCheckCriterion` and `LLMJudgeCriterion`: - -- Remove the custom `__init__`. -- Remove the `Any` import if it becomes unused. -- Remove the `ScoreScale` import if it becomes unused. -- Set a class model config that forbids unknown constructor fields while preserving the base model's arbitrary type allowance: - -```python -from pydantic import ConfigDict - - -model_config = ConfigDict(arbitrary_types_allowed=True, frozen=False, extra="forbid") -``` - -This makes `score_spec=ScoreScale(max_score=...)` the only supported construction format for max-score configuration. Raw HF fields named `max_score` are handled by Task 3 before criterion construction. - -- [x] **Step 4: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_criteria_factories.py ergon_builtins/tests/unit/state/test_llm_judge_runtime_injection.py -q -uv run ruff check ergon_builtins/ergon_builtins/benchmarks/gdpeval/criteria --output-format concise -``` - -Expected: pass. - ---- - -### Task 3: Type GDPEval Rubric Loader Output - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/task_schemas.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/loader.py` -- Test: `ergon_builtins/tests/unit/benchmarks/test_gdpeval_loader_types.py` - -- [x] **Step 1: Write tests using a local JSONL fixture** - -Create `ergon_builtins/tests/unit/benchmarks/test_gdpeval_loader_types.py`: - -```python -from pathlib import Path - -from ergon_builtins.benchmarks.gdpeval.loader import load_rubric_data, load_single_rubric -from ergon_builtins.benchmarks.gdpeval.task_schemas import GDPRubricData - - -def test_load_rubric_data_returns_typed_rubric_records( - tmp_path: Path, - monkeypatch, -) -> None: - rubric_file = tmp_path / "rubrics.jsonl" - rubric_file.write_text( - '{"task_id": "task-1", "category_name": "docs", ' - '"max_total_score": 1.0, "stages": [], "rationale": "fixture"}\n', - encoding="utf-8", - ) - monkeypatch.setattr( - "ergon_builtins.benchmarks.gdpeval.loader.hf_hub_download", - lambda **_: str(rubric_file), - ) - - rubrics = load_rubric_data() - - assert isinstance(rubrics["task-1"], GDPRubricData) - assert rubrics["task-1"].task_id == "task-1" - assert rubrics["task-1"].category_name == "docs" - - -def test_loader_normalizes_hf_max_score_to_score_spec( - tmp_path: Path, - monkeypatch, -) -> None: - rubric_file = tmp_path / "rubrics.jsonl" - rubric_file.write_text( - '{"task_id": "task-1", "category_name": "docs", ' - '"max_total_score": 1.0, "stages": [{"name": "stage", ' - '"max_points": 2.0, "criteria": [{"name": "criterion", ' - '"code_template": "True", "max_score": 2.0}]}]}\n', - encoding="utf-8", - ) - monkeypatch.setattr( - "ergon_builtins.benchmarks.gdpeval.loader.hf_hub_download", - lambda **_: str(rubric_file), - ) - - rubric = load_single_rubric("task-1") - - criterion = rubric.stages[0].criteria[0] - assert criterion.score_spec.max_score == 2.0 - assert "max_score" not in criterion.model_dump() - - -def test_load_single_rubric_returns_typed_record( - tmp_path: Path, - monkeypatch, -) -> None: - rubric_file = tmp_path / "rubrics.jsonl" - rubric_file.write_text( - '{"task_id": "task-1", "category_name": "docs", ' - '"max_total_score": 1.0, "stages": []}\n', - encoding="utf-8", - ) - monkeypatch.setattr( - "ergon_builtins.benchmarks.gdpeval.loader.hf_hub_download", - lambda **_: str(rubric_file), - ) - - rubric = load_single_rubric("task-1") - - assert isinstance(rubric, GDPRubricData) - assert rubric.task_id == "task-1" -``` - -- [x] **Step 2: Run tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_loader_types.py -q -``` - -Expected: fail because `GDPRubricData` does not exist and loader returns raw dicts. - -- [x] **Step 3: Add typed models** - -Append to `task_schemas.py`: - -```python -class GDPRubricCriterionData(BaseModel): - """Raw GDPEval criterion entry from the HuggingFace rubric JSONL.""" - - name: str | None = None - description: str | None = None - type: str | None = None - code_template: str | None = None - prompt_template: str | None = None - weight: float = 1.0 - score_spec: ScoreScale = Field(default_factory=ScoreScale) - - @model_validator(mode="before") - @classmethod - def _normalize_hf_max_score(cls, data: Any) -> Any: # slopcop: ignore[no-typing-any] - if isinstance(data, dict) and "max_score" in data and "score_spec" not in data: - data = dict(data) - data["score_spec"] = {"max_score": data.pop("max_score")} - return data - - model_config = {"extra": "allow"} - - -class GDPRubricStageData(BaseModel): - """Raw GDPEval staged-rubric entry from the HuggingFace rubric JSONL.""" - - name: str - description: str | None = None - is_required: bool = True - max_points: float = 1.0 - min_score_to_pass: float = 0.0 - on_failure_action: str = "skip_remaining" - on_failure_score: float = 0.0 - criteria: list[GDPRubricCriterionData] = Field(default_factory=list) - - model_config = {"extra": "allow"} - - -class GDPRubricData(BaseModel): - """Raw GDPEval rubric record keyed by task_id.""" - - task_id: str - category_name: str | None = None - max_total_score: float | None = None - stages: list[GDPRubricStageData] = Field(default_factory=list) - rationale: str | None = None - - model_config = {"extra": "allow"} -``` - -Add imports to `task_schemas.py`: - -```python -from typing import Any - -from ergon_core.api.criterion import ScoreScale -from pydantic import BaseModel, Field, model_validator -``` - -- [x] **Step 4: Use the typed model in the loader** - -In `loader.py`, import `GDPRubricData` and update signatures: - -```python -from ergon_builtins.benchmarks.gdpeval.task_schemas import GDPRubricData - - -def load_rubric_data( - split: str = "train", - repo_id: str = HF_REPO_ID, -) -> dict[str, GDPRubricData]: - ... - rubrics: dict[str, GDPRubricData] = {} - with open(path) as f: - for line in f: - rubric = GDPRubricData.model_validate_json(line) - rubrics[rubric.task_id] = rubric - return rubrics - - -def load_single_rubric( - task_id: str, - split: str = "train", - repo_id: str = HF_REPO_ID, -) -> GDPRubricData: - ... -``` - -- [x] **Step 5: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_loader_types.py ergon_builtins/tests/unit/state/test_gdpeval_benchmark.py ergon_builtins/tests/unit/test_gdpeval_v2_definition.py -q -``` - -Expected: pass. - ---- - -### Task 4: Move Tool Response Models Out Of Tool Builders - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/tools/response_models.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/tools/response_models.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/tools/response_models.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/tools/response_models.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/tools/tool_builder.py` -- Test: `ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py` - -- [x] **Step 1: Write architecture tests** - -Create `ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py`: - -```python -from pathlib import Path - - -ROOT = Path("ergon_builtins/ergon_builtins/benchmarks") - - -def test_tool_response_models_do_not_import_tool_builders() -> None: - for path in ROOT.glob("*/tools/response_models.py"): - text = path.read_text() - assert ".tools.tool_builder import" not in text, path - - -def test_tool_builders_import_response_models() -> None: - for path in ROOT.glob("*/tools/tool_builder.py"): - text = path.read_text() - assert ".tools.response_models import" in text, path - - -def test_empty_operations_modules_are_deleted() -> None: - assert not list(ROOT.glob("*/tools/operations.py")) -``` - -- [x] **Step 2: Run tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py -q -``` - -Expected: fail because `response_models.py` re-exports from builders and empty `operations.py` files exist. - -- [x] **Step 3: Move response model classes** - -For each benchmark: - -- Copy response `BaseModel` classes from `tools/tool_builder.py` into `tools/response_models.py`. -- Import those classes back into `tools/tool_builder.py`. -- Remove direct `BaseModel` and `Field` imports from builders when no longer needed. -- Keep `__all__` in `response_models.py`. - -For MiniF2F specifically, move: - -```python -WriteLeanResponse -LeanCheckResponse -LeanVerificationResponse -SearchLemmasResponse -``` - -For GDPEval, move: - -```python -BashResponse -EditorResponse -RunPythonResponse -``` - -For ResearchRubrics, move: - -```python -ReportWriteResult -ReportReadResult -BashResult -``` - -For SWE-Bench, move: - -```python -BashResponse -EditorResponse -``` - -- [x] **Step 4: Delete empty operations modules** - -Delete: - -```bash -ergon_builtins/ergon_builtins/benchmarks/gdpeval/tools/operations.py -ergon_builtins/ergon_builtins/benchmarks/minif2f/tools/operations.py -ergon_builtins/ergon_builtins/benchmarks/researchrubrics/tools/operations.py -ergon_builtins/ergon_builtins/benchmarks/swebench_verified/tools/operations.py -``` - -Use `apply_patch` delete hunks, not shell `rm`. - -- [x] **Step 5: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py ergon_builtins/tests/unit/benchmarks/test_minif2f_task_shape.py ergon_builtins/tests/unit/test_gdpeval_v2_definition.py ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py -q -``` - -Expected: pass. - ---- - -### Task 5: Move GDPEval Sandbox Operation DTOs Out Of Task Schemas - -**Files:** -- Create: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/tools/sandbox_response_models.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/task_schemas.py` -- Test: `ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py` - -- [x] **Step 1: Extend architecture test** - -Append: - -```python -def test_gdpeval_task_schemas_do_not_contain_tool_response_dtos() -> None: - text = Path("ergon_builtins/ergon_builtins/benchmarks/gdpeval/task_schemas.py").read_text() - assert "ReadPDFResponse" not in text - assert "CreateDocxResponse" not in text - assert "OcrImageResponse" not in text -``` - -- [x] **Step 2: Move DTOs** - -Move these classes from `task_schemas.py` into `tools/sandbox_response_models.py`: - -```python -ReadPDFResponse -CreateDocxResponse -ReadExcelResponse -CreateExcelResponse -ReadCsvResponse -CreateCsvResponse -OcrImageResponse -RunPythonResponse -``` - -Keep class names unchanged to preserve existing imports if consumers choose the new semantic module. - -- [x] **Step 3: Verify no stale imports** - -Run: - -```bash -rg "ReadPDFResponse|CreateDocxResponse|ReadExcelResponse|CreateExcelResponse|ReadCsvResponse|CreateCsvResponse|OcrImageResponse" ergon_builtins ergon_core tests -``` - -Expected: only `tools/sandbox_response_models.py` and the architecture test mention them. - -- [x] **Step 4: Verify tests** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py ergon_builtins/tests/unit/test_builtin_task_definition_roundtrip.py -q -``` - -Expected: pass. - ---- - -### Task 6: Type Tool Builder And Toolkit Runtime Parameters - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/minif2f/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/tools/tool_builder.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/benchmarks/swebench_verified/tools/tool_builder.py` - -- [x] **Step 1: Replace `Any` on sandbox/task runtime params** - -Use public contracts: - -```python -from typing import Any - -from ergon_core.api import Task -from ergon_core.api.sandbox import Sandbox -from pydantic_ai.tools import Tool -``` - -Use signatures: - -```python -def tools(self, sandbox: Sandbox, task: Task[Any]) -> list[Tool]: - ... - - -def build_tools( - toolkit: GDPEvalToolkit, - *, - sandbox: Sandbox, - task: Task[Any], -) -> list[Tool]: - ... -``` - -Apply the same shape to MiniF2F, ResearchRubrics, and SWE-Bench. - -- [x] **Step 2: Remove unused `task` warnings cleanly** - -If a builder does not use `task`, add: - -```python -del task -``` - -near the top of the function body. - -- [x] **Step 3: Verify** - -Run: - -```bash -uv run ty check ergon_builtins/ergon_builtins/benchmarks/gdpeval ergon_builtins/ergon_builtins/benchmarks/minif2f ergon_builtins/ergon_builtins/benchmarks/researchrubrics ergon_builtins/ergon_builtins/benchmarks/swebench_verified -uv run ruff check ergon_builtins/ergon_builtins/benchmarks --output-format concise -``` - -Expected: pass. - ---- - -### Task 7: Clean ResearchRubrics Judge Construction - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/criteria/judge.py` -- Test: `ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py` - -- [x] **Step 1: Add explicit tests for defaulting behavior** - -Append to `test_research_rubrics_benchmark.py`: - -```python -def test_researchrubrics_judge_defaults_from_rubric_without_constructor_override() -> None: - rubric = RubricCriterion(criterion="Cite sources", axis="evidence", weight=2.0) - - criterion = ResearchRubricsJudgeCriterion(slug="evidence", rubric=rubric) - - assert criterion.description == "Cite sources" - assert criterion.weight == 2.0 - assert criterion.score_spec.max_score == 2.0 - assert criterion.rubric_text == "Cite sources" -``` - -- [x] **Step 2: Run the targeted test** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py::test_researchrubrics_judge_defaults_from_rubric_without_constructor_override -q -``` - -Expected before implementation: pass because the constructor currently does it. This locks behavior before removing the constructor. - -- [x] **Step 3: Replace `__init__` with a before validator** - -In `ResearchRubricsJudgeCriterion`, remove `__init__` and use: - -```python -@model_validator(mode="before") -@classmethod -def _default_fields_from_rubric(cls, data: Any) -> Any: # slopcop: ignore[no-typing-any] - if not isinstance(data, dict): - return data - rubric = data.get("rubric") - if not isinstance(rubric, RubricCriterion): - return data - data = dict(data) - data.setdefault("description", rubric.criterion) - data.setdefault("weight", rubric.weight) - data.setdefault("score_spec", ScoreScale(max_score=abs(rubric.weight))) - data.setdefault("rubric_text", rubric.criterion) - return data -``` - -Keep `_reject_model_alias`; either leave it as a separate validator or fold the alias check into the new validator. - -- [x] **Step 4: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py ergon_builtins/tests/unit/test_research_rubrics_v2_definition.py -q -uv run ruff check ergon_builtins/ergon_builtins/benchmarks/researchrubrics/criteria/judge.py --output-format concise -``` - -Expected: pass. - ---- - -### Task 8: Move ResearchRubrics Settings Import To Module Scope - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/researchrubrics/benchmark.py` -- Test: `ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py` - -- [x] **Step 1: Move import** - -At module top: - -```python -from ergon_core.core.shared.settings import settings -``` - -Remove the local import from `_load_rows`. - -- [x] **Step 2: Verify monkeypatch behavior still works** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/state/test_research_rubrics_benchmark.py -q -``` - -Expected: pass. Existing tests monkeypatch `load_dataset`, not the local import, so this should be safe. - ---- - -### Task 9: Delete Builtins Logfire Observability Hook - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/workers/react_worker.py` -- Delete: `ergon_builtins/ergon_builtins/observability/__init__.py` -- Delete: `ergon_builtins/ergon_builtins/observability/pydantic_ai_logfire.py` -- Delete: `ergon_builtins/tests/unit/builtins/test_logfire_pydantic_ai.py` -- Test: `ergon_builtins/tests/unit/workers/test_react_worker_contract.py` - -- [x] **Step 1: Write/adjust architecture check** - -Add to `ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py`: - -```python -def test_builtins_do_not_configure_logfire_observability() -> None: - assert not Path("ergon_builtins/ergon_builtins/observability").exists() - react_worker = Path("ergon_builtins/ergon_builtins/workers/react_worker.py").read_text() - assert "configure_pydantic_ai_logfire" not in react_worker - assert "logfire" not in react_worker.lower() -``` - -- [x] **Step 2: Remove call site** - -In `react_worker.py`: - -- Delete `from ergon_builtins.observability.pydantic_ai_logfire import configure_pydantic_ai_logfire`. -- Delete `configure_pydantic_ai_logfire()` from the execution path. - -- [x] **Step 3: Delete module and tests** - -Delete: - -```bash -ergon_builtins/ergon_builtins/observability/__init__.py -ergon_builtins/ergon_builtins/observability/pydantic_ai_logfire.py -ergon_builtins/tests/unit/builtins/test_logfire_pydantic_ai.py -``` - -Use `apply_patch` delete hunks. - -- [x] **Step 4: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/workers/test_react_worker_contract.py ergon_builtins/tests/unit/architecture/test_builtin_tool_boundaries.py -q -rg "logfire|configure_pydantic_ai_logfire|ergon_builtins\\.observability" ergon_builtins ergon_core tests -``` - -Expected: tests pass, `rg` returns no builtins Logfire references. - ---- - -### Task 10: Clean StagedRubric Validation Hook - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/benchmarks/gdpeval/rubric/staged_rubric.py` -- Test: `ergon_builtins/tests/unit/benchmarks/test_gdpeval_staged_rubric.py` - -- [x] **Step 1: Add tests for Pydantic-owned shape validation** - -Create `ergon_builtins/tests/unit/benchmarks/test_gdpeval_staged_rubric.py`: - -```python -import pytest -from pydantic import ValidationError - -from ergon_builtins.benchmarks.gdpeval.rubric import EvaluationStage, StagedRubric -from ergon_builtins.benchmarks.gdpeval.criteria.factories import make_code_check - - -def test_staged_rubric_runtime_validation_only_checks_dependencies() -> None: - rubric = StagedRubric( - name="gdpeval", - category_name="docs", - max_total_score=1.0, - stages=[], - ) - - rubric.validate_runtime_deps() - - -def test_evaluation_stage_requires_at_least_one_criterion() -> None: - with pytest.raises(ValidationError): - EvaluationStage( - name="empty", - description="empty stage", - max_points=1.0, - criteria=[], - ) - - -def test_staged_rubric_still_rejects_stage_max_above_total() -> None: - criterion = make_code_check( - name="has-output", - code_template="True", - description="checks output", - ) - - with pytest.raises(ValidationError): - StagedRubric( - name="gdpeval", - category_name="docs", - max_total_score=0.5, - stages=[ - EvaluationStage( - name="format", - description="format", - max_points=1.0, - criteria=[criterion], - ) - ], - ) -``` - -- [x] **Step 2: Remove redundant override** - -Delete `StagedRubric.validate_runtime_deps`. The inherited `Rubric.validate_runtime_deps` still checks evaluator/criterion package dependencies. Stage shape is already handled by Pydantic: - -- `EvaluationStage.criteria` uses `min_length=1`. -- `_materialise_stage_state` rejects total stage max above total score. - -- [x] **Step 3: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/benchmarks/test_gdpeval_staged_rubric.py ergon_builtins/tests/unit/test_gdpeval_v2_definition.py -q -``` - -Expected: pass. - ---- - -### Task 11: Remove Stale Subtask Containment TODO - -**Files:** -- Modify: `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` -- Modify: `ergon_builtins/tests/unit/tools/test_subtask_lifecycle_toolkit_containment.py` - -- [x] **Step 1: Strengthen existing test wording/coverage** - -Update `test_worker_toolkit_returns_failure_when_context_blocks_target` to cover all target-taking tools: - -```python -@pytest.mark.asyncio -async def test_worker_toolkit_returns_failure_when_context_blocks_target() -> None: - context = _FakeContext() - toolkit = SubtaskLifecycleToolkit(context=context) - - for tool_name, args in [ - ("get_subtask", (str(uuid4()),)), - ("cancel_task", (str(uuid4()),)), - ("restart_task", (str(uuid4()),)), - ("refine_task", (str(uuid4()), "new description")), - ]: - tool = next(tool for tool in toolkit.get_tools() if tool.__name__ == tool_name) - result = await tool(*args) - assert isinstance(result, ToolFailure) -``` - -Change `_FakeContext.cancel_task`, `_FakeContext.refine_task`, and `_FakeContext.restart_task` to raise `RuntimeError("not contained")` when `task_id != self.allowed_id`. - -- [x] **Step 2: Update docstring** - -Replace the stale note/TODO with: - -```python - Target-taking tools delegate containment to ``WorkerContext``. That - facade verifies each supplied node id is this context's task id or a - descendant before calling runtime task services, so the toolkit keeps - only response-shaping and UUID parsing logic here. -``` - -- [x] **Step 3: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/tools/test_subtask_lifecycle_toolkit_containment.py -q -``` - -Expected: pass. - ---- - -### Task 12: Restructure ReAct And Training Worker Domains - -**Files:** -- Create: `ergon_builtins/ergon_builtins/agents/__init__.py` -- Create: `ergon_builtins/ergon_builtins/agents/react/__init__.py` -- Create: `ergon_builtins/ergon_builtins/agents/react/worker.py` -- Create: `ergon_builtins/ergon_builtins/agents/react/output.py` -- Create: `ergon_builtins/ergon_builtins/agents/training/__init__.py` -- Create: `ergon_builtins/ergon_builtins/agents/training/synthetic_worker.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/__init__.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/common/__init__.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/common/base.py` -- Modify: `ergon_builtins/ergon_builtins/workers/__init__.py` -- Modify: `ergon_builtins/ergon_builtins/workers/react_worker.py` -- Modify: `ergon_builtins/ergon_builtins/workers/react_output.py` -- Modify: `ergon_builtins/ergon_builtins/workers/toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/workers/training_stub_worker.py` -- Modify imports in benchmark worker factories/toolkits/tests. -- Test: `ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py` - -- [x] **Step 1: Add layout tests** - -Create `ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py`: - -```python -from pathlib import Path - - -def test_react_implementation_lives_under_agents_react() -> None: - assert Path("ergon_builtins/ergon_builtins/agents/react/worker.py").exists() - assert Path("ergon_builtins/ergon_builtins/agents/react/output.py").exists() - - -def test_training_stub_lives_under_agents_training() -> None: - assert Path("ergon_builtins/ergon_builtins/agents/training/synthetic_worker.py").exists() - - -def test_toolkit_base_lives_under_toolkits_common() -> None: - assert Path("ergon_builtins/ergon_builtins/toolkits/common/base.py").exists() - - -def test_old_worker_modules_are_compatibility_shims_only() -> None: - for path in [ - Path("ergon_builtins/ergon_builtins/workers/react_worker.py"), - Path("ergon_builtins/ergon_builtins/workers/react_output.py"), - Path("ergon_builtins/ergon_builtins/workers/toolkit.py"), - Path("ergon_builtins/ergon_builtins/workers/training_stub_worker.py"), - ]: - text = path.read_text() - assert "compatibility import" in text - assert "\nclass " not in text - assert "\ndef " not in text -``` - -- [x] **Step 2: Move implementation modules** - -Move file contents: - -```text -workers/react_worker.py -> agents/react/worker.py -workers/react_output.py -> agents/react/output.py -workers/toolkit.py -> toolkits/common/base.py -workers/training_stub_worker.py -> agents/training/synthetic_worker.py -``` - -Update imports inside moved files: - -```python -from ergon_builtins.agents.react.output import worker_output_from_chunks -from ergon_builtins.llm.resolution import resolve_model_target -from ergon_builtins.toolkits.common.base import Toolkit -``` - -If Task 14 has not run yet, keep `resolve_model_target` imported from `ergon_builtins.models.resolution` temporarily and update it in Task 14. - -- [x] **Step 3: Add compatibility shims** - -Replace old worker modules with import-only shims, for example: - -```python -"""Deprecated compatibility import for ``ergon_builtins.agents.react.worker``.""" - -from ergon_builtins.agents.react.worker import ReActWorker - -__all__ = ["ReActWorker"] -``` - -Apply the same pattern for `react_output.py`, `toolkit.py`, and `training_stub_worker.py`. - -- [x] **Step 4: Update internal imports** - -Replace builtins-internal imports: - -```text -ergon_builtins.workers.react_worker -> ergon_builtins.agents.react.worker -ergon_builtins.workers.react_output -> ergon_builtins.agents.react.output -ergon_builtins.workers.toolkit -> ergon_builtins.toolkits.common.base -ergon_builtins.workers.training_stub_worker -> ergon_builtins.agents.training.synthetic_worker -``` - -Keep tests that explicitly assert old compatibility imports if they are testing public import stability. - -- [x] **Step 5: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/workers ergon_builtins/tests/unit/benchmarks ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py -q -``` - -Expected: pass. - ---- - -### Task 13: Restructure Toolkit Domains - -**Files:** -- Create: `ergon_builtins/ergon_builtins/toolkits/subagents/__init__.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/subagents/toolkit.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/subagents/models.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/subagents/sandbox_bash.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/workflow_cli/__init__.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/workflow_cli/tool.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/resources/__init__.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/resources/toolkit.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/resources/models.py` -- Create: `ergon_builtins/ergon_builtins/toolkits/common/budgets.py` -- Modify: `ergon_builtins/ergon_builtins/tools/bash_sandbox_tool.py` -- Modify: `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/tools/graph_toolkit_types.py` -- Modify: `ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py` -- Modify: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Modify: `ergon_builtins/ergon_builtins/workers/tool_budget.py` -- Modify imports in tests and CLI tests. -- Test: `ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py` - -- [x] **Step 1: Extend layout tests** - -Append: - -```python -def test_toolkit_implementations_live_under_named_toolkit_packages() -> None: - for rel in [ - "toolkits/common/budgets.py", - "toolkits/subagents/toolkit.py", - "toolkits/subagents/models.py", - "toolkits/subagents/sandbox_bash.py", - "toolkits/workflow_cli/tool.py", - "toolkits/resources/toolkit.py", - "toolkits/resources/models.py", - ]: - assert Path(f"ergon_builtins/ergon_builtins/{rel}").exists() - - -def test_old_tool_modules_are_compatibility_shims_only() -> None: - for path in [ - Path("ergon_builtins/ergon_builtins/tools/bash_sandbox_tool.py"), - Path("ergon_builtins/ergon_builtins/tools/graph_toolkit.py"), - Path("ergon_builtins/ergon_builtins/tools/graph_toolkit_types.py"), - Path("ergon_builtins/ergon_builtins/tools/subtask_lifecycle_toolkit.py"), - Path("ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py"), - Path("ergon_builtins/ergon_builtins/workers/tool_budget.py"), - ]: - text = path.read_text() - assert "compatibility import" in text - assert "\nclass " not in text - assert "\ndef " not in text -``` - -- [x] **Step 2: Move implementation modules** - -Move file contents: - -```text -workers/tool_budget.py -> toolkits/common/budgets.py -tools/subtask_lifecycle_toolkit.py -> toolkits/subagents/toolkit.py -tools/bash_sandbox_tool.py -> toolkits/subagents/sandbox_bash.py -tools/workflow_cli_tool.py -> toolkits/workflow_cli/tool.py -tools/graph_toolkit.py -> toolkits/resources/toolkit.py -tools/graph_toolkit_types.py -> toolkits/resources/models.py -``` - -Update imports within moved files: - -```python -from ergon_builtins.toolkits.common.budgets import ... -from ergon_builtins.toolkits.resources.models import ResourceRef -from ergon_builtins.toolkits.subagents.sandbox_bash import make_sandbox_bash_tool -``` - -While moving `subtask_lifecycle_toolkit.py`, split response DTOs into -`toolkits/subagents/models.py` and import them from `toolkits/subagents/toolkit.py`. -This gives the subagent toolkit its own local schema file instead of keeping -models and tool construction in one large module. - -- [x] **Step 3: Add compatibility shims** - -Replace old modules with import-only shims, for example: - -```python -"""Deprecated compatibility import for ``ergon_builtins.toolkits.subagents.toolkit``.""" - -from ergon_builtins.toolkits.subagents.models import ( - AddSubtaskToolResponse, - AddSubtaskToolSuccess, - CancelTaskToolResponse, - CancelTaskToolSuccess, - GetSubtaskToolResponse, - GetSubtaskToolSuccess, - ListSubtasksToolResponse, - ListSubtasksToolSuccess, - PlanSubtasksToolResponse, - PlanSubtasksToolSuccess, - RefineTaskToolResponse, - RefineTaskToolSuccess, - RestartTaskToolResponse, - RestartTaskToolSuccess, - ToolFailure, -) -from ergon_builtins.toolkits.subagents.toolkit import ( - SubtaskLifecycleToolkit, - build_subtask_lifecycle_tools, -) - -__all__ = [ - "AddSubtaskToolResponse", - "AddSubtaskToolSuccess", - "CancelTaskToolResponse", - "CancelTaskToolSuccess", - "GetSubtaskToolResponse", - "GetSubtaskToolSuccess", - "ListSubtasksToolResponse", - "ListSubtasksToolSuccess", - "PlanSubtasksToolResponse", - "PlanSubtasksToolSuccess", - "RefineTaskToolResponse", - "RefineTaskToolSuccess", - "RestartTaskToolResponse", - "RestartTaskToolSuccess", - "SubtaskLifecycleToolkit", - "ToolFailure", - "build_subtask_lifecycle_tools", -] -``` - -Use shorter equivalent shims for the other old modules. - -- [x] **Step 4: Update internal imports** - -Replace builtins-internal imports: - -```text -ergon_builtins.workers.tool_budget -> ergon_builtins.toolkits.common.budgets -ergon_builtins.tools.subtask_lifecycle_toolkit -> ergon_builtins.toolkits.subagents.toolkit -ergon_builtins.tools.bash_sandbox_tool -> ergon_builtins.toolkits.subagents.sandbox_bash -ergon_builtins.tools.workflow_cli_tool -> ergon_builtins.toolkits.workflow_cli.tool -ergon_builtins.tools.graph_toolkit -> ergon_builtins.toolkits.resources.toolkit -ergon_builtins.tools.graph_toolkit_types -> ergon_builtins.toolkits.resources.models -``` - -- [x] **Step 5: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/tools ergon_cli/tests/unit/state/test_workflow_cli_tool.py ergon_cli/tests/unit/state/test_subtask_lifecycle_toolkit.py ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py -q -``` - -Expected: pass. - ---- - -### Task 14: Rename Model Infrastructure To LLM Providers - -**Files:** -- Create: `ergon_builtins/ergon_builtins/llm/__init__.py` -- Create: `ergon_builtins/ergon_builtins/llm/resolution.py` -- Create: `ergon_builtins/ergon_builtins/llm/capture_settings.py` -- Create: `ergon_builtins/ergon_builtins/llm/providers/__init__.py` -- Create: `ergon_builtins/ergon_builtins/llm/providers/openrouter.py` -- Create: `ergon_builtins/ergon_builtins/llm/providers/openrouter_responses.py` -- Create: `ergon_builtins/ergon_builtins/llm/providers/transformers.py` -- Create: `ergon_builtins/ergon_builtins/llm/providers/vllm.py` -- Modify: `ergon_builtins/ergon_builtins/models/*.py` -- Modify imports in workers, common LLM judge, registry local models, and tests. -- Test: `ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py` - -- [x] **Step 1: Extend layout tests** - -Append: - -```python -def test_llm_provider_implementation_lives_under_llm_package() -> None: - for rel in [ - "llm/resolution.py", - "llm/capture_settings.py", - "llm/providers/openrouter.py", - "llm/providers/openrouter_responses.py", - "llm/providers/transformers.py", - "llm/providers/vllm.py", - ]: - assert Path(f"ergon_builtins/ergon_builtins/{rel}").exists() - - -def test_old_models_modules_are_compatibility_shims_only() -> None: - for path in [ - Path("ergon_builtins/ergon_builtins/models/resolution.py"), - Path("ergon_builtins/ergon_builtins/models/openrouter_backend.py"), - Path("ergon_builtins/ergon_builtins/models/openrouter_responses_backend.py"), - Path("ergon_builtins/ergon_builtins/models/transformers_backend.py"), - Path("ergon_builtins/ergon_builtins/models/vllm_backend.py"), - ]: - text = path.read_text() - assert "compatibility import" in text - assert "\nclass " not in text - assert "\ndef " not in text -``` - -- [x] **Step 2: Move implementation modules** - -Move file contents: - -```text -models/resolution.py -> llm/resolution.py and llm/capture_settings.py -models/openrouter_backend.py -> llm/providers/openrouter.py -models/openrouter_responses_backend.py -> llm/providers/openrouter_responses.py -models/transformers_backend.py -> llm/providers/transformers.py -models/vllm_backend.py -> llm/providers/vllm.py -``` - -Split provider-agnostic capture settings out of `resolution.py` into `llm/capture_settings.py`. -Update imports in moved files from `ergon_builtins.models.resolution` to `ergon_builtins.llm.resolution`. - -- [x] **Step 3: Add compatibility shims** - -Example: - -```python -"""Deprecated compatibility import for ``ergon_builtins.llm.resolution``.""" - -from ergon_builtins.llm.capture_settings import capture_model_settings_for -from ergon_builtins.llm.resolution import ( - ResolvedModel, - register_model_backend, - registered_model_backend_prefixes, - resolve_model_target, -) - -__all__ = [ - "ResolvedModel", - "capture_model_settings_for", - "register_model_backend", - "registered_model_backend_prefixes", - "resolve_model_target", -] -``` - -- [x] **Step 4: Update internal imports** - -Replace builtins-internal imports: - -```text -ergon_builtins.models.resolution -> ergon_builtins.llm.resolution / ergon_builtins.llm.capture_settings -ergon_builtins.models.transformers_backend -> ergon_builtins.llm.providers.transformers -ergon_builtins.models.openrouter_backend -> ergon_builtins.llm.providers.openrouter -ergon_builtins.models.openrouter_responses_backend -> ergon_builtins.llm.providers.openrouter_responses -ergon_builtins.models.vllm_backend -> ergon_builtins.llm.providers.vllm -``` - -- [x] **Step 5: Verify** - -Run: - -```bash -uv run pytest ergon_builtins/tests/unit/builtins/common/test_capture_settings.py ergon_builtins/tests/unit/workers/test_react_worker_contract.py ergon_builtins/tests/unit/architecture/test_builtin_domain_layout.py -q -``` - -Expected: pass. - ---- - -### Task 15: Final TODO Sweep And PR 4 Verification - -**Files:** -- All files touched above. - -- [x] **Step 1: Confirm TODOs are gone or intentionally outside PR 4** - -Run: - -```bash -rg -n "TODO|todo|FIXME|XXX" ergon_builtins/ergon_builtins -``` - -Expected: no TODOs from the inventory remain. If a new TODO is intentionally retained, replace it with a tracked issue/RFC reference; do not leave casual cleanup comments in builtins. - -- [x] **Step 2: Run focused builtins tests** - -Run: - -```bash -uv run pytest \ - ergon_builtins/tests/unit/architecture \ - ergon_builtins/tests/unit/benchmarks \ - ergon_builtins/tests/unit/state \ - ergon_builtins/tests/unit/tools \ - ergon_builtins/tests/unit/workers \ - -q -``` - -Expected: pass. - -- [x] **Step 3: Run stack-level checks** - -Run: - -```bash -uv run ruff check ergon_builtins ergon_core ergon_cli ergon_infra ergon_ingestion tests scripts --output-format concise -uv run ty check ergon_core/ergon_core ergon_builtins/ergon_builtins ergon_cli/ergon_cli ergon_infra ergon_ingestion/ergon_ingestion -``` - -Expected: pass. - -- [x] **Step 4: Commit on PR 4** - -Commit only on `codex/builtins-pr04-benchmark-domains-v2`: - -```bash -git add ergon_builtins/ergon_builtins ergon_builtins/tests/unit docs/superpowers/plans/2026-05-19-builtins-pr04-todo-cleanup.md -git commit --amend --no-edit -``` - -- [x] **Step 5: Push PR 4** - -Run: - -```bash -git push --force-with-lease origin codex/builtins-pr04-benchmark-domains-v2:codex/builtins-pr04-benchmark-domains -``` - -Expected: PR #87 updates only; PRs #84-#86 remain unchanged. - ---- - -## Self-Review - -**Spec coverage:** Every TODO found in the supplied worktree maps to a task above. The subtask containment TODO is resolved as stale because core `WorkerContext` already enforces containment; the plan strengthens the builtins test and updates docs instead of duplicating core checks. - -**Placeholder scan:** The plan contains no implementation placeholders; each task gives concrete target files, expected tests, and the intended code shape. - -**Type consistency:** Runtime tool builders use `Sandbox` and `Task[Any]` consistently. Criterion construction uses canonical `score_spec`; raw HF `max_score` fields are normalized only by GDPEval dataset-loading models. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/00-program.md b/docs/superpowers/plans/mas-run-visual-debugger/00-program.md deleted file mode 100644 index 8c25fe0fd..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/00-program.md +++ /dev/null @@ -1,110 +0,0 @@ -# MAS Run Visual Debugger Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the current unreadable MAS run view with a visual debugger that shows the whole recursive graph at selected time `T`, an overlap-based bottom activity stack, and a task-scoped workspace drawer. - -**Architecture:** Keep graph mutation replay as the source of topology truth and derive the activity stack from existing run state plus graph mutations. Avoid fixed agent lanes: agents can join/leave dynamically, while tasks, events, and wall-clock timestamps are stable. Introduce small frontend domain modules for activity derivation/layout and keep backend DTO changes additive and narrow. - -**Tech Stack:** Next.js/React, TypeScript, React Flow, Tailwind CSS, Zod contracts, Playwright e2e, FastAPI test harness DTOs. - ---- - -## 1. Goals and non-goals - -**Goals** - -- Render the full recursive task graph as it existed at selected sequence/time `T`. -- Move the timeline into a bottom dock that visualizes concurrency by stacking overlapping activity bars. -- Keep the right-hand workspace task-scoped and time-aware: selecting a node at `T` shows resources, executions, messages, context events, and evaluations available at `T`. -- Make activity layout independent of agent cardinality. Agent/worker names are labels and filter metadata only. -- Preserve the live mode. Timeline mode must be opt-in and must not make the live dashboard feel stale. -- Add focused Playwright coverage that proves the graph canvas, activity stack, sequence scrubber, and workspace drawer are all usable on canonical MAS smoke runs. -- Use the mockup `ergon-dashboard/mockups/mas-activity-stack-debugger.html` as the UX target, not as code to copy directly. - -**Non-goals** - -- No rewrite of backend execution/control flow. -- No persistent "agent timeline" DTO. -- No new graph database model. -- No attempt to solve arbitrary huge-graph navigation in the first PR. The first PR should make the existing 9-leaf smoke and representative MAS samples readable. -- No replacing React Flow. -- No pixel-perfect visual snapshot testing in phase 1. Screenshot artifacts are review aids; assertions target stable structure and visibility. - ---- - -## 2. UX invariants - -- **Whole graph at T:** timeline scrub changes graph state, not graph scope. Collapsed containers are allowed for readability, but nodes are not silently omitted due to focus. -- **Concurrency by overlap:** overlapping work appears stacked vertically in the bottom dock. Vertical position means "needed another row because time overlaps", not "agent N". -- **Stable categories, unstable actors:** kind chips (`Execution`, `Graph`, `Talk`, `Artifact`, `Evaluation`, `Context`) are stable; worker/agent labels are secondary. -- **Task identity everywhere:** clicking an activity with `taskId` selects the graph node and opens the workspace. Clicking a graph node highlights related activity. -- **Replay is deterministic:** the same snapshot + mutation list + selected sequence produces the same graph and activity view. -- **Missing duration is explicit:** instant events render as markers; spans render as bars. Do not fake long durations for resources/messages/evaluations. - ---- - -## 3. DTO stance - -Production DTO changes should be avoided in the first phase unless implementation proves a real gap. - -Existing production data already gives the frontend enough to build the first activity stack: - -- `RunSnapshotDto` -> tasks, executions, resources, sandboxes, threads, evaluations. -- `dashboard/graph.mutation` + `/api/runs/{runId}/mutations` -> sequence, mutation kind, actor, reason, `created_at`. -- `context.event` state -> task execution, task node, event type, created/started/completed times where available. -- `task_evaluation_updated` -> task-scoped evaluation marker. - -Additive DTO work is still planned for testability and future precision: - -- Extend the **test harness** run-state DTO with activity-stack facts that Playwright can assert without reverse-engineering layout from pixels: mutation count, execution spans, context-event count, evaluation task IDs, and graph node IDs already exist; add `activity_event_count`, `activity_span_count`, and `max_concurrency` in Phase C if needed. -- Add production REST fields only if the current generated `RunSnapshotDto` lacks a timestamp needed for an honest bar. The likely candidate is evaluation duration (`startedAt`/`completedAt`) if evaluations become spans rather than instant markers. - ---- - -## 4. File map - -**New frontend domain files** - -- `ergon-dashboard/src/features/activity/types.ts` — `RunActivity`, `ActivityKind`, `ActivityStackRow`, layout result types. -- `ergon-dashboard/src/features/activity/buildRunActivities.ts` — pure derivation from `WorkflowRunState`, `RunEvent[]`, and `GraphMutationDto[]`. -- `ergon-dashboard/src/features/activity/stackLayout.ts` — overlap packing algorithm. -- `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` — bottom dock UI. -- `ergon-dashboard/src/features/activity/components/ActivityBar.tsx` — single bar/marker renderer. - -**Modified frontend files** - -- `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` — three-pane debugger shell, timeline mode wiring, selection/highlight coordination. -- `ergon-dashboard/src/components/dag/DAGCanvas.tsx` — accept highlight props and expose stable graph container/node test IDs. -- `ergon-dashboard/src/features/graph/components/MutationTimeline.tsx` — either retire after Phase B or reduce to sequence controls reused by `ActivityStackTimeline`. -- `ergon-dashboard/src/components/workspace/TaskWorkspace.tsx` — filter task-scoped collections to selected sequence/time when timeline mode is active. -- `ergon-dashboard/src/lib/runEvents.ts` — keep flat event stream derivation, but do not make it own activity packing. - -**Modified tests** - -- `ergon-dashboard/tests/helpers/dashboardFixtures.ts` — add a concurrent MAS fixture with overlapping executions/context events. -- `ergon-dashboard/tests/e2e/_shared/smoke.ts` — assert activity stack presence and screenshots. -- `ergon-dashboard/tests/helpers/backendHarnessClient.ts` — add narrow test harness fields only if backend exposes them. - ---- - -## 5. Merge checklist - -- [ ] `pnpm --dir ergon-dashboard test` or the repository's frontend unit command is green for activity derivation/layout tests. -- [ ] `pnpm --dir ergon-dashboard run check` or current frontend type/lint command is green. -- [ ] Playwright smoke opens a canonical MAS run, enters timeline mode, sees `activity-stack-region`, scrubs sequence, opens workspace from a graph node, and captures run screenshots. -- [ ] Activity stack never creates rows from agent names. -- [ ] E2E screenshot shows full recursive graph at selected `T`, not focus-filtered branch-only graph. -- [ ] Implementation handoff includes PNGs of every new UI panel: full debugger page, graph canvas, activity stack bottom dock, and workspace drawer open on a selected task. -- [ ] Existing live run updates still render without requiring mutation fetch success. -- [ ] No production backend DTO changes unless justified in `01-contracts-and-state.md`. - ---- - -## 6. Open decisions - -1. **Activity source for graph mutations:** default to using `/api/runs/{runId}/mutations` in timeline mode. If live mode needs graph mutation bars before entering timeline, also retain recent socket mutation events in `useRunState`. -2. **Evaluation duration:** default to instant marker at `evaluation.createdAt`. Upgrade to span only if backend has real start/end timestamps. -3. **Viewport fit:** default to React Flow `fitView` on initial load and sequence changes only when user has not manually panned/zoomed. -4. **Saved layout state:** defer persistence of pane sizes/zoom to a follow-up. -5. **Virtualization:** defer until the activity count in a smoke run or real run demonstrably causes UI lag. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/01-contracts-and-state.md b/docs/superpowers/plans/mas-run-visual-debugger/01-contracts-and-state.md deleted file mode 100644 index 827393e5d..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/01-contracts-and-state.md +++ /dev/null @@ -1,232 +0,0 @@ -# 01 — Contracts and State - -**Status:** draft. -**Scope:** DTO inventory, frontend activity model, deterministic replay contract, and exact places where additive DTO changes may be needed. - -Cross-refs: program goals in [`00-program.md`](00-program.md), UI work in [`02-frontend-implementation.md`](02-frontend-implementation.md), e2e contract in [`03-tests-and-e2e.md`](03-tests-and-e2e.md). - ---- - -## 1. Existing contract inventory - -### Production snapshot/state - -- `ergon-dashboard/src/lib/contracts/rest.ts` - - `RunSnapshot` already includes tasks, executions, resources, sandboxes, threads, and evaluations via generated schemas. - - `RunExecutionAttempt` has `startedAt` and `completedAt`, which are true span endpoints. - - `RunSandbox` has `createdAt` and `closedAt`, which are true span endpoints. - - `RunSandboxCommand` has `timestamp` and `durationMs`, which can render as short command spans. - - `RunTaskEvaluation` currently behaves like an instant marker unless start/end timestamps are present in generated schema. - -- `ergon-dashboard/src/lib/types.ts` - - `WorkflowRunState` is the in-memory source for current display state. - - `TaskState.history` records task transitions with sequence/time/actor/reason. - -### Graph mutations - -- `ergon-dashboard/src/features/graph/contracts/graphMutations.ts` - - `GraphMutationDto` has `sequence`, `mutation_type`, `target_id`, `actor`, `reason`, `created_at`. - - This is sufficient for graph mutation markers and sequence scrubbing. - -- `ergon-dashboard/src/features/graph/state/graphMutationReducer.ts` - - `replayToSequence` is the topology/status replay engine. - - Activity derivation should consume its result; it should not duplicate graph replay. - -### Unified event stream - -- `ergon-dashboard/src/lib/runEvents.ts` - - `buildRunEvents()` already flattens workflow lifecycle, task transitions, sandbox events, messages, evaluations, resources, context events, and unhandled mutations. - - Keep this useful for event rows and activity markers, but implement span packing in a separate `features/activity` module. - ---- - -## 2. Frontend domain model - -Create `ergon-dashboard/src/features/activity/types.ts`. - -```typescript -import type { RunEventKind } from "@/lib/runEvents"; - -export type ActivityKind = - | "execution" - | "graph" - | "message" - | "artifact" - | "evaluation" - | "context" - | "sandbox"; - -export interface RunActivity { - id: string; - kind: ActivityKind; - label: string; - taskId: string | null; - sequence: number | null; - startAt: string; - endAt: string | null; - isInstant: boolean; - actor: string | null; - sourceKind: RunEventKind | "execution.span" | "sandbox.span" | "graph.mutation"; - metadata: Record<string, string | number | boolean | null>; -} - -export interface ActivityStackItem { - activity: RunActivity; - row: number; - leftPct: number; - widthPct: number; -} - -export interface ActivityStackLayout { - items: ActivityStackItem[]; - rowCount: number; - startMs: number; - endMs: number; - maxConcurrency: number; -} -``` - -Rules: - -- `startAt` is always required. -- `endAt` is `null` for markers. -- `isInstant` is true when `endAt === null` or when duration is below the render minimum. -- `taskId` can be null for workflow-level events. -- `actor` is metadata only; it must not become a lane key. - ---- - -## 3. Activity derivation - -Create `ergon-dashboard/src/features/activity/buildRunActivities.ts`. - -Inputs: - -```typescript -import type { GraphMutationDto } from "@/features/graph/contracts/graphMutations"; -import type { RunEvent } from "@/lib/runEvents"; -import type { WorkflowRunState } from "@/lib/types"; -import type { RunActivity } from "./types"; - -export interface BuildRunActivitiesInput { - runState: WorkflowRunState | null; - events: RunEvent[]; - mutations: GraphMutationDto[]; - currentSequence: number | null; -} - -export function buildRunActivities(input: BuildRunActivitiesInput): RunActivity[] { - if (!input.runState) return []; - return [ - ...executionActivities(input.runState), - ...sandboxActivities(input.runState), - ...contextActivities(input.runState), - ...eventMarkerActivities(input.events), - ...graphMutationActivities(input.mutations), - ].sort(compareActivity); -} -``` - -Derivation rules: - -- Executions: one span per `ExecutionAttemptState` with non-null `startedAt`; use `completedAt` when available, otherwise render open span through selected/current time. -- Sandboxes: one span per `SandboxState`; use `closedAt` when available. -- Sandbox commands: marker or short span using `timestamp + durationMs`. -- Context events: span if both `startedAt` and `completedAt` exist; otherwise marker at `createdAt`. -- Thread messages, resources, evaluations, workflow lifecycle: marker activities from `RunEvent`. -- Graph mutations: marker activities from `GraphMutationDto`. -- Duplicate suppression: do not render both a `task.transition` event and a `graph.mutation` marker as identical labels if they share the same sequence/task/status. Prefer the graph mutation marker for sequence navigation and keep task transition in the event stream. - ---- - -## 4. Stack layout - -Create `ergon-dashboard/src/features/activity/stackLayout.ts`. - -```typescript -import type { ActivityStackLayout, RunActivity } from "./types"; - -export interface StackActivityOptions { - minMarkerWidthPct: number; - minSpanWidthPct: number; -} - -export function stackActivities( - activities: RunActivity[], - options: StackActivityOptions = { minMarkerWidthPct: 0.35, minSpanWidthPct: 0.75 }, -): ActivityStackLayout { - const timed = activities - .map((activity) => toTimedActivity(activity)) - .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs || a.activity.id.localeCompare(b.activity.id)); - - if (timed.length === 0) { - return { items: [], rowCount: 0, startMs: 0, endMs: 0, maxConcurrency: 0 }; - } - - const startMs = Math.min(...timed.map((a) => a.startMs)); - const endMs = Math.max(...timed.map((a) => a.endMs)); - const spanMs = Math.max(1, endMs - startMs); - const rowEnds: number[] = []; - let maxConcurrency = 0; - - const items = timed.map(({ activity, startMs: itemStartMs, endMs: itemEndMs }) => { - const row = firstFreeRow(rowEnds, itemStartMs); - rowEnds[row] = itemEndMs; - maxConcurrency = Math.max(maxConcurrency, rowEnds.filter((rowEnd) => rowEnd > itemStartMs).length); - - const leftPct = ((itemStartMs - startMs) / spanMs) * 100; - const rawWidthPct = ((itemEndMs - itemStartMs) / spanMs) * 100; - const widthPct = activity.isInstant - ? options.minMarkerWidthPct - : Math.max(options.minSpanWidthPct, rawWidthPct); - - return { activity, row, leftPct, widthPct }; - }); - - return { items, rowCount: rowEnds.length, startMs, endMs, maxConcurrency }; -} -``` - -Acceptance rules: - -- Two overlapping spans must be placed on different rows. -- Adjacent non-overlapping spans can reuse the same row. -- Instant markers should not force every later item onto a new row; give them a small render interval only for collision. -- Layout must be deterministic for identical inputs. - ---- - -## 5. DTO change decision tree - -Use this decision tree before editing backend schema files: - -1. Can the UI derive the fact from `WorkflowRunState`, `RunEvent[]`, or `GraphMutationDto[]` without lying about time? If yes, do not change production DTOs. -2. Is the missing fact only needed by Playwright? If yes, add it to `ergon_core/core/api/test_harness.py` and `ergon-dashboard/tests/helpers/backendHarnessClient.ts`, not production REST. -3. Is the missing fact needed by users and already persisted? If yes, add it to the production API schema and generated frontend contracts. -4. Is the missing fact not persisted? Stop and design the backend persistence change separately; do not smuggle fake frontend fields into the UI. - -Likely first-PR DTO edits: - -- **Test harness only:** add `activity_event_count`, `activity_span_count`, `max_concurrency` after the frontend derivation is stable enough to calculate the same values in backend or harness queries. -- **No production DTO edit:** keep evaluations as markers unless persisted evaluation span timestamps already exist. - ---- - -## 6. Unit test checklist - -Create `ergon-dashboard/src/features/activity/buildRunActivities.test.ts`. - -- [ ] Execution with start/end becomes a span with `kind: "execution"`. -- [ ] Running execution with no end becomes open span using current selected time. -- [ ] Resource event becomes instant `kind: "artifact"` marker. -- [ ] Evaluation event becomes instant `kind: "evaluation"` marker. -- [ ] Graph mutation becomes instant `kind: "graph"` marker with sequence. -- [ ] Actor names appear in metadata but not in row assignment input. - -Create `ergon-dashboard/src/features/activity/stackLayout.test.ts`. - -- [ ] Non-overlapping spans reuse one row. -- [ ] Overlapping spans use two rows. -- [ ] Three-way overlap reports `maxConcurrency === 3`. -- [ ] Instant markers do not permanently block a row. -- [ ] Same input order-independent set produces identical `row` assignments after sorting. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/02-frontend-implementation.md b/docs/superpowers/plans/mas-run-visual-debugger/02-frontend-implementation.md deleted file mode 100644 index 5c4e44b27..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/02-frontend-implementation.md +++ /dev/null @@ -1,321 +0,0 @@ -# 02 — Frontend Implementation - -**Status:** draft. -**Scope:** component boundaries, UI behavior, and task-by-task implementation plan for the visual debugger shell. - -Cross-refs: contracts in [`01-contracts-and-state.md`](01-contracts-and-state.md), tests in [`03-tests-and-e2e.md`](03-tests-and-e2e.md), phase order in [`04-phases.md`](04-phases.md). - ---- - -## 1. Target layout - -The run page becomes a three-region visual debugger: - -- Header/status strip remains at the top with run status, cohort breadcrumb, live/timeline toggle, and connection state. -- Main region is the React Flow recursive graph, showing the whole graph at selected `T`. -- Bottom dock is `ActivityStackTimeline`, always horizontal time, vertical rows allocated by overlap. -- Right drawer is `TaskWorkspace`, opened by graph node or activity click. - -The accepted mockup is `ergon-dashboard/mockups/mas-activity-stack-debugger.html`. - ---- - -## 2. Component map - -### New components - -- `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` - - Props: activities, current sequence, selected task, selected activity, callbacks. - - Owns time ruler, row rendering, legend, scrubber controls, and empty state. - -- `ergon-dashboard/src/features/activity/components/ActivityBar.tsx` - - Props: stack item, selected/highlight booleans, click handler. - - Renders span or marker using kind-specific styling. - -### Modified components - -- `RunWorkspacePage.tsx` - - Replaces old bottom `MutationTimeline` region with activity stack. - - Creates `activities = buildRunActivities({ runState: displayState, events, mutations, currentSequence })`. - - Tracks `selectedActivityId`. - - Activity click sets current sequence if present and selects `taskId` if present. - - Graph node click selects task and highlights related activities. - -- `DAGCanvas.tsx` - - Accepts `highlightedTaskIds?: Set<string>`. - - Passes selected/highlight information through node data. - - Keeps depth expansion controls, search, minimap, and React Flow controls. - - Ensures canvas has `data-testid="graph-canvas"` and individual graph elements keep `graph-node-{taskId}` / `graph-container-{taskId}`. - -- `TaskWorkspace.tsx` - - Accepts `selectedTime?: string | null` or `currentSequence?: number | null`. - - Filters task collections for timeline mode only: - - resources with `createdAt <= selectedTime` - - executions with `startedAt <= selectedTime` - - sandbox commands with `timestamp <= selectedTime` - - thread messages with `createdAt <= selectedTime` - - context events with `createdAt <= selectedTime` - - evaluation with `createdAt <= selectedTime` - - Live mode keeps current behavior. - ---- - -## 3. Task 1: Activity domain module - -**Files:** - -- Create: `ergon-dashboard/src/features/activity/types.ts` -- Create: `ergon-dashboard/src/features/activity/buildRunActivities.ts` -- Create: `ergon-dashboard/src/features/activity/stackLayout.ts` -- Test: `ergon-dashboard/src/features/activity/buildRunActivities.test.ts` -- Test: `ergon-dashboard/src/features/activity/stackLayout.test.ts` - -- [ ] **Step 1: Write tests for activity derivation** - -```typescript -import { describe, expect, it } from "vitest"; -import { buildRunActivities } from "./buildRunActivities"; - -describe("buildRunActivities", () => { - it("renders execution attempts as spans and graph mutations as sequence markers", () => { - const activities = buildRunActivities({ - runState: makeRunStateWithExecution({ - taskId: "task-a", - startedAt: "2026-04-26T10:00:00.000Z", - completedAt: "2026-04-26T10:00:05.000Z", - }), - events: [], - mutations: [ - makeGraphMutation({ - sequence: 12, - target_id: "task-a", - mutation_type: "node.status_changed", - created_at: "2026-04-26T10:00:01.000Z", - }), - ], - currentSequence: 12, - }); - - expect(activities).toEqual( - expect.arrayContaining([ - expect.objectContaining({ kind: "execution", taskId: "task-a", isInstant: false }), - expect.objectContaining({ kind: "graph", taskId: "task-a", sequence: 12, isInstant: true }), - ]), - ); - }); -}); -``` - -- [ ] **Step 2: Write tests for stack packing** - -```typescript -import { describe, expect, it } from "vitest"; -import { stackActivities } from "./stackLayout"; - -describe("stackActivities", () => { - it("puts overlapping spans on separate rows and reuses rows after overlap ends", () => { - const layout = stackActivities([ - activity("a", "2026-04-26T10:00:00.000Z", "2026-04-26T10:00:10.000Z"), - activity("b", "2026-04-26T10:00:05.000Z", "2026-04-26T10:00:12.000Z"), - activity("c", "2026-04-26T10:00:12.000Z", "2026-04-26T10:00:15.000Z"), - ]); - - expect(layout.rowCount).toBe(2); - expect(layout.maxConcurrency).toBe(2); - expect(layout.items.find((item) => item.activity.id === "c")?.row).toBe(0); - }); -}); -``` - -- [ ] **Step 3: Implement derivation and packing** - -Implement the interfaces and functions from [`01-contracts-and-state.md`](01-contracts-and-state.md). Keep the implementation pure and free of React. - -- [ ] **Step 4: Run unit tests** - -Run: `pnpm --dir ergon-dashboard test src/features/activity` - -Expected: activity tests pass; no browser required. - ---- - -## 4. Task 2: Activity stack UI - -**Files:** - -- Create: `ergon-dashboard/src/features/activity/components/ActivityStackTimeline.tsx` -- Create: `ergon-dashboard/src/features/activity/components/ActivityBar.tsx` -- Modify: `ergon-dashboard/src/lib/statusTokens.ts` only if existing colors cannot cover activity kinds. - -- [ ] **Step 1: Add render contract** - -`ActivityStackTimeline` must expose: - -- `data-testid="activity-stack-region"` on the dock root. -- `data-testid="activity-stack-row"` per rendered row. -- `data-testid="activity-bar-{activity.id}"` per activity. -- `data-kind` and `data-task-id` on activity bars. -- `data-testid="activity-current-sequence"` for the visible selected sequence. - -- [ ] **Step 2: Implement timeline controls** - -Controls required in first pass: - -- Step back/forward by available graph mutation sequence. -- Play/pause using mutation timestamps, preserving current `MutationTimeline` min/max delay behavior. -- Drag/scrub range input using sequence numbers. -- Kind legend showing counts. - -- [ ] **Step 3: Implement click behavior** - -Activity click behavior: - -```typescript -function handleActivityClick(activity: RunActivity) { - setSelectedActivityId(activity.id); - if (activity.sequence !== null) setCurrentSequence(activity.sequence); - if (activity.taskId) setSelectedTaskId(activity.taskId); -} -``` - -- [ ] **Step 4: Add empty and partial-data states** - -Empty states: - -- No run state: "Run state is still loading." -- Run has no activities: "No activity has been recorded for this run yet." -- Timeline mode has no mutations: show activities from snapshot timestamps but disable sequence scrub. - ---- - -## 5. Task 3: Wire into `RunWorkspacePage` - -**Files:** - -- Modify: `ergon-dashboard/src/components/run/RunWorkspacePage.tsx` -- Modify: `ergon-dashboard/src/features/graph/components/MutationTimeline.tsx` only if extracting reusable controls. - -- [ ] **Step 1: Build activities from display state** - -Add: - -```typescript -const activities = useMemo( - () => - buildRunActivities({ - runState: displayState, - events, - mutations, - currentSequence: timelineMode === "timeline" ? currentSequence : null, - }), - [displayState, events, mutations, timelineMode, currentSequence], -); -``` - -- [ ] **Step 2: Replace timeline region** - -Replace the old `MutationTimeline` bottom panel with: - -```tsx -<section data-testid="timeline-region" className="border-t border-slate-200 bg-white"> - <ActivityStackTimeline - activities={activities} - mutations={mutations} - currentSequence={currentSequence} - selectedTaskId={selectedTaskId} - selectedActivityId={selectedActivityId} - isPlaying={isPlaying} - speed={playbackSpeed} - onSequenceChange={setCurrentSequence} - onTogglePlay={() => setIsPlaying((prev) => !prev)} - onSpeedChange={setPlaybackSpeed} - onActivityClick={handleActivityClick} - /> -</section> -``` - -- [ ] **Step 3: Preserve event stream** - -Keep `UnifiedEventStream` as a collapsible secondary inspector, not the primary bottom timeline. - -- [ ] **Step 4: Run frontend check** - -Run: `pnpm --dir ergon-dashboard run check` - -Expected: TypeScript and lint pass. - ---- - -## 6. Task 4: Time-aware workspace - -**Files:** - -- Modify: `ergon-dashboard/src/components/workspace/TaskWorkspace.tsx` -- Test: add or extend component/unit tests near existing workspace tests if present. - -- [ ] **Step 1: Add selected time prop** - -`RunWorkspacePage` computes: - -```typescript -const selectedTimelineTime = useMemo(() => { - if (timelineMode !== "timeline") return null; - return mutations.find((mutation) => mutation.sequence === currentSequence)?.created_at ?? null; -}, [timelineMode, mutations, currentSequence]); -``` - -- [ ] **Step 2: Filter visible task evidence** - -Inside `TaskWorkspace`, apply filtering only when `selectedTimelineTime` is non-null. Use ISO string comparison after converting both sides to milliseconds with `Date.parse`. - -- [ ] **Step 3: Show time badge** - -Add a small badge in the workspace header: - -`Viewing evidence available at seq {currentSequence}` - -Only render in timeline mode. - ---- - -## 7. Task 5: Graph highlighting and readability - -**Files:** - -- Modify: `ergon-dashboard/src/components/dag/DAGCanvas.tsx` -- Modify: `ergon-dashboard/src/components/dag/TaskNode.tsx` -- Modify: `ergon-dashboard/src/features/graph/components/ContainerNode.tsx` -- Modify: `ergon-dashboard/src/features/graph/components/LeafNode.tsx` -- Modify: `ergon-dashboard/src/features/graph/layout/hierarchicalLayout.ts` only for collision/readability fixes. - -- [ ] **Step 1: Add highlight data** - -Pass node data flags: - -```typescript -isSelected: task.id === selectedTaskId, -isHighlighted: highlightedTaskIds.has(task.id), -``` - -- [ ] **Step 2: Keep whole graph at T** - -Do not filter nodes by selected activity/task. Highlight related nodes while preserving full topology. - -- [ ] **Step 3: Improve fit and spacing only where measured** - -If overlap persists in the 9-leaf smoke graph, tune `MIN_CONTAINER_WIDTH`, `CONTAINER_PADDING`, and dagre separation constants in `layoutTypes.ts` / `hierarchicalLayout.ts`. Do not introduce a second graph layout engine in this PR. - ---- - -## 8. Task 6: Remove or demote old mutation strip - -**Files:** - -- Modify or delete: `ergon-dashboard/src/features/graph/components/MutationTimeline.tsx` - -Decision after Task 2: - -- If controls are reused, rename to `SequenceControls.tsx`. -- If no code is reused, delete the component and update imports. - -Acceptance: the only bottom timeline users see is activity-stack based. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/03-tests-and-e2e.md b/docs/superpowers/plans/mas-run-visual-debugger/03-tests-and-e2e.md deleted file mode 100644 index 1afc4ed57..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/03-tests-and-e2e.md +++ /dev/null @@ -1,275 +0,0 @@ -# 03 — Tests and E2E - -**Status:** draft. -**Scope:** frontend unit tests, dashboard fixture tests, Playwright smoke assertions, screenshot capture points, and optional backend harness DTO additions. - -Cross-refs: test-refactor north star in `docs/superpowers/plans/test-refactor/03-dashboard-and-playwright.md`, implementation tasks in [`02-frontend-implementation.md`](02-frontend-implementation.md). - ---- - -## 1. Test strategy - -Use five layers: - -- **Pure unit tests:** prove activity derivation and stack packing without React or browser layout. -- **Golden fixture semantic tests:** pump realistic serialized MAS run data through replay, activity derivation, stack layout, and graph layout. -- **Coarse browser geometry checks:** assert catastrophic overlaps do not happen without pinning exact pixels. -- **Dashboard fixture e2e:** seed a deterministic concurrent run through dashboard harness routes and assert the visual debugger contract quickly. -- **Canonical smoke e2e:** run against real backend state and capture screenshots for graph + activity stack review. - -Do not assert pixel-perfect bar positions. Assert stable structure, counts, selected state, and task/sequence coordination. -Use local PNG dumps for human visual review while building; do not make PNG diffs a CI gate in the first PR. - ---- - -## 2. Unit tests - -### Activity derivation tests - -File: `ergon-dashboard/src/features/activity/buildRunActivities.test.ts` - -Required cases: - -- `ExecutionAttemptState.startedAt/completedAt` -> execution span. -- open running execution -> execution span ending at selected timeline time. -- resource event -> artifact marker. -- thread message -> message marker. -- task evaluation -> evaluation marker. -- context event with start/end -> context span. -- graph mutation -> graph marker with sequence. -- no agent lane key is emitted. - -### Stack layout tests - -File: `ergon-dashboard/src/features/activity/stackLayout.test.ts` - -Required cases: - -- non-overlap reuses row. -- overlap allocates rows. -- three-way overlap reports max concurrency. -- instant marker has minimum render width. -- deterministic order independent of input order. - -### Golden fixture semantic tests - -Files: - -- `ergon-dashboard/tests/fixtures/mas-runs/concurrent-mas-run.json` -- `ergon-dashboard/src/features/activity/goldenFixture.test.ts` -- `ergon-dashboard/src/features/graph/layout/goldenLayout.test.ts` -- `ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.test.ts` - -Required cases: - -- replaying fixture mutations to checkpoint sequence `T` yields the whole expected graph at `T`. -- graph layout for the fixture has no overlapping node/container boxes using coarse rectangle checks. -- activity stack reports expected max concurrency for overlapping executions. -- activity rows are not grouped by agent or worker identity. -- task evidence filtering hides resources/messages/evaluations created after selected time. - -Full details live in [`06-fast-feedback-and-visual-review.md`](06-fast-feedback-and-visual-review.md). - ---- - -## 3. Dashboard fixture update - -Modify `ergon-dashboard/tests/helpers/dashboardFixtures.ts`. - -Add a fixture run with: - -- root task plus at least 5 child tasks. -- two executions overlapping between `12:00:10` and `12:00:20`. -- one sandbox command inside an execution span. -- one thread message marker. -- one resource marker. -- one evaluation marker attached to a non-root task. -- graph mutations with sequences spanning node add/status events. - -Suggested helper shape: - -```typescript -export function concurrentMasRunState(): SerializedWorkflowRunState { - return serializedRunState({ - scenario: "concurrent-mas-debugger", - }); -} -``` - -If `serializedRunState` is not currently parameterized, extract current fixture setup into small helpers first. Keep old fixture behavior unchanged for existing specs. - ---- - -## 4. Playwright dashboard fixture spec - -Create `ergon-dashboard/tests/e2e/activity-stack.spec.ts`. - -Core assertions: - -```typescript -test("run visual debugger shows recursive graph, activity stack, and time-aware workspace", async ({ page }) => { - const client = new DashboardHarnessClient(page); - const { cohortId, runId } = await client.seedConcurrentMasRun(); - - await page.goto(`/cohorts/${cohortId}/runs/${runId}`); - - await expect(page.getByTestId("run-header")).toBeVisible(); - await expect(page.getByTestId("graph-canvas")).toBeVisible(); - await expect(page.getByTestId("activity-stack-region")).toBeVisible(); - await expect(page.getByTestId("activity-stack-row")).toHaveCountGreaterThan(1); - expect( - await overlappingPairsFor(page, '[data-testid^="graph-node-"]'), - ).toEqual([]); - - const firstExecution = page.locator('[data-testid^="activity-bar-"][data-kind="execution"]').first(); - await expect(firstExecution).toBeVisible(); - await firstExecution.click(); - - await expect(page.getByTestId("workspace-region")).toBeVisible(); - await expect(page.getByTestId("workspace-header")).toBeVisible(); - - await page.getByTestId("activity-step-forward").click(); - await expect(page.getByTestId("activity-current-sequence")).toContainText(/seq/i); -}); -``` - -If Playwright's matcher set lacks `toHaveCountGreaterThan`, replace with: - -```typescript -expect(await page.getByTestId("activity-stack-row").count()).toBeGreaterThan(1); -``` - -Add coarse geometry helpers in the spec or shared helper: - -```typescript -async function overlappingPairsFor(page: Page, selector: string): Promise<[number, number][]> { - const boxes = await page.locator(selector).evaluateAll((elements) => - elements.map((element) => { - const rect = element.getBoundingClientRect(); - return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; - }), - ); - return overlappingPairs(boxes, { tolerancePx: 2 }); -} -``` - -The overlap assertion is intentionally coarse. It catches the broken layout class we care about without becoming a pixel-perfect visual test. - -### Local-only PNG dumps - -The fixture spec should dump review screenshots only when explicitly requested: - -```bash -VISUAL_DEBUGGER_SCREENSHOTS=1 pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts --project=chromium -``` - -Output: - -```text -ergon-dashboard/tmp/visual-debugger/run-full.png -ergon-dashboard/tmp/visual-debugger/graph-canvas.png -ergon-dashboard/tmp/visual-debugger/activity-stack.png -ergon-dashboard/tmp/visual-debugger/workspace-open.png -``` - -These PNGs are for local human review while building. They should not run in CI and should not be committed. - ---- - -## 5. Canonical smoke e2e changes - -Modify `ergon-dashboard/tests/e2e/_shared/smoke.ts`. - -Add to `assertRunWorkspace` after `graph-canvas` assertion: - -```typescript -await expect(page.getByTestId("activity-stack-region")).toBeVisible(); - -const activityBars = page.locator('[data-testid^="activity-bar-"]'); -await expect(activityBars.first()).toBeVisible(); - -if (state.mutation_count > 0) { - await page.getByTestId("mode-timeline").click(); - await expect(page.getByTestId("timeline-region")).toBeVisible(); - await expect(page.getByTestId("activity-current-sequence")).toContainText(/seq/i); -} -``` - -Screenshot additions: - -- `<env>/<run_id>-visual-debugger-full.png` — full run page. -- `<env>/<run_id>-activity-stack.png` — bottom dock if Playwright can screenshot locator reliably. -- Keep existing happy/sad screenshots until the new ones prove stable. - ---- - -## 6. Optional backend harness DTO additions - -Only add these after frontend derivation is implemented and the e2e test needs backend truth for concurrency: - -Modify backend `/api/test/read/run/{run_id}/state` DTO to include: - -```json -{ - "activity_event_count": 37, - "activity_span_count": 12, - "max_concurrency": 4 -} -``` - -Modify `ergon-dashboard/tests/helpers/backendHarnessClient.ts`: - -```typescript -export interface BackendRunState { - activity_event_count?: number; - activity_span_count?: number; - max_concurrency?: number; -} -``` - -Rules: - -- These fields are optional in TypeScript while the backend branch catches up. -- Do not block the visual debugger UI on these fields. -- If added, Playwright may assert `max_concurrency >= 2` for the smoke run. - ---- - -## 7. Accessibility and stable selectors - -Required test IDs: - -- `activity-stack-region` -- `activity-stack-row` -- `activity-bar-{activityId}` -- `activity-current-sequence` -- `activity-step-back` -- `activity-step-forward` -- `activity-play-toggle` -- `activity-speed-control` -- `graph-canvas` -- `graph-node-{taskId}` -- `graph-container-{taskId}` -- `workspace-region` -- `workspace-header` - -Required ARIA labels: - -- Activity bar button: `Open activity {label}`. -- Sequence scrubber: `Run timeline sequence`. -- Play/pause: `Play timeline` / `Pause timeline`. - ---- - -## 8. Acceptance gate - -- [ ] Pure activity tests pass. -- [ ] Golden fixture semantic/layout tests pass. -- [ ] Dashboard fixture e2e passes locally. -- [ ] Fixture e2e coarse graph overlap check passes. -- [ ] Local PNG dump works when `VISUAL_DEBUGGER_SCREENSHOTS=1` is set. -- [ ] Canonical smoke e2e still passes locally. -- [ ] Screenshots show more than one activity row for concurrent samples. -- [ ] Clicking an activity with a task opens the workspace for that task. -- [ ] Scrubbing sequence updates graph status/topology via existing replay. -- [ ] No assertion relies on agent lane count. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/04-phases.md b/docs/superpowers/plans/mas-run-visual-debugger/04-phases.md deleted file mode 100644 index fdcb4ba0e..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/04-phases.md +++ /dev/null @@ -1,230 +0,0 @@ -# 04 — Phases, Deliverables, Acceptance Gates - -**Status:** draft. -**Scope:** delivery order for the frontend visual debugger branch. One PR is preferred if phases stay small; split after Phase C if review size gets uncomfortable. - -Cross-refs: program in [`00-program.md`](00-program.md), frontend tasks in [`02-frontend-implementation.md`](02-frontend-implementation.md), test contract in [`03-tests-and-e2e.md`](03-tests-and-e2e.md). - ---- - -## Delivery shape - -Each phase should be a clean commit with: - -- Scope: files touched. -- Deliverables: what now works. -- Acceptance gate: exact tests/commands before moving on. - -Do not start the next phase while the current phase is red. - ---- - -## Phase A — Plan and branch scaffold - -**Scope** - -- Create branch `feature/mas-run-visual-debugger-plan`. -- Add this plan folder. -- Keep mockups unmodified except as design reference. - -**Deliverables** - -- `docs/superpowers/plans/mas-run-visual-debugger/` exists. -- Branch records the implementation approach before app edits. - -**Acceptance gate** - -- `git branch --show-current` prints `feature/mas-run-visual-debugger-plan`. -- Plan docs are readable and self-contained. - ---- - -## Phase B — Pure activity model - -**Scope** - -- `ergon-dashboard/tests/fixtures/mas-runs/concurrent-mas-run.json` -- `ergon-dashboard/src/features/activity/types.ts` -- `ergon-dashboard/src/features/activity/buildRunActivities.ts` -- `ergon-dashboard/src/features/activity/stackLayout.ts` -- Unit tests for both modules. -- Golden fixture semantic tests from [`06-fast-feedback-and-visual-review.md`](06-fast-feedback-and-visual-review.md). - -**Deliverables** - -- Activity derivation from `WorkflowRunState`, `RunEvent[]`, and `GraphMutationDto[]`. -- Deterministic overlap stack layout. -- Realistic MAS fixture replay proves concurrency is derived from overlap, not agent lanes. -- No React component changes yet. - -**Acceptance gate** - -- `pnpm --dir ergon-dashboard test src/features/activity` -- Golden fixture tests pass locally. -- `pnpm --dir ergon-dashboard run check` - -**Not in this phase** - -- No UI replacement. -- No backend DTO changes. - ---- - -## Phase C — Bottom activity stack UI - -**Scope** - -- `ActivityStackTimeline.tsx` -- `ActivityBar.tsx` -- Wire into `RunWorkspacePage.tsx` behind existing timeline/live mode controls. -- Keep old `MutationTimeline` available until this phase is green. - -**Deliverables** - -- Bottom dock renders activity rows and bars. -- Sequence controls still work. -- Activity click selects task/sequence. -- Empty states are clear. - -**Acceptance gate** - -- `pnpm --dir ergon-dashboard run check` -- Local dashboard fixture page renders without runtime errors. -- Manual browser check against seeded run: graph visible, activity stack visible, workspace opens from activity. - -**Not in this phase** - -- No graph layout tuning unless the new dock breaks existing graph rendering. -- No smoke e2e assertions yet. - ---- - -## Phase D — Time-aware workspace and graph highlights - -**Scope** - -- `TaskWorkspace.tsx` filters task evidence by selected timeline time. -- `DAGCanvas.tsx`/node components accept selected and highlighted task IDs. -- Preserve whole graph at selected `T`. - -**Deliverables** - -- Selecting an activity highlights graph task and opens workspace. -- Selecting a graph node highlights related activity bars. -- Workspace indicates timeline time/sequence. -- Evidence that did not exist at selected time is hidden in timeline mode. - -**Acceptance gate** - -- `pnpm --dir ergon-dashboard run check` -- Component/unit tests for time filtering pass. -- Manual check: scrub backward before a resource appears; workspace no longer shows that resource. - -**Not in this phase** - -- No persisted UI preferences. -- No virtualization. - ---- - -## Phase E — Dashboard fixture e2e - -**Scope** - -- Add concurrent MAS dashboard fixture in `tests/helpers/dashboardFixtures.ts`. -- Add `tests/e2e/activity-stack.spec.ts`. -- Add selectors/ARIA labels required by [`03-tests-and-e2e.md §7`](03-tests-and-e2e.md). -- Add coarse browser geometry checks for graph node overlap. - -**Deliverables** - -- Fast deterministic Playwright test proves the visual debugger contract without real backend execution. -- Screenshot artifact captures the accepted layout shape when `VISUAL_DEBUGGER_SCREENSHOTS=1` is set locally. -- Browser geometry check catches catastrophic overlapping graph boxes without pixel-perfect assertions. -- Local-only PNG dump path works behind `VISUAL_DEBUGGER_SCREENSHOTS=1`. - -**Acceptance gate** - -- `pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts` -- Coarse graph overlap check passes. -- `VISUAL_DEBUGGER_SCREENSHOTS=1 pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts --project=chromium` writes PNGs under `ergon-dashboard/tmp/visual-debugger/`. -- Local PNG review command shows at least two activity rows and full graph canvas. - -**Not in this phase** - -- No backend harness DTO additions unless the fixture spec cannot cover a critical contract. - ---- - -## Phase F — Canonical smoke e2e hardening - -**Scope** - -- Update `tests/e2e/_shared/smoke.ts`. -- Extend screenshot capture points. -- Optionally extend `BackendRunState` and backend harness DTO with `activity_event_count`, `activity_span_count`, `max_concurrency`. - -**Deliverables** - -- Real smoke run opens the new visual debugger. -- Playwright proves graph, activity stack, sequence controls, and workspace are usable. -- Screenshots are useful for PR review. -- No CI visual-diff gate is introduced. - -**Acceptance gate** - -- Local smoke Playwright spec green for at least one benchmark. -- Full e2e matrix remains green before merge. -- Smoke screenshot artifacts are generated as review aids only. -- If harness DTO fields are added, backend unit/integration harness tests pass. - -**Not in this phase** - -- No production DTO expansion unless a user-facing timestamp gap is proven. - ---- - -## Phase G — Cleanup and docs - -**Scope** - -- Delete or rename obsolete `MutationTimeline.tsx`. -- Update dashboard architecture docs if they describe the old event stream/timeline split. -- Add a short note in PR description linking to the accepted mockup and this plan folder. - -**Deliverables** - -- No dead imports/components. -- Standing docs match the shipped dashboard behavior. - -**Acceptance gate** - -- `pnpm --dir ergon-dashboard run check` -- `rg -n "MutationTimeline" ergon-dashboard/src` returns either no matches or only the intentional renamed/reused sequence-control component. -- Final Playwright screenshots attached to PR. -- Final implementation review presents local PNGs for all new UI panels: full debugger page, graph canvas, activity stack bottom dock, and workspace drawer open on a selected task. - ---- - -## Phase size estimates - -| Phase | Scope | Est. diff size | -|---|---|---| -| A | Plan folder | ~500 lines docs | -| B | Activity pure model + golden fixture tests | ~650 LoC | -| C | Activity UI + RunWorkspace wiring + local PNG dump | ~750 LoC | -| D | Workspace filtering + graph highlights | ~300 LoC | -| E | Fixture e2e + browser geometry checks | ~350 LoC | -| F | Smoke hardening + optional harness DTO | ~200-500 LoC | -| G | Cleanup/docs | ~100 LoC | - ---- - -## Failure modes - -- **Activity bars look like lanes:** remove any row grouping by actor/agent. Rows are only collision rows. -- **Graph disappears while scrubbing:** inspect `replayToSequence` input state and current sequence; do not filter by selected task. -- **Workspace shows future evidence:** compare evidence timestamps to selected mutation `created_at`. -- **PNG review reveals cramped layout:** tune spacing/styling, then keep semantic and geometry tests green. Do not add pixel-perfect screenshot diffs in the first PR. -- **E2E flakes on exact counts:** assert minimum visibility and backend DTO truth, not pixel geometry. -- **Backend DTO temptation:** use the decision tree in `01-contracts-and-state.md`; most first-pass needs are frontend-derived. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/05-implementation-shape.md b/docs/superpowers/plans/mas-run-visual-debugger/05-implementation-shape.md deleted file mode 100644 index 5fd6a4a30..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/05-implementation-shape.md +++ /dev/null @@ -1,302 +0,0 @@ -# 05 — Implementation Shape, File Ownership, and Refactor Boundaries - -**Status:** draft for review. -**Scope:** the reviewer-facing "how" plan: what domains the frontend should have after this work, which files are added, which files are refactored, which files are deleted or deliberately left alone, and how tests are laid out. - -Cross-refs: product/DTO stance in [`00-program.md`](00-program.md), activity contracts in [`01-contracts-and-state.md`](01-contracts-and-state.md), phase gates in [`04-phases.md`](04-phases.md). - ---- - -## 1. Target domain map - -After the implementation, the run dashboard should have these frontend domains: - -| Domain | Responsibility | Owns | Must not own | -|---|---|---|---| -| `features/activity` | Turn run state into time-based activity, pack overlaps into stack rows, render bottom dock. | Activity types, derivation, overlap layout, activity timeline UI. | Graph replay, workspace evidence rendering, backend fetching. | -| `features/graph` | Reconstruct and render recursive task topology at selected sequence/time. | Graph mutation contracts, replay reducer, React Flow layout, node components. | Activity stacking, agent lanes, workspace filtering. | -| `components/workspace` | Show task-scoped evidence for the selected task. | Resources, executions, sandbox commands, messages, context events, evaluations for one task. | Timeline packing, graph topology. | -| `components/run` | Page orchestration and cross-panel selection state. | Live/timeline mode, selected task, selected activity, selected sequence, panel composition. | Pure derivation algorithms. | -| `lib/runEvents` | Normalize existing state into a chronological event stream. | Event union, event labels/colors, stream rows. | Visual timeline row allocation. | -| `tests/e2e` + `tests/helpers` | Prove the visual debugger contract with fixture and smoke runs. | Stable selectors, seeded concurrent fixture, screenshot capture, harness assertions. | Pixel-perfect visual diffs. | - -The most important boundary: **activity stack rows are not a domain concept**. They are a layout result. The domain concept is a `RunActivity` with task/time/kind metadata. - ---- - -## 2. Intended folder layout - -Target new files: - -```text -ergon-dashboard/src/features/activity/ - types.ts - buildRunActivities.ts - stackLayout.ts - goldenFixture.test.ts - buildRunActivities.test.ts - stackLayout.test.ts - components/ - ActivityStackTimeline.tsx - ActivityBar.tsx - ActivityKindLegend.tsx - SequenceControls.tsx -``` - -Target modified existing files: - -```text -ergon-dashboard/src/components/run/ - RunWorkspacePage.tsx - -ergon-dashboard/src/components/dag/ - DAGCanvas.tsx - TaskNode.tsx - -ergon-dashboard/src/features/graph/components/ - ContainerNode.tsx - LeafNode.tsx - MutationTimeline.tsx - -ergon-dashboard/src/features/graph/layout/ - hierarchicalLayout.ts - layoutTypes.ts - -ergon-dashboard/src/components/workspace/ - TaskWorkspace.tsx - -ergon-dashboard/src/lib/ - runEvents.ts - statusTokens.ts -``` - -Target test files: - -```text -ergon-dashboard/tests/helpers/ - dashboardFixtures.ts - testHarnessClient.ts - backendHarnessClient.ts - -ergon-dashboard/tests/fixtures/mas-runs/ - concurrent-mas-run.json - nested-delegation-run.json - README.md - -ergon-dashboard/tests/e2e/ - activity-stack.spec.ts - _shared/smoke.ts -``` - -Optional backend files if the e2e harness needs additive DTO truth: - -```text -ergon_core/ergon_core/core/api/test_harness.py -tests/unit/test_test_harness.py -tests/integration/smokes/test_smoke_harness.py -``` - ---- - -## 3. Add, refactor, delete, leave alone - -### Add - -| File | Why it exists | -|---|---| -| `features/activity/types.ts` | Shared activity vocabulary: `RunActivity`, `ActivityKind`, `ActivityStackLayout`, `ActivityStackItem`. | -| `features/activity/buildRunActivities.ts` | Pure state-to-activity derivation. Lets tests verify semantics without React. | -| `features/activity/stackLayout.ts` | Pure overlap packing. Keeps "concurrency stack" independent from rendering. | -| `features/activity/components/ActivityStackTimeline.tsx` | Bottom dock shell: time ruler, rows, controls, legend, selection. | -| `features/activity/components/ActivityBar.tsx` | Single activity marker/span renderer. Keeps bar styling out of the dock shell. | -| `features/activity/components/ActivityKindLegend.tsx` | Small count/filter legend if `ActivityStackTimeline.tsx` gets too large. | -| `features/activity/components/SequenceControls.tsx` | Reusable play/step/speed controls extracted from old mutation timeline behavior. | -| `features/activity/buildRunActivities.test.ts` | Unit coverage for event/span semantics. | -| `features/activity/stackLayout.test.ts` | Unit coverage for overlap packing and max concurrency. | -| `features/activity/goldenFixture.test.ts` | Pumps realistic MAS fixture data through replay/activity/stack derivation. | -| `tests/fixtures/mas-runs/concurrent-mas-run.json` | Stable local fixture for semantic layout and browser visual review. | -| `tests/fixtures/mas-runs/nested-delegation-run.json` | Optional second fixture for deeper recursive nesting once the first path is green. | -| `tests/e2e/activity-stack.spec.ts` | Fast fixture-driven UI contract for the new debugger. | - -### Refactor - -| File | Refactor | -|---|---| -| `RunWorkspacePage.tsx` | Becomes the cross-panel coordinator. It should compute display state, activities, selected time, selected task/activity, and pass props down. It should not implement activity derivation inline. | -| `DAGCanvas.tsx` | Adds highlight props and preserves graph-level controls. No activity logic here. | -| `TaskNode.tsx`, `ContainerNode.tsx`, `LeafNode.tsx` | Add selected/highlight styling and stable test IDs. Avoid redesigning node semantics. | -| `TaskWorkspace.tsx` | Adds time-aware filtering by selected sequence time. Keep the existing evidence sections. | -| `MutationTimeline.tsx` | Either deleted after replacement, or split so reusable sequence controls move to `features/activity/components/SequenceControls.tsx`. | -| `hierarchicalLayout.ts`, `layoutTypes.ts` | Only tune spacing if smoke screenshots still show overlap. Keep dagre and current recursive container model. | -| `runEvents.ts` | Remains event normalization. It may gain helper exports, but it should not pack visual rows. | -| `dashboardFixtures.ts` | Adds a deterministic concurrent MAS fixture, preserving existing fixture exports. | -| `_shared/smoke.ts` | Adds activity stack assertions and screenshots without making visual pixel claims. | -| `activity-stack.spec.ts` | Adds coarse DOM bounding-box overlap checks and optional local screenshot dumping behind `VISUAL_DEBUGGER_SCREENSHOTS=1`. | - -### Delete - -Delete only after the activity stack is wired and tested: - -| File | Delete condition | -|---|---| -| `features/graph/components/MutationTimeline.tsx` | Delete if no code is reused by `SequenceControls.tsx`. | - -No other deletions are planned for the first visual debugger PR. - -### Leave alone - -| Area | Reason | -|---|---| -| Backend execution/control-flow services | The UI problem is representational; backend task orchestration does not need to change. | -| Graph mutation persistence model | Existing sequence/time mutation contract is the right replay primitive. | -| React Flow dependency | The current rendering stack already supports recursive graph rendering. | -| Cohort pages | This work is scoped to run detail pages and smoke screenshots. | -| Production REST schemas | Avoid production DTO expansion unless a real user-facing timestamp gap is proven. | - ---- - -## 4. Data flow after refactor - -```text -REST snapshot / socket updates - | - v -useRunState(runId) --------------------+ - | | - v | -WorkflowRunState | - | | - +--> replayToSequence() ----> displayState at T ----> DAGCanvas - | | - +--> buildRunEvents() ---------+ - | | -/api/runs/{runId}/mutations -----------+ - | - v -buildRunActivities(displayState, events, mutations, currentSequence) - | - v -stackActivities(activities) - | - v -ActivityStackTimeline - | - +--> select task/activity/sequence - | - v -RunWorkspacePage state - | - +--> DAGCanvas highlight/selection - +--> TaskWorkspace selected task + selected time -``` - -Selection rules: - -- Graph node click sets `selectedTaskId`. -- Activity click sets `selectedActivityId`, sets `selectedTaskId` when present, and jumps to `activity.sequence` when present. -- Sequence scrub changes `currentSequence`; it does not clear task selection unless the selected task does not exist at that sequence. -- Workspace reads selected task from `displayState`, not live state, when timeline mode is active. - ---- - -## 5. Test layout - -### Pure unit tests - -```text -ergon-dashboard/src/features/activity/buildRunActivities.test.ts -ergon-dashboard/src/features/activity/stackLayout.test.ts -``` - -These tests should use small inline fixture builders. Do not import Playwright, React, or browser APIs. - -### Component-level tests if local harness exists - -If the dashboard already has React component tests, add: - -```text -ergon-dashboard/src/features/activity/components/ActivityStackTimeline.test.tsx -``` - -This test should assert: - -- rows render from layout items. -- clicking a bar calls `onActivityClick`. -- controls call `onSequenceChange`. - -If the project does not have component-test infrastructure, skip this and rely on pure unit + Playwright. - -### Fixture e2e - -```text -ergon-dashboard/tests/e2e/activity-stack.spec.ts -``` - -This is the fast UI contract: - -- seed concurrent MAS fixture. -- open run page. -- assert graph, stack, and workspace regions. -- assert more than one stack row. -- assert no catastrophic graph-node bounding-box overlaps. -- click activity -> workspace opens. -- scrub sequence -> current sequence indicator changes. -- dump local PNGs only when `VISUAL_DEBUGGER_SCREENSHOTS=1`. - -### Golden fixture semantic tests - -```text -ergon-dashboard/src/features/activity/goldenFixture.test.ts -ergon-dashboard/src/features/graph/layout/goldenLayout.test.ts -ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.test.ts -``` - -These are the fast feedback loop for the exact failure mode we want to avoid: - -- replay fixture to selected sequence `T`; -- assert whole graph expected at `T`; -- assert no overlapping graph boxes in pure layout output; -- assert activity stack max concurrency; -- assert row assignment does not depend on agent/worker identity; -- assert future task evidence is hidden in timeline mode. - -### Local PNG review - -```text -ergon-dashboard/tmp/visual-debugger/ - run-full.png - graph-canvas.png - activity-stack.png - workspace-open.png -``` - -Generated only by local command: - -```bash -VISUAL_DEBUGGER_SCREENSHOTS=1 pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts --project=chromium -``` - -These files are for human review while building. They should not be committed and should not be required in CI. - -### Smoke e2e - -```text -ergon-dashboard/tests/e2e/_shared/smoke.ts -``` - -This is the real integration contract: - -- backend harness proves graph/resources/evaluations are real. -- dashboard proves visual debugger renders real run state. -- screenshots capture full page and activity stack. - ---- - -## 6. Review questions before implementation - -1. Should `features/activity` own `SequenceControls.tsx`, or should sequence controls live under `features/graph` because sequences come from graph mutations? -2. Should `ActivityStackTimeline` support filtering by kind in the first PR, or only render the legend counts? -3. Should `TaskWorkspace` hide future evidence in timeline mode, or show it disabled with "after selected time" labels? -4. Should the old event stream stay visible by default, or be collapsed once the activity stack exists? -5. Should fixture e2e be required before smoke e2e changes, or can smoke drive the first UI contract directly? -6. Should `nested-delegation-run.json` ship in the first PR, or should the first PR use only `concurrent-mas-run.json` and add the deeper fixture after the UI stabilizes? diff --git a/docs/superpowers/plans/mas-run-visual-debugger/06-fast-feedback-and-visual-review.md b/docs/superpowers/plans/mas-run-visual-debugger/06-fast-feedback-and-visual-review.md deleted file mode 100644 index d7a16b529..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/06-fast-feedback-and-visual-review.md +++ /dev/null @@ -1,309 +0,0 @@ -# 06 — Fast Feedback, TDD, and Local Visual Review - -**Status:** draft. -**Scope:** the feedback loop that prevents another unreadable MAS layout from landing: test-first semantic layout checks, coarse browser geometry assertions, and local-only PNG dumps for human visual review. - -Cross-refs: test contract in [`03-tests-and-e2e.md`](03-tests-and-e2e.md), implementation shape in [`05-implementation-shape.md`](05-implementation-shape.md). - ---- - -## 1. Why this exists - -The prior UI failure was not mainly a data-fetching failure. It was a semantics/layout failure: - -- recursive task containers were hard to read; -- graph state at selected time `T` was not clearly represented; -- timeline lanes implied stable agents even though agents/workers can join and leave; -- overlapping work was not represented as concurrency; -- visual density problems were not caught by tests. - -This plan adds a fast feedback loop before full e2e smoke: - -1. Pure TDD tests for semantics and layout algorithms. -2. Coarse browser geometry checks for catastrophic overlap. -3. Local-only PNG dumps that humans inspect while building the UI. - -PNG review is required for development/review discipline, but it is **not** a CI gate in the first PR. - ---- - -## 2. Test-first policy for this feature - -Use TDD for the core behavior: - -- write the failing semantic/layout test; -- run it and confirm it fails for the expected reason; -- implement the smallest code to pass; -- keep the test as a regression guard. - -Do this for: - -- activity derivation; -- activity overlap packing; -- graph snapshot at sequence `T`; -- no graph node overlap for the golden fixture; -- activity click -> task/sequence selection; -- workspace time filtering. - -Do not use TDD for throwaway visual CSS tweaking. For CSS, use local PNG review and coarse browser checks. - ---- - -## 3. Golden fixture data - -Add deterministic fixture data that represents the MAS case we care about. - -Target files: - -```text -ergon-dashboard/tests/fixtures/mas-runs/ - concurrent-mas-run.json - nested-delegation-run.json - README.md -``` - -`concurrent-mas-run.json` should include: - -- full serialized run snapshot; -- graph mutations sorted by sequence; -- expected sequence checkpoints; -- expected graph node IDs/slugs at each checkpoint; -- expected activity concurrency facts. - -Example shape: - -```json -{ - "name": "concurrent-mas-run", - "runState": {}, - "mutations": [], - "checkpoints": [ - { - "sequence": 12, - "expectedTaskSlugs": ["root", "d_root", "d_left", "d_right", "d_join", "l_1"], - "expectedVisibleResourceNames": [], - "expectedMaxConcurrency": 3 - } - ] -} -``` - -Rules: - -- Keep fixture JSON small enough to review. -- Prefer real captured run shape when available, then minimize it. -- Do not include secrets, model outputs, or large artifacts. -- If the fixture comes from a real run/VCR capture, sanitize IDs only if tests do not depend on specific UUID shape. - ---- - -## 4. Pure semantic layout tests - -Create: - -```text -ergon-dashboard/src/features/activity/goldenFixture.test.ts -ergon-dashboard/src/features/graph/layout/goldenLayout.test.ts -ergon-dashboard/src/components/workspace/timeFiltering.test.ts -``` - -### Activity fixture test - -This test pumps fixture data through pure functions: - -```typescript -import fixture from "../../../tests/fixtures/mas-runs/concurrent-mas-run.json"; -import { parseGraphMutationDtoArray } from "@/features/graph/contracts/graphMutations"; -import { replayToSequence } from "@/features/graph/state/graphMutationReducer"; -import { buildRunActivities } from "./buildRunActivities"; -import { stackActivities } from "./stackLayout"; -import { buildRunEvents } from "@/lib/runEvents"; -import { deserializeRunState } from "@/lib/runState"; - -it("derives concurrency from overlapping activity rather than agent lanes", () => { - const liveState = deserializeRunState(fixture.runState); - const mutations = parseGraphMutationDtoArray(fixture.mutations); - const checkpoint = fixture.checkpoints.find((c) => c.sequence === 12)!; - const displayState = replayToSequence(mutations, checkpoint.sequence, emptyRunStateFrom(liveState), new Map()); - const events = buildRunEvents(displayState); - const activities = buildRunActivities({ runState: displayState, events, mutations, currentSequence: checkpoint.sequence }); - const stack = stackActivities(activities); - - expect(stack.maxConcurrency).toBe(checkpoint.expectedMaxConcurrency); - expect(new Set(activities.map((activity) => activity.kind))).toEqual( - expect.arrayContaining(["execution", "graph", "artifact", "evaluation"]), - ); - expect(stack.items.some((item) => item.activity.actor && item.row === Number(item.activity.actor))).toBe(false); -}); -``` - -The exact helper names can change during implementation, but the assertion intent must stay: - -- concurrency comes from overlap; -- activities are not grouped by agent/worker lane; -- graph mutations remain sequence-addressable. - -### Graph layout fixture test - -This test runs the same fixture through replay + layout and asserts no overlapping rendered boxes. - -```typescript -it("lays out the whole recursive graph at sequence T without overlapping node boxes", () => { - const displayState = replayFixtureToSequence("concurrent-mas-run", 12); - const result = computeHierarchicalLayout( - displayState.tasks, - calculateExpandedContainers(displayState.tasks, Infinity), - "", - undefined, - null, - "LR", - new Set(), - ); - - expect(new Set(result.nodes.map((node) => node.id))).toEqual(expectedWholeGraphNodeIdsAtSequence(12)); - expect(findOverlappingNodeBoxes(result.nodes)).toEqual([]); -}); -``` - -`findOverlappingNodeBoxes` should compare coarse rectangles from React Flow node `position`, `width`, and `height`. This is not a pixel-perfect visual diff; it catches catastrophic overlap. - -### Workspace time filtering test - -Extract filtering into a pure helper if `TaskWorkspace.tsx` is otherwise hard to test: - -```text -ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.ts -ergon-dashboard/src/components/workspace/filterTaskEvidenceForTime.test.ts -``` - -Assert: - -- resource created after selected time is hidden; -- execution started before selected time is visible; -- message created after selected time is hidden; -- live mode returns unfiltered evidence. - ---- - -## 5. Browser geometry checks - -Add coarse checks to `ergon-dashboard/tests/e2e/activity-stack.spec.ts`. - -Use DOM bounding boxes for rendered elements: - -```typescript -async function boxesFor(page: Page, selector: string) { - return page.locator(selector).evaluateAll((elements) => - elements.map((element) => { - const rect = element.getBoundingClientRect(); - return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; - }), - ); -} - -function overlappingPairs(boxes: { x: number; y: number; width: number; height: number }[]) { - const pairs: [number, number][] = []; - for (let i = 0; i < boxes.length; i++) { - for (let j = i + 1; j < boxes.length; j++) { - if (boxesOverlap(boxes[i], boxes[j])) pairs.push([i, j]); - } - } - return pairs; -} - -expect(overlappingPairs(await boxesFor(page, '[data-testid^="graph-node-"]'))).toEqual([]); -``` - -Rules: - -- Use coarse overlap checks only. -- Do not assert exact coordinates. -- Ignore tiny overlaps below 2px if React Flow transform/subpixel rendering creates false positives. -- Keep these checks on fixture e2e first; only add to real smoke if stable. - ---- - -## 6. Local-only PNG dump - -Add a developer-only screenshot command/spec path. This is for us while building and reviewing. It does **not** need to run in CI. - -Target output: - -```text -ergon-dashboard/tmp/visual-debugger/ - run-full.png - graph-canvas.png - activity-stack.png - workspace-open.png -``` - -Suggested command: - -```bash -pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts --project=chromium -``` - -The spec should write screenshots when `VISUAL_DEBUGGER_SCREENSHOTS=1`: - -```typescript -const shouldDumpScreenshots = process.env.VISUAL_DEBUGGER_SCREENSHOTS === "1"; - -if (shouldDumpScreenshots) { - await page.screenshot({ - path: "tmp/visual-debugger/run-full.png", - fullPage: true, - }); - await page.getByTestId("graph-canvas").screenshot({ - path: "tmp/visual-debugger/graph-canvas.png", - }); - await page.getByTestId("activity-stack-region").screenshot({ - path: "tmp/visual-debugger/activity-stack.png", - }); - await page.getByTestId("workspace-region").screenshot({ - path: "tmp/visual-debugger/workspace-open.png", - }); -} -``` - -Recommended local command: - -```bash -VISUAL_DEBUGGER_SCREENSHOTS=1 pnpm --dir ergon-dashboard exec playwright test tests/e2e/activity-stack.spec.ts --project=chromium -``` - -Review rules: - -- Inspect PNGs locally during development. -- Look for cramped graph, overlapping containers, unreadable labels, poor activity row density, confusing color hierarchy, and workspace clipping. -- Treat final implementation review as incomplete until the implementer presents the four panel PNGs to the user/reviewer: `run-full.png`, `graph-canvas.png`, `activity-stack.png`, and `workspace-open.png`. -- Do not commit PNGs from `tmp/visual-debugger/`. -- Do not block CI on PNG generation or screenshot diffs in the first PR. - ---- - -## 7. What becomes a hard gate - -Hard gates: - -- pure semantic tests pass; -- fixture e2e renders graph/stack/workspace; -- coarse graph node overlap check passes for golden fixture; -- no test asserts fixed agent lane counts; -- local screenshot command works when run manually. - -Not hard gates in first PR: - -- pixel-perfect screenshot diff; -- exact `x/y` coordinate assertions; -- local PNG files existing in CI; -- visual comparison against the HTML mockup. - ---- - -## 8. Phase impact - -This adds work to the phase plan: - -- Phase B adds golden fixture semantic tests before implementing activity/layout code. -- Phase E adds browser geometry overlap checks and local screenshot dumping behind `VISUAL_DEBUGGER_SCREENSHOTS=1` to fixture e2e. -- Phase F keeps screenshot artifacts for real smoke/PR review, but no CI visual-diff gate. diff --git a/docs/superpowers/plans/mas-run-visual-debugger/README.md b/docs/superpowers/plans/mas-run-visual-debugger/README.md deleted file mode 100644 index f3677b195..000000000 --- a/docs/superpowers/plans/mas-run-visual-debugger/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# MAS Run Visual Debugger — plan folder - -**Status:** draft for review — branch planning only; no frontend implementation landed yet. -**Date:** 2026-04-26. -**Branch:** `feature/mas-run-visual-debugger-plan`. -**Design reference:** `ergon-dashboard/mockups/mas-activity-stack-debugger.html`. - -## Read order - -1. [`00-program.md`](00-program.md) — product goal, non-goals, UX invariants, DTO stance, merge checklist. -2. [`05-implementation-shape.md`](05-implementation-shape.md) — reviewer-facing "how": domains, file ownership, add/refactor/delete plan, test layout. -3. [`01-contracts-and-state.md`](01-contracts-and-state.md) — event/DTO inventory, activity-stack domain model, replay rules, and where backend contract changes are actually needed. -4. [`02-frontend-implementation.md`](02-frontend-implementation.md) — component and layout plan for the three-pane visual debugger. -5. [`03-tests-and-e2e.md`](03-tests-and-e2e.md) — unit/component/e2e coverage, screenshot contract, and harness DTO additions. -6. [`06-fast-feedback-and-visual-review.md`](06-fast-feedback-and-visual-review.md) — TDD fixture loop, coarse layout geometry checks, and local-only PNG review workflow. -7. [`04-phases.md`](04-phases.md) — phased delivery order with acceptance gates. - -## Principle - -The dashboard should be a visual debugger for a MAS run, not an agent swimlane view. The durable axes are graph state, task-scoped events, and wall-clock overlap. Agents/workers are labels on events, not layout anchors. - -When documents disagree, `00-program.md` wins. When `00-program.md` and code reality disagree, update `00-program.md` first and re-review before implementing. diff --git a/docs/superpowers/plans/test-refactor/00-program.md b/docs/superpowers/plans/test-refactor/00-program.md deleted file mode 100644 index a0544010e..000000000 --- a/docs/superpowers/plans/test-refactor/00-program.md +++ /dev/null @@ -1,163 +0,0 @@ -# 00 — Program, goals, tier model - -**Status:** draft -**Date:** 2026-04-23 -**Entry-point index:** [`README.md`](README.md) -**Authoritative tech content:** this folder. Everything else is superseded (see [`05-deletions.md`](05-deletions.md)). - -This is the "why + what". Technical "how" lives in [`01-fixtures.md`](01-fixtures.md), [`02-drivers-and-asserts.md`](02-drivers-and-asserts.md), [`03-dashboard-and-playwright.md`](03-dashboard-and-playwright.md), [`04-ci-and-workflows.md`](04-ci-and-workflows.md). Delivery ordering in [`06-phases.md`](06-phases.md). - ---- - -## 1. Goals and non-goals - -**Goals** - -- Four test tiers with strict, non-overlapping roles (unit / integration / e2e-smoke / real-llm). -- Every PR runs unit + integration against **real Postgres + real Inngest** (no SQLite). -- Every PR runs an **e2e smoke matrix** of 3 benchmarks, each submitting **3 cohort-parallel workflow runs** of its own smoke worker, each spawning a **9-leaf DAG** on its own sandbox image — **9 top-level runs per PR, 80 leaf sandboxes per PR**. The researchrubrics cohort's third slot is a **sad-path run** that deterministically fails `l_2` to exercise static-sibling cascade + partial-work-persists-on-failure. -- Dashboard exercised end-to-end via Playwright on every PR, with screenshots pushed back as PR artifacts. -- Zero LLM calls in unit / integration / e2e-smoke. LLM behaviour lives in `tests/real_llm/` only. -- Test-only stubs (workers, criteria, benchmarks, rubrics) live under `tests/e2e/_fixtures/`, not `ergon_builtins`. -- No "canonical smoke" shared worker, no separate `smoke-test` benchmark, no shared smoke-only sandbox image. Smoke is **always per environment**, against that env's own benchmark. - -**Non-goals** - -- Sub-60s wall-clock targets for smoke. 5-min hard ceiling per matrix leg is the contract. -- New benchmark domains beyond `researchrubrics`, `minif2f`, `swebench-verified`. -- Relaxing "graph shape identical everywhere" — it is enforced by `SmokeWorkerBase.execute()` being `@final`. -- Replacing the real-LLM harness — it keeps its current shape under `tests/real_llm/`. - ---- - -## 2. Tier model - -Path-based only. `@pytest.mark.slow` is available for local dev ergonomics; CI runs everything in-tier. - -| Tier | Path | Infra | CI trigger | What it proves | -|---|---|---|---|---| -| **Unit** | `tests/unit/` | No I/O | every PR (`ci-fast`) | Pure logic: Pydantic, validators, registry, pure functions, static lints | -| **Integration** | `tests/integration/` | Real Postgres 15 + Inngest dev server (Docker) | every PR (`ci-fast`) | Graph / service / persistence; API boundaries; harness round-trips | -| **E2E smoke** | `tests/e2e/` | Full Docker stack + **real E2B** + dashboard + Playwright | every PR (`e2e-benchmarks` matrix) | Cross-service + cross-process + UI truth; sandbox provisioning at volume; cohort-parallel scheduling | -| **Real-LLM** | `tests/real_llm/` | As e2e + real model calls, budget-gated | on demand + nightly | Non-deterministic model behaviour | - -**`tests/state/` is retired.** Pure files moved to `tests/unit/state/`. Graph/service files migrate to `tests/integration/`. Migration status is tracked in [`06-phases.md`](06-phases.md). - -**Decision rule.** Pure function / validator / pydantic model / registry key → unit. Exercises graph, persistence, or HTTP boundary → integration. Needs sandbox + dashboard + UI → e2e. Needs LLM → real-llm. - ---- - -## 3. Canonical smoke program — invariants - -### 3.1 Shape — immutable 9-leaf DAG - -Every per-env smoke run produces **exactly** this graph, enforced by `SmokeWorkerBase.execute()` via `@final`. Subclasses supply only the leaf slug; topology is unmodifiable. - -``` -Diamond (4): Line (3): Singletons (2): - d_root l_1 → l_2 → l_3 s_a s_b - ↙ ↘ -d_left d_right - ↘ ↙ - d_join -``` - -```python -EXPECTED_SUBTASK_SLUGS = ( - "d_root", "d_left", "d_right", "d_join", - "l_1", "l_2", "l_3", - "s_a", "s_b", -) -``` - -Exported once; imported by `SmokeWorkerBase`, `SmokeCriterionBase`, every pytest driver, every Playwright spec. Changing the tuple is the only way to change topology. - -**Invariants each shape proves:** diamond → fan-out + fan-in + two-parent deps; line → strict sequential cascade; singletons → multi-terminal leaves + `wait_all` termination. - -### 3.2 Cohort-parallel submission - -Each matrix leg submits **3 runs of its benchmark's smoke worker as one cohort**. Proves cohort parallelism and concurrent workflow submission at volume on top of graph-propagation invariants. **The researchrubrics leg replaces its third cohort slot with a sad-path run** that uses `ResearchRubricsSadPathSmokeWorker`; the line-cascade failure invariants are exercised here, without adding a 10th top-level run. - -| Leg | Slot 1 | Slot 2 | Slot 3 | -|---|---|---|---| -| `researchrubrics` | happy | happy | **sad** — `l_2` deterministic FAIL | -| `minif2f` | happy | happy | happy | -| `swebench-verified` | happy | happy | happy | - -Per PR: 3 matrix legs × 3 cohort-parallel runs = **9 top-level runs**. Sandbox acquisitions: 8 happy runs × 9 leaves + 1 sad run × 8 leaves (`l_3` never provisioned because its dep failed) = **80 E2B leaf sandbox acquisitions**. - -Per leg: 26 or 27 leaf sandbox acquisitions on that env's image. No shared smoke sandbox image. - -If E2B cost becomes an issue, step down to 1 run per leg (9 sandboxes/PR total); the researchrubrics sad slot stays, minif2f and swebench-verified collapse to 1 happy run each. Do **not** reduce leaf count. - -### 3.3 Registry surface — 9 entries, test-only - -Nine rows, all under `tests/e2e/_fixtures/`, none in `ergon_builtins`. Code sketches in [`01-fixtures.md`](01-fixtures.md). - -| Slug | Kind | -|---|---| -| `{env}-smoke-worker` × 3 | Worker (parent) | -| `{env}-smoke-leaf` × 3 | Worker (leaf) | -| `{env}-smoke-criterion` × 3 | Criterion | - -`{env} ∈ {researchrubrics, minif2f, swebench}`. Shared base classes (`SmokeWorkerBase`, `BaseSmokeLeafWorker`, `SmokeSubworker` Protocol, `SmokeCriterionBase`) are **not registered** — composed via inheritance. - -### 3.4 Test stubs out of `ergon_builtins` - -`ergon_builtins/` contains **only production baselines**. All smoke infrastructure moves or gets deleted (see [`05-deletions.md`](05-deletions.md) for the manifest). - -Test-only discovery: `tests/e2e/_fixtures/__init__.py` registers via import side-effect. The e2e pytest session imports it at session start. Production CLI paths do not import `tests/`; builtins stays clean. - ---- - -## 4. Budgets - -| Measure | Value | -|---|---| -| Per matrix leg | 10-min job timeout; 5-min pytest timeout (hard ceiling) | -| Leaf-subtask sandbox acquisitions per leg | happy legs: 3 runs × 9 leaves = **27**; researchrubrics leg: 2×9 + 1×8 = **26** (sad slot skips `l_3`) | -| Leaf-subtask sandbox acquisitions per PR | (minif2f 27) + (swebench 27) + (researchrubrics 26) = **80** across 3 images | -| Parent-task sandbox per run | 1 (used by parent worker + attached to by the criterion per `sandbox-lifetime-covers-criteria` RFC). Not additional at evaluation time. | -| Sad-path residency | Folded into researchrubrics cohort slot 3 — **zero additional top-level runs**. | -| Parallel workflow runs per PR | **9** (3 legs × 3-run cohort) | -| Warm wall-clock per leg | 1–3 min (post-Docker cache) | -| Cold wall-clock per leg | up to 5 min | - -E2B API key is required on every PR — acceptable for private-repo phase; revisit before open-source. - ---- - -## 5. Acceptance gate (merge checklist) - -- [ ] All 3 `e2e-benchmarks` matrix legs green on the PR. (Researchrubrics leg contains 2 happy + 1 sad-path cohort runs; all must be green.) -- [ ] `ci-fast` unit + integration green. -- [ ] No `ergon_builtins/**` file contains `stub` or `smoke` in its name (case-insensitive), **except** the production-baseline exception list: `workers/baselines/training_stub_worker.py` (synthetic-trajectory RL baseline per accepted RFC `2026-04-22-worker-interface-and-artifact-routing`). Anything else that still matches is a miss. -- [ ] Registry contains no `canonical-smoke`, `smoke-test`, or generic `smoke-leaf` slug. -- [ ] Production `pnpm build` of dashboard does not bundle `/api/test/*` routes (harness-mount-gate test enforces). -- [ ] No LLM references on the smoke path: grep `OPENROUTER`, `anthropic`, `openai`, `pydantic_ai` in `tests/e2e/` → zero. -- [ ] Screenshots land on `screenshots/pr-{N}` and PR comment renders inline images. -- [ ] Docker layer cache in effect (cold leg < 5 min; warm < 3 min). -- [ ] Per-run assertions green on every run: graph, resources, turn counts, sandbox command WAL, sandbox lifecycle events, thread messages (9 happy / 8 sad), blob round-trip, temporal ordering, evaluation, env content. -- [ ] Cohort membership assertion green for each matrix leg (3 runs visible on `/cohort/{key}` via harness DTO). -- [ ] Sad-path assertions green: graph cascade (l_1 COMPLETED, l_2 FAILED, l_3 BLOCKED/CANCELLED; diamond + singletons COMPLETED), partial artifact persisted as `RunResource`, pre-fail `wc -l` WAL entry persisted, completion-message suppression (8 msgs on `smoke-completion` thread, `l_2` missing), evaluation score=0.0 with `l_2` named in feedback. -- [ ] `docs/architecture/07_testing.md` points at this folder; all superseded RFCs/plans listed in [`05-deletions.md`](05-deletions.md) have been removed in the same PR. - ---- - -## 6. Open decisions to pin - -1. **Training-stub residency — RESOLVED: keep in `ergon_builtins/workers/baselines/`.** `TrainingStubWorker` is a production baseline: RL operators invoke it (`ergon benchmark run smoke-test --worker training-stub`) to exercise the trajectory-extraction pipeline without spending on a real model. Accepted RFC [`2026-04-22-worker-interface-and-artifact-routing.md`](../../../rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md) already treats it as a first-class baseline alongside `ReActWorker`; the "stub" in its name is a technical term (synthetic trajectories / fake logprobs), not "test fixture." This refactor does **not** touch the file. Renaming it to drop "stub" is an optional follow-up, not blocking. -2. **Cohort size.** Default **3** per leg (81/PR). Step down to 1/leg (27/PR) if E2B cost is unacceptable. -3. **Dashboard fidelity vs speed.** Default prod build (`pnpm build && start`); dev-server fallback if 5-min budget blown. -4. **Existing Next.js `/api/test/dashboard/*` routes.** Keep as-is, consolidate into backend `/api/test/*`, or remove? Covered in [`03-dashboard-and-playwright.md §4`](03-dashboard-and-playwright.md). -5. **Harness test-secret rotation.** `TEST_HARNESS_SECRET` read from env; CI passes a random per-run value. Document mechanism in [`03-dashboard-and-playwright.md §2`](03-dashboard-and-playwright.md). - ---- - -## 7. Changelog - -| Date | Change | -|---|---| -| 2026-04-23 | Initial single-plan draft. | -| 2026-04-23 (rev 1) | Drop `canonical-smoke` slug + `smoke-test` benchmark. Smoke is per-env self-contained. 9 registry rows. | -| 2026-04-23 (rev 2) | Split into folder `test-refactor/`. This file becomes the "program" entry; technical detail moves to `01-` through `06-`. | diff --git a/docs/superpowers/plans/test-refactor/01-fixtures.md b/docs/superpowers/plans/test-refactor/01-fixtures.md deleted file mode 100644 index 65448ae79..000000000 --- a/docs/superpowers/plans/test-refactor/01-fixtures.md +++ /dev/null @@ -1,1191 +0,0 @@ -# 01 — Fixtures: shared bases + per-env workers, leaves, criteria - -**Status:** draft -**Scope:** every production-code change in `tests/e2e/_fixtures/` and the corresponding deletions in `ergon_builtins/`. Code sketches are representative, not literal copy-paste — the actual PR must conform to current `Worker` / `Criterion` ABC signatures. - -For deletion of the equivalent files from `ergon_builtins/`, see [`05-deletions.md`](05-deletions.md). - ---- - -## 1. Directory layout - -``` -tests/ -└── e2e/ - ├── __init__.py - ├── conftest.py - ├── _fixtures/ - │ ├── __init__.py ← registration hook; see §2.7 - │ ├── smoke_base/ - │ │ ├── __init__.py - │ │ ├── constants.py ← EXPECTED_SUBTASK_SLUGS, SUBTASK_GRAPH - │ │ ├── worker_base.py ← SmokeWorkerBase (non-registered, @final execute) - │ │ ├── leaf_base.py ← BaseSmokeLeafWorker (non-registered) - │ │ ├── subworker.py ← SmokeSubworker Protocol + SubworkerResult - │ │ └── criterion_base.py ← SmokeCriterionBase (non-registered) - │ ├── workers/ - │ │ ├── __init__.py - │ │ ├── researchrubrics_smoke.py ← parent + leaf + subworker for researchrubrics - │ │ ├── minif2f_smoke.py ← parent + leaf + subworker for minif2f - │ │ └── swebench_smoke.py ← parent + leaf + subworker for swebench - │ └── criteria/ - │ ├── __init__.py - │ ├── researchrubrics_smoke.py ← ResearchRubricsSmokeCriterion - │ ├── minif2f_smoke.py ← MiniF2FSmokeCriterion - │ └── swebench_smoke.py ← SweBenchSmokeCriterion - ├── test_researchrubrics_smoke.py - ├── test_minif2f_smoke.py - └── test_swebench_smoke.py -``` - -Rationale for a single file per env in `workers/`: the env-specific parent, leaf, and subworker are cohesive and changing one usually requires changing the others. Bundle per env, not per role. - ---- - -## 2. Shared bases (not registered) - -### 2.1 `smoke_base/constants.py` - -```python -# tests/e2e/_fixtures/smoke_base/constants.py - -from collections.abc import Sequence - -EXPECTED_SUBTASK_SLUGS: tuple[str, ...] = ( - "d_root", "d_left", "d_right", "d_join", - "l_1", "l_2", "l_3", - "s_a", "s_b", -) - -# (slug, depends_on_slugs, description) — shape of the DAG in one place. -SUBTASK_GRAPH: Sequence[tuple[str, tuple[str, ...], str]] = ( - ("d_root", (), "Diamond root"), - ("d_left", ("d_root",), "Diamond left arm"), - ("d_right", ("d_root",), "Diamond right arm"), - ("d_join", ("d_left", "d_right"), "Diamond join"), - ("l_1", (), "Line node 1"), - ("l_2", ("l_1",), "Line node 2"), - ("l_3", ("l_2",), "Line node 3"), - ("s_a", (), "Singleton A"), - ("s_b", (), "Singleton B"), -) -``` - -### 2.2 `smoke_base/subworker.py` - -```python -# tests/e2e/_fixtures/smoke_base/subworker.py - -from dataclasses import dataclass -from typing import Protocol, runtime_checkable - -from ergon_core.core.providers.sandbox.manager import AsyncSandbox - - -@dataclass(frozen=True) -class SubworkerResult: - file_path: str - probe_stdout: str - probe_exit_code: int - - -@runtime_checkable -class SmokeSubworker(Protocol): - """Env-specific deterministic work inside a sandbox. - - MUST NOT call an LLM. MUST NOT make external network calls. - MUST complete within 20 seconds under normal sandbox conditions. - """ - - async def work(self, node_id: str, sandbox: AsyncSandbox) -> SubworkerResult: ... -``` - -### 2.3 `smoke_base/worker_base.py` - -**Key difference from the retired `CanonicalSmokeWorker`:** no `type_slug` at the base (so it is not registered), and `execute()` is `@final`. Subclasses set `leaf_slug` only. - -```python -# tests/e2e/_fixtures/smoke_base/worker_base.py - -from collections.abc import AsyncGenerator -from typing import ClassVar, final -from uuid import UUID - -from ergon_core.api import BenchmarkTask, Worker, WorkerContext -from ergon_core.api.generation import GenerationTurn, TextPart -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.shared.types import ( - AssignedWorkerSlug, NodeId, RunId, TaskSlug, -) -from ergon_core.core.runtime.services.task_management_dto import ( - PlanSubtasksCommand, SubtaskSpec, -) -from ergon_core.core.runtime.services.task_management_service import ( - TaskManagementService, -) - -from tests.e2e._fixtures.smoke_base.constants import SUBTASK_GRAPH - - -class SmokeWorkerBase(Worker): - """Abstract parent. Subclasses set `type_slug` and `leaf_slug`. - - Topology is locked by `@final` on `execute` and by using SUBTASK_GRAPH - directly (no subclass hook for altering the shape). - """ - - leaf_slug: ClassVar[str] # e.g. "researchrubrics-smoke-leaf" - - def __init__( - self, *, name: str, model: str | None, task_id: UUID, sandbox_id: str, - ) -> None: - super().__init__(name=name, model=model, task_id=task_id, sandbox_id=sandbox_id) - - # Parent execution yields 3 turns so the incremental turn persistence - # path is exercised on every smoke run. See §2.6 ("Fidelity") for why. - PARENT_TURN_COUNT: ClassVar[int] = 3 - - @final - async def execute( - self, - task: BenchmarkTask, - *, - context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - if context.node_id is None: - raise ValueError(f"{type(self).__name__} requires context.node_id") - - # --- Turn 1: planning announcement (pre-service-call) --- - yield GenerationTurn( - response_parts=[ - TextPart( - content=( - f"{type(self).__name__}: planning 9 subtasks " - f"(diamond+line+singletons) → leaf slug={self.leaf_slug}" - ), - ), - ], - ) - - # Per-slug spec construction goes through _spec_for so sad-path - # subclasses can route specific slugs to a different leaf worker - # without overriding execute() (which stays @final). - specs = [self._spec_for(slug, deps, desc) for slug, deps, desc in SUBTASK_GRAPH] - with get_session() as session: - result = await TaskManagementService().plan_subtasks( - session, - PlanSubtasksCommand( - run_id=RunId(context.run_id), - parent_node_id=NodeId(context.node_id), - subtasks=specs, - ), - ) - - # --- Turn 2: plan result (post-service-call) --- - summary = "\n".join( - f"{slug}: planned (node_id={result.nodes[TaskSlug(slug)]})" - for slug, _deps, _desc in SUBTASK_GRAPH - ) - yield GenerationTurn( - response_parts=[ - TextPart( - content=( - f"{type(self).__name__}: 9 subtasks planned " - f"(roots={sorted(result.roots)}):\n{summary}" - ), - ), - ], - ) - - # --- Turn 3: awaiting children (terminal) --- - yield GenerationTurn( - response_parts=[ - TextPart( - content=( - f"{type(self).__name__}: awaiting 9 children → " - "runtime will mark parent COMPLETED once wait_all resolves" - ), - ), - ], - ) - - def _spec_for( - self, slug: str, deps: tuple[str, ...], desc: str, - ) -> SubtaskSpec: - """Overridable per-slug → (assigned_worker_slug, deps) mapping. - - Default routes every slug to `self.leaf_slug`. Sad-path subclasses - (see §3.4) override this to route specific slugs to a failing leaf - while keeping the 9-subtask topology identical. Execute() stays - @final so topology is never changed; only the leaf binding is. - """ - return SubtaskSpec( - task_slug=TaskSlug(slug), - description=desc, - assigned_worker_slug=AssignedWorkerSlug(self.leaf_slug), - depends_on=[TaskSlug(d) for d in deps], - ) -``` - -**No pydantic-ai toolkits, no tool-loop, no generation model.** `execute()` runs deterministic Python that plans nine subtasks in one `TaskManagementService.plan_subtasks` call and yields one terminal `GenerationTurn`. `model=None` is passed through `__init__` and ignored. - -### 2.4 `smoke_base/leaf_base.py` - -Unchanged in shape from current `ergon_builtins/workers/stubs/base_smoke_leaf.py`, just re-homed and sanded down for the new import paths. Subclasses set `type_slug` (registered) and `subworker_cls` (composition). - -```python -# tests/e2e/_fixtures/smoke_base/leaf_base.py - -from collections.abc import AsyncGenerator -from typing import ClassVar -from uuid import UUID - -from e2b_code_interpreter import AsyncSandbox -from ergon_core.api import BenchmarkTask, Worker, WorkerContext -from ergon_core.api.generation import GenerationTurn, TextPart -from ergon_core.api.results import WorkerOutput - -from tests.e2e._fixtures.smoke_base.subworker import SmokeSubworker, SubworkerResult - - -class BaseSmokeLeafWorker(Worker): - """Abstract leaf. Subclasses set `type_slug` and `subworker_cls`.""" - - subworker_cls: ClassVar[type[SmokeSubworker]] - - # Each leaf yields 2 turns. Driver asserts on per-run totals to catch - # silent regressions in incremental turn persistence. See §2.6. - LEAF_TURN_COUNT: ClassVar[int] = 2 - - def __init__( - self, *, name: str, model: str | None, task_id: UUID, sandbox_id: str, - ) -> None: - super().__init__(name=name, model=model, task_id=task_id, sandbox_id=sandbox_id) - self._last_result: SubworkerResult | None = None - - async def execute( - self, task: BenchmarkTask, *, context: WorkerContext, - ) -> AsyncGenerator[GenerationTurn, None]: - node_hex = context.node_id.hex[:8] if context.node_id else "unknown" - - # --- Turn 1: attaching + starting --- - yield GenerationTurn( - response_parts=[ - TextPart( - content=( - f"{type(self).__name__}: attaching to sandbox " - f"{context.sandbox_id} for node={node_hex}" - ), - ), - ], - ) - - sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - result = await self.subworker_cls().work(node_id=node_hex, sandbox=sandbox) - self._last_result = result - - # Post a one-line completion message to the shared "smoke-completion" - # thread. Every happy-path leaf sends exactly one message; sad-path - # leaves that raise before this point do NOT send — the driver asserts - # on that shape. Uses the existing (today unused) CommunicationService. - await self._send_completion_message(context, result) - - # --- Turn 2: done + result summary --- - yield GenerationTurn( - response_parts=[ - TextPart( - content=( - f"{type(self).__name__}: done node={node_hex} " - f"file={result.file_path} probe_exit={result.probe_exit_code}" - ), - ), - ], - ) - - async def _send_completion_message( - self, context: WorkerContext, result: SubworkerResult, - ) -> None: - """One ThreadMessage per leaf on the smoke-completion thread. - - Structure asserted by driver (`_assert_thread_messages_ordered`): - - - Thread topic == "smoke-completion" - - Agents: `leaf-{task_slug}` → `parent` - - 9 messages per happy-path run (sequence_num 1..9, per-thread monotonic) - - Creation timestamps monotonically non-decreasing in leaf-completion order - - Sad-path leaf does NOT reach this call because `AlwaysFailSubworker` - raises inside `work()`. Driver asserts 8 messages in the sad-path run. - """ - from ergon_core.core.runtime.services.communication_service import ( - communication_service, - ) - from ergon_core.core.runtime.services.communication_schemas import ( - CreateMessageRequest, - ) - - await communication_service.save_message( - CreateMessageRequest( - run_id=context.run_id, - task_execution_id=context.execution_id, - from_agent_id=f"leaf-{context.task_slug}", - to_agent_id="parent", - thread_topic="smoke-completion", - content=( - f"{context.task_slug}: done exit={result.probe_exit_code} " - f"file={result.file_path}" - ), - ), - ) - - def get_output(self, context: WorkerContext) -> WorkerOutput: - r = self._last_result - if r is None: - return WorkerOutput(output="", success=False, metadata={"error": "no_result"}) - return WorkerOutput( - output=r.probe_stdout, - success=r.probe_exit_code == 0, - metadata={"probe_exit_code": r.probe_exit_code, "file_path": r.file_path}, - ) -``` - -### 2.5 `smoke_base/criterion_base.py` - -**Change from the current file:** `_pull_children` and `_pull_probe_results` must be concretely implemented, not raise `NotImplementedError`. The current file has pointer comments — those get filled in here. - -```python -# tests/e2e/_fixtures/smoke_base/criterion_base.py - -from typing import Any -from uuid import UUID - -from sqlmodel import select - -from ergon_core.api import ( - Criterion, CriterionResult, CriteriaCheckError, EvaluationContext, -) -from ergon_core.core.persistence.graph.models import RunGraphNode -from ergon_core.core.persistence.graph.status_conventions import COMPLETED -from ergon_core.core.persistence.resources.models import RunResource -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.providers.blob.client import BlobClient # or canonical import - -from tests.e2e._fixtures.smoke_base.constants import EXPECTED_SUBTASK_SLUGS - - -class SmokeCriterionBase(Criterion): - """Structural + probe checks. Env subclasses override `_verify_env_content`.""" - - async def evaluate(self, context: EvaluationContext) -> CriterionResult: - try: - # 1. Artifact-side checks (no sandbox; reads blob storage only) - children = await self._pull_children(context) - self._check_graph_shape(children) - self._check_children_completed(children) - probes = await self._pull_probe_results(context, children) - self._check_probes_succeeded(probes, children) - await self._verify_env_content(context, children, probes) - - # 2. Sandbox-side check: attach to the parent task's OWN - # sandbox (kept alive by the runtime per RFC - # `sandbox-lifetime-covers-criteria`) and run a trivial - # env-specific command. Proves the image / toolchain is - # healthy; independent of what the leaves produced. - # No fresh sandbox acquisition — zero extra E2B cost. - await self._verify_sandbox_setup(context) - except CriteriaCheckError as e: - return CriterionResult( - name=self.name, score=0.0, passed=False, - weight=self.weight, - feedback=f"smoke criterion failed: {e}", - ) - return CriterionResult( - name=self.name, score=1.0, passed=True, - weight=self.weight, feedback="smoke passed", - ) - - async def _pull_children(self, context: EvaluationContext) -> list[RunGraphNode]: - with get_session() as s: - parent = s.exec( - select(RunGraphNode).where( - RunGraphNode.task_execution_id == context.execution_id, - ), - ).one() - children = s.exec( - select(RunGraphNode) - .where(RunGraphNode.parent_id == parent.id) - .order_by(RunGraphNode.task_slug), - ).all() - return list(children) - - async def _pull_probe_results( - self, context: EvaluationContext, children: list[RunGraphNode], - ) -> dict[UUID, dict[str, Any]]: - results: dict[UUID, dict[str, Any]] = {} - with get_session() as s: - for child in children: - resource = s.exec( - select(RunResource) - .where(RunResource.node_id == child.id) - .where(RunResource.name.like("probe_%.json")), - ).first() - if resource is None: - raise CriteriaCheckError(f"{child.task_slug}: no probe_*.json resource") - blob_bytes = await BlobClient.default().get(resource.content_hash) - results[child.id] = _parse_probe_json(blob_bytes) - return results - - def _check_graph_shape(self, children: list[RunGraphNode]) -> None: - actual = {c.task_slug for c in children} - expected = set(EXPECTED_SUBTASK_SLUGS) - if actual != expected: - raise CriteriaCheckError( - f"graph shape mismatch: actual={sorted(actual)} expected={sorted(expected)}", - ) - - def _check_children_completed(self, children: list[RunGraphNode]) -> None: - for c in children: - if c.status != COMPLETED: - raise CriteriaCheckError( - f"child {c.task_slug} not completed (status={c.status!r})", - ) - - def _check_probes_succeeded( - self, probes: dict[UUID, dict[str, Any]], children: list[RunGraphNode], - ) -> None: - by_id = {c.id: c for c in children} - for child_id, probe in probes.items(): - slug = by_id[child_id].task_slug - code = probe.get("exit_code") - if code != 0: - raise CriteriaCheckError( - f"probe for {slug} exited {code}, stdout={probe.get('stdout', '')!r}", - ) - - async def _verify_env_content( - self, context: EvaluationContext, - children: list[RunGraphNode], - probes: dict[UUID, dict[str, Any]], - ) -> None: - """Subclass hook: read artifacts and check env-specific file shape.""" - raise NotImplementedError - - async def _verify_sandbox_setup(self, context: EvaluationContext) -> None: - """Subclass hook: run a trivial env-specific command in the parent - task's sandbox to prove the toolchain is healthy. - - Canonical shape for subclasses — goes through the landed - `CriterionRuntime` DI API (RFC `criterion-runtime-di-container`, - accepted). The runtime owns sandbox lifecycle; the criterion never - calls `AsyncSandbox.connect` directly. - - await context.runtime.ensure_sandbox() - result = await context.runtime.run_command( - "<env-health-probe>", timeout=20, - ) - if result.exit_code != 0: - raise CriteriaCheckError( - f"<env> health probe failed: exit={result.exit_code} " - f"stdout={(result.stdout or '')[:200]!r}", - ) - - Why the runtime is the right surface: - - - `CriterionRuntime.ensure_sandbox()` (`runtime/evaluation/criterion_runtime.py`) - attaches via `sandbox_manager.get_sandbox(run_id)` for in-process - criteria and will use `BaseSandboxManager.reconnect(sandbox_id)` - for cross-process criteria once Phase G lands that method. Either - way, criteria keep calling `context.runtime.run_command(...)` — - no code change on the criterion side. - - RFC `sandbox-lifetime-covers-criteria` (active) guarantees the - task's sandbox is kept alive through criterion execution. - - Criteria MUST NOT construct sandboxes directly — the anti-pattern - that `bugs/fixed/2026-04-18-swebench-criterion-spawns-sandbox.md` - fixed. - """ - raise NotImplementedError -``` - -`_parse_probe_json` is a pure helper colocated beside the class. - -### 2.6 Fidelity — what smoke proves, and what it deliberately skips - -Smoke fixtures are not miniature production workers; they are the minimal subclass of `Worker` that exercises the runtime plumbing we care about without requiring an LLM. Being explicit about the gap is what lets smoke stay fast and deterministic. - -#### What smoke preserves (same code path as production) - -- `Worker` ABC signature (`__init__(*, name, model, task_id, sandbox_id)`, `execute`, `get_output`). -- `AsyncGenerator[GenerationTurn, None]` yield protocol — **smoke yields multiple turns per execution** (3 from the parent, 2 per leaf; §2.3, §2.4), so incremental turn persistence, context event sequencing, and thread-message emission per-turn are exercised on every smoke run. -- `AsyncSandbox.connect(sandbox_id=...)` attach contract — used by **leaf workers** (during execution) **and** by smoke criteria (during evaluation, via the parent task's sandbox kept alive per `sandbox-lifetime-covers-criteria`). -- `WorkerOutput(output, success, metadata)` shape. -- `type_slug` registration flow (via `tests/e2e/_fixtures/__init__.py`). -- `TaskManagementService.plan_subtasks` service call — this is the same service-layer entry point a production worker eventually reaches, after its toolkit tool is invoked. -- Per-leaf sandbox acquisition, resource publication from `/workspace/final_output/`, per-criterion evaluation row write. -- **Criterion-side sandbox attach** (not acquisition) — proves the env toolchain is healthy at evaluation time without violating [`rfcs/accepted/2026-04-17-criterion-runtime-di-container.md`](../../../rfcs/accepted/2026-04-17-criterion-runtime-di-container.md) (criteria never construct sandboxes). - -#### What smoke deliberately skips - -- **LLM agent loop.** Zero calls to any model; `model=None` is accepted and ignored. -- **Pydantic-AI toolkits** (`GraphToolkit`, `ResearchRubricsToolkit`, `SubtaskLifecycleToolkit`, …). Toolkits exist to mediate between an LLM's tool calls and Ergon services. With no LLM, a toolkit buys nothing and would force us to invent a deterministic fake agent loop. `SmokeWorkerBase` calls `TaskManagementService.plan_subtasks` **directly** instead — one call hop below where a toolkit tool would land. -- **Tool-call message shape.** Real workers stream tool-call / tool-return turns interleaved with text. Smoke yields text-only `GenerationTurn`s. Code paths that specifically parse tool-call turn parts are not exercised here. -- **LLM response parsing / retry / budget logic.** Owned by `tests/real_llm/`. - -#### Coverage owner matrix - -Use this to decide where a regression should be caught. - -| Code path | Smoke (this tier) | Integration | Real-LLM | -|---|:---:|:---:|:---:| -| `Worker.execute` async generator consumed | ✅ | ✅ | ✅ | -| Multi-turn persistence (`GenerationTurn` rows) | ✅ (3 parent + 2 × 9 = 21 turns/run) | partial (services called directly) | ✅ | -| `TaskManagementService.plan_subtasks` DB effects | ✅ | ✅ | ✅ | -| Per-subtask sandbox acquisition at volume | ✅ (81/PR) | ❌ | ✅ | -| **Env toolchain health in sandbox** (Lean / Python / bash) | ✅ (per-env `_verify_sandbox_setup`) | ❌ | partial (only exercised if LLM happens to hit it) | -| Sandbox lifetime covers criterion (RFC) | ✅ (criterion attaches to task's sandbox) | ❌ | ✅ | -| **Sandbox command WAL emits per `sandbox.commands.run`** | ✅ (`_assert_sandbox_command_wal` per run) | partial | ✅ | -| **Sandbox lifecycle events** (`sandbox_created`/`_command`/`_closed`) fire end-to-end | ✅ (`_assert_sandbox_lifecycle_events`) | ❌ | ✅ | -| **`CommunicationService.save_message` round-trips to `ThreadMessage` rows** | ✅ (first caller of a currently-dead service) | ❌ | possible | -| **ThreadMessage `sequence_num` per-thread monotonicity** | ✅ (9 per happy run / 8 per sad run) | ❌ | ❌ | -| **Partial work persists on FAILED leaf** (partial file + pre-fail WAL) | ✅ (sad-path driver) | ❌ | ❌ | -| **Static-sibling failure cascade** (l_3 blocked when l_2 fails) | ✅ (sad-path driver) | partial | ❌ | -| **Blob-store round-trip byte fidelity** | ✅ (`_assert_blob_roundtrip` on 1 leaf/run) | ❌ | ❌ | -| **Temporal ordering honours DAG deps** (d_join.started ≥ max(d_left, d_right).completed) | ✅ (`_assert_temporal_ordering`) | ❌ | ❌ | -| **Cohort-key membership** (3 runs visible on `/cohort/{key}`) | ✅ (`_assert_cohort_membership`) | ❌ | partial | -| Graph toolkit tool → service wiring | ❌ | ✅ (toolkit unit/integration tests) | ✅ | -| LLM tool-call turn part persisted correctly | ❌ | ❌ | ✅ | -| LLM retry / budget / provider failover | ❌ | ❌ | ✅ | - -The ❌ cells are not gaps in smoke's design; they are the *reason* we run three tiers instead of one. - -#### Why multi-turn matters (and why we didn't just yield one) - -A prior revision of this plan had `SmokeWorkerBase.execute` yield a single terminal turn. That would have silently skipped incremental turn persistence — `GenerationTurnRepository`, `RunContextEvent` sequence numbers, and per-turn dashboard deltas only trigger once per turn. Bumping to 3 parent + 2 leaf turns is the cheapest way to put those paths on the hot smoke path without introducing a fake agent loop. - -Driver-side invariant (asserted per run in [`02-drivers-and-asserts.md`](02-drivers-and-asserts.md)): - -> For a run with 1 parent × 3 turns + 9 leaves × 2 turns, total `GenerationTurn` rows = **21**. - -If we ever change `PARENT_TURN_COUNT` or `LEAF_TURN_COUNT`, the driver assertion changes in lock-step. These are `ClassVar`s on the base classes (§2.3, §2.4) so the driver imports them rather than hardcoding `21`. - -### 2.7 Data flow — how criteria get their inputs - -Two independent channels feed the criterion. Understanding which channel carries which check is how we avoid conflating "leaf output correct" with "env toolchain healthy." - -#### Channel A: blob storage (for leaf artifact checks) - -``` -Leaf worker in sandbox - │ - │ writes /workspace/final_output/<file> (report_<node>.md, probe_<node>.json, …) - ▼ -Runtime persist_outputs step (Inngest) - │ - │ scans /workspace/final_output/, hashes bytes, uploads blob, writes - │ RunResource(run_id, node_id, name, content_hash) row - ▼ -Sandbox is torn down (or repurposed) - │ - ▼ -Criterion.evaluate() - │ - │ _pull_probe_results: SELECT RunResource WHERE name LIKE 'probe_%.json' - │ → BlobClient.default().get(content_hash) → bytes - │ → json.loads → exit code + stdout - │ - │ _verify_env_content: SELECT RunResource WHERE name LIKE '<env>_%.<ext>' - │ → BlobClient.default().get(content_hash) → bytes - │ → content assertions -``` - -No sandbox access needed. Works even if every sandbox is already gone. - -#### Channel B: sandbox attach (for env-health check) - -``` -Parent task's sandbox is kept alive by the runtime, per - docs/rfcs/accepted/2026-04-17-sandbox-lifetime-covers-criteria.md - │ - │ (leaves had their own sandboxes; those are gone. - │ The parent's sandbox persists through evaluation.) - ▼ -Criterion.evaluate() → _verify_sandbox_setup() - │ - │ sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - │ # context.sandbox_id comes from the CriterionRuntime DI container per - │ # docs/rfcs/accepted/2026-04-17-criterion-runtime-di-container.md - │ - │ await sandbox.commands.run("<env-specific health probe>") - │ # Lean: lean --check /tmp/smoke_health.lean - │ # Python: python /tmp/smoke_health.py && python -c 'import pytest; …' - │ # Bash: echo … | wc -l - │ - │ assert exit_code == 0 - ▼ -Criterion returns CriterionResult(score=1.0) iff both channels passed. -``` - -The criterion does not own the sandbox's lifecycle. It attaches, runs a command, and returns. The runtime tears the sandbox down after `evaluate()` completes. - -#### Which channel catches which regression? - -| Regression | Channel A (artifacts) | Channel B (sandbox) | -|---|:---:|:---:| -| Leaf wrote wrong content | ✅ | — | -| Leaf didn't write the file at all | ✅ | — | -| Probe JSON missing / malformed | ✅ | — | -| Lean toolchain silently broken (probe's `\|\| true` masked it) | ❌ | ✅ | -| Python interpreter missing `pytest` | ❌ | ✅ | -| Sandbox image regressed (PATH wrong, `/tmp` read-only) | partial | ✅ | -| Sandbox-lifetime RFC regressed (sandbox torn down too early) | — | ✅ (criterion fails attach) | - -Having both channels means a silent leaf probe failure cannot mask a toolchain break, and a toolchain break cannot be disguised by a leaf writing a plausible-looking artifact. - -#### Lifecycle dependency — and the final step of this program - -Channel B relies on the parent task's sandbox staying alive through criterion execution. That guarantee is RFC [`2026-04-17-sandbox-lifetime-covers-criteria`](../../../rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md), which ships as two coordinated changes: - -1. **Timeout split** — `BaseSandboxManager.create()` takes `task_timeout_minutes + max_criterion_timeout_minutes`, provisions their sum to E2B. Ensures the sandbox is not killed by E2B mid-criterion. -2. **`BaseSandboxManager.reconnect(sandbox_id)`** — the blessed cross-process reconnect path. `CriterionRuntime.ensure_sandbox()` uses it to hand a live sandbox handle to the criterion, so the criterion can run its commands with the sandbox held open for the entire evaluation. - -In the sketches above, `_verify_sandbox_setup` calls `AsyncSandbox.connect(sandbox_id=context.sandbox_id)` directly. That works end-to-end and is fine as an interim, but it violates [`sandbox_lifecycle.md`](../../../architecture/cross_cutting/sandbox_lifecycle.md) invariant 3 ("Criteria MUST reconnect via the manager"). The invariant exists so event emission, template pinning, and expiry-error translation all flow through one path. - -**The final step of this test-refactor program is landing the `reconnect` path so smoke criteria — and every production criterion — reconnect to the task's sandbox through the manager and hold it open for the duration of evaluation.** That's Phase G in [`06-phases.md`](06-phases.md). Once it ships, the only change in smoke criteria is swapping the one-liner: - -```python -# before (interim, used in Phases C–F): -sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - -# after (Phase G): -sandbox = await context.get_sandbox() # or whatever the DI accessor is named -``` - -And `sandbox_lifecycle.md` invariant 3 flips from "pending enforcement" to "enforced end-to-end by smoke on every PR." - -### 2.7 `_fixtures/__init__.py` — registration hook - -```python -# tests/e2e/_fixtures/__init__.py -"""Test-only registration hook. - -Importing this package registers the 9 smoke workers + criteria into the -process-level builtins registry. Production CLI paths do not import this -package, so registrations are confined to test runtimes. -""" - -from ergon_core.api.registry import registry - -from tests.e2e._fixtures.workers.researchrubrics_smoke import ( - ResearchRubricsSmokeWorker, ResearchRubricsSmokeLeafWorker, -) -from tests.e2e._fixtures.workers.minif2f_smoke import ( - MiniF2FSmokeWorker, MiniF2FSmokeLeafWorker, -) -from tests.e2e._fixtures.workers.swebench_smoke import ( - SweBenchSmokeWorker, SweBenchSmokeLeafWorker, -) -from tests.e2e._fixtures.criteria.researchrubrics_smoke import ( - ResearchRubricsSmokeCriterion, -) -from tests.e2e._fixtures.criteria.minif2f_smoke import MiniF2FSmokeCriterion -from tests.e2e._fixtures.criteria.swebench_smoke import SweBenchSmokeCriterion - - -def register_smoke_fixtures() -> None: - for cls in ( - ResearchRubricsSmokeWorker, ResearchRubricsSmokeLeafWorker, - MiniF2FSmokeWorker, MiniF2FSmokeLeafWorker, - SweBenchSmokeWorker, SweBenchSmokeLeafWorker, - ): - registry.register_worker(cls) - for cls in ( - ResearchRubricsSmokeCriterion, - MiniF2FSmokeCriterion, - SweBenchSmokeCriterion, - ): - registry.register_criterion(cls) - - -register_smoke_fixtures() -``` - -Exact call names (`registry.register_worker`, `register_criterion`) must match the current registry API in `ergon_core/api/registry.py` — if today's API uses decorators, the hook adapts to that. This is a shape not a contract. - -**Session-entry:** `tests/e2e/conftest.py` does `import tests.e2e._fixtures # noqa: F401` at module scope. - ---- - -## 3. Per-env fixtures - -### 3.1 ResearchRubrics - -`tests/e2e/_fixtures/workers/researchrubrics_smoke.py` - -```python -import json - -from e2b_code_interpreter import AsyncSandbox - -from tests.e2e._fixtures.smoke_base.leaf_base import BaseSmokeLeafWorker -from tests.e2e._fixtures.smoke_base.subworker import SmokeSubworker, SubworkerResult -from tests.e2e._fixtures.smoke_base.worker_base import SmokeWorkerBase - - -class ResearchRubricsSmokeWorker(SmokeWorkerBase): - type_slug = "researchrubrics-smoke-worker" - leaf_slug = "researchrubrics-smoke-leaf" - - -class ResearchRubricsSubworker: # implements SmokeSubworker structurally - """Writes a deterministic markdown report + runs `wc -l` as the probe.""" - - async def work(self, node_id: str, sandbox: AsyncSandbox) -> SubworkerResult: - path = f"/workspace/final_output/report_{node_id}.md" - contents = ( - f"# Research report {node_id}\n\n" - "Deterministic smoke output. Non-empty body required.\n" - ) - await sandbox.files.write(path, contents) - probe = await sandbox.commands.run(f"wc -l {path}", timeout=10) - probe_stdout = probe.stdout.strip() - # Persist probe JSON as an artifact the criterion will read. - probe_path = f"/workspace/final_output/probe_{node_id}.json" - await sandbox.files.write( - probe_path, - json.dumps({"exit_code": probe.exit_code, "stdout": probe_stdout}), - ) - return SubworkerResult( - file_path=path, - probe_stdout=probe_stdout, - probe_exit_code=probe.exit_code, - ) - - -class ResearchRubricsSmokeLeafWorker(BaseSmokeLeafWorker): - type_slug = "researchrubrics-smoke-leaf" - subworker_cls = ResearchRubricsSubworker -``` - -`tests/e2e/_fixtures/criteria/researchrubrics_smoke.py` - -```python -from typing import Any -from uuid import UUID - -from ergon_core.api import CriteriaCheckError -from ergon_core.core.persistence.graph.models import RunGraphNode -from ergon_core.core.persistence.resources.models import RunResource -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.providers.blob.client import BlobClient - -from tests.e2e._fixtures.smoke_base.criterion_base import SmokeCriterionBase - - -class ResearchRubricsSmokeCriterion(SmokeCriterionBase): - type_slug = "researchrubrics-smoke-criterion" - - async def _verify_env_content( - self, context, children: list[RunGraphNode], - probes: dict[UUID, dict[str, Any]], - ) -> None: - for child in children: - report = await _read_artifact(child.id, name_like="report_%.md") - if not report.startswith(b"# Research report"): - raise CriteriaCheckError( - f"{child.task_slug}: report missing `# Research report` header", - ) - if len(report.strip()) < 20: - raise CriteriaCheckError( - f"{child.task_slug}: report body too short ({len(report)} bytes)", - ) - - async def _verify_sandbox_setup(self, context) -> None: - """Trivial env probe: bash + coreutils are present and produce - expected output. Catches researchrubrics sandbox image regressions - (e.g. missing `wc`, broken PATH, no /tmp write access). - - Attaches to the parent task's sandbox — same sandbox the parent - worker ran in, kept alive by the runtime for criterion execution. - """ - sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - # Exercise the 3 things a researchrubrics leaf actually needs: - # write to tmp, run wc, and sh exit-code propagation. - result = await sandbox.commands.run( - "set -e; " - "echo '# hello world' > /tmp/smoke_health.md && " - "test \"$(wc -l < /tmp/smoke_health.md)\" = '1' && " - "echo OK", - timeout=10, - ) - if result.exit_code != 0 or "OK" not in result.stdout: - raise CriteriaCheckError( - f"researchrubrics sandbox health failed: " - f"exit={result.exit_code} stdout={result.stdout[:200]!r}", - ) -``` - -`_read_artifact` is a small helper colocated in the criterion module. - -### 3.2 MiniF2F - -`tests/e2e/_fixtures/workers/minif2f_smoke.py` - -```python -LEAN_SOURCE = """\ -theorem smoke_trivial : 1 + 1 = 2 := by norm_num -""" - - -class MiniF2FSmokeWorker(SmokeWorkerBase): - type_slug = "minif2f-smoke-worker" - leaf_slug = "minif2f-smoke-leaf" - - -class MiniF2FSubworker: - async def work(self, node_id: str, sandbox: AsyncSandbox) -> SubworkerResult: - path = f"/workspace/final_output/proof_{node_id}.lean" - await sandbox.files.write(path, LEAN_SOURCE) - probe = await sandbox.commands.run( - f"lean --check {path} || true", # smoke-level: we want file parse, not full kernel - timeout=20, - ) - probe_path = f"/workspace/final_output/probe_{node_id}.json" - await sandbox.files.write( - probe_path, - json.dumps({"exit_code": probe.exit_code, "stdout": probe.stdout[:4096]}), - ) - return SubworkerResult( - file_path=path, - probe_stdout=probe.stdout.strip()[:4096], - probe_exit_code=probe.exit_code, - ) - - -class MiniF2FSmokeLeafWorker(BaseSmokeLeafWorker): - type_slug = "minif2f-smoke-leaf" - subworker_cls = MiniF2FSubworker -``` - -`tests/e2e/_fixtures/criteria/minif2f_smoke.py` - -```python -from e2b_code_interpreter import AsyncSandbox - -HEALTH_THEOREM = """\ -theorem health_check : 1 + 1 = 2 := by norm_num -""" - - -class MiniF2FSmokeCriterion(SmokeCriterionBase): - type_slug = "minif2f-smoke-criterion" - - async def _verify_env_content(self, context, children, probes) -> None: - for child in children: - source = await _read_artifact(child.id, name_like="proof_%.lean") - text = source.decode("utf-8") - if "theorem smoke_trivial" not in text: - raise CriteriaCheckError( - f"{child.task_slug}: lean source missing theorem marker", - ) - if ":=" not in text: - raise CriteriaCheckError( - f"{child.task_slug}: lean source missing proof term `:=`", - ) - - async def _verify_sandbox_setup(self, context) -> None: - """Compile a trivial theorem. Proves the Lean toolchain, Mathlib - (for `norm_num`), and the sandbox's elan / `lean` wrapper are all - wired up. This is the difference between "a file with `.lean` got - written" and "Lean can actually typecheck in this image." - - Attaches to the parent task's sandbox; Lean is probably warm by - now since the 9 leaves just ran against the same toolchain. - """ - sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - path = "/tmp/smoke_health.lean" - await sandbox.files.write(path, HEALTH_THEOREM) - result = await sandbox.commands.run( - f"lean --check {path}", # no `|| true` — we want the real exit code - timeout=60, # generous; Lean should be warm post-leaves - ) - if result.exit_code != 0: - raise CriteriaCheckError( - f"minif2f sandbox health failed: lean --check " - f"exit={result.exit_code} " - f"stdout={result.stdout[:300]!r} " - f"stderr={result.stderr[:300]!r}", - ) -``` - -**Probe choice note (leaf side):** the `MiniF2FSubworker` leaf probe uses `lean --check || true` to keep leaf probe exit deterministic during first-boot toolchain warmup; the *criterion-side* health probe runs without `|| true` so a genuinely broken toolchain fails loudly. If the criterion-side probe is consistently slow, trim `HEALTH_THEOREM` to avoid `norm_num` / `Mathlib` — `theorem health_check : True := trivial` imports nothing and compiles in under a second. - -### 3.3 SWE-Bench Verified - -`tests/e2e/_fixtures/workers/swebench_smoke.py` - -```python -PY_SOURCE = """\ -def add(a, b): - return a + b - -if __name__ == "__main__": - assert add(2, 3) == 5 -""" - - -class SweBenchSmokeWorker(SmokeWorkerBase): - type_slug = "swebench-smoke-worker" - leaf_slug = "swebench-smoke-leaf" - - -class SweBenchSubworker: - async def work(self, node_id: str, sandbox: AsyncSandbox) -> SubworkerResult: - path = f"/workspace/final_output/patch_{node_id}.py" - await sandbox.files.write(path, PY_SOURCE) - probe = await sandbox.commands.run( - f"python -m py_compile {path} && python {path}", timeout=20, - ) - probe_path = f"/workspace/final_output/probe_{node_id}.json" - await sandbox.files.write( - probe_path, - json.dumps({"exit_code": probe.exit_code, "stdout": probe.stdout[:4096]}), - ) - return SubworkerResult( - file_path=path, - probe_stdout=probe.stdout.strip()[:4096], - probe_exit_code=probe.exit_code, - ) - - -class SweBenchSmokeLeafWorker(BaseSmokeLeafWorker): - type_slug = "swebench-smoke-leaf" - subworker_cls = SweBenchSubworker -``` - -`tests/e2e/_fixtures/criteria/swebench_smoke.py` - -```python -import ast - - -HEALTH_PY = """\ -import sys -assert sys.version_info >= (3, 10), sys.version_info -print("HEALTH_OK") -""" - - -class SweBenchSmokeCriterion(SmokeCriterionBase): - type_slug = "swebench-smoke-criterion" - - async def _verify_env_content(self, context, children, probes) -> None: - for child in children: - source = await _read_artifact(child.id, name_like="patch_%.py") - try: - tree = ast.parse(source.decode("utf-8")) - except SyntaxError as e: - raise CriteriaCheckError(f"{child.task_slug}: python AST parse failed: {e}") - func_names = [ - node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) - ] - if "add" not in func_names: - raise CriteriaCheckError( - f"{child.task_slug}: expected function `add`, got {func_names}", - ) - - async def _verify_sandbox_setup(self, context) -> None: - """Proves Python 3.10+ is present, exits cleanly, and `pytest` is - importable. These are the three things every swebench-verified leaf - assumes about its sandbox image; surface regressions here instead of - at evaluation time deep inside the runner. - - Attaches to the parent task's sandbox (kept alive for the criterion - per RFC `sandbox-lifetime-covers-criteria`). - """ - sandbox = await AsyncSandbox.connect(sandbox_id=context.sandbox_id) - path = "/tmp/smoke_health.py" - await sandbox.files.write(path, HEALTH_PY) - # Two cheap checks rolled into one command: - # 1. python runs and reports HEALTH_OK - # 2. pytest is importable (no test discovery) - result = await sandbox.commands.run( - f"python {path} && python -c 'import pytest; print(pytest.__version__)'", - timeout=15, - ) - if result.exit_code != 0 or "HEALTH_OK" not in result.stdout: - raise CriteriaCheckError( - f"swebench sandbox health failed: " - f"exit={result.exit_code} " - f"stdout={result.stdout[:300]!r} " - f"stderr={result.stderr[:300]!r}", - ) -``` - -### 3.4 Sad-path fixtures — researchrubrics cohort slot 3 - -The researchrubrics leg's cohort-of-3 is 2 happy + 1 sad. The sad slot uses `ResearchRubricsSadPathSmokeWorker`, which routes `l_2` through a failing leaf; the rest of the 9-subtask topology is unchanged. No separate matrix leg; no additional top-level run; no separate driver file. MiniF2F + SWE-bench cohorts stay 3-happy. - -Invariants the sad slot exercises: - -- **Partial work persists on FAILED leaf.** The failing subworker does real work (file write + sandbox command) **before** raising. Driver asserts the partial artifact landed as a `RunResource` and the pre-failure command emitted a WAL entry. -- **Static sibling failure semantics.** Line cascade (`l_1 → l_2 → l_3`) with l_2 forced to FAIL; `l_3` must end up BLOCKED/CANCELLED per [`rfcs/active/2026-04-17-static-sibling-failure-semantics.md`](../../../rfcs/active/2026-04-17-static-sibling-failure-semantics.md). Diamond + singletons in the same run must be unaffected. -- **Completion message suppression.** Sad-path leaf raises before reaching `_send_completion_message`, so exactly 8 ThreadMessages (not 9) exist on the smoke-completion thread for this run. - -Fixtures live under `tests/e2e/_fixtures/`: - -```python -# tests/e2e/_fixtures/workers/researchrubrics_smoke_sadpath.py - -from e2b_code_interpreter import AsyncSandbox - -from tests.e2e._fixtures.smoke_base.leaf_base import BaseSmokeLeafWorker -from tests.e2e._fixtures.smoke_base.subworker import SmokeSubworker, SubworkerResult -from tests.e2e._fixtures.smoke_base.worker_base import SmokeWorkerBase -from ergon_core.core.persistence.shared.types import AssignedWorkerSlug, TaskSlug -from ergon_core.core.runtime.services.task_management_dto import SubtaskSpec - - -class AlwaysFailSubworker: - """Does TWO units of real work, then raises. - - Purpose: exercise the partial-work-persists-on-failure path. We want to - prove that when a leaf fails mid-execution: - - 1. Files it already wrote to /workspace/final_output/ still become - RunResource rows (runtime persist step runs regardless of worker - exit outcome). - 2. Sandbox commands it already ran still emit sandbox_command events - + WAL entries (the command path writes synchronously). - 3. The leaf's own task row ends up FAILED (exception propagates). - 4. Downstream static siblings (l_3) end up BLOCKED/CANCELLED. - """ - - async def work(self, node_id: str, sandbox: AsyncSandbox) -> SubworkerResult: - # Action 1: write a partial artifact — must land as RunResource. - partial_path = f"/workspace/final_output/partial_{node_id}.md" - await sandbox.files.write( - partial_path, - ( - f"# Partial work {node_id}\n\n" - "This content was written before a deliberate failure. " - "If smoke sees this as a RunResource row, partial serialization works.\n" - ), - ) - - # Action 2: run a sandbox command — must emit sandbox_command + WAL row. - pre_check = await sandbox.commands.run( - f"wc -l {partial_path}", - timeout=5, - ) - if pre_check.exit_code != 0: - raise RuntimeError( - f"AlwaysFailSubworker: precondition failed — expected wc to succeed " - f"but got exit={pre_check.exit_code}. The sad-path design assumes " - f"partial work completes cleanly before the deliberate raise.", - ) - - # Action 3: deliberate failure. Raises instead of returning exit_code=1 - # so the leaf task row is marked FAILED (not merely produces a failed - # WorkerOutput). - raise RuntimeError( - f"SmokeSadPathError: deliberate failure of {node_id} after " - f"writing {partial_path} and running probe (exit={pre_check.exit_code}). " - "Smoke asserts the partial file + probe WAL survive.", - ) - - -class ResearchRubricsFailingLeafWorker(BaseSmokeLeafWorker): - """Registered leaf that always fails after doing 2 units of work.""" - type_slug = "researchrubrics-smoke-leaf-failing" - subworker_cls = AlwaysFailSubworker - - -class ResearchRubricsSadPathSmokeWorker(SmokeWorkerBase): - """Parent worker that routes `l_2` to the failing leaf, everything else normal. - - Topology stays identical (still 9 subtasks, same deps); only the leaf - binding for l_2 differs. `execute()` is still @final; the hook is - `_spec_for`. - """ - type_slug = "researchrubrics-sadpath-smoke-worker" - leaf_slug = "researchrubrics-smoke-leaf" # default for everything EXCEPT l_2 - - FAILING_SLUGS = frozenset({"l_2"}) - FAILING_LEAF_SLUG = "researchrubrics-smoke-leaf-failing" - - def _spec_for(self, slug: str, deps: tuple[str, ...], desc: str) -> SubtaskSpec: - leaf_slug = ( - self.FAILING_LEAF_SLUG if slug in self.FAILING_SLUGS else self.leaf_slug - ) - return SubtaskSpec( - task_slug=TaskSlug(slug), - description=desc, - assigned_worker_slug=AssignedWorkerSlug(leaf_slug), - depends_on=[TaskSlug(d) for d in deps], - ) -``` - -Register these in `_fixtures/__init__.py` alongside the happy-path fixtures: - -```python -from tests.e2e._fixtures.workers.researchrubrics_smoke_sadpath import ( - ResearchRubricsSadPathSmokeWorker, - ResearchRubricsFailingLeafWorker, -) - -# inside register_smoke_fixtures(): -registry.register_worker(ResearchRubricsSadPathSmokeWorker) -registry.register_worker(ResearchRubricsFailingLeafWorker) -``` - -**No new criterion** for sad-path. The existing `ResearchRubricsSmokeCriterion` is reused; it will return a failed `CriterionResult` because `_check_children_completed` sees `l_2 != COMPLETED`, `l_3 != COMPLETED`. That's the correct outcome — we don't invent a new criterion, we prove the happy-path criterion correctly rejects a partially-failed run. - ---- - -## 4. Benchmarks - -No changes to the three production benchmarks' `Benchmark` subclasses. The existing `ResearchRubricsBenchmark`, `MiniF2FBenchmark`, `SweBenchVerifiedBenchmark` are used as-is. - -Smoke submission passes `--worker {env}-smoke-worker --evaluator {env}-smoke-criterion` against the existing benchmark slug. The benchmark's own sandbox image is used; smoke does not provision a separate image. - -The three existing `smoke_rubric.py` files in `ergon_builtins/benchmarks/{env}/` are **deleted**; smoke no longer uses rubric composition. Criterion is passed directly by slug. - ---- - -## 5. Worker ABC contract expectations - -Double-check against current code before coding: - -- `Worker.__init__(self, *, name, model, task_id, sandbox_id)`. -- `Worker.execute(self, task, *, context)` returns `AsyncGenerator[GenerationTurn, None]`. -- `Worker.get_output(self, context)` returns `WorkerOutput(output, success, metadata)`. -- `WorkerContext` exposes `run_id`, `node_id`, `sandbox_id`, `execution_id`. -- `Criterion` subclasses set `type_slug: ClassVar[str]` and implement `async evaluate(context) -> CriterionResult`. -- `EvaluationContext` exposes `execution_id` (plus whatever `_pull_children` needs). - -If any of these has moved by the time of the PR, the sketches must be updated; the invariants (no LLM, 9-subtask plan, sandbox connect by id, probe JSON artifact, env content check) remain. - ---- - -## 6. Unit test coverage for these fixtures - -Every class above gets a unit test under `tests/unit/` (no fixtures from `_fixtures/__init__.py` — unit tests construct classes directly): - -| Test | What it asserts | -|---|---| -| `tests/unit/test_smoke_worker_base_final.py` | `SmokeWorkerBase.execute` is `@final`; subclass cannot override | -| `tests/unit/test_smoke_worker_plans_9_subtasks.py` | Parent plans exactly the 9 expected slugs with correct `depends_on` (using a fake `plan_subtasks` recorder) | -| `tests/unit/test_smoke_worker_turn_count.py` | Parent yields exactly `PARENT_TURN_COUNT` (3) turns, in order: planning → planned → awaiting | -| `tests/unit/test_smoke_leaf_turn_count.py` | Leaf yields exactly `LEAF_TURN_COUNT` (2) turns, in order: attaching → done | -| `tests/unit/test_base_smoke_leaf.py` | Existing test — retarget to new import path | -| `tests/unit/test_smoke_criterion_shape.py` | `_check_graph_shape` raises on missing/extra slugs | -| `tests/unit/test_smoke_criterion_completed.py` | `_check_children_completed` raises on non-COMPLETED child | -| `tests/unit/test_smoke_criterion_probe.py` | `_check_probes_succeeded` raises on non-zero exit | -| `tests/unit/test_env_criterion_verify_content.py` × 3 | `_verify_env_content` per env (fixture bytes in, pass/fail out) | -| `tests/unit/test_env_criterion_sandbox_setup.py` × 3 | `_verify_sandbox_setup` per env using a fake `AsyncSandbox` that records commands and returns canned `CommandResult`s. Asserts: (a) criterion calls `connect(sandbox_id=...)` with `context.sandbox_id`, (b) passes if exit_code==0 + expected marker present, (c) raises `CriteriaCheckError` on non-zero exit. **Never** exercises real E2B — kept to unit tier. | -| `tests/unit/test_registry_smoke_entries.py` | After import of `tests.e2e._fixtures`, registry contains exactly the 11 expected slugs (9 happy-path + `researchrubrics-sadpath-smoke-worker` + `researchrubrics-smoke-leaf-failing`) and none of the retired slugs | -| `tests/unit/test_smoke_worker_spec_for_override.py` | `SmokeWorkerBase._spec_for` is called for each of 9 slugs; default returns `self.leaf_slug`; sad-path subclass routes only `l_2` to failing leaf | -| `tests/unit/test_always_fail_subworker.py` | `AlwaysFailSubworker.work` — using a fake `AsyncSandbox` recorder — (a) writes partial file, (b) runs `wc -l`, (c) raises `RuntimeError` naming the node. Order of operations must be write-then-probe-then-raise (not probe-then-write) so partial artifact is guaranteed persisted before the raise | -| `tests/unit/test_leaf_sends_completion_message.py` | `BaseSmokeLeafWorker._send_completion_message` is invoked exactly once per happy-path `execute`, with `from_agent_id == f"leaf-{task_slug}"`, `to_agent_id == "parent"`, `thread_topic == "smoke-completion"`. Uses a fake `communication_service.save_message` recorder | -| `tests/unit/test_failing_leaf_skips_message.py` | `BaseSmokeLeafWorker.execute` raises before reaching `_send_completion_message` when subworker raises — asserts recorder has ZERO calls on the failing leaf path | - -These unit tests run in `ci-fast` and don't need Docker. They catch 80% of regressions before we pay for an e2e matrix leg. diff --git a/docs/superpowers/plans/test-refactor/02-drivers-and-asserts.md b/docs/superpowers/plans/test-refactor/02-drivers-and-asserts.md deleted file mode 100644 index 3aaf9c3ef..000000000 --- a/docs/superpowers/plans/test-refactor/02-drivers-and-asserts.md +++ /dev/null @@ -1,829 +0,0 @@ -# 02 — Pytest drivers + per-env Postgres assertions - -**Status:** draft -**Scope:** three `tests/e2e/test_{env}_smoke.py` files. How they submit, how they wait, exactly what they assert on Postgres, and where Playwright is invoked. - -Cross-refs: shared fixtures in [`01-fixtures.md`](01-fixtures.md); Playwright spec shape in [`03-dashboard-and-playwright.md`](03-dashboard-and-playwright.md). - ---- - -## 1. Driver template (shared shape, 3 files) - -Every `test_{env}_smoke.py` follows this shape. Each driver defines a **cohort recipe** — a list of `(worker_slug, criterion_slug, kind)` per cohort slot. Minif2f + swebench-verified pass 3 × `"happy"`; researchrubrics passes 2 × `"happy"` + 1 × `"sad"`. The per-run assertion loop dispatches on `kind`. - -```python -# tests/e2e/test_researchrubrics_smoke.py -"""Researchrubrics canonical smoke — 3-run cohort (2 happy + 1 sad) against real E2B.""" - -from __future__ import annotations - -import asyncio -import os -import pathlib -import subprocess -import uuid -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Literal - -import pytest -from sqlmodel import select - -import tests.e2e._fixtures # noqa: F401 (registration hook) - -from ergon_cli.submit import submit_cohort -from ergon_core.core.persistence.graph.models import RunGraphNode, RunGraphMutation -from ergon_core.core.persistence.resources.models import RunResource -from ergon_core.core.persistence.evaluation.models import RunTaskEvaluation -from ergon_core.core.persistence.shared.db import get_session -from ergon_core.core.persistence.graph.status_conventions import COMPLETED - -from tests.e2e._fixtures.smoke_base.constants import EXPECTED_SUBTASK_SLUGS - -ENV = "researchrubrics" -HAPPY_WORKER = f"{ENV}-smoke-worker" -SAD_WORKER = f"{ENV}-sadpath-smoke-worker" -CRITERION = f"{ENV}-smoke-criterion" -PER_RUN_TIMEOUT = 270 # seconds; < pytest's 300s --timeout - - -@dataclass(frozen=True) -class CohortSlot: - worker_slug: str - criterion_slug: str - kind: Literal["happy", "sad"] - - -# Researchrubrics cohort: 2 happy + 1 sad. MiniF2F and SWE-bench drivers -# pass 3 × CohortSlot(HAPPY_WORKER, CRITERION, "happy"). -COHORT: tuple[CohortSlot, ...] = ( - CohortSlot(HAPPY_WORKER, CRITERION, "happy"), - CohortSlot(HAPPY_WORKER, CRITERION, "happy"), - CohortSlot(SAD_WORKER, CRITERION, "sad"), -) - - -@pytest.mark.e2e -@pytest.mark.asyncio -async def test_smoke_cohort(tmp_path: pathlib.Path) -> None: - cohort_key = f"ci-smoke-{ENV}-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S')}" - - # ── Phase 1: submit the cohort (may be mixed worker slugs) ───────── - run_ids: list[uuid.UUID] = await submit_cohort( - benchmark_slug=ENV, - slots=[(s.worker_slug, s.criterion_slug) for s in COHORT], - cohort_key=cohort_key, - ) - assert len(run_ids) == len(COHORT) - - # Pair each run_id with the slot that submitted it (preserves order). - slotted: list[tuple[CohortSlot, uuid.UUID]] = list(zip(COHORT, run_ids)) - - # ── Phase 2: wait for terminal state ─────────────────────────────── - await asyncio.gather(*( - wait_for_terminal(rid, timeout_seconds=PER_RUN_TIMEOUT) for rid in run_ids - )) - - # ── Phase 3: Postgres assertions (per run, dispatched on kind) ───── - for slot, rid in slotted: - if slot.kind == "happy": - _assert_happy_run(rid) - else: - _assert_sad_run(rid) - - # ── Phase 3b: cohort-level invariant (works for any mix) ─────────── - _assert_cohort_membership(cohort_key, run_ids) - - # ── Phase 4: Playwright subprocess (screenshots per run) ─────────── - _invoke_playwright( - env=ENV, - cohort_key=cohort_key, - cohort=[ - {"run_id": str(rid), "kind": slot.kind} - for slot, rid in slotted - ], - screenshot_dir=tmp_path / "playwright", - ) - - # Phase 5 (finalizer) is attached via fixture; see §4. - - -def _assert_happy_run(rid: uuid.UUID) -> None: - _assert_run_graph(rid) - _assert_run_resources(rid) - _assert_run_turn_counts(rid) # incremental turn persistence - _assert_sandbox_command_wal(rid) # §2.5a — bash cmds land in WAL - _assert_sandbox_lifecycle_events(rid) # §2.5b — created/command/closed - _assert_thread_messages_ordered(rid) # §2.5c — 9 per-leaf completion msgs - _assert_blob_roundtrip(rid) # §2.5d — bytes-in == bytes-out - _assert_temporal_ordering(rid) # §2.5e — DAG deps honoured in time - _assert_run_evaluation(rid) - _assert_env_content(rid) # per-env extension; §3 - - -def _assert_sad_run(rid: uuid.UUID) -> None: - # Sad-path-specific invariants (§10) - _assert_sadpath_graph_cascade(rid) - _assert_sadpath_partial_artifact(rid) - _assert_sadpath_partial_wal(rid) - _assert_sadpath_thread_messages(rid) # 8 msgs (l_2 missing) - _assert_sadpath_evaluation(rid) - # Shared helpers that still apply on the sad path: - _assert_sandbox_command_wal(rid) - _assert_sandbox_lifecycle_events(rid) # 9 created / 9 closed (l_3 never provisioned) - _assert_temporal_ordering(rid) # only edges whose child reached "started" state -``` - -The driver is intentionally boring. All structural assertions live in the named helpers below; `kind`-dispatch keeps happy and sad run invariants cleanly separated. **MiniF2F + SWE-bench drivers are the same file with `COHORT = (CohortSlot(HAPPY_WORKER, CRITERION, "happy"),) * 3`** — no sad slot. - ---- - -## 2. Shared assertion helpers (used by all 3 envs) - -Live in `tests/e2e/_asserts.py` (not under `_fixtures/`; these helpers are test-side, not registry-side). - -### 2.1 `_assert_run_graph(run_id)` — graph shape + completion - -```python -def _assert_run_graph(run_id: UUID) -> None: - with get_session() as s: - nodes = s.exec( - select(RunGraphNode) - .where(RunGraphNode.run_id == run_id) - .order_by(RunGraphNode.depth, RunGraphNode.task_slug), - ).all() - - # 1 root + 9 subtasks = 10 total nodes - assert len(nodes) == 10, f"expected 10 nodes, got {len(nodes)}" - root = next(n for n in nodes if n.depth == 0) - leaves = [n for n in nodes if n.depth > 0] - assert len(leaves) == 9 - assert sorted(n.task_slug for n in leaves) == sorted(EXPECTED_SUBTASK_SLUGS) - assert all(n.status == COMPLETED for n in nodes), \ - f"non-completed nodes: {[(n.task_slug, n.status) for n in nodes if n.status != COMPLETED]}" - assert root.status == COMPLETED - - # Structural dep check: d_join depends on d_left + d_right; l_3 depends on l_2. - by_slug = {n.task_slug: n for n in leaves} - _assert_deps(by_slug, "d_join", {"d_left", "d_right"}) - _assert_deps(by_slug, "d_left", {"d_root"}) - _assert_deps(by_slug, "d_right", {"d_root"}) - _assert_deps(by_slug, "l_2", {"l_1"}) - _assert_deps(by_slug, "l_3", {"l_2"}) - for roots in ("d_root", "l_1", "s_a", "s_b"): - _assert_deps(by_slug, roots, set()) -``` - -`_assert_deps` reads `RunGraphEdge` (or whatever the edge table is called today) and compares parent-sets. Sketch only; must match the current schema. - -### 2.2 `_assert_run_resources(run_id)` — 9 `RunResource` rows + optional artifact probes - -```python -def _assert_run_resources(run_id: UUID) -> None: - with get_session() as s: - resources = s.exec( - select(RunResource).where(RunResource.run_id == run_id), - ).all() - - # Each leaf writes at minimum: 1 output file + 1 probe.json = 18 resources - # (9 outputs + 9 probes). The parent does not publish resources. - assert len(resources) >= 18, \ - f"expected >= 18 resources (9 outputs + 9 probes), got {len(resources)}" - probes = [r for r in resources if r.name.startswith("probe_") and r.name.endswith(".json")] - assert len(probes) == 9 - assert all(r.content_hash for r in resources), \ - "some resources missing content_hash" -``` - -### 2.3 `_assert_run_turn_counts(run_id)` — multi-turn persistence invariant - -Protects the multi-turn fidelity choice documented in [`01-fixtures.md §2.6`](01-fixtures.md). Imports the `ClassVar`s off the base classes so the numbers can change in lock-step without editing this driver. - -```python -from tests.e2e._fixtures.smoke_base.worker_base import SmokeWorkerBase -from tests.e2e._fixtures.smoke_base.leaf_base import BaseSmokeLeafWorker - - -def _assert_run_turn_counts(run_id: UUID) -> None: - """1 parent × PARENT_TURN_COUNT + 9 leaves × LEAF_TURN_COUNT GenerationTurn rows.""" - expected = ( - SmokeWorkerBase.PARENT_TURN_COUNT - + 9 * BaseSmokeLeafWorker.LEAF_TURN_COUNT - ) # currently 3 + 18 = 21 - - with get_session() as s: - turns = s.exec( - select(GenerationTurn) - .where(GenerationTurn.run_id == run_id) - .order_by(GenerationTurn.sequence), - ).all() - - assert len(turns) == expected, ( - f"turn count mismatch: expected {expected} " - f"(parent={SmokeWorkerBase.PARENT_TURN_COUNT}, " - f"leaves=9×{BaseSmokeLeafWorker.LEAF_TURN_COUNT}), got {len(turns)}" - ) - - # Sequence must be strictly monotonic and dense (no gaps). - seqs = [t.sequence for t in turns] - assert seqs == sorted(seqs), f"turn sequences not monotonic: {seqs}" - assert seqs == list(range(seqs[0], seqs[0] + len(seqs))), \ - f"turn sequences not dense: {seqs}" -``` - -Sequence-density check catches regressions where `GenerationTurnRepository` silently drops a turn mid-stream. - -### 2.4 `_assert_run_evaluation(run_id)` — exactly 1 evaluation, score 1.0 - -```python -def _assert_run_evaluation(run_id: UUID) -> None: - with get_session() as s: - evals = s.exec( - select(RunTaskEvaluation).where(RunTaskEvaluation.run_id == run_id), - ).all() - assert len(evals) == 1, f"expected 1 evaluation, got {len(evals)}" - assert evals[0].score == 1.0, \ - f"expected score 1.0, got {evals[0].score} ({evals[0].feedback!r})" - assert evals[0].passed is True -``` - -### 2.5a `_assert_sandbox_command_wal(run_id)` — bash commands recorded - -Each leaf runs at least one `sandbox.commands.run(...)` (its probe); the `BaseSandboxManager._emit_wal_entry` path writes one WAL row per command. Happy-path assertion: **at least 9 probe commands in the WAL for the run** (one per leaf), all with `exit_code == 0`. - -```python -from ergon_core.core.persistence.sandbox.models import SandboxCommandWalEntry # or current path - -def _assert_sandbox_command_wal(run_id: UUID) -> None: - with get_session() as s: - entries = s.exec( - select(SandboxCommandWalEntry) - .where(SandboxCommandWalEntry.run_id == run_id) - .order_by(SandboxCommandWalEntry.created_at), - ).all() - - # Each leaf runs at least 1 probe command. Env-specific health check in - # the criterion adds 1 more on the parent. Minimum = 9 (leaves) + 1 (criterion). - probes = [e for e in entries if "probe" in e.command or "wc" in e.command - or "lean --check" in e.command or "py_compile" in e.command] - assert len(probes) >= 9, f"expected ≥9 probe commands in WAL, got {len(probes)}" - - # Happy path: every recorded exit_code is 0 (or None for commands we didn't - # instrument). Any non-zero is a real probe failure, not a silent pass. - non_zero = [(e.command, e.exit_code) for e in entries if e.exit_code not in (0, None)] - assert not non_zero, f"non-zero exit_codes in WAL (happy path): {non_zero}" -``` - -Schema path is a sketch — replace with the current WAL table name. If WAL entries are stored alongside sandbox events rather than in a dedicated table, read from wherever `_emit_wal_entry` actually persists. - -### 2.5b `_assert_sandbox_lifecycle_events(run_id)` — created/command/closed trio - -The `SandboxEventSink` Protocol has three methods: `sandbox_created`, `sandbox_command`, `sandbox_closed`. `BaseSandboxManager.close()` fires `sandbox_closed` in a `finally` block, so the happy path must produce one `created` + one `closed` per sandbox, with `sandbox_command`s interleaved. - -```python -from ergon_core.core.persistence.sandbox.models import SandboxEvent # or current path - -def _assert_sandbox_lifecycle_events(run_id: UUID) -> None: - with get_session() as s: - events = s.exec( - select(SandboxEvent) - .where(SandboxEvent.run_id == run_id) - .order_by(SandboxEvent.created_at), - ).all() - - # Sandboxes this run: 1 parent + 9 leaves = 10 - created = [e for e in events if e.kind == "sandbox_created"] - closed = [e for e in events if e.kind == "sandbox_closed"] - assert len(created) == 10, f"expected 10 sandbox_created events, got {len(created)}" - assert len(closed) == 10, f"expected 10 sandbox_closed events, got {len(closed)}" - - # Each (sandbox_id, created) must have a matching (sandbox_id, closed) - created_ids = {e.sandbox_id for e in created} - closed_ids = {e.sandbox_id for e in closed} - assert created_ids == closed_ids, ( - f"created/closed sandbox_id mismatch: " - f"only created = {created_ids - closed_ids}, only closed = {closed_ids - created_ids}" - ) - - # Ordering: for each sandbox_id, created.created_at < all its commands < closed.created_at. - by_id: dict[str, list[SandboxEvent]] = {} - for e in events: - by_id.setdefault(e.sandbox_id, []).append(e) - for sid, group in by_id.items(): - first, *_, last = sorted(group, key=lambda e: e.created_at) - assert first.kind == "sandbox_created", f"{sid}: first event is {first.kind}" - assert last.kind == "sandbox_closed", f"{sid}: last event is {last.kind}" -``` - -Kind-name strings (`"sandbox_created"`, `"sandbox_command"`, `"sandbox_closed"`) must match the SandboxEventSink persistence layer's current convention. - -### 2.5c `_assert_thread_messages_ordered(run_id)` — inter-agent messaging - -Each happy-path leaf calls `BaseSmokeLeafWorker._send_completion_message` exactly once. Expect 9 `ThreadMessage` rows on topic `"smoke-completion"` for this run. - -```python -from ergon_core.core.persistence.telemetry.models import Thread, ThreadMessage - -def _assert_thread_messages_ordered(run_id: UUID) -> None: - with get_session() as s: - threads = s.exec( - select(Thread) - .where(Thread.run_id == run_id) - .where(Thread.topic == "smoke-completion"), - ).all() - assert len(threads) == 1, ( - f"expected 1 smoke-completion thread for run, got {len(threads)}" - ) - thread = threads[0] - assert thread.agent_a_id in ("parent",), f"unexpected agent_a_id: {thread.agent_a_id}" - - msgs = s.exec( - select(ThreadMessage) - .where(ThreadMessage.thread_id == thread.id) - .order_by(ThreadMessage.sequence_num), - ).all() - - # 9 messages, one per leaf. - assert len(msgs) == 9, f"expected 9 completion messages, got {len(msgs)}" - - # sequence_num is per-thread monotonic starting at 1. - seqs = [m.sequence_num for m in msgs] - assert seqs == list(range(1, 10)), f"sequence_num not 1..9: {seqs}" - - # Each message is from a distinct leaf, targeting the parent. - from_ids = [m.from_agent_id for m in msgs] - assert set(from_ids) == {f"leaf-{slug}" for slug in EXPECTED_SUBTASK_SLUGS}, ( - f"from_agent_id set mismatch: {sorted(from_ids)}" - ) - assert all(m.to_agent_id == "parent" for m in msgs), ( - f"non-parent to_agent_id: {[m.to_agent_id for m in msgs if m.to_agent_id != 'parent']}" - ) - - # Creation timestamps must not go backwards (sequence_num already enforces - # thread-local order; this pins wall-clock order too). - ts = [m.created_at for m in msgs] - assert ts == sorted(ts), f"message timestamps not monotonic: {ts}" - - # task_execution_id FK present on every message. - assert all(m.task_execution_id is not None for m in msgs), ( - "some messages missing task_execution_id FK" - ) -``` - -Sad-path counterpart asserts **8** messages with `l_2` missing from `from_ids`. See §10. - -### 2.5d `_assert_blob_roundtrip(run_id)` — bytes-in == bytes-out - -Picks one leaf's output artifact, re-reads it from blob storage, and confirms the bytes match what the leaf wrote. Cheap catch for blob-store truncation or content-hash drift. - -```python -def _assert_blob_roundtrip(run_id: UUID) -> None: - with get_session() as s: - r = s.exec( - select(RunResource) - .where(RunResource.run_id == run_id) - .where(RunResource.name.like("probe_%.json")) - .order_by(RunResource.created_at) - .limit(1), - ).first() - assert r is not None, "no probe_*.json to round-trip" - assert r.content_hash, "resource missing content_hash" - - bytes_from_blob = BlobClient.default().get_sync(r.content_hash) - # Minimal content-shape check: parses as JSON and has an exit_code key. - parsed = json.loads(bytes_from_blob) - assert "exit_code" in parsed, f"probe JSON missing exit_code: {parsed!r}" - - # Bytes should be identical on two fetches (idempotent read). - bytes_again = BlobClient.default().get_sync(r.content_hash) - assert bytes_from_blob == bytes_again, "blob read non-deterministic" -``` - -### 2.5e `_assert_temporal_ordering(run_id)` — DAG deps honoured in time - -Graph dep check (§2.1) is structural. This one asserts **schedule**: `d_join.started_at >= max(d_left.completed_at, d_right.completed_at)`, etc. - -```python -def _assert_temporal_ordering(run_id: UUID) -> None: - with get_session() as s: - leaves = s.exec( - select(RunGraphNode) - .where(RunGraphNode.run_id == run_id) - .where(RunGraphNode.depth > 0), - ).all() - by_slug = {n.task_slug: n for n in leaves} - - def _after(child: str, parents: list[str]) -> None: - child_started = by_slug[child].started_at - latest_parent_done = max(by_slug[p].completed_at for p in parents) - assert child_started >= latest_parent_done, ( - f"{child}.started_at ({child_started}) < " - f"max({parents}).completed_at ({latest_parent_done})" - ) - - _after("d_join", ["d_left", "d_right"]) - _after("d_left", ["d_root"]) - _after("d_right", ["d_root"]) - _after("l_2", ["l_1"]) - _after("l_3", ["l_2"]) -``` - -Only asserts ordering across dependency edges; zero-dep slugs (`d_root`, `l_1`, `s_a`, `s_b`) are free to run in any order. - -### 2.5f `_assert_cohort_membership(cohort_key, run_ids)` — cohort view on BE side - -Playwright checks the UI; this pins the same invariant in Python against the harness read DTO. - -```python -def _assert_cohort_membership(cohort_key: str, run_ids: list[UUID]) -> None: - import requests - r = requests.get( - f"{os.environ['ERGON_API_BASE_URL']}/api/test/read/cohort/{cohort_key}/runs", - headers={"X-Test-Secret": os.environ["TEST_HARNESS_SECRET"]}, - timeout=10, - ) - r.raise_for_status() - cohort_runs = r.json() - returned_ids = {row["run_id"] for row in cohort_runs} - expected_ids = {str(rid) for rid in run_ids} - assert returned_ids == expected_ids, ( - f"cohort membership mismatch: " - f"only returned = {returned_ids - expected_ids}, " - f"only expected = {expected_ids - returned_ids}" - ) -``` - -### 2.5 `_assert_mutations_ordered(run_id)` — optional extra - -Not every assertion helper needs to run for every PR. `_assert_mutations_ordered` reads `RunGraphMutation` and asserts that: - -- First mutation is `add_subtask` for either a diamond root, line root, or singleton (any of the four zero-dep slugs). -- `d_join` add precedes `d_join` start. -- All 9 `add_subtask` mutations exist with `parent_id == root.id`. - -Include this in the driver only once the cheaper asserts are green in CI — it's a deeper probe of the mutation log that catches sequence regressions, not completion regressions. - ---- - -## 3. Per-env `_assert_env_content` - -### 3.1 ResearchRubrics - -```python -def _assert_env_content(run_id: UUID) -> None: - """One markdown report per leaf, non-empty, correct header.""" - outputs = _pull_artifacts(run_id, name_like="report_%.md") - assert len(outputs) == 9 - for r in outputs: - body = BlobClient.default().get_sync(r.content_hash).decode("utf-8") - assert body.startswith("# Research report"), \ - f"{r.node_id}: missing `# Research report` header" - assert len(body.strip()) >= 20, \ - f"{r.node_id}: body shorter than 20 bytes" -``` - -### 3.2 MiniF2F - -```python -def _assert_env_content(run_id: UUID) -> None: - """One .lean file per leaf, parses, contains `theorem smoke_trivial`.""" - outputs = _pull_artifacts(run_id, name_like="proof_%.lean") - assert len(outputs) == 9 - for r in outputs: - src = BlobClient.default().get_sync(r.content_hash).decode("utf-8") - assert "theorem smoke_trivial" in src - assert ":=" in src -``` - -### 3.3 SWE-Bench Verified - -```python -import ast - -def _assert_env_content(run_id: UUID) -> None: - """One .py file per leaf, valid AST, contains function `add`.""" - outputs = _pull_artifacts(run_id, name_like="patch_%.py") - assert len(outputs) == 9 - for r in outputs: - src = BlobClient.default().get_sync(r.content_hash).decode("utf-8") - tree = ast.parse(src) - funcs = {n.name for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)} - assert "add" in funcs, f"{r.node_id}: expected `add`, got {sorted(funcs)}" -``` - -These assertions duplicate what `SmokeCriterionBase._verify_env_content` does on the criterion side — that is intentional. The **criterion runs inside the workflow and writes an evaluation row**; the **driver reruns the check out-of-band against the same artifacts**. If the criterion regresses silently, the driver catches it (and vice versa). - ---- - -## 4. Phase 4: Playwright subprocess - -```python -def _invoke_playwright( - env: str, cohort_key: str, run_ids: list[UUID], screenshot_dir: pathlib.Path, -) -> None: - screenshot_dir.mkdir(parents=True, exist_ok=True) - result = subprocess.run( - [ - "pnpm", "--dir", "ergon-dashboard", "exec", "playwright", - "test", f"tests/e2e/{env}.smoke.spec.ts", - "--project=chromium", - ], - env={ - **os.environ, - "COHORT_KEY": cohort_key, - "RUN_IDS": ",".join(str(r) for r in run_ids), - "SMOKE_ENV": env, - "SCREENSHOT_DIR": str(screenshot_dir), - "PLAYWRIGHT_LIVE": "1", - "PLAYWRIGHT_BASE_URL": "http://127.0.0.1:3000", - "ERGON_API_BASE_URL": "http://127.0.0.1:9000", - "TEST_HARNESS_SECRET": os.environ["TEST_HARNESS_SECRET"], - }, - check=False, # do not raise — screenshots still need uploading on fail - ) - if result.returncode != 0: - pytest.fail(f"playwright spec failed (returncode={result.returncode})") -``` - -Always runs, even if phase-3 assertions failed (so the dashboard state at time of failure is captured). Pytest own CI pass/fail — Playwright failure fails the test via `pytest.fail`. - ---- - -## 5. Phase 5: screenshot finalizer - -Implemented as a session-scoped pytest finalizer fixture (autouse in `tests/e2e/conftest.py`). - -```python -# tests/e2e/conftest.py (excerpt) - -@pytest.fixture(autouse=True, scope="session") -def _screenshot_uploader(): - yield - pr_number = os.environ.get("GITHUB_PR_NUMBER") - screenshot_dir = os.environ.get("SCREENSHOT_DIR", "/tmp/playwright") - env = os.environ.get("SMOKE_ENV", "unknown") - if not pr_number: - return # local run — skip - subprocess.run( - ["bash", "ci/push_screenshots.sh", pr_number, env, screenshot_dir], - check=False, - ) -``` - -`ci/push_screenshots.sh` is specified in [`04-ci-and-workflows.md §4`](04-ci-and-workflows.md). - ---- - -## 6. `submit_cohort` helper - -Not yet a CLI surface. Minimal implementation — accepts a list of `(worker_slug, criterion_slug)` tuples so cohorts can be heterogeneous (e.g. 2 happy + 1 sad): - -```python -# ergon_cli/submit.py (or wherever current CLI submit lives) - -async def submit_cohort( - *, benchmark_slug: str, - slots: list[tuple[str, str]], # [(worker_slug, criterion_slug), …] - cohort_key: str, -) -> list[UUID]: - """Submit one run per slot, all with the same cohort_key. - - Returns run_ids in the same order as `slots`. Parallel submission is via - asyncio.gather. Under the hood this calls whatever today does `ergon run`; - the cross-cutting additions are (a) heterogeneous worker slugs per slot - and (b) stable cohort_key. - """ - ... -``` - -If the CLI already supports cohort submission, extend it for heterogeneous slots. Otherwise a 20-line wrapper over the existing single-run submit path is fine — it is test-only. - ---- - -## 7. `wait_for_terminal` - -Polls `/runs/{run_id}` (or the equivalent service call) every 2 seconds until status ∈ {completed, failed, cancelled}. Returns the terminal status. Raises `TimeoutError` on timeout. - -```python -TERMINAL = {"completed", "failed", "cancelled"} - -async def wait_for_terminal(run_id: UUID, timeout_seconds: int) -> str: - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - status = await _get_run_status(run_id) - if status in TERMINAL: - return status - await asyncio.sleep(2) - raise TimeoutError(f"run {run_id} did not reach terminal status within {timeout_seconds}s") -``` - -The driver does **not** assert on the terminal status value here — it asserts after via `_assert_run_evaluation` (score == 1.0 implies completed-clean). This keeps "did we finish?" and "did we finish correctly?" separate. - ---- - -## 8. Failure modes worth naming - -| Failure | Which assertion catches it | -|---|---| -| Inngest function crashes mid-run | `wait_for_terminal` times out | -| Graph edge table wrong | `_assert_deps` in `_assert_run_graph` | -| A leaf subtask stuck in `IN_PROGRESS` | `_assert_run_graph` node status check | -| Sandbox provision fails partway | `wait_for_terminal` returns `failed`; `_assert_run_evaluation` fails (score != 1.0) | -| Leaf writes file but probe JSON missing | `_assert_run_resources` probe-count check | -| `GenerationTurn` rows dropped / sequence gap | `_assert_run_turn_counts` | -| Parent yields wrong number of turns | `_assert_run_turn_counts` (catches silent divergence from `PARENT_TURN_COUNT`) | -| Sandbox `commands.run` doesn't emit WAL row | `_assert_sandbox_command_wal` | -| `sandbox_closed` not fired on teardown | `_assert_sandbox_lifecycle_events` (mismatched created/closed counts) | -| `CommunicationService.save_message` regression | `_assert_thread_messages_ordered` (message count / ordering / FK) | -| Blob-store returns wrong bytes for a hash | `_assert_blob_roundtrip` | -| DAG dep violated at scheduling time (child runs before parent done) | `_assert_temporal_ordering` | -| Cohort view misses a run | `_assert_cohort_membership` | -| Criterion writes evaluation with score 0 | `_assert_run_evaluation` | -| Partial artifacts lost when leaf fails | `_assert_sadpath_partial_artifact` (sad-path driver) | -| Sandbox command WAL not written for pre-failure commands | `_assert_sadpath_partial_wal` | -| Failed leaf still marked COMPLETED | `_assert_sadpath_graph_cascade` | -| Static-sibling failure cascade missing on l_3 | `_assert_sadpath_graph_cascade` | -| Failed leaf sends completion message anyway | `_assert_sadpath_thread_messages` | -| Dashboard missing graph UI | Playwright spec | -| Dashboard rerender broken on cohort index | Playwright spec ([`03-dashboard-and-playwright.md §5`](03-dashboard-and-playwright.md)) | - -The driver's assertion order is chosen so the first failure points squarely at a layer (runtime, graph, resource, eval, UI) instead of at a cascade. - ---- - -## 9. Migration note — 3-run cohort asserts - -Every helper above takes a **single `run_id`** and is called in a loop for all 3 cohort runs. This is intentional: each run must independently pass every check. No "at least one passed" fallback; no cohort-level aggregation that could hide a per-run regression. - -If a cohort-level invariant becomes useful later (e.g. "all 3 runs produced identical resource counts"), add it as a separate helper after the per-run loop — do not merge into the per-run check. - ---- - -## 10. Sad-run assertion helpers (researchrubrics cohort slot 3) - -The researchrubrics cohort's third slot uses `ResearchRubricsSadPathSmokeWorker`. It is submitted through the same `submit_cohort` call as the 2 happy slots (§1) and runs in parallel with them under the same `cohort_key`. Only the per-run **assertion dispatch** differs: `_assert_sad_run` below replaces the happy-path block. - -There is **no separate driver file** and **no fourth matrix leg**. The sad run lives inside `tests/e2e/test_researchrubrics_smoke.py`. MiniF2F + SWE-bench drivers have no sad slot. - -### 10.1 Sad-path specific helpers - -```python -def _assert_sadpath_graph_cascade(run_id: UUID) -> None: - """Line cascade failure: l_1 done, l_2 failed, l_3 blocked/cancelled. - Diamond + singletons unaffected.""" - with get_session() as s: - leaves = s.exec( - select(RunGraphNode) - .where(RunGraphNode.run_id == run_id) - .where(RunGraphNode.depth > 0), - ).all() - by_slug = {n.task_slug: n for n in leaves} - - # Line: cascade of deterministic failure - assert by_slug["l_1"].status == COMPLETED, by_slug["l_1"].status - assert by_slug["l_2"].status == FAILED, by_slug["l_2"].status - assert by_slug["l_3"].status in {BLOCKED, CANCELLED}, ( - f"l_3 expected BLOCKED or CANCELLED per static-sibling-failure-semantics RFC, " - f"got {by_slug['l_3'].status}" - ) - - # Diamond + singletons: independent branches, must still all COMPLETED. - for slug in ("d_root", "d_left", "d_right", "d_join", "s_a", "s_b"): - assert by_slug[slug].status == COMPLETED, ( - f"{slug} expected COMPLETED (independent branch), got {by_slug[slug].status}" - ) - - -def _assert_sadpath_partial_artifact(run_id: UUID) -> None: - """AlwaysFailSubworker writes `partial_<node>.md` BEFORE raising. - The runtime's persist step must still serialize it as a RunResource.""" - with get_session() as s: - partials = s.exec( - select(RunResource) - .where(RunResource.run_id == run_id) - .where(RunResource.name.like("partial_%.md")), - ).all() - # Only l_2 fails, so exactly 1 partial artifact. - assert len(partials) == 1, ( - f"expected 1 partial artifact from l_2 (partial work must persist on " - f"FAILED leaf), got {len(partials)}" - ) - r = partials[0] - assert r.content_hash, f"partial resource missing content_hash" - body = BlobClient.default().get_sync(r.content_hash).decode("utf-8") - assert body.startswith("# Partial work"), ( - f"partial artifact body unexpected: {body[:80]!r}" - ) - - -def _assert_sadpath_partial_wal(run_id: UUID) -> None: - """The pre-failure `wc -l` command must land as a sandbox_command WAL row - even though the leaf ultimately failed.""" - with get_session() as s: - entries = s.exec( - select(SandboxCommandWalEntry) - .where(SandboxCommandWalEntry.run_id == run_id), - ).all() - wc_entries = [e for e in entries if "wc -l" in e.command and "partial_" in e.command] - assert len(wc_entries) >= 1, ( - f"expected ≥1 'wc -l partial_*' WAL entry from pre-failure probe; " - f"sandbox_command path did not persist the command before the raise" - ) - # That command should have succeeded (exit 0) — the failure is the raise - # AFTER the command, not the command itself. - assert all(e.exit_code == 0 for e in wc_entries), ( - f"pre-failure wc probe non-zero exit: {[(e.command, e.exit_code) for e in wc_entries]}" - ) - - -def _assert_sadpath_thread_messages(run_id: UUID) -> None: - """Happy path sends 9 completion messages; l_2 raises before sending. - Expect 8 on the smoke-completion thread, l_2 missing.""" - with get_session() as s: - thread = s.exec( - select(Thread) - .where(Thread.run_id == run_id) - .where(Thread.topic == "smoke-completion"), - ).first() - assert thread is not None, "no smoke-completion thread created" - msgs = s.exec( - select(ThreadMessage) - .where(ThreadMessage.thread_id == thread.id) - .order_by(ThreadMessage.sequence_num), - ).all() - assert len(msgs) == 8, ( - f"expected 8 completion messages (l_2 raises before sending), got {len(msgs)}" - ) - from_slugs = {m.from_agent_id.removeprefix("leaf-") for m in msgs} - assert "l_2" not in from_slugs, ( - f"l_2 sent a completion message despite raising: {from_slugs}" - ) - assert from_slugs == set(EXPECTED_SUBTASK_SLUGS) - {"l_2"} - - -def _assert_sadpath_evaluation(run_id: UUID) -> None: - """Reusing happy-path criterion on sad-path run MUST return score 0. - feedback should name the failing slug so operators see it quickly.""" - with get_session() as s: - evals = s.exec( - select(RunTaskEvaluation).where(RunTaskEvaluation.run_id == run_id), - ).all() - assert len(evals) == 1 - assert evals[0].score == 0.0 - assert evals[0].passed is False - # Criterion's `_check_children_completed` raises with "l_2 not completed" - # or similar — just assert the slug appears for operator-visibility. - assert "l_2" in (evals[0].feedback or ""), ( - f"sad-path evaluation feedback should mention l_2; got: {evals[0].feedback!r}" - ) -``` - -### 10.2 Playwright delta for the sad run - -The driver passes a `cohort: [{ run_id, kind }, …]` array to Playwright (see `_invoke_playwright` in §1). The existing `researchrubrics.smoke.spec.ts` — built from the shared factory in `_shared/smoke.ts` — iterates that list and branches on `kind`: - -```typescript -// ergon-dashboard/tests/e2e/_shared/smoke.ts (excerpt, added inside defineSmokeSpec) - -for (const { run_id, kind } of cohort) { - test(`run ${run_id} (${kind})`, async ({ page }) => { - if (kind === "happy") { - // existing happy-path assertions (§5 of 03-dashboard-and-playwright.md) - } else { - // sad: l_2 FAILED, l_3 BLOCKED/CANCELLED, everything else COMPLETED - await page.goto(`/run/${run_id}`); - await expect(page.getByTestId("run-status")).toHaveText(/failed/i); - await expect(page.getByTestId("task-node-l_2")) - .toHaveAttribute("data-status", "failed"); - await expect(page.getByTestId("task-node-l_3")) - .toHaveAttribute("data-status", /blocked|cancelled/); - for (const slug of ["d_root", "d_left", "d_right", "d_join", "s_a", "s_b", "l_1"]) { - await expect(page.getByTestId(`task-node-${slug}`)) - .toHaveAttribute("data-status", "completed"); - } - await page.screenshot({ - path: `${screenshotDir}/${run_id}-sad-failed.png`, fullPage: true, - }); - } - }); -} -``` - -No separate spec file. No separate Playwright project. The sad run is visible in the cohort index alongside the 2 happy runs — and the Playwright spec's cohort assertion (3 runs listed) still passes. - -### 10.3 CI sequencing - -No fourth matrix leg. The researchrubrics leg's cohort-of-3 naturally contains the sad run in slot 3; `e2e-benchmarks.yml` matrix stays at `[researchrubrics, minif2f, swebench-verified]`. The sad run adds zero top-level runs and replaces one happy leaf-count-9 with a leaf-count-8 (l_3 not provisioned), for a net **-1** sandbox vs an all-happy matrix. - -### 10.4 What the sad run specifically catches - -| Regression | Caught by | -|---|---| -| Partial artifacts dropped when a leaf fails | `_assert_sadpath_partial_artifact` | -| Sandbox command WAL not written until success | `_assert_sadpath_partial_wal` | -| Leaf exception silently converted to COMPLETED | `_assert_sadpath_graph_cascade` (l_2 must be FAILED) | -| Static-sibling failure cascade not implemented | `_assert_sadpath_graph_cascade` (l_3 check) | -| Dynamic/static subtask semantics conflated (l_3 COMPLETED anyway) | same | -| Independent branches accidentally cancelled when a sibling fails | `_assert_sadpath_graph_cascade` (diamond + singletons) | -| Completion message sent from a failed leaf | `_assert_sadpath_thread_messages` (l_2 must NOT appear) | -| Evaluation silently returns score=1.0 on a half-failed run | `_assert_sadpath_evaluation` | diff --git a/docs/superpowers/plans/test-refactor/03-dashboard-and-playwright.md b/docs/superpowers/plans/test-refactor/03-dashboard-and-playwright.md deleted file mode 100644 index 4005eacc2..000000000 --- a/docs/superpowers/plans/test-refactor/03-dashboard-and-playwright.md +++ /dev/null @@ -1,278 +0,0 @@ -# 03 — Dashboard harness + Playwright specs + screenshots - -**Status:** draft -**Scope:** every HTTP boundary and UI assertion the smoke tier depends on. Existing wiring catalogued in §1; new contract nailed down in §2–§3; Playwright specs in §5. - -Cross-refs: driver calls Playwright from [`02-drivers-and-asserts.md §4`](02-drivers-and-asserts.md); CI invocation in [`04-ci-and-workflows.md`](04-ci-and-workflows.md). - ---- - -## 1. What exists today (catalogue, before edits) - -### 1.1 Backend harness - -- `ergon_core/core/api/test_harness.py` — FastAPI router mounted conditionally on `ENABLE_TEST_HARNESS=1`. -- Mount gate covered by `tests/unit/test_app_mounts_harness_conditionally.py`. - -### 1.2 Next.js dashboard routes (under `/api/test/*`) - -- `src/app/api/test/dashboard/seed/route.ts` — seed cohort/run state for UI fixtures. -- `src/app/api/test/dashboard/reset/route.ts` — reset in-memory state. -- `src/app/api/test/dashboard/events/run-complete/route.ts` — inject run-completion event. -- `src/app/api/test/dashboard/events/thread-message/route.ts` — inject chat/turn event. -- `src/app/api/test/dashboard/events/task-evaluation/route.ts` — inject evaluation event. - -### 1.3 TypeScript client - -- `ergon-dashboard/tests/helpers/testHarnessClient.ts` — wraps the dashboard `/api/test/*` routes. - -### 1.4 Real-LLM harness - -- `tests/real_llm/fixtures/harness_client.py` — Python client for the backend harness. -- `tests/real_llm/fixtures/playwright_client.py` — subprocess wrapper. -- `tests/real_llm/fixtures/stack.py` — Docker stack orchestration. - ---- - -## 2. Harness contract (backend, consumed by smoke) - -Smoke drivers + Playwright specs agree on **one** backend harness contract. Everything else is dashboard-internal. - -### 2.1 Gates - -- **Mount:** `ENABLE_TEST_HARNESS=1`. -- **Per-request:** every `/api/test/*` request must carry `X-Test-Secret: ${TEST_HARNESS_SECRET}` or `?test_secret=...`. Requests without a matching secret 401. -- **Secret rotation:** CI generates a fresh per-job random value, exports it to both backend and Playwright subprocess. Local dev pins via `.env.test`. - -### 2.2 Read endpoints (what Playwright calls) - -| Method | Path | Returns | -|---|---|---| -| GET | `/api/test/read/run/{run_id}/state` | Narrow DTO: `{ run_id, status, node_count, leaf_statuses: [{slug, status}], evaluation: {score, passed} \| null }` | -| GET | `/api/test/read/cohort/{cohort_key}/runs` | `[{run_id, status}]` | -| GET | `/api/test/read/run/{run_id}/resources` | `[{name, content_hash, size}]` — metadata only, no blob bytes | - -Narrow DTOs — **not** raw SQL row dumps. These are contract types; the dashboard or Playwright must not depend on schema columns it doesn't need. - -### 2.3 Write endpoints (optional, secret-gated) - -Already-implemented Next.js routes (`events/run-complete` etc.) exist for dashboard-fixture injection. Not used by the smoke path — Playwright reads real state via §2.2. - -**Decision (pinned in `00-program.md §6.4`):** keep the Next.js write routes as-is. They serve dashboard dev loop (seed state without standing up a full backend); smoke doesn't rely on them. - -### 2.4 `BackendHarnessClient` (TS) - -```typescript -// ergon-dashboard/tests/helpers/backendHarnessClient.ts -// -// One client, one contract. Reads only. Throws on non-2xx so Playwright -// specs don't silently assert against undefined. - -export interface RunState { - run_id: string; - status: "completed" | "failed" | "cancelled" | "in_progress"; - node_count: number; - leaf_statuses: { slug: string; status: string }[]; - evaluation: { score: number; passed: boolean } | null; -} - -export class BackendHarnessClient { - constructor( - private readonly baseUrl: string, - private readonly secret: string, - ) {} - - async getRunState(runId: string): Promise<RunState> { - const r = await fetch(`${this.baseUrl}/api/test/read/run/${runId}/state`, { - headers: { "X-Test-Secret": this.secret }, - }); - if (!r.ok) throw new Error(`harness ${r.status}: ${await r.text()}`); - return r.json(); - } - - async getCohortRuns(cohortKey: string) { - const r = await fetch( - `${this.baseUrl}/api/test/read/cohort/${cohortKey}/runs`, - { headers: { "X-Test-Secret": this.secret } }, - ); - if (!r.ok) throw new Error(`harness ${r.status}`); - return r.json() as Promise<{ run_id: string; status: string }[]>; - } -} -``` - -The existing `tests/helpers/testHarnessClient.ts` targets the Next.js `/api/test/dashboard/*` routes (dashboard-side harness). The new `backendHarnessClient.ts` targets the backend `/api/test/*` router. Two clients, two contracts; smoke specs import `BackendHarnessClient` only. - ---- - -## 3. Playwright spec template (shared across 3 envs) - -`ergon-dashboard/tests/e2e/_shared/smoke.ts` — factory: - -```typescript -import { test, expect, Page } from "@playwright/test"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; - -import { BackendHarnessClient } from "../../helpers/backendHarnessClient"; - -export interface SmokeSpecConfig { - env: string; - expectedSubtaskSlugs: string[]; -} - -export function defineSmokeSpec(cfg: SmokeSpecConfig) { - const cohortKey = process.env.COHORT_KEY!; - const runIds = process.env.RUN_IDS!.split(","); - const screenshotDir = process.env.SCREENSHOT_DIR!; - const secret = process.env.TEST_HARNESS_SECRET!; - const apiBase = process.env.ERGON_API_BASE_URL!; - - const client = new BackendHarnessClient(apiBase, secret); - - test.describe(`${cfg.env} canonical smoke`, () => { - for (const runId of runIds) { - test(`run ${runId}`, async ({ page }) => { - // 1. Backend truth via harness - const state = await client.getRunState(runId); - expect(state.status).toBe("completed"); - expect(state.node_count).toBe(10); // root + 9 - expect(state.leaf_statuses.length).toBe(9); - expect(new Set(state.leaf_statuses.map(l => l.slug))) - .toEqual(new Set(cfg.expectedSubtaskSlugs)); - expect(state.evaluation?.score).toBe(1.0); - - // 2. Run page - await page.goto(`/run/${runId}`); - await expect(page.getByTestId("run-status")).toHaveText(/completed/i); - const nodes = page.getByTestId("task-node"); - await expect(nodes).toHaveCount(9); - for (const slug of cfg.expectedSubtaskSlugs) { - await expect(page.getByTestId(`task-node-${slug}`)).toBeVisible(); - await expect(page.getByTestId(`task-node-${slug}`)) - .toHaveAttribute("data-status", "completed"); - } - await screenshot(page, path.join(screenshotDir, cfg.env, `${runId}-run-full.png`), - { fullPage: true }); - await screenshot(page.getByTestId("graph-canvas"), - path.join(screenshotDir, cfg.env, `${runId}-graph.png`)); - }); - } - - test("cohort index lists all runs", async ({ page }) => { - await page.goto(`/cohort/${cohortKey}`); - await expect(page.getByTestId("cohort-run-row")) - .toHaveCount(runIds.length); - await screenshot(page, path.join(screenshotDir, cfg.env, `cohort-${cohortKey}.png`), - { fullPage: true }); - }); - }); -} - -async function screenshot(target: Page | ReturnType<Page["getByTestId"]>, out: string, opts?: any) { - await fs.mkdir(path.dirname(out), { recursive: true }); - if ("screenshot" in target) { - await (target as any).screenshot({ path: out, ...opts }); - } -} -``` - -Per-env specs are 3-liners: - -```typescript -// ergon-dashboard/tests/e2e/researchrubrics.smoke.spec.ts -import { defineSmokeSpec } from "./_shared/smoke"; -import { EXPECTED_SUBTASK_SLUGS } from "./_shared/expected"; - -defineSmokeSpec({ env: "researchrubrics", expectedSubtaskSlugs: EXPECTED_SUBTASK_SLUGS }); -``` - -```typescript -// ergon-dashboard/tests/e2e/_shared/expected.ts -export const EXPECTED_SUBTASK_SLUGS = [ - "d_root", "d_left", "d_right", "d_join", - "l_1", "l_2", "l_3", - "s_a", "s_b", -]; -``` - -(Same tuple as Python's `EXPECTED_SUBTASK_SLUGS` — duplicated because cross-language share is more work than it's worth; both exports live in small files and diverge loudly.) - ---- - -## 4. Per-env Playwright deltas - -Minimal — content checks happen on the Python side via artifact reads. Playwright asserts the UI contract only. Two additions over a naïve per-run loop: - -### 4.1 Cohort label on the cohort index - -```typescript -await expect(page.getByTestId("cohort-env-label")) - .toHaveText(new RegExp(cfg.env, "i")); -``` - -### 4.2 Mixed happy/sad cohort dispatch (researchrubrics only) - -The Python driver passes `cohort: [{ run_id, kind }, …]` via env var (JSON-encoded). The shared factory iterates it and branches on `kind`. Happy-path assertions stay as in §3; sad-path assertions (for researchrubrics slot 3) run the block specified in [`02-drivers-and-asserts.md §10.2`](02-drivers-and-asserts.md): - -```typescript -// inside defineSmokeSpec, per-run loop: -for (const { run_id, kind } of cohort) { - test(`run ${run_id} (${kind})`, async ({ page }) => { - if (kind === "happy") { - // happy-path assertions (§3) - } else { - // sad-run: l_2 FAILED, l_3 BLOCKED/CANCELLED, rest COMPLETED (§10.2 of 02-...) - } - }); -} -``` - -MiniF2F and SWE-bench cohorts pass 3 × `{ run_id, kind: "happy" }`, so the sad branch is dead code for those envs. - -### 4.3 Env-specific content (when needed) - -If an env needs a richer UI check later (e.g. MiniF2F theorem preview), extend `SmokeSpecConfig` with an optional `extraRunAssertions(page, runId): Promise<void>` and call it from the happy-path block. - ---- - -## 5. Screenshots captured (per leg) - -Per matrix leg, after all 3 cohort runs: - -| File | Contents | -|---|---| -| `<env>/<run_id>-run-full.png` × 3 | Full `/run/{id}` page | -| `<env>/<run_id>-graph.png` × 3 | `graph-canvas` element only (zoomed on DAG) | -| `<env>/cohort-<cohort_key>.png` | Cohort index with all 3 runs listed | - -7 PNGs per leg × 3 legs = 21 per PR. File naming is stable so the `gh pr comment` can reference them by path. - ---- - -## 6. Dashboard `data-testid` contract (frontend-side work) - -The spec assumes these test IDs exist: - -| testid | Element | -|---|---| -| `run-status` | Text showing run status on `/run/{id}` | -| `task-node` (list) | Each DAG node in the graph canvas | -| `task-node-{slug}` | A specific node by task slug | -| `graph-canvas` | The graph container element | -| `cohort-run-row` | Each row on `/cohort/{key}` listing one run | -| `cohort-env-label` | Env display on cohort index | - -If today's dashboard uses different attributes (classes, aria-labels), add `data-testid` attributes as part of this PR. `data-testid` is the contract; the Playwright spec does not fall back to text-matching. - ---- - -## 7. Harness unit tests (kept, aligned) - -| Test | What it asserts | Lives | -|---|---|---| -| `test_app_mounts_harness_conditionally.py` | `/api/test/*` only mounts if `ENABLE_TEST_HARNESS=1` | `tests/unit/` | -| `test_test_harness.py` (existing) | Secret-gate 401 behaviour | `tests/unit/` | -| `test_smoke_harness.py` (existing) | Harness round-trip against real Postgres | `tests/integration/smokes/` | - -These stay and are aligned to the catalogued read DTOs in §2.2 — if a DTO changes, these tests change with it. diff --git a/docs/superpowers/plans/test-refactor/04-ci-and-workflows.md b/docs/superpowers/plans/test-refactor/04-ci-and-workflows.md deleted file mode 100644 index 9f5d6c13c..000000000 --- a/docs/superpowers/plans/test-refactor/04-ci-and-workflows.md +++ /dev/null @@ -1,321 +0,0 @@ -# 04 — CI workflows, Docker caching, screenshots - -**Status:** draft -**Scope:** `.github/workflows/ci-fast.yml`, `.github/workflows/e2e-benchmarks.yml`, `docker-compose.ci.yml`, screenshot push/cleanup scripts, PR comment format. - -Cross-refs: assertions + driver in [`02-drivers-and-asserts.md`](02-drivers-and-asserts.md); Playwright in [`03-dashboard-and-playwright.md`](03-dashboard-and-playwright.md). - ---- - -## 1. `ci-fast.yml` — unit + integration, every PR - -Jobs (parallel): - -```yaml -name: ci-fast -on: - pull_request: - push: - branches: [main] - -jobs: - lint-and-type-check: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: { node-version: 20, cache: pnpm } - - uses: astral-sh/setup-uv@v4 - - run: pnpm install --frozen-lockfile - - run: pnpm run check:be - - run: pnpm run check:fe - - unit-tests: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - run: uv sync --frozen - - run: pnpm install --frozen-lockfile - - run: pnpm run test:be:unit - - integration-tests: - runs-on: ubuntu-latest - timeout-minutes: 10 - services: - postgres: - image: postgres:15 - env: - POSTGRES_USER: ergon - POSTGRES_PASSWORD: ci_test - POSTGRES_DB: ergon - ports: ["5433:5432"] - options: >- - --health-cmd "pg_isready -U ergon" - --health-interval 2s --health-retries 20 - env: - ERGON_DATABASE_URL: postgresql://ergon:ci_test@localhost:5433/ergon - INNGEST_DEV: "1" - INNGEST_EVENT_KEY: dev - INNGEST_API_BASE_URL: http://localhost:8289 - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v4 - - run: uv sync --frozen - - run: pnpm install --frozen-lockfile - - name: Start Inngest dev - run: docker compose -f docker-compose.ci.yml up -d inngest-dev - - run: pnpm run test:be:integration -``` - -**No SQLite anywhere** — the fixture in `tests/integration/conftest.py` skips the suite if `ERGON_DATABASE_URL` is missing or points to SQLite (existing behaviour, keep it). - ---- - -## 2. `e2e-benchmarks.yml` — every PR, 3-leg matrix - -```yaml -name: e2e-benchmarks -on: - pull_request: - workflow_dispatch: - -permissions: - contents: write # for pushing screenshots to screenshots/pr-{N} - pull-requests: write # for gh pr comment - -jobs: - smoke: - strategy: - fail-fast: false - matrix: - env: [researchrubrics, minif2f, swebench-verified] - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - SMOKE_ENV: ${{ matrix.env }} - ENABLE_TEST_HARNESS: "1" - TEST_HARNESS_SECRET: ${{ secrets.TEST_HARNESS_SECRET }} - E2B_API_KEY: ${{ secrets.E2B_API_KEY }} - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 0 } - - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: { node-version: 20, cache: pnpm } - - uses: astral-sh/setup-uv@v4 - - - run: pnpm install --frozen-lockfile - - run: uv sync --frozen - - # ── Docker layer cache (see §3) ────────────────────────────── - - uses: docker/setup-buildx-action@v3 - - name: Bring up stack with cache - run: docker compose -f docker-compose.ci.yml up -d --build - env: - DOCKER_BUILDKIT: "1" - COMPOSE_DOCKER_CLI_BUILD: "1" - - - name: Wait for Postgres + API + Inngest - run: bash ci/wait_for_stack.sh - - - name: Build + start dashboard - run: | - pnpm --dir ergon-dashboard build - pnpm --dir ergon-dashboard start & - bash ci/wait_for_dashboard.sh - - # ── Playwright browser (cached) ────────────────────────────── - - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: pw-${{ hashFiles('ergon-dashboard/pnpm-lock.yaml') }} - - run: pnpm --dir ergon-dashboard exec playwright install --with-deps chromium - - # ── Smoke for this matrix env ──────────────────────────────── - - name: Run smoke - run: | - uv run pytest tests/e2e/test_${SMOKE_ENV//-/_}_smoke.py -v \ - --timeout=270 --tb=short - - # ── Screenshot push (runs on success and failure) ──────────── - - name: Push screenshots - if: always() - run: bash ci/push_screenshots.sh "$GITHUB_PR_NUMBER" "$SMOKE_ENV" /tmp/playwright - - # ── PR comment (on job completion) ─────────────────────────── - - name: Comment on PR - if: always() && github.event.pull_request.number - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: bash ci/pr_comment_screenshots.sh "$GITHUB_PR_NUMBER" "$SMOKE_ENV" -``` - -Note the matrix value translates `swebench-verified` → `swebench_verified` for the pytest file path. The slug in Python is `swebench` (see [`01-fixtures.md`](01-fixtures.md)) and the benchmark slug in Ergon is `swebench-verified`; the pytest filename uses the Ergon slug. - ---- - -## 3. Docker layer caching - -Blocks on [`docs/bugs/open/2026-04-18-ci-docker-caching.md`](../../../bugs/open/2026-04-18-ci-docker-caching.md). Without the cache, cold legs blow the 10-min job budget and pay for a full rebuild on every PR. - -### 3.1 `docker-compose.ci.yml` changes - -```yaml -services: - api: - build: - context: . - dockerfile: Dockerfile - cache_from: - - type=gha,scope=api - cache_to: - - type=gha,scope=api,mode=max - image: ergon-api:ci - - postgres: - image: postgres:15@sha256:<pinned-digest> - # ... unchanged - - inngest-dev: - image: inngest/inngest:<pinned-tag>@sha256:<pinned-digest> -``` - -Pin the `postgres` and `inngest` image digests so a vendor tag rotation doesn't invalidate the cache mid-PR. - -### 3.2 Cold vs warm expectations - -| Scenario | Per-leg wall clock | -|---|---| -| Cold PR (cache miss) | 4–5 min — near the hard ceiling | -| Warm PR (cache hit) | 1–3 min | -| Cache rebuild on `main` nightly | acceptable one-shot | - -If warm consistently exceeds 3 min, the dashboard prod build is the next budget suspect — revisit the open decision in [`00-program.md §6.3`](00-program.md) to fall back to dev-server for smoke only. - ---- - -## 4. Screenshot delivery - -Two shell scripts, both committed under `ci/`. - -### 4.1 `ci/push_screenshots.sh` - -```bash -#!/usr/bin/env bash -# Usage: push_screenshots.sh <pr_number> <env> <screenshot_dir> -set -euo pipefail -pr="$1"; env="$2"; dir="$3" -branch="screenshots/pr-${pr}" - -git config user.name "github-actions[bot]" -git config user.email "github-actions[bot]@users.noreply.github.com" - -# Fetch or init branch -if git ls-remote --heads origin "$branch" | grep -q "$branch"; then - git fetch origin "$branch:$branch" - git checkout "$branch" -else - git checkout --orphan "$branch" - git rm -rf . || true - echo "Screenshots for PR #$pr" > README.md - git add README.md - git commit -m "ci: init screenshots/pr-${pr}" -fi - -mkdir -p "${env}" -cp -r "${dir}/${env}"/*.png "${env}/" 2>/dev/null || echo "no screenshots for ${env}" - -git add "${env}/" -git diff --cached --quiet || git commit -m "ci: screenshots ${env} $(date -u +%Y%m%dT%H%M%SZ)" -git push origin "$branch" -``` - -### 4.2 `ci/pr_comment_screenshots.sh` - -```bash -#!/usr/bin/env bash -# Usage: pr_comment_screenshots.sh <pr_number> <env> -set -euo pipefail -pr="$1"; env="$2" -repo="${GITHUB_REPOSITORY:-DeepFlow-research/ergon}" -base="https://raw.githubusercontent.com/${repo}/screenshots/pr-${pr}/${env}" - -# Find the images pushed for this env on this leg -imgs=$(git ls-tree -r --name-only "screenshots/pr-${pr}" -- "${env}" || true) - -comment=$(cat <<EOF -## E2E smoke — \`${env}\` - -![run-full](${base}/$(echo "$imgs" | grep -m1 run-full.png)) -![graph](${base}/$(echo "$imgs" | grep -m1 graph.png)) -![cohort](${base}/$(echo "$imgs" | grep -m1 cohort)) -EOF -) - -gh pr comment "$pr" --body "$comment" -``` - -Minimal. Graceful degradation when images are missing; no hard failure on comment errors (screenshots in the branch are the primary artifact). - -### 4.3 Cleanup on PR close - -Separate workflow: - -```yaml -# .github/workflows/cleanup-screenshots.yml -name: cleanup-screenshots -on: - pull_request: - types: [closed] - -permissions: - contents: write - -jobs: - delete-branch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: | - branch="screenshots/pr-${{ github.event.pull_request.number }}" - git push origin --delete "$branch" || true -``` - ---- - -## 5. `wait_for_stack.sh` and `wait_for_dashboard.sh` - -Small bash scripts with 60-second deadlines. Poll: - -- Postgres: `pg_isready -h localhost -p 5433 -U ergon`. -- Inngest: `curl -sf http://localhost:8289/health`. -- API: `curl -sf http://localhost:9000/healthz`. -- Dashboard: `curl -sf http://localhost:3000`. - -Exit nonzero on timeout. Keeps the workflow log clean — failures point at a specific service. - ---- - -## 6. Secrets required - -| Secret | Where set | Used by | -|---|---|---| -| `E2B_API_KEY` | repo secrets | e2e-benchmarks leg | -| `TEST_HARNESS_SECRET` | repo secrets | e2e-benchmarks leg (exported to backend + Playwright) | -| `GITHUB_TOKEN` | default | gh pr comment | - -No separate database or Inngest credentials needed in CI (they stand up inside the job). - ---- - -## 7. Non-goals for the CI file - -- No nightly full `tests/real_llm/` run wired here; that has its own workflow file ([`docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md`](../2026-04-21-real-llm-debug-harness.md)). -- No cross-matrix aggregation (the matrix legs are independent; a green PR requires all 3). -- No retry-on-flake logic. If a leg flakes, fix the flake or raise the issue — silent retries mask regressions. diff --git a/docs/superpowers/plans/test-refactor/05-deletions.md b/docs/superpowers/plans/test-refactor/05-deletions.md deleted file mode 100644 index ee92c5b47..000000000 --- a/docs/superpowers/plans/test-refactor/05-deletions.md +++ /dev/null @@ -1,210 +0,0 @@ -# 05 — Deletion manifest - -**Status:** draft -**Scope:** every file, registration, and doc that must be removed in the same PR as the new code. Nothing listed here survives the landing PR. - -**Phase ordering:** deletions in this doc are tagged to Phase F in [`06-phases.md`](06-phases.md). Don't delete until the new code is green — a PR that breaks both old and new simultaneously is unreviewable. - ---- - -## 1. `ergon_builtins/` deletions - -### 1.1 Benchmark stubs (delete outright) - -- `ergon_builtins/benchmarks/smoke_test/` — whole directory. -- `ergon_builtins/benchmarks/researchrubrics/smoke.py` -- `ergon_builtins/benchmarks/researchrubrics/smoke_rubric.py` -- `ergon_builtins/benchmarks/minif2f/smoke_rubric.py` -- `ergon_builtins/benchmarks/swebench_verified/smoke_rubric.py` - -### 1.2 Stub workers (delete outright) - -- `ergon_builtins/workers/stubs/canonical_smoke_worker.py` — retired slug. -- `ergon_builtins/workers/baselines/stub_worker.py` -- ~~`ergon_builtins/workers/baselines/training_stub_worker.py`~~ — **KEPT.** Production baseline per accepted RFC `2026-04-22-worker-interface-and-artifact-routing`. See §6 for rationale. -- `ergon_builtins/workers/research_rubrics/stub_worker.py` - -### 1.3 Stub workers (move then delete) - -- `ergon_builtins/workers/stubs/base_smoke_leaf.py` → `tests/e2e/_fixtures/smoke_base/leaf_base.py`. -- `ergon_builtins/workers/stubs/smoke_subworker.py` → `tests/e2e/_fixtures/smoke_base/subworker.py`. -- `ergon_builtins/workers/stubs/` — delete the whole directory once the two files above are moved. - -### 1.4 Stub criteria (delete outright) - -- `ergon_builtins/evaluators/criteria/stub_criterion.py` -- `ergon_builtins/evaluators/criteria/stub_report_exists.py` -- `ergon_builtins/evaluators/criteria/varied_stub_criterion.py` - -### 1.5 Shared smoke criterion (move then delete) - -- `ergon_builtins/evaluators/criteria/smoke_criterion.py` → `tests/e2e/_fixtures/smoke_base/criterion_base.py` (as `SmokeCriterionBase`; env subclasses move to `tests/e2e/_fixtures/criteria/{env}_smoke.py`, see [`01-fixtures.md §3`](01-fixtures.md)). - -### 1.6 Stub rubrics (delete outright) - -- `ergon_builtins/evaluators/rubrics/stub_rubric.py` -- `ergon_builtins/evaluators/rubrics/varied_stub_rubric.py` - -### 1.7 Builtins `__init__` imports - -Scrub every `__init__.py` under `ergon_builtins/` that re-exports the above. Missed re-exports cause import-time failures in CI post-deletion — treat this as part of the PR checklist, not a cleanup for later. - ---- - -## 2. Registry rows to expunge - -After registration hook runs in `tests/e2e/_fixtures/__init__.py`, the process-level registry must **not** contain any of these slugs: - -| Slug | Kind | -|---|---| -| `canonical-smoke` | Worker | -| `smoke-leaf` (generic) | Worker | -| `smoke-test` | Benchmark | -| `researchrubrics-smoke` | Benchmark | -| `stub-worker` | Worker | -| `researcher` | Worker (alias) | -| `manager-researcher` | Worker | -| `researchrubrics-manager` | Worker | -| ~~`training-stub`~~ | — (kept: production baseline, see §6) | -| `researchrubrics-stub` | Worker | -| `stub-criterion` | Criterion | -| `varied-stub-criterion` | Criterion | -| `stub-report-exists` | Criterion | -| `stub-rubric` | Rubric | -| `varied-stub-rubric` | Rubric | - -Unit test in [`01-fixtures.md §6`](01-fixtures.md) (`test_registry_smoke_entries.py`) asserts the **present** set exactly; any leftover slug fails the test. - ---- - -## 3. `tests/` deletions - -### 3.1 Old e2e tests (confirm existence, delete) - -- `tests/e2e/test_benchmarks_stubbed.py` (pycache implies it existed) -- `tests/e2e/test_researchrubrics_smoke_e2b.py` (pycache implies it existed) -- Any other `tests/e2e/test_*.py` not in {`test_researchrubrics_smoke.py`, `test_minif2f_smoke.py`, `test_swebench_smoke.py`} - -### 3.2 `tests/state/` — fully retired - -`tests/state/` was the SQLite-backed fast tier; migration to `tests/unit/state/` + `tests/integration/` is already underway. Confirm migration complete and delete: - -- `tests/state/` — whole directory, including `conftest.py`, `factories.py`, `mocks.py`, and every `test_*.py` not yet moved. - -If anything still lives in `tests/state/` at PR-land time, classify per [`00-program.md §2`](00-program.md) decision rule and migrate in this same PR. - -### 3.3 Integration tier SQLite holdovers (audit, migrate, delete) - -- `tests/integration/test_full_lifecycle.py` -- `tests/integration/test_full_lifecycle_with_eval.py` - -These currently use SQLite + direct service calls per the old `testing-posture-reset` RFC. Rewrite against real Postgres + Inngest **or** delete; they must not survive as SQLite-backed tests post-landing. - ---- - -## 4. Docs to delete (plans + RFCs) - -All under `docs/` in the Ergon repo. - -### 4.1 Accepted RFCs (superseded by this folder) - -- `docs/rfcs/accepted/2026-04-18-testing-posture-reset.md` -- `docs/rfcs/accepted/2026-04-21-e2e-smoke-coverage-rewrite.md` - -### 4.2 Rejected RFCs (already dead, housekeep) - -- `docs/rfcs/rejected/2026-04-18-test-harness-endpoints.md` -- `docs/rfcs/rejected/2026-04-18-fixed-delegation-stub-worker.md` - -### 4.3 Superseded plans - -- `docs/superpowers/plans/2026-04-21-e2e-smoke-coverage-rewrite.md` -- `docs/superpowers/plans/2026-04-22-phase-2-canonical-smoke-spec.md` -- `docs/superpowers/plans/2026-04-22-unified-testing-e2e-smoke-plan.md` - -### 4.4 Brainstorms to audit - -- `docs/superpowers/brainstorms/2026-04-21-real-llm-debug-harness.md` — keep only if the real-LLM plan at `docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md` references it. Otherwise delete. - -### 4.5 NOT deleted (out of scope) - -- `docs/rfcs/accepted/2026-04-21-real-llm-debug-harness.md` -- `docs/superpowers/plans/2026-04-21-real-llm-debug-harness.md` - -Real-LLM is a separate tier — it has its own design authority and is not restructured by this refactor. - -### 4.6 This folder itself - -After the rebuild lands and `docs/architecture/07_testing.md` fully describes the standing system: - -- Delete `docs/superpowers/plans/test-refactor/`. - -Planning docs are not documentation. Leaving them around after landing creates the same fragmentation we just cleaned up. - ---- - -## 5. Architecture docs — updated, not deleted - -These get edits, not deletions: - -- `docs/architecture/07_testing.md` — rewritten to describe the 4 tiers, canonical-smoke-shape-enforced-by-inheritance, `/api/test/*` contract, screenshot flow. Post-landing, `07_testing.md` is the documentation; this folder is the planning trail. -- `docs/architecture/06_builtins.md` — remove any mention of stubs/smoke that contradicts §1. Add pointer to `tests/e2e/_fixtures/` for test-only registry entries. -- `docs/architecture/05_dashboard.md` — add `/api/test/*` route contract summary + `data-testid` contract. -- `docs/architecture/01_public_api.md` — no change expected (Worker/Criterion ABCs unchanged). - ---- - -## 6. Training-stub audit — RESOLVED (2026-04-23) - -Audit run with `grep -Rn "training-stub\|training_stub\|TrainingStubWorker" ergon_core/ ergon_cli/ ergon_infra/ ergon_builtins/` surfaced: - -- **`ergon_builtins/registry_core.py`** registers `"training-stub": TrainingStubWorker` — production registry entry. -- **`ergon_builtins/AGENTS.md`** documents it as the RL-logprobs fixture operators invoke via `ergon benchmark run smoke-test --worker training-stub`. -- **Accepted RFC [`2026-04-22-worker-interface-and-artifact-routing.md`](../../../rfcs/accepted/2026-04-22-worker-interface-and-artifact-routing.md)** treats `TrainingStubWorker` as a first-class baseline alongside `ReActWorker` and migrates it through the worker-interface refactor. - -**Outcome:** `training_stub_worker.py` is the **only** remaining stub exception in `ergon_builtins/workers/baselines/` — kept untouched. The "stub" in its name is a technical term (synthetic trajectory / fake logprobs), not "test fixture." `StubWorker`, `ManagerResearcherWorker`, `StubResearchRubricsWorker`, and `ResearchRubricsManagerWorker` have all been deleted as part of the `assigned_worker_slug` default removal (see §1.2 and §2). Renaming `training_stub_worker.py` is an optional follow-up, not blocking this refactor. - -The merge checklist in `00-program.md §5` carries this exception explicitly. - ---- - -## 7. Code-path follow-ups - -Things that are not strict deletions but must not break post-PR: - -- `ergon_cli/` subcommands that default to `stub-worker` — update defaults to a real baseline or remove the subcommand. -- `ergon_infra/` config templates that pre-populate `stub-*` slugs — update or remove. -- `docker-compose.ci.yml` comments referencing `smoke-test` benchmark — update. -- Any `README.md` or `ONBOARDING.md` walking through `stub-*` usage — rewrite with real baselines. - -Run `git grep -n 'stub\|canonical-smoke\|smoke-test'` pre-commit; each surviving match is either (a) intentionally in `tests/e2e/_fixtures/`, (b) the preserved `training_stub_worker.py` production baseline (§6), or (c) a miss to fix. - ---- - -## 8. One-shot verification command - -Before marking the PR ready: - -```bash -# Should return zero: -rg -n 'canonical-smoke|smoke-test|ResearchRubricsSmokeTestBenchmark' \ - ergon_core/ ergon_cli/ ergon_builtins/ ergon_infra/ ergon-dashboard/src - -# Should return only tests/e2e/_fixtures/: -rg -l 'SmokeWorkerBase|SmokeSubworker|BaseSmokeLeafWorker|SmokeCriterionBase' \ - | grep -v '^tests/e2e/_fixtures/' - -# Training-stub baseline is ALLOWED to remain in ergon_builtins (see §6). -# This command should return exactly ONE match — the baseline itself: -rg -l 'TrainingStubWorker' ergon_builtins/ | head - -# "researcher" slug, StubWorker, ManagerResearcherWorker, and -# ResearchRubricsManagerWorker must all be gone from production code. -# Should return zero matches: -rg -n '"researcher"' ergon_core/ ergon_cli/ ergon_infra/ ergon_builtins/ -rg -n '"stub-worker"' ergon_core/ ergon_cli/ ergon_infra/ ergon_builtins/ -rg -n '"manager-researcher"' ergon_core/ ergon_cli/ ergon_infra/ ergon_builtins/ -rg -n '"researchrubrics-manager"' ergon_core/ ergon_cli/ ergon_infra/ ergon_builtins/ -``` - -All should be empty outputs. If not, the deletion pass is incomplete. diff --git a/docs/superpowers/plans/test-refactor/06-phases.md b/docs/superpowers/plans/test-refactor/06-phases.md deleted file mode 100644 index 685f36dc7..000000000 --- a/docs/superpowers/plans/test-refactor/06-phases.md +++ /dev/null @@ -1,286 +0,0 @@ -# 06 — Phases, deliverables, acceptance gates - -**Status:** draft -**Scope:** internal branch / commit ordering for the single landing PR. Reviewers see one PR; the branch is layered Phase-B through Phase-F so each commit is independently reviewable. - -Cross-refs: program in [`00-program.md`](00-program.md); deletions in [`05-deletions.md`](05-deletions.md). - ---- - -## Delivery shape - -**One PR** against `main`, with commits ordered B → F. Each phase has: - -- **Scope:** what code + docs move in that commit. -- **Deliverables:** the concrete outputs. -- **Acceptance gate:** the test(s) / command(s) that must be green before proceeding to the next phase. - -Phase A was Phase A from the prior plan — it already landed. Phase B onward is new work. - ---- - -## Phase A — Tier command hygiene (DONE) - -- `package.json`: `test:be:unit`, `test:be:integration`, `test:be:e2e`, `test:be:real-llm`, `test:be:all`, `test:be:coverage`, `test:be:fast` aligned to the 4-tier model. -- `.github/workflows/ci-fast.yml`: unit job runs `tests/unit/` with coverage. - -No work remaining. - ---- - -## Phase B — Fixtures scaffolding + test-only registration - -**Scope** - -- Create `tests/e2e/_fixtures/` package with the structure in [`01-fixtures.md §1`](01-fixtures.md). -- `smoke_base/constants.py` with `EXPECTED_SUBTASK_SLUGS` + `SUBTASK_GRAPH`. -- `smoke_base/subworker.py` — `SmokeSubworker` Protocol + `SubworkerResult` (moved from builtins). -- `smoke_base/worker_base.py` — **new** `SmokeWorkerBase` class with `@final execute()` and `_spec_for(slug, deps, desc)` override hook. -- `smoke_base/leaf_base.py` — `BaseSmokeLeafWorker` (moved from builtins, imports rewired) plus `_send_completion_message` helper invoking `CommunicationService.save_message`. -- `smoke_base/criterion_base.py` — `SmokeCriterionBase` with `_pull_children` + `_pull_probe_results` **concretely implemented** (the current file's `NotImplementedError` stubs filled in). -- `_fixtures/__init__.py` registration hook (empty registry calls until Phase C registers real classes). -- `tests/e2e/conftest.py` imports `_fixtures` at session start. - -**Deliverables** - -- `_fixtures/` importable; unit tests for each base class added under `tests/unit/` per [`01-fixtures.md §6`](01-fixtures.md): - - `test_smoke_worker_base_final.py` - - `test_smoke_worker_spec_for_override.py` - - `test_base_smoke_leaf.py` (retarget existing) - - `test_leaf_sends_completion_message.py` - - `test_failing_leaf_skips_message.py` - - `test_smoke_criterion_shape.py` - - `test_smoke_criterion_completed.py` - - `test_smoke_criterion_probe.py` - -**Acceptance gate** - -- `pnpm run test:be:unit` green. -- `pnpm run check:be` green (types pass with new imports). -- No production code imports `tests/e2e/_fixtures/`. - -**Not in this phase:** deletions, env-specific code, CI workflow changes. - ---- - -## Phase C — ResearchRubrics end-to-end (one env, full pipe) - -Proves the pipe end-to-end on one env before replicating twice. - -**Scope** - -- `tests/e2e/_fixtures/workers/researchrubrics_smoke.py` — `ResearchRubricsSmokeWorker`, `ResearchRubricsSubworker`, `ResearchRubricsSmokeLeafWorker`. -- `tests/e2e/_fixtures/criteria/researchrubrics_smoke.py` — `ResearchRubricsSmokeCriterion._verify_env_content` + `_verify_sandbox_setup`. -- `tests/e2e/_fixtures/workers/researchrubrics_smoke_sadpath.py` — `AlwaysFailSubworker`, `ResearchRubricsFailingLeafWorker`, `ResearchRubricsSadPathSmokeWorker` (see [`01-fixtures.md §3.4`](01-fixtures.md)). -- Register all five new classes in `_fixtures/__init__.py`. -- `tests/e2e/_asserts.py` — shared pytest assertion helpers from [`02-drivers-and-asserts.md §2`](02-drivers-and-asserts.md): `_assert_run_graph`, `_assert_run_resources`, `_assert_run_turn_counts`, `_assert_sandbox_command_wal`, `_assert_sandbox_lifecycle_events`, `_assert_thread_messages_ordered`, `_assert_blob_roundtrip`, `_assert_temporal_ordering`, `_assert_cohort_membership`, `_assert_run_evaluation`. -- `tests/e2e/test_researchrubrics_smoke.py` — cohort-of-3 driver: 2 happy + 1 sad slot. Per-run assertion dispatch on `kind` ([`02-drivers-and-asserts.md §1`](02-drivers-and-asserts.md), sad helpers at [§10](02-drivers-and-asserts.md)). -- `submit_cohort` helper in `ergon_cli/` — takes `slots: list[tuple[worker_slug, criterion_slug]]` so cohorts can be heterogeneous ([`02-drivers-and-asserts.md §6`](02-drivers-and-asserts.md)). -- `wait_for_terminal` helper. -- `ergon-dashboard/tests/e2e/_shared/smoke.ts` + `_shared/expected.ts` + `helpers/backendHarnessClient.ts`. -- `ergon-dashboard/tests/e2e/researchrubrics.smoke.spec.ts` — single spec file, iterates `cohort: [{run_id, kind}]` and dispatches per-kind assertions ([`03-dashboard-and-playwright.md §4.2`](03-dashboard-and-playwright.md)). -- `data-testid` attributes added to the relevant dashboard components (per [`03-dashboard-and-playwright.md §6`](03-dashboard-and-playwright.md)). -- Unit tests: - - `test_env_criterion_verify_content.py` (researchrubrics flavour) - - `test_env_criterion_sandbox_setup.py` (researchrubrics flavour — see [`01-fixtures.md §6`](01-fixtures.md)) - - `test_always_fail_subworker.py` — asserts write→probe→raise ordering - - `test_registry_smoke_entries.py` — researchrubrics happy + sad slugs registered; no retired slugs present -- Backend `/api/test/read/run/{id}/state` + `/api/test/read/cohort/{key}/runs` endpoints if not already shaped this way. - -**Deliverables** - -- Running `pnpm run test:be:e2e -- tests/e2e/test_researchrubrics_smoke.py` against a local Docker stack produces a green 3-run cohort (2 happy + 1 sad) with screenshots under `/tmp/playwright/researchrubrics/`. The sad slot's partial artifact (`partial_l_2.md`) and pre-fail `wc -l` WAL entry are both present in Postgres. - -**Acceptance gate** - -- Local e2e test green: 3 runs on the researchrubrics leg (2 happy with all happy-path assertions + 1 sad with all sad-path assertions), 26 leaf sandbox acquisitions on the researchrubrics image (2×9 + 1×8). -- Playwright spec green: happy runs render COMPLETED nodes + cohort index lists all 3; sad run renders l_2 FAILED + l_3 BLOCKED/CANCELLED + rest COMPLETED. -- All unit tests green. -- `pnpm run check:be` + `check:fe` green. -- The 6 new assertion helpers (`_assert_sandbox_command_wal`, `_assert_sandbox_lifecycle_events`, `_assert_thread_messages_ordered`, `_assert_blob_roundtrip`, `_assert_temporal_ordering`, `_assert_cohort_membership`) all green on the happy-path run. - -**Not in this phase:** CI workflow changes (still runs locally only), other two envs, deletions. - ---- - -## Phase D — MiniF2F + SWE-Bench Verified - -Replicate Phase C pattern twice. With Phase C as template, this should be mechanical. - -**Scope** - -- `tests/e2e/_fixtures/workers/minif2f_smoke.py` + criterion. -- `tests/e2e/_fixtures/workers/swebench_smoke.py` + criterion. -- Register both in `_fixtures/__init__.py`. -- `tests/e2e/test_minif2f_smoke.py`. -- `tests/e2e/test_swebench_smoke.py`. -- Playwright specs: `minif2f.smoke.spec.ts`, `swebench-verified.smoke.spec.ts`. -- Unit tests: - - `test_env_criterion_verify_content.py` — extend with minif2f + swebench cases. - - `test_registry_smoke_entries.py` — updated to expect all 9 slugs exactly. - -**Deliverables** - -- All three drivers pass locally. -- Playwright specs pass locally. -- Full registry set correct. - -**Acceptance gate** - -- `pnpm run test:be:e2e` green on all three files locally. -- `pnpm run test:be:unit` green. -- `test_registry_smoke_entries.py` asserts exactly 9 slugs, no leftovers. - -**Not in this phase:** CI workflow, deletions. - ---- - -## Phase E — CI matrix + Docker cache + screenshot delivery - -**Scope** - -- `docker-compose.ci.yml` — Buildx cache config from [`04-ci-and-workflows.md §3.1`](04-ci-and-workflows.md); pinned image digests. -- `ci/wait_for_stack.sh` + `ci/wait_for_dashboard.sh`. -- `ci/push_screenshots.sh` + `ci/pr_comment_screenshots.sh`. -- `.github/workflows/e2e-benchmarks.yml` rewritten per [`04-ci-and-workflows.md §2`](04-ci-and-workflows.md) — PR-triggered, 3-leg matrix. -- `.github/workflows/cleanup-screenshots.yml` — branch cleanup on PR close. -- `.github/workflows/ci-fast.yml` integration job: ensure it stands up Postgres service (unit job already aligned in Phase A). -- Repo secrets set: `E2B_API_KEY`, `TEST_HARNESS_SECRET`. - -**Deliverables** - -- First PR to hit this branch runs the full matrix + produces screenshots + posts a PR comment. - -**Acceptance gate** - -- `e2e-benchmarks` workflow green on the PR opening this phase's commit. -- All three matrix legs under 5 min warm, under 10 min cold. -- Screenshots visible in `screenshots/pr-{N}` branch + PR comment. -- `cleanup-screenshots.yml` deletes branch on PR close (tested by closing and reopening a throwaway PR, or by merging). - -**Not in this phase:** deletions of old builtins / RFCs / plans. - ---- - -## Phase F — Deletions + architecture doc rewrite - -The heaviest phase for file count, but mechanical — if Phase B–E passed, nothing here should break functionality. - -**Scope** - -- Every deletion listed in [`05-deletions.md`](05-deletions.md): - - `ergon_builtins/` stub workers, criteria, rubrics, benchmarks, smoke_test dir. - - `tests/` orphaned files (`tests/state/` residue, old e2e tests, old SQLite integration tests). - - `docs/rfcs/accepted/{testing-posture-reset,e2e-smoke-coverage-rewrite}.md`. - - `docs/rfcs/rejected/{test-harness-endpoints,fixed-delegation-stub-worker}.md`. - - `docs/superpowers/plans/{e2e-smoke-coverage-rewrite,phase-2-canonical-smoke-spec,unified-testing-e2e-smoke-plan}.md`. -- Training-stub audit outcome (grep from [`05-deletions.md §6`](05-deletions.md)) — either delete or rehome. -- `docs/architecture/07_testing.md` rewritten: 4 tiers, canonical-smoke-shape-enforced-by-inheritance, `/api/test/*` contract, screenshot flow. -- `docs/architecture/06_builtins.md` + `05_dashboard.md` edits. -- Verification commands from [`05-deletions.md §8`](05-deletions.md) return empty. - -**Deliverables** - -- Branch diff shows: - - ~15 new files under `tests/e2e/_fixtures/` + `tests/e2e/test_*_smoke.py`. - - 3 new Playwright specs + 2 shared modules + 1 backend harness client. - - ~30 deletions across `ergon_builtins/`, `tests/state/`, old `tests/e2e/*`, superseded docs. - - `07_testing.md`, `06_builtins.md`, `05_dashboard.md` edits. - - Root `README.md` / onboarding doc updates if they reference stubs. - -**Acceptance gate (final, = merge gate)** - -Exactly the checklist in [`00-program.md §5`](00-program.md): - -- [ ] All 3 `e2e-benchmarks` matrix legs green on the PR. -- [ ] `ci-fast` unit + integration green. -- [ ] No `ergon_builtins/**` file contains `stub` or `smoke` in its name. -- [ ] Registry contains no `canonical-smoke` / `smoke-test` / generic `smoke-leaf` slug. -- [ ] Production dashboard build does not bundle `/api/test/*` routes. -- [ ] Zero LLM references on smoke path: `rg -n 'OPENROUTER|anthropic|openai|pydantic_ai' tests/e2e/` empty. -- [ ] Screenshots land on `screenshots/pr-{N}` + PR comment inline images. -- [ ] Docker cache in effect (warm < 3 min; cold < 5 min per leg). -- [ ] `07_testing.md` points at the standing system; `test-refactor/` folder deleted in this same PR (or in an immediately-following housekeeping PR — see §"Deleting this folder" below). - ---- - -## Phase G — Final: wire `BaseSandboxManager.reconnect` through CriterionRuntime - -**Why this is the last step.** Phases B–F ship smoke end-to-end with criteria attaching via direct `AsyncSandbox.connect(sandbox_id=...)`. That works but violates [`architecture/cross_cutting/sandbox_lifecycle.md`](../../../architecture/cross_cutting/sandbox_lifecycle.md) invariant 3 ("criteria MUST reconnect via the manager"). Phase G closes that gap. After Phase G, criteria reconnect to the task's sandbox **through the manager** and hold it open for the entire evaluation. - -**Scope** - -- Land RFC [`2026-04-17-sandbox-lifetime-covers-criteria`](../../../rfcs/active/2026-04-17-sandbox-lifetime-covers-criteria.md) PR 2 if not already shipped: add `BaseSandboxManager.reconnect(sandbox_id)` (returns `AsyncSandbox`, raises `SandboxExpiredError` on expiry). -- Wire `CriterionRuntime.ensure_sandbox()` (or add `CriterionRuntime.get_sandbox()`) to call `manager.reconnect(context.sandbox_id)` and hold the handle for the duration of `evaluate()`. The sandbox stays open from criterion start through criterion completion — then `check_evaluators` teardown closes it as today. -- Migrate the 3 smoke criteria's `_verify_sandbox_setup` from `AsyncSandbox.connect(sandbox_id=context.sandbox_id)` to `context.get_sandbox()` (or the final DI accessor name). One-line change per criterion — no behavioural change; just routing through the blessed path. -- Add `SandboxExpiredError` handling: smoke criterion subclasses let it bubble; `CriterionRuntime` or `check_evaluators` translates it to a `"sandbox-expired"` evaluation outcome per the RFC. -- Flip invariant 3 in `cross_cutting/sandbox_lifecycle.md` from "pending enforcement" to "enforced end-to-end by smoke tier on every PR". - -**Deliverables** - -- `BaseSandboxManager.reconnect` shipped with unit tests (`tests/unit/test_sandbox_reconnect.py`: success path, `SandboxExpiredError` on not-found/404/expired, re-raise on other errors). -- `CriterionRuntime` DI accessor for the task's sandbox, holding it open for the criterion's run. -- 3 smoke criteria migrated to the DI accessor. -- `cross_cutting/sandbox_lifecycle.md` invariant 3 updated. -- Optional canary e2e test from the RFC (`tests/e2e/test_sandbox_criterion_timeout_canary.py`) — slow criterion still reaches sandbox when timeout is provisioned correctly. - -**Acceptance gate** - -- E2E matrix green across all 3 envs on the migration PR — smoke is now the regression test for this invariant on every PR. -- `tests/unit/test_sandbox_reconnect.py` green. -- `rg -n 'AsyncSandbox.connect' ergon_builtins/ tests/e2e/_fixtures/criteria/` returns only the expected production paths (criteria no longer call it directly). -- `cross_cutting/sandbox_lifecycle.md` invariant 3 no longer says "pending". - -**Not in this phase:** anything from Phases B–F. Phase G is strictly a migration of the attach mechanism, not a feature addition to smoke. - -**Sequencing note.** Phase G is the one phase that can, if necessary, ship as a follow-up PR rather than in the same landing PR as B–F. Smoke is already proving its end-to-end invariant with direct `AsyncSandbox.connect`; the migration to `manager.reconnect` is a tightening, not a correctness fix. If the RFC's PR 2 is still in flight when B–F is merge-ready, ship B–F first and land Phase G within the following week. Mark the PR description explicitly: "interim `AsyncSandbox.connect` in smoke criteria; follow-up Phase G tightens to `manager.reconnect`". - ---- - -## Deleting this folder - -`docs/superpowers/plans/test-refactor/` is planning, not documentation. Two options for when it goes away: - -1. **In the same PR as Phase F.** Landing PR also removes this folder; `07_testing.md` is the single source of post-landing truth. Cleanest but biggest diff. -2. **Immediately-following housekeeping PR (same day).** Makes Phase F's diff smaller and keeps the plan available for review during the PR window. - -Default: **option 2.** If PR review feedback flags option 1 as cleaner, switch. - -Either way, do not leave this folder around past the week of landing. - ---- - -## Phase size estimates (for PR structuring) - -| Phase | Scope | Est. diff size | -|---|---|---| -| A | Done | — | -| B | 5 base files + 5 unit tests | ~600 LoC add | -| C | 1 env + driver + Playwright + helpers + 2 dashboard endpoints | ~1500 LoC add | -| D | 2 envs × (worker + criterion + driver + spec) | ~1000 LoC add | -| E | CI + cache + scripts | ~400 LoC add | -| F | Deletions + doc rewrites | ~1500 LoC delete, ~500 LoC add (docs) | -| G | `BaseSandboxManager.reconnect` + `CriterionRuntime` DI wiring + 3 one-liner criterion migrations | ~200 LoC add, ~10 LoC replace | - -Total (B–F bundle): **~3500 LoC add, ~1500 LoC delete**. Phase G is a small follow-up (or inline if the sandbox-lifetime RFC's PR 2 is ready to land together). - ---- - -## Phase transition discipline - -Do not start Phase C until Phase B's acceptance gate is green. Do not start Phase D until Phase C is fully green end-to-end on one env. Phase E requires Phase D (otherwise 2/3 legs fail on the first CI run). Phase F requires Phase E (otherwise there's nothing to prove deletions didn't break). Phase G requires Phase F **only in the sense that it targets the new smoke criteria** — it can ship inline with F or as an immediate follow-up PR once the sandbox-lifetime RFC's PR 2 (`manager.reconnect`) is ready. - -This is a chain; each phase's gate protects the next phase from being based on broken scaffolding. - ---- - -## If a phase fails - -- **Phase B:** fundamental misunderstanding of current `Worker` / `Criterion` ABCs. Re-read code, update base classes, re-run unit tests. -- **Phase C:** first exposure to real Docker + E2B. Likely: missing env var, sandbox image missing, harness endpoint mismatch. Fix root cause, not the test. -- **Phase D:** env-specific probe choice wrong (e.g. MiniF2F `lean --check` too slow). Widen probe-exit semantics or switch probe command; update `03-dashboard-and-playwright.md §3.2` to match. -- **Phase E:** CI shape issue — Docker cache miss, dashboard port conflict, screenshot push permission. Fix workflow, not the test. -- **Phase F:** deletion missed an import. `git grep` until clean. -- **Phase G:** `manager.reconnect` raises on an unexpected error shape (E2B SDK quirk). Tighten the exception-classification block in `reconnect`; add a regression unit test. Do not swallow unknown errors — let them propagate. - -No retry-in-sleep-loop hacks. No `|| true` fallbacks that hide failures. Fix the root cause or escalate. diff --git a/docs/superpowers/plans/test-refactor/README.md b/docs/superpowers/plans/test-refactor/README.md deleted file mode 100644 index 927364c9c..000000000 --- a/docs/superpowers/plans/test-refactor/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Test refactor — plan folder - -**Status:** draft for review — nothing landed yet. -**Date:** 2026-04-23. -**Scope:** complete rebuild of the testing posture, smoke program, and CI integration. Single merge-ready PR when all nine docs are reviewed and the code matches. - -## Read order - -1. [`00-program.md`](00-program.md) — goals, non-goals, tier model, invariants, budgets, merge checklist. The "why + what." -2. [`01-fixtures.md`](01-fixtures.md) — directory layout + code sketches for every shared base, per-env worker/leaf/criterion, registration hook. No LLM, no pydantic-ai toolkits; deterministic Python only. -3. [`02-drivers-and-asserts.md`](02-drivers-and-asserts.md) — pytest driver template + per-env Postgres assertion catalogs. Describes every SQLModel row read after a smoke run. -4. [`03-dashboard-and-playwright.md`](03-dashboard-and-playwright.md) — `/api/test/*` harness contract (what's there, what's new), `BackendHarnessClient` TS shape, Playwright spec template, per-env deltas, screenshot capture points. -5. [`04-ci-and-workflows.md`](04-ci-and-workflows.md) — `ci-fast.yml` job layout, `e2e-benchmarks.yml` PR-trigger matrix, docker-layer-cache fix, screenshot ref push + PR comment, cleanup on PR close. -6. [`05-deletions.md`](05-deletions.md) — full manifest of files / RFCs / plans / registry slugs / brainstorms to delete on the landing PR. -7. [`06-phases.md`](06-phases.md) — phased delivery plan inside the single PR (Phase A done; B–F ahead; Phase G as the concluding step that wires `BaseSandboxManager.reconnect` through `CriterionRuntime` so criteria attach via the manager and hold the task's sandbox open throughout evaluation). Each phase has scope, deliverables, acceptance gate. - -## Principle - -Each doc is **self-contained for its audience**. If a reviewer only reads one doc, that doc must be enough to execute its part. Cross-references are explicit links, not vague gestures. When two docs disagree, `00-program.md` wins; when `00-program.md` and reality disagree, we amend `00-program.md` and re-review. - -## When this plan lands - -1. Delete every file listed in [`05-deletions.md`](05-deletions.md) in the same PR as the new code. -2. Update `docs/architecture/07_testing.md` to point at `docs/superpowers/plans/test-refactor/`. -3. Delete `docs/superpowers/plans/test-refactor/` itself once the rebuild is done and `07_testing.md` fully documents the standing system. - -Keeping this folder around after landing is an anti-pattern — it's planning, not documentation. diff --git a/docs/superpowers/plans/workflow-agent-cli/00-program.md b/docs/superpowers/plans/workflow-agent-cli/00-program.md deleted file mode 100644 index 315b0b8bd..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/00-program.md +++ /dev/null @@ -1,131 +0,0 @@ -# 00 — Program, Scope, File Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` or `superpowers:executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build `ergon workflow ...`, an agent-local command surface for benchmark-task workers to inspect workflow topology/resources and explicitly materialize useful visible resources into their workspace without exposing raw SQL. - -**Architecture:** Put scoped Postgres reads and resource materialization policy in one core workflow service, command parsing in `ergon_cli`, and the agent-callable wrapper in `ergon_builtins/tools`. The command runs in the local worker/API process, reads local Postgres via `get_session()`, and uses the existing sandbox manager to copy approved resources into the current E2B workspace. - -**Tech Stack:** Python, argparse, SQLModel, Alembic migrations, pydantic-ai `Tool`, pytest, existing sandbox manager APIs. - ---- - -## Package Placement - -- Core workflow logic: `ergon_core/ergon_core/core/runtime/services/workflow_service.py` -- DTOs: `ergon_core/ergon_core/core/runtime/services/workflow_dto.py` -- CLI handlers: `ergon_cli/ergon_cli/commands/workflow.py` -- CLI registration: `ergon_cli/ergon_cli/main.py` -- Agent wrapper: `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py` -- Proof-of-concept worker: `ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py` - -Keep `ergon_builtins/ergon_builtins/tools/graph_toolkit.py` intact. `workflow` is the more general command-shaped interface; the existing graph toolkit remains a research-specific resource discovery toolkit. - -## Added Files - -```text -ergon_core/ - migrations/ - versions/ - <revision>_add_copied_from_resource_id.py - ergon_core/ - core/ - runtime/ - services/ - workflow_dto.py - workflow_service.py - -ergon_cli/ - ergon_cli/ - commands/ - workflow.py - -ergon_builtins/ - ergon_builtins/ - tools/ - workflow_cli_tool.py - workers/ - research_rubrics/ - workflow_cli_react_worker.py - -tests/ - unit/ - runtime/ - test_workflow_service.py - test_workflow_input_resource_semantics.py - cli/ - test_workflow_cli.py - state/ - test_workflow_cli_tool.py - integration/ - runtime/ - test_workflow_cli_materialize_resource.py -``` - -## Edited Files - -```text -ergon_cli/ - ergon_cli/ - main.py - -ergon_core/ - ergon_core/ - core/ - persistence/ - telemetry/ - models.py - queries.py - -ergon_builtins/ - ergon_builtins/ - registry_data.py - -tests/ - unit/ - state/ - test_research_rubrics_workers.py - e2e/ - _asserts.py - test_researchrubrics_smoke.py - test_minif2f_smoke.py - test_swebench_smoke.py - real_llm/ - benchmarks/ - test_researchrubrics.py -``` - -`ergon_builtins/ergon_builtins/workers/research_rubrics/researcher_worker.py` is intentionally not edited for the POC path. - -## Deleted Files - -```text -(none) -``` - -## Optional Later Files - -Automatic input materialization at sandbox creation time is separate from `workflow manage materialize-resource`. - -```text -ergon_core/ - ergon_core/ - api/ - worker_context.py - core/ - runtime/ - inngest/ - execute_task.py - services/ - orchestration_dto.py - task_execution_service.py -``` - -## Non-Goals - -- Do not expose raw SQL. -- Do not expose resources from outside the injected current run. If that happens, it is a CLI bug. -- Do not expose eval/private/system data. -- Do not mutate source resources. -- Do not add E2B-side CLI execution or require E2B-to-localhost networking. -- Do not modify dashboard UI in v1, except optional future display of resource lineage. diff --git a/docs/superpowers/plans/workflow-agent-cli/01-resource-semantics.md b/docs/superpowers/plans/workflow-agent-cli/01-resource-semantics.md deleted file mode 100644 index 757741b45..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/01-resource-semantics.md +++ /dev/null @@ -1,84 +0,0 @@ -# 01 — Resource Semantics - -## Core Rule - -Resources are immutable published artifacts. A task may read a visible resource from another task in the same run, including a resource from a divergent branch of the control DAG, but it must never mutate the source row or source bytes. - -The workflow CLI treats copying as a fork: - -- **Read is context.** `inspect resource-content` does not change graph state. -- **Materialize is fork.** Copying a resource into the current agent workspace creates a new `RunResource` row owned by the current task execution. -- **Publish is ownership.** If the current task edits the copied file and publishes it later, the edited artifact is another new resource owned by the current task execution. -- **Lineage is evidence.** The copied resource row records `copied_from_resource_id=<source_resource_id>`. -- **Control edges schedule work; resource lineage explains information flow.** Materializing a resource from a divergent DAG branch must not add a control dependency edge. - -## Current-Run Invariant - -All workflow CLI commands are current-run scoped by invariant. The agent wrapper injects `run_id`; the model cannot select a run. - -`visible` is not a cross-run permission mode. It is the broadest resource discovery scope **inside the injected current run**. It exists so agents can find useful resources from divergent DAG branches. Any cross-run result is a correctness/security bug. - -## Copy Example - -```text -task_a publishes: - resource_id=res_a - task_execution_id=task_a_execution - name="paper.pdf" - content_hash=abc - -task_b materializes res_a: - resource_id=res_b_copy - task_execution_id=task_b_execution - name="paper (copy).pdf" - content_hash=abc - copied_from_resource_id=res_a - metadata.sandbox_destination="/workspace/imported/task-a/paper (copy).pdf" - -task_b edits and publishes: - resource_id=res_b_edited - task_execution_id=task_b_execution - name="paper_annotated.pdf" - content_hash=def - metadata.derived_from_resource_ids=["res_a", "res_b_copy"] -``` - -If task A later republishes a newer `paper.pdf`, task B's copy remains pinned to the old `resource_id` and `content_hash`. Rerun/staleness logic can use resource lineage to flag B as potentially stale, but A never mutates B's copy. - -## Materialization Ordering - -`workflow manage materialize-resource` must: - -1. Resolve the source resource by ID inside the injected current run. -2. Validate visibility and destination before side effects. -3. Write bytes to the sandbox destination. -4. Append the copied `RunResource` row only after the sandbox write succeeds. -5. Update `/workspace/.ergon/resource_imports.json` after the copied row exists. - -If the manifest update fails after the file copy/resource row succeeds, return a structured warning rather than pretending the source was not materialized. - -## Naming - -Default copied names should make ownership clear: - -```text -paper.pdf -> paper (copy).pdf -paper (copy).pdf -> paper (copy 2).pdf -``` - -Identity comes from `resource_id` and `content_hash`, not names. The name suffix is for human and agent readability. - -## V1 Lineage Boundary - -V1 adds a nullable self-reference on `RunResource`: `copied_from_resource_id`. - -That is enough for one-resource materialization. Do not build a many-to-many lineage table in v1. If synthesis from multiple copied resources becomes central, add `run_resource_lineage` later. - -For arbitrary later edits, it is enough in v1 that the run has: - -- the copied `RunResource` row -- `copied_from_resource_id` -- the import manifest -- tool-call context events - -Do not pretend to infer every arbitrary transformation automatically. diff --git a/docs/superpowers/plans/workflow-agent-cli/02-schema-and-services.md b/docs/superpowers/plans/workflow-agent-cli/02-schema-and-services.md deleted file mode 100644 index fec9871bd..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/02-schema-and-services.md +++ /dev/null @@ -1,159 +0,0 @@ -# 02 — Schema and Services - -## Schema - -### Migration - -Create: - -```text -ergon_core/migrations/versions/<revision>_add_copied_from_resource_id.py -``` - -Migration behavior: - -- `upgrade()` adds nullable `copied_from_resource_id` UUID column to `run_resources` -- creates a self-referential foreign key to `run_resources.id` -- creates an index on `run_resources.copied_from_resource_id` -- `downgrade()` drops index, foreign key, and column - -### Model Updates - -Modify `ergon_core/ergon_core/core/persistence/telemetry/models.py`: - -```python -class RunResourceKind(StrEnum): - OUTPUT = "output" - REPORT = "report" - ARTIFACT = "artifact" - SEARCH_CACHE = "search_cache" - NOTE = "note" - IMPORT = "import" - """Copied snapshot materialized from another RunResource into a task workspace.""" - -class RunResource(SQLModel, table=True): - # ... - copied_from_resource_id: UUID | None = Field( - default=None, - foreign_key="run_resources.id", - index=True, - ) -``` - -Modify `ergon_core/ergon_core/core/persistence/queries.py`: - -```python -def append( - self, - *, - run_id: UUID, - task_execution_id: UUID, - kind: str, - name: str, - mime_type: str, - file_path: str, - size_bytes: int, - error: str | None, - content_hash: str | None, - metadata: JsonObject | None = None, - copied_from_resource_id: UUID | None = None, -) -> RunResource: - ... -``` - -Pass `copied_from_resource_id` into `RunResource(...)`. - -## DTOs - -Create `ergon_core/ergon_core/core/runtime/services/workflow_dto.py` with frozen Pydantic models: - -- `WorkflowTaskRef` -- `WorkflowExecutionRef` -- `WorkflowResourceRef` -- `WorkflowDependencyRef` -- `WorkflowBlockerRef` -- `WorkflowNextActionRef` -- `WorkflowMaterializedResourceRef` - -`WorkflowResourceRef` includes: - -```python -resource_id: UUID -run_id: UUID -task_execution_id: UUID | None -node_id: UUID | None -task_slug: str | None -kind: str -name: str -mime_type: str -size_bytes: int -file_path: str -content_hash: str | None = None -copied_from_resource_id: UUID | None = None -created_at: datetime -``` - -`WorkflowMaterializedResourceRef` includes: - -```python -source_resource_id: UUID -copied_resource_id: UUID | None -copied_from_resource_id: UUID -source_name: str -copied_name: str -source_content_hash: str | None -copied_content_hash: str | None -sandbox_path: str -dry_run: bool = False -source_mutated: bool = False -``` - -## Workflow Service - -Create `ergon_core/ergon_core/core/runtime/services/workflow_service.py`. - -This single service owns both read-only inspection and resource materialization policy. Keep it as one file for v1 because resource reads, visibility checks, and materialization share the same scope logic. Split later only if the implementation becomes hard to hold in context. - -Required methods: - -```python -class WorkflowService: - def list_tasks(self, session: Session, *, run_id: UUID, parent_node_id: UUID | None = None) -> list[WorkflowTaskRef]: ... - def get_task(self, session: Session, *, run_id: UUID, node_id: UUID | None, task_slug: str | None) -> WorkflowTaskRef: ... - def get_latest_execution(self, session: Session, *, node_id: UUID) -> RunTaskExecution | None: ... - def list_dependencies(self, session: Session, *, run_id: UUID, node_id: UUID, direction: Literal["upstream", "downstream", "both"]) -> list[WorkflowDependencyRef]: ... - def list_resources(self, session: Session, *, run_id: UUID, node_id: UUID | None, scope: Literal["input", "upstream", "own", "children", "descendants", "visible"], kind: str | None = None, max_depth: int = 3, limit: int = 50) -> list[WorkflowResourceRef]: ... - def read_resource_bytes(self, session: Session, *, run_id: UUID, resource_id: UUID, max_bytes: int) -> bytes: ... - def get_task_blockers(self, session: Session, *, run_id: UUID, node_id: UUID) -> list[WorkflowBlockerRef]: ... - def get_next_actions(self, session: Session, *, run_id: UUID, node_id: UUID, manager_capable: bool) -> list[WorkflowNextActionRef]: ... - async def materialize_resource( - self, - session: Session, - *, - run_id: UUID, - current_node_id: UUID, - current_execution_id: UUID, - sandbox_task_key: UUID, - benchmark_type: str, - resource_id: UUID, - destination: str | None, - dry_run: bool, - ) -> WorkflowMaterializedResourceRef: ... -``` - -For `input` and `upstream`, use incoming edges to the current node, get each source node's latest completed execution, then collect `RunResource` rows for those execution IDs. - -For `visible`, list current-run resources allowed by policy, including resources from divergent DAG branches. This must still exclude eval/private/system resources and must be capped by `limit`. - -`materialize_resource(...)` should use the benchmark's sandbox manager class and existing `BaseSandboxManager.upload_file(...)` to write into the live E2B sandbox. Do not create a new low-level E2B upload primitive. - -The service must reject: - -- resource IDs not in the injected current run -- invisible resources -- eval/private/system resources -- absolute destinations -- `..` path traversal -- symlink escapes -- paths outside `/workspace` -- destination collisions unless explicit versioning/overwrite behavior is implemented diff --git a/docs/superpowers/plans/workflow-agent-cli/03-cli-command-surface.md b/docs/superpowers/plans/workflow-agent-cli/03-cli-command-surface.md deleted file mode 100644 index d16657c5e..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/03-cli-command-surface.md +++ /dev/null @@ -1,242 +0,0 @@ -# 03 — CLI Command Surface - -## Global Rules - -Build two command groups: - -- `workflow inspect ...`: read-only task topology, dependency, resource, and workspace inspection. -- `workflow manage ...`: state-changing commands. Graph lifecycle commands require a manager-capable profile; `materialize-resource` is available to task agents for visible resources. - -Every command is scoped by injected runtime context: - -- `run_id` -- `node_id` -- `execution_id` when needed -- `sandbox_id` / `sandbox_task_key` when materializing - -All commands support `--format text|json` and `--explain`. Commands that can produce large output support `--limit`, `--max-chars`, or `--max-bytes`. - -All `manage ...` commands support `--dry-run`. - -## Inspect Commands - -### `workflow inspect task-list` - -Lists task nodes in a run. - -Examples: - -```bash -workflow("inspect task-list") -workflow("inspect task-list --children") -workflow("inspect task-list --level 2") -workflow("inspect task-list --under d_root --level 3") -workflow("inspect task-list --format json") -``` - -### `workflow inspect task-tree` - -Shows a recursive task subtree, grouped by level. This is the default orientation command for agents. - -Examples: - -```bash -workflow("inspect task-tree") -workflow("inspect task-tree --from current --depth 2") -workflow("inspect task-tree --from root --depth 3") -workflow("inspect task-tree --from d_root --level 2") -workflow("inspect task-tree --from d_root --status pending") -``` - -### `workflow inspect task-details` - -Shows one task node plus latest execution summary. - -Examples: - -```bash -workflow("inspect task-details") -workflow("inspect task-details --task-slug d_left --include-output") -workflow("inspect task-details --task-slug d_left --format json") -``` - -### `workflow inspect task-dependencies` - -Shows graph dependencies for a task. - -Examples: - -```bash -workflow("inspect task-dependencies") -workflow("inspect task-dependencies --task-slug d_join --direction upstream") -workflow("inspect task-dependencies --task-slug d_left --direction downstream --format json") -``` - -### `workflow inspect task-blockers` - -Explains why a task is not ready, not completed, or cannot proceed. - -Examples: - -```bash -workflow("inspect task-blockers") -workflow("inspect task-blockers --task-slug d_join") -workflow("inspect task-blockers --task-slug l_3 --format json") -``` - -### `workflow inspect next-actions` - -Gives a concise recovery/orientation summary. - -Examples: - -```bash -workflow("inspect next-actions") -workflow("inspect next-actions --include-completed") -``` - -### `workflow inspect resource-list` - -Lists current-run resources visible under the selected scope. - -Examples: - -```bash -workflow("inspect resource-list --scope input") -workflow("inspect resource-list --scope upstream") -workflow("inspect resource-list --scope children") -workflow("inspect resource-list --scope descendants --max-depth 3") -workflow("inspect resource-list --scope visible --limit 20") -workflow("inspect resource-list --scope own --kind report") -workflow("inspect resource-list --scope input --format json") -``` - -Scopes: - -- `input`: latest successful resources from immediate upstream dependency nodes. -- `upstream`: alias for `input` in v1. -- `own`: resources produced by the current node's latest execution. -- `children`: resources produced by direct child task executions. -- `descendants`: resources produced by descendants up to `--max-depth`. -- `visible`: broadest current-run discovery scope for resources the profile may see, including divergent DAG branches. This is not cross-run access. - -### `workflow inspect resource-content` - -Reads resource content from `RunResource.file_path`. - -Examples: - -```bash -workflow("inspect resource-content --resource-id $RESOURCE_ID") -workflow("inspect resource-content --resource-id $RESOURCE_ID --max-bytes 20000") -``` - -### `workflow inspect resource-location` - -Returns resource metadata and local blob path without dumping content. - -```bash -workflow("inspect resource-location --resource-id $RESOURCE_ID") -``` - -### `workflow inspect task-workspace` - -Shows a compact current task workspace snapshot: task, execution, upstream dependencies, downstream dependents, input resources, own resources, children, and next commands. - -```bash -workflow("inspect task-workspace") -workflow("inspect task-workspace --task-slug d_join") -workflow("inspect task-workspace --task-slug d_join --include-output") -``` - -## Manage Commands - -### `workflow manage create-task` - -Adds one dynamic subtask under the current node via `TaskManagementService.add_subtask`. - -```bash -workflow("manage create-task --task-slug summarize_left --worker researchrubrics-smoke-leaf --description 'Summarize left branch'") -workflow("manage create-task --task-slug join --worker researchrubrics-smoke-leaf --description 'Join summaries' --depends-on summarize_left --dry-run") -``` - -### `workflow manage create-task-plan` - -Adds multiple dynamic subtasks in one transaction via `TaskManagementService.plan_subtasks`. - -```bash -workflow("manage create-task-plan --json '[{\"task_slug\":\"a\",\"description\":\"Do A\",\"assigned_worker_slug\":\"researchrubrics-smoke-leaf\"}]' --dry-run") -``` - -### `workflow manage create-dependency` - -Adds a dependency edge between two existing visible tasks. - -```bash -workflow("manage create-dependency --source summarize_left --target join --dry-run") -``` - -### `workflow manage restart-task` - -Resets a terminal task back to pending via `TaskManagementService.restart_task`. - -```bash -workflow("manage restart-task --task-slug l_2 --dry-run") -``` - -### `workflow manage abandon-task` - -Cancels a task via `TaskManagementService.cancel_task`. - -```bash -workflow("manage abandon-task --task-slug l_3 --dry-run") -``` - -### `workflow manage update-task-description` - -Updates a non-running task description via `TaskManagementService.refine_task`. - -```bash -workflow("manage update-task-description --task-slug l_3 --description 'Retry with corrected input file' --dry-run") -``` - -### `workflow manage materialize-resource` - -Copies one immutable visible `RunResource` into the current agent's E2B workspace and records the copy as a new current-task-owned resource. - -```bash -workflow("manage materialize-resource --resource-id $RESOURCE_ID") -workflow("manage materialize-resource --resource-id $RESOURCE_ID --destination imported/task-a/paper.pdf") -workflow("manage materialize-resource --resource-id $RESOURCE_ID --destination imported/task-a/paper.pdf --dry-run") -``` - -Text output: - -```text -materialized resource - source: res_a paper.pdf sha256:abc... - copy: res_b paper (copy).pdf kind=import - copied_from_resource_id: res_a - sandbox_path: /workspace/imported/task-a/paper (copy).pdf - note: source resource was not modified -``` - -JSON output: - -```json -{ - "source_resource_id": "res_a", - "copied_resource_id": "res_b", - "copied_from_resource_id": "res_a", - "source_content_hash": "abc", - "copied_content_hash": "abc", - "sandbox_path": "/workspace/imported/task-a/paper (copy).pdf", - "source_mutated": false -} -``` - -## Deferred Commands - -- `workflow messages send` -- `workflow manage remove-dependency` -- many-to-many resource lineage management diff --git a/docs/superpowers/plans/workflow-agent-cli/04-agent-tool-and-worker.md b/docs/superpowers/plans/workflow-agent-cli/04-agent-tool-and-worker.md deleted file mode 100644 index 5c8b9c0ef..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/04-agent-tool-and-worker.md +++ /dev/null @@ -1,127 +0,0 @@ -# 04 — Agent Tool and Worker POC - -## Agent Invocation Model - -The agent runs locally in the worker process. The environment runs in E2B. Therefore `workflow(...)` should read local Postgres directly, then use existing sandbox manager APIs for approved workspace materialization. - -Do not put this command inside the E2B sandbox. - -## Tool Factory - -Create `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py`. - -```python -def make_workflow_cli_tool( - *, - context: WorkerContext, - allowed_scopes: frozenset[str] = frozenset({"input", "own", "upstream", "children", "descendants", "visible"}), - manager_capable: bool = False, -) -> Callable[..., Awaitable[WorkflowCliToolResponse]]: - ... -``` - -Tool signature: - -```python -async def workflow(command: str, timeout_s: int = 10) -> WorkflowCliToolResponse: - """Run a scoped workflow navigation command.""" -``` - -Examples the prompt can show: - -```text -inspect task-tree -inspect task-workspace -inspect resource-list --scope input -inspect resource-list --scope visible --limit 20 -inspect resource-content --resource-id ... -manage materialize-resource --resource-id ... --dry-run -manage restart-task --task-slug l_2 --dry-run -``` - -## Wrapper Rules - -The wrapper: - -- receives only the model's subcommand string -- prepends `workflow` -- injects `--run-id <context.run_id>` -- injects `--node-id <context.node_id>` -- injects `--execution-id <context.execution_id>` when supported -- injects `--sandbox-id <context.sandbox_id>` when supported -- injects `--sandbox-task-key <context.task_id or context.node_id>` when supported -- rejects user-supplied `--run-id`, `--node-id`, `--execution-id`, `--sandbox-id`, `--sandbox-task-key`, `--definition-id`, `--experiment-id`, or `--cohort-id` -- calls `ergon_cli.main._main(argv)` in-process with stdout/stderr captured -- rejects multiline input and unrelated top-level commands - -## Permissions - -All workflow CLI commands are current-run scoped by invariant. Cross-run output is a CLI bug. - -Leaf agents: - -- can inspect `input`, `own`, `upstream`, `children`, `descendants`, and capped `visible` resources when allowed by profile -- can run `manage materialize-resource` for visible current-run resources -- cannot run graph lifecycle mutations - -Manager-capable agents: - -- can inspect the same scopes -- can run `manage materialize-resource` -- can run graph lifecycle commands: `create-task`, `create-task-plan`, `create-dependency`, `restart-task`, `abandon-task`, `update-task-description` - -No command queries eval tables or accepts raw SQL. - -## Prompt Guidance - -Use this instruction in the POC worker: - -```text -Use the `workflow` tool to inspect task topology and resources. Start with -`inspect task-tree`, `inspect task-workspace`, or -`inspect resource-list --scope input`. If a visible resource from another task -is useful, run `manage materialize-resource --resource-id ... --dry-run` -before importing it into your workspace. Use `--dry-run` before any graph -lifecycle `manage ...` mutation command. -``` - -## POC Worker - -Create: - -```text -ergon_builtins/ergon_builtins/workers/research_rubrics/workflow_cli_react_worker.py -``` - -Class: - -```python -class ResearchRubricsWorkflowCliReActWorker(ResearchRubricsResearcherWorker): - """Research-rubrics ReAct worker with the workflow CLI tool enabled.""" -``` - -Inside `execute()`, mirror the existing research-rubrics runtime tool composition and add: - -```python -workflow_tool = make_workflow_cli_tool(context=context) -self.tools = [*rr_tools, *graph_tools, Tool(function=workflow_tool, takes_ctx=False)] -``` - -If pydantic-ai expects the callable directly rather than prewrapped `Tool`, mirror the existing toolkit pattern. - -## Registry - -Modify `ergon_builtins/ergon_builtins/registry_data.py`: - -```python -from ergon_builtins.workers.research_rubrics.workflow_cli_react_worker import ( - ResearchRubricsWorkflowCliReActWorker, -) - -WORKERS: dict[str, Callable[..., Worker]] = { - "researchrubrics-researcher": ResearchRubricsResearcherWorker, - "researchrubrics-workflow-cli-react": ResearchRubricsWorkflowCliReActWorker, -} -``` - -Do not alter the existing `"researchrubrics-researcher"` worker behavior for this POC. diff --git a/docs/superpowers/plans/workflow-agent-cli/05-tests-and-acceptance.md b/docs/superpowers/plans/workflow-agent-cli/05-tests-and-acceptance.md deleted file mode 100644 index c3f8a6172..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/05-tests-and-acceptance.md +++ /dev/null @@ -1,268 +0,0 @@ -# 05 — Tests and Acceptance - -## Unit Tests - -### Workflow Service - -Create `tests/unit/runtime/test_workflow_service.py`. - -Cover: - -- `list_tasks(run_id)` returns all run nodes ordered by level/task slug -- `list_tasks(run_id, parent_node_id=...)` returns direct children only -- `list_dependencies(..., direction="upstream")` returns incoming edges with source/target summaries -- `list_resources(..., scope="input")` returns latest resources from immediate upstream nodes only -- `list_resources(..., scope="visible")` can include resources from divergent branches inside the same run -- `read_resource_bytes(...)` rejects resources outside the injected run -- blockers and next actions return useful suggested commands -- copied row has a new resource ID -- copied row has `kind="import"` -- copied row keeps the same `content_hash` and blob `file_path` -- copied row has `copied_from_resource_id=<source id>` -- source row is unchanged -- default name appends `(copy)` -- destination path is normalized under `/workspace` -- `dry_run=True` performs no DB or sandbox write -- absolute paths, `..`, cross-run resources, invisible resources, and destination collisions fail -- import manifest write is requested with source/copy/destination metadata - -### CLI Parser and Handler - -Create `tests/unit/cli/test_workflow_cli.py`. - -Cover: - -- `inspect task-list` -- `inspect task-details --include-output` -- `inspect task-dependencies --direction upstream` -- `inspect resource-list --scope input` -- `inspect resource-list --scope visible --limit 20` -- `inspect resource-content` -- `inspect task-blockers` -- `inspect next-actions` -- `manage materialize-resource --dry-run` -- every graph lifecycle `manage ... --dry-run` -- invalid UUID exits non-zero -- duplicate task slug exits non-zero with a helpful message -- JSON output has stable fields - -### Agent Wrapper - -Create `tests/unit/state/test_workflow_cli_tool.py`. - -Cover: - -- injects `run_id` -- injects `node_id` -- injects `execution_id`, `sandbox_id`, and `sandbox_task_key` for materialization -- denies user-supplied scope/context arguments -- allows `inspect resource-list --scope visible --limit 20` -- allows `manage materialize-resource ... --dry-run` for leaf wrappers -- denies graph lifecycle mutations for leaf wrappers -- allows graph lifecycle dry-runs for manager wrappers -- multiline/bad top-level commands fail structurally, not by subprocess shell behavior - -### Worker Wiring - -Modify `tests/unit/state/test_research_rubrics_workers.py`. - -Cover: - -- `researchrubrics-workflow-cli-react` is registered -- `ResearchRubricsWorkflowCliReActWorker` exposes `workflow` -- prompt recommends `inspect task-workspace`, `inspect resource-list --scope input`, `manage materialize-resource --dry-run`, and dry-run before graph lifecycle mutation -- existing `ResearchRubricsResearcherWorker` behavior remains unchanged - -## Integration Tests - -Yes, this feature needs integration tests because the unit tests can mock too much of the persistence/sandbox boundary. - -Create: - -```text -tests/integration/runtime/test_workflow_cli_materialize_resource.py -``` - -Use real Postgres and the normal SQLModel session setup for integration tests. Use a fake or stub sandbox manager registered for the test benchmark so the test can assert `upload_file(...)` calls without real E2B. - -Cover: - -- current-run invariant: a resource from another run cannot be listed, read, or materialized -- `visible` lists resources from divergent branches inside the same run -- `materialize-resource` creates a copied `RunResource` row and leaves the source unchanged -- copied row has `task_execution_id` for the consuming task -- copied row has `copied_from_resource_id` -- copied row reuses the same content-addressed blob path/hash -- sandbox upload receives the copied bytes and normalized `/workspace/...` destination -- import manifest update is attempted after the copied row exists -- failed sandbox upload does not append a copied resource row - -Integration command: - -```bash -pytest tests/integration/runtime/test_workflow_cli_materialize_resource.py -v -``` - -## E2E Tests - -Do not add a fourth standalone e2e test file for this feature. The existing benchmark smoke e2e tests are already slow and already exercise the full orchestration path. Extend the existing stub/smoke workers and assertions instead. - -Modify: - -```text -tests/e2e/test_researchrubrics_smoke.py -tests/e2e/test_minif2f_smoke.py -tests/e2e/test_swebench_smoke.py -tests/e2e/_asserts.py -ergon-dashboard/tests/e2e/_shared/smoke.ts -ergon-dashboard/tests/e2e/researchrubrics.smoke.spec.ts -ergon-dashboard/tests/e2e/minif2f.smoke.spec.ts -ergon-dashboard/tests/e2e/swebench.smoke.spec.ts -``` - -The e2e smoke workers for each benchmark should receive the workflow CLI tool and perform a small deterministic workflow exercise during the existing run. Keep this no-LLM. - -Recommended deterministic e2e behavior inside the existing smoke topology: - -```text -producer leaf: - publishes useful artifact - -consumer leaf or parent smoke worker: - calls workflow("inspect task-tree") - calls workflow("inspect resource-list --scope visible --limit 20") - calls workflow("manage materialize-resource --resource-id <producer-resource> --dry-run") - calls workflow("manage materialize-resource --resource-id <producer-resource>") - reads/edits the materialized workspace file - publishes a consumer-owned artifact - -manager-capable smoke path: - calls one graph lifecycle command with --dry-run, such as create-task or create-dependency - asserts dry-run returns the planned mutation without changing graph shape -``` - -Assertions: - -- run reaches terminal status -- producer has original published `RunResource` -- consumer has `kind=import` copied `RunResource` -- copied row points to producer row via `copied_from_resource_id` -- consumer has a later output resource owned by consumer execution -- source producer row is unchanged -- context events include the `workflow` tool call and materialization result -- no control edge is added between producer and consumer unless explicitly in the test graph -- graph lifecycle dry-run command did not create nodes/edges -- existing smoke assertions still pass for all three benchmark e2e files - -Playwright assertions: - -- The run/task trace UI shows a `workflow` tool call made by the smoke worker. -- The visible trace includes command text for `inspect task-tree`. -- The visible trace includes command text for `inspect resource-list --scope visible`. -- The visible trace includes command text for `manage materialize-resource`. -- The visible trace shows a successful tool result for materialization. -- The UI does not need to render lineage-specific fields like `copied_from_resource_id` in v1; PG assertions own lineage correctness. - -If the existing dashboard smoke spec already has a shared per-run assertion helper, add this as a small optional assertion there rather than duplicating per benchmark. If test IDs are missing, add stable test IDs around tool-call rows/results rather than relying on brittle text layout selectors. - -E2E commands: - -```bash -pytest tests/e2e/test_researchrubrics_smoke.py -v -pytest tests/e2e/test_minif2f_smoke.py -v -pytest tests/e2e/test_swebench_smoke.py -v -pnpm --dir ergon-dashboard test:e2e -- researchrubrics.smoke.spec.ts -pnpm --dir ergon-dashboard test:e2e -- minif2f.smoke.spec.ts -pnpm --dir ergon-dashboard test:e2e -- swebench.smoke.spec.ts -``` - -This saves a full extra e2e run while still proving the workflow CLI works through real orchestration, real sandbox execution, and each benchmark's smoke path. - -## Input Resource Contract - -Create `tests/unit/runtime/test_workflow_input_resource_semantics.py`. - -Graph: - -```text -d_root -> d_left -d_root -> d_right -d_left -> d_join -d_right -> d_join -l_1 -> l_2 -> l_3 -``` - -Assert: - -- `d_join` input resources are latest resources from `d_left` and `d_right` -- `l_3` input resources are latest resources from `l_2`, not `l_1` -- roots and singletons have empty input resources - -This is separate from on-demand `materialize-resource`. - -## Real-LLM Acceptance - -Modify `tests/real_llm/benchmarks/test_researchrubrics.py` so the rollout can parameterize worker and sample limit: - -```python -model = os.environ.get("ERGON_REAL_LLM_MODEL", _DEFAULT_MODEL) -benchmark = os.environ.get("ERGON_REAL_LLM_BENCHMARK", "researchrubrics") -worker = os.environ.get("ERGON_REAL_LLM_WORKER", "researchrubrics-researcher") -evaluator = os.environ.get("ERGON_REAL_LLM_EVALUATOR", "research-rubric") -limit = os.environ.get("ERGON_REAL_LLM_LIMIT", "1") -``` - -Final acceptance rollout: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=5 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s -``` - -Expected: - -- real-LLM test reaches terminal status: `completed`, `failed`, or `cancelled` -- artifacts are written under `tests/real_llm/.rollouts/<timestamp>-<run_id>/` -- manifest records `worker=researchrubrics-workflow-cli-react` and `limit=5` -- report and dumped rows show whether the agent invoked `workflow(...)` -- report and dumped rows show whether it materialized copied resources -- reviewer can inspect whether CLI usage helped the agent orient around topology/resources - -## Verification Bundle - -Run focused tests: - -```bash -pytest tests/unit/runtime/test_workflow_service.py tests/unit/cli/test_workflow_cli.py tests/unit/state/test_workflow_cli_tool.py -v -``` - -Run integration: - -```bash -pytest tests/integration/runtime/test_workflow_cli_materialize_resource.py -v -``` - -Run e2e: - -```bash -pytest tests/e2e/test_researchrubrics_smoke.py -v -pytest tests/e2e/test_minif2f_smoke.py -v -pytest tests/e2e/test_swebench_smoke.py -v -``` - -Run worker wiring: - -```bash -pytest tests/unit/state/test_research_rubrics_workers.py -v -``` - -Run final real-LLM rollout on demand: - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=5 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s -``` diff --git a/docs/superpowers/plans/workflow-agent-cli/06-phases.md b/docs/superpowers/plans/workflow-agent-cli/06-phases.md deleted file mode 100644 index 831df2447..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/06-phases.md +++ /dev/null @@ -1,143 +0,0 @@ -# 06 — Phases, Deliverables, Acceptance Gates - -## Delivery Shape - -One PR, layered so each phase is independently reviewable. Each phase should leave tests green before moving on. - -## Phase A — Schema and Persistence - -**Scope** - -- Add `RunResourceKind.IMPORT`. -- Add `RunResource.copied_from_resource_id`. -- Extend `ResourcesQueries.append(...)`. -- Add migration `<revision>_add_copied_from_resource_id.py`. - -**Acceptance Gate** - -```bash -pytest tests/unit/runtime/test_workflow_service.py -v -``` - -At this phase, only schema/helper tests need to pass; workflow service tests can be marked/structured around the implemented subset. - -## Phase B — Workflow DTOs and Read Service - -**Scope** - -- Create `workflow_dto.py`. -- Create `workflow_service.py`. -- Implement task listing, tree traversal, dependency inspection, resource scopes, blockers, next actions, and resource content reads. - -**Acceptance Gate** - -```bash -pytest tests/unit/runtime/test_workflow_service.py -v -``` - -## Phase C — Workflow Materialization - -**Scope** - -- Implement current-run resource resolution. -- Implement visible-resource policy. -- Normalize destinations under `/workspace`. -- Copy bytes into sandbox via existing `BaseSandboxManager.upload_file(...)`. -- Append current-task-owned `kind=import` resource row after successful upload. -- Update `/workspace/.ergon/resource_imports.json`. - -**Acceptance Gate** - -```bash -pytest tests/unit/runtime/test_workflow_service.py -v -pytest tests/integration/runtime/test_workflow_cli_materialize_resource.py -v -``` - -## Phase D — CLI Command Surface - -**Scope** - -- Create `ergon_cli/ergon_cli/commands/workflow.py`. -- Register `workflow` in `ergon_cli/ergon_cli/main.py`. -- Add all `inspect` commands. -- Add graph lifecycle `manage` commands. -- Add `manage materialize-resource`. -- Support `--format`, `--explain`, output caps, and `--dry-run`. - -**Acceptance Gate** - -```bash -pytest tests/unit/cli/test_workflow_cli.py -v -pytest tests/integration/runtime/test_workflow_cli_materialize_resource.py -v -``` - -## Phase E — Agent Wrapper - -**Scope** - -- Create `ergon_builtins/ergon_builtins/tools/workflow_cli_tool.py`. -- Inject runtime scope from `WorkerContext`. -- Reject user-supplied scope/context arguments. -- Enforce leaf vs manager permissions. -- Capture stdout/stderr from in-process CLI calls. - -**Acceptance Gate** - -```bash -pytest tests/unit/state/test_workflow_cli_tool.py -v -``` - -## Phase F — ResearchRubrics POC Worker - -**Scope** - -- Create `ResearchRubricsWorkflowCliReActWorker`. -- Register `researchrubrics-workflow-cli-react`. -- Add prompt guidance for inspect/materialize/dry-run behavior. -- Keep existing `ResearchRubricsResearcherWorker` unchanged. - -**Acceptance Gate** - -```bash -pytest tests/unit/state/test_research_rubrics_workers.py -v -``` - -## Phase G — Deterministic E2E - -**Scope** - -- Extend the existing benchmark smoke e2e paths instead of adding a fourth e2e test file. -- Give the smoke/stub workers enough workflow CLI access to exercise `inspect`, `visible` resource discovery, `materialize-resource`, and graph lifecycle dry-run behavior. -- Add assertions to existing e2e test files/helpers for copied resources, lineage, tool-call context events, and unchanged graph shape after dry-runs. -- Extend the existing dashboard Playwright smoke specs to assert the workflow CLI tool calls/results are visible in the run/task trace UI. -- No real LLM. - -**Acceptance Gate** - -```bash -pytest tests/e2e/test_researchrubrics_smoke.py -v -pytest tests/e2e/test_minif2f_smoke.py -v -pytest tests/e2e/test_swebench_smoke.py -v -pnpm --dir ergon-dashboard test:e2e -- researchrubrics.smoke.spec.ts -pnpm --dir ergon-dashboard test:e2e -- minif2f.smoke.spec.ts -pnpm --dir ergon-dashboard test:e2e -- swebench.smoke.spec.ts -``` - -## Phase H — Real-LLM Rollout - -**Scope** - -- Parameterize `tests/real_llm/benchmarks/test_researchrubrics.py`. -- Run `researchrubrics-workflow-cli-react` for 5 samples. -- Inspect generated rollout artifacts. - -**Acceptance Gate** - -```bash -ERGON_REAL_LLM=1 \ -ERGON_REAL_LLM_WORKER=researchrubrics-workflow-cli-react \ -ERGON_REAL_LLM_LIMIT=5 \ -uv run pytest tests/real_llm/benchmarks/test_researchrubrics.py -v -s -``` - -The run reaching terminal status is enough for the test harness. The product is the artifact bundle and review of actual agent behavior. diff --git a/docs/superpowers/plans/workflow-agent-cli/README.md b/docs/superpowers/plans/workflow-agent-cli/README.md deleted file mode 100644 index 20e2186a5..000000000 --- a/docs/superpowers/plans/workflow-agent-cli/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Workflow Agent CLI — Plan Folder - -**Status:** draft for review. -**Date:** 2026-04-26. -**Scope:** build `ergon workflow ...`, an agent-local command surface for workflow topology/resource inspection, controlled resource materialization, and a ResearchRubrics ReAct proof of concept. - -## Read Order - -1. [`00-program.md`](00-program.md) — goal, architecture, non-goals, file plan. -2. [`01-resource-semantics.md`](01-resource-semantics.md) — immutable resources, copy/fork semantics, current-run invariant, lineage. -3. [`02-schema-and-services.md`](02-schema-and-services.md) — schema migration, DTOs, and the single workflow service. -4. [`03-cli-command-surface.md`](03-cli-command-surface.md) — `workflow inspect ...` and `workflow manage ...` commands. -5. [`04-agent-tool-and-worker.md`](04-agent-tool-and-worker.md) — pydantic-ai wrapper, permissions, ResearchRubrics POC worker. -6. [`05-tests-and-acceptance.md`](05-tests-and-acceptance.md) — unit, integration, e2e, and real-LLM acceptance gates. -7. [`06-phases.md`](06-phases.md) — phased implementation order and acceptance gates. - -## Principle - -Each document should be self-contained for its implementation area. If two docs disagree, [`00-program.md`](00-program.md) wins for scope and ownership, while [`01-resource-semantics.md`](01-resource-semantics.md) wins for resource semantics. - -## Supersedes - -This folder supersedes [`../2026-04-26-mas-navigation-cli.md`](../2026-04-26-mas-navigation-cli.md). Keep the old file only as a pointer while reviewers migrate.