From 2ca648965aa4ea5c7660c04ec825102b2349afd0 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 15 May 2026 10:24:23 +0200 Subject: [PATCH] Expose coordinator retry controls --- README.md | 7 + docs/http-contract.md | 3 + pyproject.toml | 2 +- src/loopy_loop/cli.py | 4 + src/loopy_loop/config.py | 38 ++++++ src/loopy_loop/harness_runner.py | 40 +++++- src/loopy_loop/models.py | 3 + .../loopy_loop_config.yaml | 4 + .../inner_outer_eval/loopy_loop_config.yaml | 4 + src/tests/test_config.py | 33 +++++ src/tests/test_harness_runner.py | 121 ++++++++++++++++++ uv.lock | 8 +- 12 files changed, 261 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 901fb7c..e4e4855 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,10 @@ team_harness_agent_models: codex: "gpt-5.5" team_harness_agent_reasoning_efforts: codex: "high" +# Optional coordinator retry controls. Omit to use team-harness defaults. +# team_harness_max_retries: 8 +# team_harness_retry_base_delay_s: 2.0 +# team_harness_retry_max_delay_s: 60.0 team_harness_api_base: "https://openrouter.ai/api/v1" team_harness_api_key_env: "OPENROUTER_API_KEY" ``` @@ -128,6 +132,9 @@ Rules: - `team_harness_api_key_env` must be set during coordinator preflight and again in the worker before `TeamHarness(...)` - `team_harness_agent_reasoning_efforts` is optional and only affects workers whose team-harness template supports a reasoning-effort flag +- `team_harness_max_retries`, `team_harness_retry_base_delay_s`, and + `team_harness_retry_max_delay_s` are optional coordinator retry controls for + transient team-harness API/network errors Workflow config: diff --git a/docs/http-contract.md b/docs/http-contract.md index 9083daf..ebc6579 100644 --- a/docs/http-contract.md +++ b/docs/http-contract.md @@ -26,6 +26,9 @@ Run response: "team_harness_agents": ["codex"], "team_harness_agent_models": {"codex": "gpt-5.5"}, "team_harness_agent_reasoning_efforts": {"codex": "high"}, + "team_harness_max_retries": null, + "team_harness_retry_base_delay_s": null, + "team_harness_retry_max_delay_s": null, "team_harness_api_base": "https://openrouter.ai/api/v1", "team_harness_api_key_env": "OPENROUTER_API_KEY", "team_harness_system_prompt_extension": "" diff --git a/pyproject.toml b/pyproject.toml index 3700831..3249fce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "pydantic>=2.11", "pyyaml>=6.0.2", "rich>=14.0", - "team-harness>=0.2.8", + "team-harness>=0.2.9", "uvicorn[standard]>=0.32.0", ] diff --git a/src/loopy_loop/cli.py b/src/loopy_loop/cli.py index 0851e6c..a17cf3e 100644 --- a/src/loopy_loop/cli.py +++ b/src/loopy_loop/cli.py @@ -50,6 +50,10 @@ - "codex" team_harness_agent_models: {{}} team_harness_agent_reasoning_efforts: {{}} +# Optional coordinator retry controls. Omit to use team-harness defaults. +# team_harness_max_retries: 8 +# team_harness_retry_base_delay_s: 2.0 +# team_harness_retry_max_delay_s: 60.0 team_harness_api_base: "https://openrouter.ai/api/v1" team_harness_api_key_env: "OPENROUTER_API_KEY" """ diff --git a/src/loopy_loop/config.py b/src/loopy_loop/config.py index 6af5131..7eeb440 100644 --- a/src/loopy_loop/config.py +++ b/src/loopy_loop/config.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import field_validator +from pydantic import model_validator from pydantic import ValidationError import yaml @@ -90,6 +91,29 @@ class RootConfig(BaseModel): "effort flag will use this value." ), ) + team_harness_max_retries: int | None = Field( + default=None, + ge=0, + description=( + "Optional coordinator retry budget passed to team-harness. Leave null " + "to use the installed team-harness default." + ), + ) + team_harness_retry_base_delay_s: float | None = Field( + default=None, + gt=0, + description=( + "Optional base delay in seconds for team-harness coordinator retry backoff." + ), + ) + team_harness_retry_max_delay_s: float | None = Field( + default=None, + gt=0, + description=( + "Optional maximum delay in seconds for team-harness coordinator retry " + "backoff." + ), + ) team_harness_api_base: str = Field( default=DEFAULT_API_BASE, description="OpenAI-compatible API base URL passed to team-harness.", @@ -125,6 +149,20 @@ def validate_non_empty_string_mapping(cls, value: dict[str, str]) -> dict[str, s def normalize_api_base_value(cls, value: str) -> str: return normalize_api_base(value=value) + @model_validator(mode="after") + def validate_retry_delay_bounds(self) -> "RootConfig": + if ( + self.team_harness_retry_base_delay_s is not None + and self.team_harness_retry_max_delay_s is not None + and self.team_harness_retry_max_delay_s + < self.team_harness_retry_base_delay_s + ): + raise ValueError( + "team_harness_retry_max_delay_s must be greater than or equal to " + "team_harness_retry_base_delay_s" + ) + return self + class RunAfterSuccesses(BaseModel): """Cadence rule that unlocks a workflow after successful runs of another.""" diff --git a/src/loopy_loop/harness_runner.py b/src/loopy_loop/harness_runner.py index b6e2428..ada3548 100644 --- a/src/loopy_loop/harness_runner.py +++ b/src/loopy_loop/harness_runner.py @@ -55,12 +55,16 @@ def run_harness_iteration( raise except TeamHarnessError as exc: traceback.print_exc() + harness_run_id, harness_output_dir = _failure_harness_paths( + detail=exc.detail, harness_output_root=harness_output_root + ) return IterationResult( success=False, text=None, error=str(exc), error_detail=exc.detail, - harness_run_id="", + harness_run_id=harness_run_id, + harness_output_dir=harness_output_dir, ) except Exception as exc: traceback.print_exc() @@ -90,6 +94,26 @@ def _build_harness_kwargs( "cwd": str(repo_root), "console_mode": "silent", } + retry_kwargs = { + "max_retries": config_snapshot.team_harness_max_retries, + "retry_base_delay_s": config_snapshot.team_harness_retry_base_delay_s, + "retry_max_delay_s": config_snapshot.team_harness_retry_max_delay_s, + } + configured_retry_kwargs = { + key: value for key, value in retry_kwargs.items() if value is not None + } + if configured_retry_kwargs: + if _supports_kwargs( + harness_factory=harness_factory, names=configured_retry_kwargs.keys() + ): + kwargs.update(configured_retry_kwargs) + else: + raise ConfigError( + "Installed team-harness does not support coordinator retry " + "controls; upgrade team-harness or remove " + "team_harness_max_retries, team_harness_retry_base_delay_s, " + "and team_harness_retry_max_delay_s from loopy_loop_config.yaml." + ) agent_override_kwargs = { "agent_models": config_snapshot.team_harness_agent_models, "agent_reasoning_efforts": config_snapshot.team_harness_agent_reasoning_efforts, @@ -125,6 +149,20 @@ def _supports_kwargs( return all(name in signature.parameters for name in names) +def _failure_harness_paths( + *, detail: dict[str, object] | None, harness_output_root: Path | None +) -> tuple[str, str]: + if not detail: + return "", "" + run_id_value = detail.get("run_id") + output_dir_value = detail.get("session_output_dir") + run_id = run_id_value if isinstance(run_id_value, str) else "" + output_dir = output_dir_value if isinstance(output_dir_value, str) else "" + if not output_dir and run_id and harness_output_root is not None: + output_dir = str(harness_output_root / run_id) + return run_id, output_dir + + def write_iteration_artifacts( *, iteration_dir: Path, rendered_prompt: str, iteration_result: IterationResult ) -> None: diff --git a/src/loopy_loop/models.py b/src/loopy_loop/models.py index 6010e21..a05d8bd 100644 --- a/src/loopy_loop/models.py +++ b/src/loopy_loop/models.py @@ -36,6 +36,9 @@ class RootConfigSnapshot(BaseModel): team_harness_agents: list[str] = Field(...) team_harness_agent_models: dict[str, str] = Field(default_factory=dict) team_harness_agent_reasoning_efforts: dict[str, str] = Field(default_factory=dict) + team_harness_max_retries: int | None = Field(default=None) + team_harness_retry_base_delay_s: float | None = Field(default=None) + team_harness_retry_max_delay_s: float | None = Field(default=None) team_harness_api_base: str = Field(...) team_harness_api_key_env: str = Field(...) team_harness_system_prompt_extension: str = Field(...) diff --git a/src/loopy_loop/templates/atomic_bomberman_lol/loopy_loop_config.yaml b/src/loopy_loop/templates/atomic_bomberman_lol/loopy_loop_config.yaml index 091c0ed..4405a0b 100644 --- a/src/loopy_loop/templates/atomic_bomberman_lol/loopy_loop_config.yaml +++ b/src/loopy_loop/templates/atomic_bomberman_lol/loopy_loop_config.yaml @@ -25,6 +25,10 @@ team_harness_agents: team_harness_agent_models: codex: "gpt-5.5" team_harness_agent_reasoning_efforts: {} +# Optional coordinator retry controls. Omit to use team-harness defaults. +# team_harness_max_retries: 8 +# team_harness_retry_base_delay_s: 2.0 +# team_harness_retry_max_delay_s: 60.0 team_harness_api_base: "https://openrouter.ai/api/v1" team_harness_api_key_env: "OPENROUTER_API_KEY" team_harness_system_prompt_extension: | diff --git a/src/loopy_loop/templates/inner_outer_eval/loopy_loop_config.yaml b/src/loopy_loop/templates/inner_outer_eval/loopy_loop_config.yaml index 64e19cf..58b1695 100644 --- a/src/loopy_loop/templates/inner_outer_eval/loopy_loop_config.yaml +++ b/src/loopy_loop/templates/inner_outer_eval/loopy_loop_config.yaml @@ -13,6 +13,10 @@ team_harness_agent_models: gemini: "gemini-3-flash" team_harness_agent_reasoning_efforts: codex: "high" +# Optional coordinator retry controls. Omit to use team-harness defaults. +# team_harness_max_retries: 8 +# team_harness_retry_base_delay_s: 2.0 +# team_harness_retry_max_delay_s: 60.0 team_harness_api_base: "https://openrouter.ai/api/v1" team_harness_api_key_env: "OPENROUTER_API_KEY" team_harness_system_prompt_extension: | diff --git a/src/tests/test_config.py b/src/tests/test_config.py index 0363c40..5871a30 100644 --- a/src/tests/test_config.py +++ b/src/tests/test_config.py @@ -22,6 +22,9 @@ def test_load_root_config_and_workflows(repo_builder: Any, monkeypatch: Any) -> assert root_config.team_harness_api_base == "https://openrouter.ai/api/v1" assert root_config.team_harness_agent_models == {} assert root_config.team_harness_agent_reasoning_efforts == {} + assert root_config.team_harness_max_retries is None + assert root_config.team_harness_retry_base_delay_s is None + assert root_config.team_harness_retry_max_delay_s is None assert root_config.goal == "Ship a minimal working landing page" assert [workflow.id for workflow in workflows] == ["goal_check", "planner"] assert workflows[0].priority == 0 @@ -61,6 +64,36 @@ def test_load_root_config_uses_goal_file_and_optional_defaults( assert root_config.team_harness_system_prompt_extension == "" +def test_load_root_config_accepts_team_harness_retry_controls( + repo_builder: Any, +) -> None: + repo_root = repo_builder( + root_config={ + "team_harness_max_retries": 8, + "team_harness_retry_base_delay_s": 2.0, + "team_harness_retry_max_delay_s": 60.0, + } + ) + + root_config = load_root_config(repo_root=repo_root) + + assert root_config.team_harness_max_retries == 8 + assert root_config.team_harness_retry_base_delay_s == 2.0 + assert root_config.team_harness_retry_max_delay_s == 60.0 + + +def test_load_root_config_rejects_invalid_retry_delay_bounds(repo_builder: Any) -> None: + repo_root = repo_builder( + root_config={ + "team_harness_retry_base_delay_s": 10.0, + "team_harness_retry_max_delay_s": 1.0, + } + ) + + with pytest.raises(ConfigError, match="team_harness_retry_max_delay_s"): + load_root_config(repo_root=repo_root) + + def test_load_root_config_rejects_inline_goal(repo_builder: Any) -> None: repo_root = repo_builder(root_config={"goal": "Inline goals are no longer used"}) diff --git a/src/tests/test_harness_runner.py b/src/tests/test_harness_runner.py index 14c7394..7244c28 100644 --- a/src/tests/test_harness_runner.py +++ b/src/tests/test_harness_runner.py @@ -106,6 +106,69 @@ async def run(self, task: str) -> TeamHarnessResult: } +def test_harness_runner_passes_configured_retry_controls( + repo_root: Any, snapshot_factory: Any, monkeypatch: Any +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "secret") + captured: dict[str, Any] = {} + + class FakeHarness: + def __init__(self, **kwargs: Any) -> None: + captured.update(kwargs) + + async def run(self, task: str) -> TeamHarnessResult: + return TeamHarnessResult(text="done", agents=[], run_id="run-123") + + run_harness_iteration( + repo_root=repo_root, + config_snapshot=snapshot_factory( + team_harness_max_retries=8, + team_harness_retry_base_delay_s=2.0, + team_harness_retry_max_delay_s=60.0, + ), + rendered_prompt="rendered prompt", + harness_factory=FakeHarness, + ) + + assert captured["max_retries"] == 8 + assert captured["retry_base_delay_s"] == 2.0 + assert captured["retry_max_delay_s"] == 60.0 + + +def test_harness_runner_rejects_retry_controls_for_old_harness( + repo_root: Any, snapshot_factory: Any, monkeypatch: Any +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "secret") + + class OldHarness: + def __init__( + self, + *, + provider: str, + model: str, + api_base: str, + api_key: str, + agents: list[str], + agent_models: dict[str, str], + agent_reasoning_efforts: dict[str, str], + system_prompt: str, + cwd: str, + console_mode: str, + ) -> None: + pass + + async def run(self, task: str) -> TeamHarnessResult: + return TeamHarnessResult(text="done", agents=[], run_id="run-123") + + with pytest.raises(ConfigError, match="coordinator retry controls"): + run_harness_iteration( + repo_root=repo_root, + config_snapshot=snapshot_factory(team_harness_max_retries=8), + rendered_prompt="rendered prompt", + harness_factory=OldHarness, + ) + + def test_harness_runner_passes_none_api_key_for_codex_provider( repo_root: Any, snapshot_factory: Any, monkeypatch: Any ) -> None: @@ -255,6 +318,64 @@ async def run(self, task: str) -> TeamHarnessResult: assert result.error_detail == detail +def test_harness_runner_preserves_failed_harness_paths_from_error_detail( + repo_root: Any, snapshot_factory: Any, monkeypatch: Any, tmp_path: Any +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "secret") + detail: dict[str, object] = { + "kind": "coordinator_api", + "run_id": "run-456", + "session_output_dir": str(tmp_path / "outputs" / "0002_inner" / "run-456"), + } + + class FakeHarness: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + + async def run(self, task: str) -> TeamHarnessResult: + raise TeamHarnessError("boom", detail=detail) + + result = run_harness_iteration( + repo_root=repo_root, + config_snapshot=snapshot_factory(), + rendered_prompt="rendered prompt", + harness_output_root=tmp_path / "outputs" / "0002_inner", + harness_factory=FakeHarness, + ) + + assert result.success is False + assert result.harness_run_id == "run-456" + assert result.harness_output_dir == str( + tmp_path / "outputs" / "0002_inner" / "run-456" + ) + + +def test_harness_runner_derives_failed_output_dir_from_run_id( + repo_root: Any, snapshot_factory: Any, monkeypatch: Any, tmp_path: Any +) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "secret") + + class FakeHarness: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + + async def run(self, task: str) -> TeamHarnessResult: + raise TeamHarnessError("boom", detail={"run_id": "run-789"}) + + result = run_harness_iteration( + repo_root=repo_root, + config_snapshot=snapshot_factory(), + rendered_prompt="rendered prompt", + harness_output_root=tmp_path / "outputs" / "0003_inner", + harness_factory=FakeHarness, + ) + + assert result.harness_run_id == "run-789" + assert result.harness_output_dir == str( + tmp_path / "outputs" / "0003_inner" / "run-789" + ) + + def test_harness_runner_writes_failure_artifacts(tmp_path: Any) -> None: write_iteration_artifacts( iteration_dir=tmp_path, diff --git a/uv.lock b/uv.lock index 9da2bd4..1ee2887 100644 --- a/uv.lock +++ b/uv.lock @@ -259,7 +259,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rich", specifier = ">=14.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, - { name = "team-harness", specifier = ">=0.2.8" }, + { name = "team-harness", specifier = ">=0.2.9" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, ] @@ -556,7 +556,7 @@ wheels = [ [[package]] name = "team-harness" -version = "0.2.8" +version = "0.2.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -567,9 +567,9 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/b1/e4d1496481edc550efa949d0ad04b7c9810d90eb6ada75e773bcc53daffb/team_harness-0.2.8.tar.gz", hash = "sha256:d29c361f5398b6553520439afb30fa3d0c6bff6e6e5468c40050e73e711c7a39", size = 1955706 } +sdist = { url = "https://files.pythonhosted.org/packages/a3/58/fc508f4e59d123b5a0647576636c2aa18d8f5f525b1e68f1e04831348b77/team_harness-0.2.9.tar.gz", hash = "sha256:c7635cc6c5546c34512493c90f0b5728e6f9c1075f0a10e91b8e224e6bd16b64", size = 1959046 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/59/8cf24202cefc3e487342dbcd5b18c95a3fe924bb3243fd86f089fc678b2b/team_harness-0.2.8-py3-none-any.whl", hash = "sha256:0f5d6551a9f4e3a4eaa9c89b09b9c9c48c1b1646965eb021da19009f027a0cd6", size = 101150 }, + { url = "https://files.pythonhosted.org/packages/26/a2/9b4d3efe0f803e6ad1d76ee2407bff73ebfd6e12bfe851285996d1861a50/team_harness-0.2.9-py3-none-any.whl", hash = "sha256:e01b3ac67fc094566855aa811ba17fa9a8926d8e8460fe86c8db9a24898c4959", size = 103561 }, ] [[package]]