Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ All notable changes to this project are documented here. The format is based on

### Fixed

- **`--epochs 0` or a negative value now exits with a guided error instead of a
raw traceback** (#145). Both `inspect-robots run` and `inspect-robots eval-set`
catch the `ConfigError` raised by `Task`'s epoch validation and surface it
through the existing `_resolve_or_exit` pattern, matching how invalid
constructor kwargs are handled for config-file components (#47).
- **Operator scoring no longer prompts twice for self-confirming embodiments**
(#53). On interactive ad-hoc runs, definitive `success` or `failure`
termination verdicts are adopted as the operator judgement, announced on the
Expand Down
18 changes: 16 additions & 2 deletions src/inspect_robots/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,12 @@ def _cmd_run(args: argparse.Namespace) -> int:
embodiment = resolved.embodiment
try:
if args.epochs is not None:
task = replace(task, epochs=args.epochs)
from inspect_robots.errors import ConfigError

try:
task = replace(task, epochs=args.epochs)
except ConfigError as exc:
raise SystemExit(f"--epochs: {exc}") from exc

_announce_components(resolved)
approver = _build_and_announce_guardrails(args, embodiment.info.action_space)
Expand Down Expand Up @@ -1005,7 +1010,16 @@ def _cmd_eval_set(args: argparse.Namespace) -> int:
defaults = load_defaults(os.environ)
tasks = [_resolve_or_exit("task", name) for name in task_names]
if args.epochs is not None:
tasks = [replace(t, epochs=args.epochs) for t in tasks]
from inspect_robots.errors import ConfigError
from inspect_robots.task import Task

patched: list[Task] = []
for t in tasks:
try:
patched.append(replace(t, epochs=args.epochs))
except ConfigError as exc:
raise SystemExit(f"--epochs (task {t.name!r}): {exc}") from exc
tasks = patched

resolved = _resolve_components(args, defaults)
embodiment = resolved.embodiment
Expand Down
54 changes: 51 additions & 3 deletions tests/test_registry_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,50 @@ def test_cli_run_epochs_fail_on_error_store_frames(
assert list((tmp_path / "frames").rglob("*.npy")) # --store-frames streamed (per-run subdir)


@pytest.mark.parametrize("epochs_value", ["0", "-1", "-5"])
def test_cli_run_zero_epochs_exits_with_guided_error(epochs_value: str) -> None:
"""--epochs 0 / negative must produce a guided SystemExit, not a raw traceback (#145)."""
with pytest.raises(SystemExit) as excinfo:
main(
[
"run",
"--task",
"cubepick-reach",
"--policy",
"scripted",
"--embodiment",
"cubepick",
"--epochs",
epochs_value,
]
)
message = str(excinfo.value)
assert "--epochs" in message
assert epochs_value in message


@pytest.mark.parametrize("epochs_value", ["0", "-1"])
def test_cli_eval_set_zero_epochs_exits_with_guided_error(epochs_value: str) -> None:
"""eval-set --epochs 0 / negative must produce a guided SystemExit naming the task (#145)."""
with pytest.raises(SystemExit) as excinfo:
main(
[
"eval-set",
"cubepick-reach",
"--policy",
"scripted",
"--embodiment",
"cubepick",
"--epochs",
epochs_value,
]
)
message = str(excinfo.value)
assert "--epochs" in message
assert epochs_value in message
assert "cubepick-reach" in message # the failing task name is named


def _register_task(name: str, *, num_scenes: int = 1, max_steps: int = 20) -> None:
from inspect_robots.registry import task as task_decorator
from inspect_robots.scene import Scene
Expand Down Expand Up @@ -2464,8 +2508,11 @@ def test_cli_run_closes_embodiment_when_validation_raises(
) -> None:
"""A failure between resolving the embodiment and eval() (here: --epochs 0
raising ConfigError) must still close the embodiment — otherwise a bad flag
leaves real arms energized."""
from inspect_robots.errors import ConfigError
leaves real arms energized.

The fix converts the raw ConfigError into a guided SystemExit; the
important invariant is that close() still fires via the outer try/finally.
"""
from inspect_robots.mock import CubePickEmbodiment

closed: list[bool] = []
Expand All @@ -2478,7 +2525,7 @@ def close(self) -> None:
monkeypatch.setitem(reg._FACTORIES["embodiment"], "tracked-cubepick", _Tracked)
monkeypatch.setenv(ENV_POLICY, "scripted")
monkeypatch.setenv(ENV_EMBODIMENT, "tracked-cubepick")
with pytest.raises(ConfigError):
with pytest.raises(SystemExit) as excinfo:
main(
[
"reach the cube",
Expand All @@ -2490,6 +2537,7 @@ def close(self) -> None:
str(tmp_path / "logs"),
]
)
assert "--epochs" in str(excinfo.value)
assert closed == [True]


Expand Down
Loading