feat(task): support seconds-based task horizons resolved against control rate (#160)

CHANGELOG.md

@@ -39,37 +39,13 @@ All notable changes to this project are documented here. The format is based on
 
 ### Added
 
-- **Policy lifecycle hook: `on_trial_end`** — policies can now hook into
-  the end of a trial to persist state or artifacts. The orchestrator calls
-  `policy.on_trial_end(record, log_dir, run_id)` and any metadata the policy
-  attaches to `record.metadata` is persisted in the final `EvalLog`. Hook
-  failures are caught and logged as trial errors, preventing them from
-  crashing the overall evaluation (#40).
-- **Agent plugin transcript persistence** — `LLMAgentPolicy` now implements
-  `on_trial_end` to persist its full conversation transcript (tool calls,
-  observations, system prompts) to a JSONL file per trial under
-  `<log-dir>/transcripts/<run_id>/<scene_id>-e<epoch>.jsonl`. Camera images
-  are stripped from the transcript to save space, as they are already
-  recorded in the frame store. The relative path to the transcript is
-  stored in the trial's metadata for easy post-hoc analysis (#40).
-- **Agent plugin:** `-P wire=responses` selects the OpenAI Responses API wire,
-  so reasoning effort works together with function tools on recent OpenAI
-  models (Chat Completions rejects the combination, observed on
-  `gpt-5.6-sol`). The chat-wire rejection now names the fix in its error
-  message (#131).
-- **`inspect-robots view LOG.json`**: render a saved eval log as a
-  self-contained HTML report with run metadata, scores, scene results,
-  collapsible policy conversations, highlighted agent notes, and the camera
-  frames the model saw in `--store-frames` runs. `--no-frames` keeps
-  placeholders and `--frames-budget` controls the inline payload limit (#132,
-  #141).
-- **`inspect-robots eval-set TASK [TASK ...]`**: run several registered tasks
-  against one resolved policy/embodiment pair in a single invocation, matching
-  task names exactly or by shell-quoted `fnmatch` glob (e.g.
-  `'kitchenbench/*'`). Thin CLI wrapper over
-  [`eval_set`][inspect_robots.eval.eval_set] that resolves the embodiment once
-  for the whole set rather than once per task, and prints one status line plus
-  a compact per-task row instead of a full summary per task (#45).
+- **Seconds-based task horizons (`Task(max_seconds=...)`)** — tasks can now
+  declare physical time horizons in seconds, which `eval()` resolves into
+  integer step budgets (`max_steps = math.ceil(max_seconds * control_hz)`)
+  after pairing with an embodiment. Preflight compatibility checks
+  (`check_compatibility`) fail fast with a `control_hz_missing` error if
+  `max_seconds` is specified on an embodiment without a declared `control_hz`
+  (#160).
 - Live agent-policy transcript rows on the Rerun `step` timeline, with
   best-effort non-blocking streaming and complete eval-log persistence (#124).
 - Remote Rerun streaming via `inspect-robots run --rerun-connect [URL]`, so

src/inspect_robots/compat.py

@@ -169,6 +169,17 @@ def check_compatibility(
         )
 
     if task is not None:
+        if task.max_seconds is not None and (
+            embodiment.info.control_hz is None or embodiment.info.control_hz <= 0
+        ):
+            issues.append(
+                CompatIssue(
+                    "error",
+                    "control_hz_missing",
+                    f"task {task.name!r} specifies max_seconds ({task.max_seconds}s) but "
+                    f"embodiment {embodiment.info.name!r} has no valid control_hz declared",
+                )
+            )
         _check_scenes_realizable(task, embodiment, issues)
 
     return report

src/inspect_robots/eval.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import math
 import os
 import subprocess
 import time
@@ -239,16 +240,28 @@ def _run_eval(
     if callable(bind):
         bind(embodiment.info)
 
+    # Resolve step horizon from explicit max_steps or max_seconds x control_hz.
+    if task.max_steps is not None:
+        max_steps: int | None = task.max_steps
+    elif embodiment.info.control_hz is not None and embodiment.info.control_hz > 0:
+        assert task.max_seconds is not None
+        max_steps = math.ceil(task.max_seconds * embodiment.info.control_hz)
+    else:
+        max_steps = None
+
     # Horizon-aware embodiments (plan 0013): an optional bind_task() hook runs
     # here too, so the adapter can learn the rollout envelope (e.g. for an
     # operator countdown) before any hardware is touched. Duck-typed —
     # bind_task is not part of the Embodiment Protocol.
     bind_task = getattr(embodiment, "bind_task", None)
-    if callable(bind_task):
-        bind_task(task.envelope)
+    if callable(bind_task) and max_steps is not None:
+        from inspect_robots.task import TaskEnvelope
+
+        bind_task(TaskEnvelope(name=task.name, max_steps=max_steps))
 
     # Fail fast on incompatible pairings before touching any hardware/sim.
     assert_compatible(policy, embodiment, task, remap=remap)
+    assert max_steps is not None
 
     epoch_spec = task.epoch_spec
     scorers = task.scorers
@@ -291,7 +304,8 @@ def _run_eval(
             "capabilities": sorted(embodiment.info.capabilities),
         },
         seed=seed,
-        max_steps=task.max_steps,
+        max_steps=max_steps,
+        max_seconds=task.max_seconds,
     )
     bus.on_eval_start(spec)
 
@@ -329,7 +343,7 @@ def _run_eval(
                     policy,
                     embodiment,
                     scene,
-                    max_steps=task.max_steps,
+                    max_steps=max_steps,
                     seed=trial_seed,
                     epoch=epoch,
                     controller=controller,

src/inspect_robots/log.py

@@ -38,6 +38,7 @@ class EvalSpec:
     embodiment_info: dict[str, Any] = field(default_factory=dict)
     seed: int | None = None
     max_steps: int | None = None
+    max_seconds: float | None = None
 
 
 @dataclass(frozen=True)

src/inspect_robots/task.py

@@ -54,19 +54,41 @@ class Task:
     """A benchmark: scenes + scorer(s) + horizon, independent of any embodiment.
 
     ``scorer`` accepts scorer objects or **registry names** (e.g.
-    ``scorer="success_at_end"``), or a sequence mixing both.
+    ``scorer="success_at_end"``), or a sequence mixing both. Specify the horizon
+    as an integer step budget (``max_steps``) or a physical duration in seconds
+    (``max_seconds``), resolved against the paired embodiment's control rate at
+    ``eval()`` time.
     """
 
     name: str
     scenes: Sequence[Scene]
     scorer: Scorer | str | Sequence[Scorer | str]
-    max_steps: int
+    max_steps: int | None = None
+    max_seconds: float | None = None
     epochs: int | Epochs = 1
     metadata: Mapping[str, Any] = field(default_factory=dict)
 
     def __post_init__(self) -> None:
-        if self.max_steps < 1:
-            raise ConfigError(f"Task {self.name!r}: max_steps must be >= 1, got {self.max_steps}")
+        if (self.max_steps is None) == (self.max_seconds is None):
+            raise ConfigError(
+                f"Task {self.name!r}: specify exactly one of max_steps or max_seconds"
+            )
+        if self.max_steps is not None and (
+            isinstance(self.max_steps, bool)
+            or not isinstance(self.max_steps, int)
+            or self.max_steps < 1
+        ):
+            raise ConfigError(
+                f"Task {self.name!r}: max_steps must be an integer >= 1, got {self.max_steps}"
+            )
+        if self.max_seconds is not None and (
+            isinstance(self.max_seconds, bool)
+            or not isinstance(self.max_seconds, (int, float))
+            or self.max_seconds <= 0
+        ):
+            raise ConfigError(
+                f"Task {self.name!r}: max_seconds must be a number > 0, got {self.max_seconds}"
+            )
         _ = self.epoch_spec  # validates an int epochs count via Epochs
 
     @property
@@ -96,4 +118,9 @@ def epoch_spec(self) -> Epochs:
     @property
     def envelope(self) -> TaskEnvelope:
         """The adapter-safe view of this task handed to ``bind_task`` hooks."""
+        if self.max_steps is None:
+            raise ConfigError(
+                f"Task {self.name!r} has max_seconds set ({self.max_seconds}s); "
+                "step envelope must be resolved at eval() time with an embodiment"
+            )
         return TaskEnvelope(name=self.name, max_steps=self.max_steps)

tests/test_task_horizon.py

@@ -0,0 +1,102 @@
+"""Tests for seconds-based task horizons resolved against embodiment control rates."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, cast
+
+import pytest
+
+from inspect_robots import eval
+from inspect_robots.compat import check_compatibility
+from inspect_robots.embodiment import EmbodimentInfo
+from inspect_robots.errors import CompatibilityError, ConfigError
+from inspect_robots.mock import CubePickEmbodiment, ScriptedPolicy
+from inspect_robots.scene import Scene
+from inspect_robots.scorer import success_at_end
+from inspect_robots.task import Task
+
+
+def _make_scene() -> Scene:
+    return Scene(id="s0", instruction="reach target", init_seed=0)
+
+
+def test_task_validation_requires_exactly_one_horizon() -> None:
+    # Neither specified
+    with pytest.raises(ConfigError, match="specify exactly one of max_steps or max_seconds"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end())
+
+    # Both specified
+    with pytest.raises(ConfigError, match="specify exactly one of max_steps or max_seconds"):
+        Task(
+            name="t",
+            scenes=[_make_scene()],
+            scorer=success_at_end(),
+            max_steps=10,
+            max_seconds=5.0,
+        )
+
+
+def test_task_validation_invalid_max_steps() -> None:
+    with pytest.raises(ConfigError, match="max_steps must be an integer >= 1"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_steps=0)
+
+    with pytest.raises(ConfigError, match="max_steps must be an integer >= 1"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_steps=cast(Any, True))
+
+
+def test_task_validation_invalid_max_seconds() -> None:
+    with pytest.raises(ConfigError, match="max_seconds must be a number > 0"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=0.0)
+
+    with pytest.raises(ConfigError, match="max_seconds must be a number > 0"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=-1.5)
+
+    with pytest.raises(ConfigError, match="max_seconds must be a number > 0"):
+        Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=cast(Any, True))
+
+
+def test_task_envelope_with_max_seconds_raises() -> None:
+    task = Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=5.0)
+    with pytest.raises(ConfigError, match="step envelope must be resolved at eval"):
+        _ = task.envelope
+
+
+def test_compat_error_when_control_hz_missing() -> None:
+    task = Task(name="t", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=5.0)
+    emb = CubePickEmbodiment()
+
+    # Modify embodiment to have no control_hz
+    emb.info = EmbodimentInfo(
+        name="no_hz_emb",
+        action_space=emb.info.action_space,
+        observation_space=emb.info.observation_space,
+        control_hz=None,
+    )
+
+    report = check_compatibility(ScriptedPolicy(), emb, task)
+    assert any(issue.code == "control_hz_missing" for issue in report.errors)
+
+    with pytest.raises(CompatibilityError, match="control_hz_missing"):
+        eval(task, ScriptedPolicy(), emb)
+
+
+def test_eval_resolves_max_seconds_with_ceil(tmp_path: Path) -> None:
+    # 5.5 seconds at 10.0 Hz control_hz => ceil(55.0) = 55 steps
+    task = Task(name="t_seconds", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=5.5)
+    emb = CubePickEmbodiment()  # 10 Hz
+
+    (log,) = eval(task, ScriptedPolicy(), emb, log_dir=str(tmp_path))
+    assert log.status == "success"
+    assert log.eval.max_seconds == 5.5
+    assert log.eval.max_steps == 55
+
+
+def test_eval_resolves_max_seconds_fractional_ceil(tmp_path: Path) -> None:
+    # 0.12 seconds at 10.0 Hz => ceil(1.2) = 2 steps
+    task = Task(name="t_short", scenes=[_make_scene()], scorer=success_at_end(), max_seconds=0.12)
+    emb = CubePickEmbodiment()  # 10 Hz
+
+    (log,) = eval(task, ScriptedPolicy(), emb, log_dir=str(tmp_path))
+    assert log.eval.max_seconds == 0.12
+    assert log.eval.max_steps == 2