diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ed75d..55de93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ### Changed +- **Add time-based `max_seconds` to task specifications.** ([#28](https://github.com/robocurve/kitchenbench/issues/28)) + + `TaskSpec` now includes a `max_seconds` field representing the real-world completion time budget in seconds (the physical protocol budget, ranging from 60s for simple pick-and-place up to 200s for multi-item sorting). Mock-scale `max_steps` is retained for backwards compatibility. + + Task versions have been bumped (nine tasks bumped to version 3, `scoop_pasta` to version 4) to reflect the metadata change. + - **Scene seeds are now derived from `instance_id` instead of the instance's sorted position.** ([#3](https://github.com/robocurve/kitchenbench/issues/3)) diff --git a/src/kitchenbench/specs.py b/src/kitchenbench/specs.py index afbacca..2c56254 100644 --- a/src/kitchenbench/specs.py +++ b/src/kitchenbench/specs.py @@ -26,13 +26,20 @@ @dataclass(frozen=True) class TaskSpec: - """One KitchenBench task and its (distribution-based) task instances.""" + """One KitchenBench task and its (distribution-based) task instances. + + ``max_seconds`` is the task's real-world physical completion time budget in seconds, + ranging from 60s for simple pick-and-place up to 200s for multi-item sorting, derived + from the physical-automation methodology. ``max_steps`` is the mock-scale step limit + retained for backward compatibility and to bound abstract mock-world evaluation. + """ key: str title: str category: str bimanual: bool max_steps: int + max_seconds: float instances: tuple[TaskInstance, ...] version: str = "1" description: str = "" @@ -1340,8 +1347,9 @@ def _sort_fixtures( category="pick_place", bimanual=False, max_steps=60, + max_seconds=60.0, instances=_PLACE_CUTLERY, - version="2", + version="3", description="Pick a single piece of cutlery and place it on a target surface.", ), TaskSpec( @@ -1350,8 +1358,9 @@ def _sort_fixtures( category="stacking", bimanual=False, max_steps=80, + max_seconds=80.0, instances=_STACK, - version="2", + version="3", description="Stack multiple like items into a single neat stack.", ), TaskSpec( @@ -1360,8 +1369,9 @@ def _sort_fixtures( category="insertion", bimanual=False, max_steps=80, + max_seconds=80.0, instances=_PLACE_IN_RACK, - version="2", + version="3", description="Drop a dish into the correct slot of a dish rack.", ), TaskSpec( @@ -1370,8 +1380,9 @@ def _sort_fixtures( category="granular", bimanual=True, max_steps=100, + max_seconds=100.0, instances=_POUR_PASTA, - version="2", + version="3", description="Pour dry pasta into a receiving vessel; one arm steadies, the other pours.", ), TaskSpec( @@ -1380,8 +1391,9 @@ def _sort_fixtures( category="articulated", bimanual=True, max_steps=120, + max_seconds=120.0, instances=_OPEN_CONTAINER, - version="2", + version="3", description="Remove or unscrew a lid — one arm braces while the other twists or pries.", ), TaskSpec( @@ -1390,8 +1402,9 @@ def _sort_fixtures( category="deformable", bimanual=True, max_steps=120, + max_seconds=120.0, instances=_FOLD_CLOTH, - version="2", + version="3", description="Deformable manipulation: grasp opposite corners and manage slack.", ), TaskSpec( @@ -1400,8 +1413,9 @@ def _sort_fixtures( category="mating", bimanual=True, max_steps=120, + max_seconds=120.0, instances=_SEAL_CONTAINER, - version="2", + version="3", description="Align and press-or-twist a matching lid onto a base while one arm holds it.", ), TaskSpec( @@ -1410,8 +1424,9 @@ def _sort_fixtures( category="coordination", bimanual=True, max_steps=80, + max_seconds=80.0, instances=_HANDOFF, - version="2", + version="3", description="A pure handover that a single arm cannot do — the must-use-both-arms anchor.", ), TaskSpec( @@ -1420,8 +1435,9 @@ def _sort_fixtures( category="classification", bimanual=False, max_steps=200, + max_seconds=200.0, instances=_SORT_CUTLERY, - version="2", + version="3", description="Sort a mixed pile into spoon/fork/knife compartments — multi-instance.", ), TaskSpec( @@ -1430,8 +1446,9 @@ def _sort_fixtures( category="granular_tool", bimanual=True, max_steps=120, + max_seconds=120.0, instances=_SCOOP_PASTA, - version="3", + version="4", description="Tool-mediated granular handling: manage fill level, then transfer.", ), ) diff --git a/src/kitchenbench/tasks.py b/src/kitchenbench/tasks.py index c42f7b3..c9521cd 100644 --- a/src/kitchenbench/tasks.py +++ b/src/kitchenbench/tasks.py @@ -115,6 +115,7 @@ def make_task(spec: TaskSpec) -> Task: "bimanual": spec.bimanual, "version": spec.version, "description": spec.description, + "max_seconds": spec.max_seconds, "k_instances": K_INSTANCES, "k_realizations": K_REALIZATIONS, }, diff --git a/tests/test_specs.py b/tests/test_specs.py index 63bbcb7..a0c28c8 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -35,6 +35,11 @@ def test_each_task_has_k_instances() -> None: assert len(spec.instances) == K_INSTANCES, spec.key +def test_max_seconds_positive() -> None: + for spec in SPECS: + assert spec.max_seconds > 0.0, spec.key + + def test_language_vars_and_placeholders_consistent() -> None: for spec in SPECS: for inst in spec.instances: diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 7e5d347..94ce7a8 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -67,6 +67,8 @@ def test_canonical_instruction_matches_epoch_zero() -> None: def test_make_task_has_two_scorers() -> None: task = make_task(SPECS[0]) assert {s.name for s in task.scorers} == {"task_success", "episode_length"} + assert task.max_steps == 60 # mock-scale step limit + assert task.metadata["max_seconds"] == 60.0 assert task.metadata["k_instances"] == 5 assert task.metadata["k_realizations"] == 5