Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion bench_env/runner/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,13 @@ def _evaluate_with_vlm(self, task, exec_result, episode) -> JudgeResult:

def _action_fingerprint(action) -> str:
"""Extract action behavioral fingerprint (type + normalized data)."""
return f"{action.action_type}|{json.dumps(action.data, sort_keys=True, ensure_ascii=False)}"
data = action.data
if action.action_type == ActionType.WAIT:
# WAIT's "value" is a duration in seconds, not behavior: two waits are the
# same repeated action for loop-detection purposes regardless of how long
# each one slept, so drop it before hashing.
data = {k: v for k, v in data.items() if k != "value"}
return f"{action.action_type}|{json.dumps(data, sort_keys=True, ensure_ascii=False)}"


def _snapshot_stopwatch(sw) -> tuple[float, dict[str, float], list[dict[str, Any]]]:
Expand Down
88 changes: 88 additions & 0 deletions bench_env/tests/common/test_loop_detect.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
from __future__ import annotations

import pytest

from bench_env.env.base import Action, ActionType, Observation, StepResult
from bench_env.env.stopwatch import StopWatch
from bench_env.runner.base import Controller


def _make_obs(step_idx: int = 0) -> Observation:
return Observation(
screenshot_base64="",
route={"app": "demo", "path": "/"},
state={"apps": {}, "os": {}},
step_idx=step_idx,
)


class _VaryingWaitAgent:
"""Emits WAIT actions with a different duration on every call, the way a real
agent does when it picks the wait length itself."""

name = "varying-wait"

def __init__(self) -> None:
self.history: list = []
self._n = 0

def reset(self, task: str) -> None:
self.task = task

def act(self, obs: Observation) -> Action:
self._n += 1
return Action.wait(seconds=self._n * 0.1)


class _WaitTrackingEnv:
def __init__(self) -> None:
self._agent_answer: str | None = None
self._agent_message: str | None = None
self.stopwatch = StopWatch()

async def get_state(self, *, required_apps: list[str] | None = None) -> dict:
return {}

async def step(self, action: Action) -> StepResult:
if action.action_type == ActionType.WAIT:
return StepResult(observation=_make_obs(1), done=False, info={})
raise AssertionError(f"unexpected action: {action.action_type}")

@property
def agent_answer(self) -> str | None:
return self._agent_answer

@property
def agent_message(self) -> str | None:
return self._agent_message


class _TaskForController:
id = "demo.WaitTask"
description = "等待"
suite = "demo"
apps: list[str] = []

def teardown(self, env) -> None:
return None


@pytest.mark.asyncio
async def test_controller_run_detects_repetitive_wait_despite_varying_duration() -> None:
env = _WaitTrackingEnv()
agent = _VaryingWaitAgent()
task = _TaskForController()

exec_result, *_ = await Controller.run(
env,
agent,
task,
_make_obs(),
max_steps=5,
recorder=None,
loop_threshold=3,
)

assert exec_result.truncated is True
assert exec_result.stop_reason == "REPETITIVE_LOOP"
assert exec_result.steps == 3