diff --git a/bench_env/runner/base.py b/bench_env/runner/base.py index 09fdc8fd..676ee5cd 100644 --- a/bench_env/runner/base.py +++ b/bench_env/runner/base.py @@ -161,7 +161,13 @@ def _evaluate_with_vlm(self, task, exec_result, episode) -> JudgeResult: def _action_fingerprint(action) -> str: """Extract action behavioral fingerprint (type + normalized data).""" - return f"{action.action_type}|{json.dumps(action.data, sort_keys=True, ensure_ascii=False)}" + data = action.data + if action.action_type == ActionType.WAIT: + # WAIT's "value" is a duration in seconds, not behavior: two waits are the + # same repeated action for loop-detection purposes regardless of how long + # each one slept, so drop it before hashing. + data = {k: v for k, v in data.items() if k != "value"} + return f"{action.action_type}|{json.dumps(data, sort_keys=True, ensure_ascii=False)}" def _snapshot_stopwatch(sw) -> tuple[float, dict[str, float], list[dict[str, Any]]]: diff --git a/bench_env/tests/common/test_loop_detect.py b/bench_env/tests/common/test_loop_detect.py new file mode 100644 index 00000000..4f4f2d58 --- /dev/null +++ b/bench_env/tests/common/test_loop_detect.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from bench_env.env.base import Action, ActionType, Observation, StepResult +from bench_env.env.stopwatch import StopWatch +from bench_env.runner.base import Controller + + +def _make_obs(step_idx: int = 0) -> Observation: + return Observation( + screenshot_base64="", + route={"app": "demo", "path": "/"}, + state={"apps": {}, "os": {}}, + step_idx=step_idx, + ) + + +class _VaryingWaitAgent: + """Emits WAIT actions with a different duration on every call, the way a real + agent does when it picks the wait length itself.""" + + name = "varying-wait" + + def __init__(self) -> None: + self.history: list = [] + self._n = 0 + + def reset(self, task: str) -> None: + self.task = task + + def act(self, obs: Observation) -> Action: + self._n += 1 + return Action.wait(seconds=self._n * 0.1) + + +class _WaitTrackingEnv: + def __init__(self) -> None: + self._agent_answer: str | None = None + self._agent_message: str | None = None + self.stopwatch = StopWatch() + + async def get_state(self, *, required_apps: list[str] | None = None) -> dict: + return {} + + async def step(self, action: Action) -> StepResult: + if action.action_type == ActionType.WAIT: + return StepResult(observation=_make_obs(1), done=False, info={}) + raise AssertionError(f"unexpected action: {action.action_type}") + + @property + def agent_answer(self) -> str | None: + return self._agent_answer + + @property + def agent_message(self) -> str | None: + return self._agent_message + + +class _TaskForController: + id = "demo.WaitTask" + description = "等待" + suite = "demo" + apps: list[str] = [] + + def teardown(self, env) -> None: + return None + + +@pytest.mark.asyncio +async def test_controller_run_detects_repetitive_wait_despite_varying_duration() -> None: + env = _WaitTrackingEnv() + agent = _VaryingWaitAgent() + task = _TaskForController() + + exec_result, *_ = await Controller.run( + env, + agent, + task, + _make_obs(), + max_steps=5, + recorder=None, + loop_threshold=3, + ) + + assert exec_result.truncated is True + assert exec_result.stop_reason == "REPETITIVE_LOOP" + assert exec_result.steps == 3