From de6bb633f4b2856419b4266114fde2f50e47283e Mon Sep 17 00:00:00 2001
From: Amir Fathi <amirfathi.me@gmail.com>
Date: Tue, 4 Aug 2026 01:11:15 +0000
Subject: [PATCH] fix(bench): exclude wait duration from repetitive-loop
 fingerprint

_action_fingerprint() hashed the full action payload, including WAIT's
"value" duration field. An agent that varies the wait length on every
call (e.g. picks a duration itself) produces a different fingerprint
each step, so Controller.run()'s loop guard never sees two identical
fingerprints and REPETITIVE_LOOP never trips; the episode runs to
max_steps instead of truncating early.

Drop the duration from the fingerprint specifically for WAIT actions
before hashing, so repeated waits are recognized as the same action
regardless of how long each one slept. Other action types are
unaffected.

Fixes #14
---
 bench_env/runner/base.py                   |  8 +-
 bench_env/tests/common/test_loop_detect.py | 88 ++++++++++++++++++++++
 2 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 bench_env/tests/common/test_loop_detect.py

diff --git a/bench_env/runner/base.py b/bench_env/runner/base.py
index 09fdc8fd..676ee5cd 100644
--- a/bench_env/runner/base.py
+++ b/bench_env/runner/base.py
@@ -161,7 +161,13 @@ def _evaluate_with_vlm(self, task, exec_result, episode) -> JudgeResult:
 
 def _action_fingerprint(action) -> str:
     """Extract action behavioral fingerprint (type + normalized data)."""
-    return f"{action.action_type}|{json.dumps(action.data, sort_keys=True, ensure_ascii=False)}"
+    data = action.data
+    if action.action_type == ActionType.WAIT:
+        # WAIT's "value" is a duration in seconds, not behavior: two waits are the
+        # same repeated action for loop-detection purposes regardless of how long
+        # each one slept, so drop it before hashing.
+        data = {k: v for k, v in data.items() if k != "value"}
+    return f"{action.action_type}|{json.dumps(data, sort_keys=True, ensure_ascii=False)}"
 
 
 def _snapshot_stopwatch(sw) -> tuple[float, dict[str, float], list[dict[str, Any]]]:
diff --git a/bench_env/tests/common/test_loop_detect.py b/bench_env/tests/common/test_loop_detect.py
new file mode 100644
index 00000000..4f4f2d58
--- /dev/null
+++ b/bench_env/tests/common/test_loop_detect.py
@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import pytest
+
+from bench_env.env.base import Action, ActionType, Observation, StepResult
+from bench_env.env.stopwatch import StopWatch
+from bench_env.runner.base import Controller
+
+
+def _make_obs(step_idx: int = 0) -> Observation:
+    return Observation(
+        screenshot_base64="",
+        route={"app": "demo", "path": "/"},
+        state={"apps": {}, "os": {}},
+        step_idx=step_idx,
+    )
+
+
+class _VaryingWaitAgent:
+    """Emits WAIT actions with a different duration on every call, the way a real
+    agent does when it picks the wait length itself."""
+
+    name = "varying-wait"
+
+    def __init__(self) -> None:
+        self.history: list = []
+        self._n = 0
+
+    def reset(self, task: str) -> None:
+        self.task = task
+
+    def act(self, obs: Observation) -> Action:
+        self._n += 1
+        return Action.wait(seconds=self._n * 0.1)
+
+
+class _WaitTrackingEnv:
+    def __init__(self) -> None:
+        self._agent_answer: str | None = None
+        self._agent_message: str | None = None
+        self.stopwatch = StopWatch()
+
+    async def get_state(self, *, required_apps: list[str] | None = None) -> dict:
+        return {}
+
+    async def step(self, action: Action) -> StepResult:
+        if action.action_type == ActionType.WAIT:
+            return StepResult(observation=_make_obs(1), done=False, info={})
+        raise AssertionError(f"unexpected action: {action.action_type}")
+
+    @property
+    def agent_answer(self) -> str | None:
+        return self._agent_answer
+
+    @property
+    def agent_message(self) -> str | None:
+        return self._agent_message
+
+
+class _TaskForController:
+    id = "demo.WaitTask"
+    description = "等待"
+    suite = "demo"
+    apps: list[str] = []
+
+    def teardown(self, env) -> None:
+        return None
+
+
+@pytest.mark.asyncio
+async def test_controller_run_detects_repetitive_wait_despite_varying_duration() -> None:
+    env = _WaitTrackingEnv()
+    agent = _VaryingWaitAgent()
+    task = _TaskForController()
+
+    exec_result, *_ = await Controller.run(
+        env,
+        agent,
+        task,
+        _make_obs(),
+        max_steps=5,
+        recorder=None,
+        loop_threshold=3,
+    )
+
+    assert exec_result.truncated is True
+    assert exec_result.stop_reason == "REPETITIVE_LOOP"
+    assert exec_result.steps == 3