From 00edb8db7715e0376079362884ee2daeee18f712 Mon Sep 17 00:00:00 2001 From: Rob Macrae Date: Fri, 3 Jul 2026 11:06:52 +0100 Subject: [PATCH] Fix: raise docker client timeout so slow evals aren't silently scored unresolved docker.from_env() uses the SDK default 60s read timeout. On the --use_local_docker path, container.wait() then raises ReadTimeout for any test suite whose exec exceeds 60s (routine for Go/JS repos), so no output.json is written and the instance is silently scored unresolved. Add DOCKER_CLIENT_TIMEOUT (default 3600s, overridable via env) and use it for the docker client. Modal path unaffected. Fixes the false-negatives observed in #54, #22, #23. --- swe_bench_pro_eval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/swe_bench_pro_eval.py b/swe_bench_pro_eval.py index cac63a62..17993d7f 100644 --- a/swe_bench_pro_eval.py +++ b/swe_bench_pro_eval.py @@ -53,6 +53,9 @@ from helper_code.image_uri import get_dockerhub_image_uri +# Docker's default 60s client read timeout is too short for long-running test suites. +DOCKER_CLIENT_TIMEOUT = int(os.environ.get("DOCKER_CLIENT_TIMEOUT", "3600")) + # Credit: prabhuteja12 def load_base_docker(iid): with open(f"dockerfiles/base_dockerfile/{iid}/Dockerfile") as fp: @@ -378,7 +381,7 @@ def eval_with_docker(patch, sample, output_dir, dockerhub_username, scripts_dir, dockerhub_image_uri = get_dockerhub_image_uri(uid, dockerhub_username, sample.get("repo", "")) print(f"Using Docker Hub image: {dockerhub_image_uri}") - client = docker.from_env() + client = docker.from_env(timeout=DOCKER_CLIENT_TIMEOUT) try: if docker_platform: client.images.pull(dockerhub_image_uri, platform=docker_platform)