fix: isolate three-system benchmark environments

scripts/compare_three_systems.py

@@ -125,6 +125,8 @@ def prepare_workspace(task: SWEBenchTask, workspace: Path, cache_dir: Path | Non
     if not repo_cache.exists():
         raise FileNotFoundError(f"repo cache not found: {repo_cache}")
 
+    _ensure_commit_history(repo_cache, task.base_commit)
+
     if workspace.exists():
         shutil.rmtree(workspace)
     shutil.copytree(repo_cache, workspace)
@@ -162,6 +164,59 @@ def prepare_workspace(task: SWEBenchTask, workspace: Path, cache_dir: Path | Non
     )
 
 
+def _ensure_commit_history(repo: Path, commit: str) -> None:
+    """Fetch enough base-commit ancestry for version derivation in local images."""
+    try:
+        exists = (
+            subprocess.run(
+                ["git", "cat-file", "-t", commit],
+                cwd=repo,
+                capture_output=True,
+                text=True,
+                timeout=5,
+            ).returncode
+            == 0
+        )
+        shallow = (
+            subprocess.run(
+                ["git", "rev-parse", "--is-shallow-repository"],
+                cwd=repo,
+                capture_output=True,
+                text=True,
+                timeout=5,
+                check=True,
+            ).stdout.strip()
+            == "true"
+        )
+        count = (
+            int(
+                subprocess.run(
+                    ["git", "rev-list", "--count", commit],
+                    cwd=repo,
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                    check=True,
+                ).stdout.strip()
+            )
+            if exists
+            else 0
+        )
+        if exists and (not shallow or count >= 200):
+            return
+    except (OSError, subprocess.SubprocessError, ValueError):
+        pass
+
+    subprocess.run(
+        ["git", "fetch", "--depth", "500", "--tags", "origin", commit],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+        timeout=300,
+        check=True,
+    )
+
+
 def run_claude(
     task: SWEBenchTask,
     task_output_dir: Path,
@@ -173,13 +228,20 @@ def run_claude(
     task_output_dir.mkdir(parents=True, exist_ok=True)
     prompt = build_goal_description(task)
 
-    env = dict(os.environ)
+    command_venv = task_output_dir / "command_venv"
+    if command_venv.exists():
+        shutil.rmtree(command_venv)
+    subprocess.run(
+        [sys.executable, "-m", "venv", "--system-site-packages", str(command_venv)],
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    env = _claude_environment(model)
+    env["VIRTUAL_ENV"] = str(command_venv)
+    env["PATH"] = f"{command_venv / 'bin'}{os.pathsep}{env.get('PATH', '')}"
     env["CLAUDE_CODE_DEBUG"] = "1"
-    # Use the same underlying deepseek-v4-flash model via cc-switch.
-    # cc-switch recognizes the Anthropic-style model id with a [1m] suffix.
-    env["ANTHROPIC_MODEL"] = model if model.endswith("[1m]") else f"{model}[1m]"
-    env["ANTHROPIC_BASE_URL"] = "http://127.0.0.1:15721/v1"
-    env["ANTHROPIC_API_KEY"] = "placeholder"
     env["ANTHROPIC_MAX_TOKENS"] = "64000"
 
     stdout_path = task_output_dir / "claude.out"
@@ -235,6 +297,49 @@ def run_claude(
     return evaluated
 
 
+def _claude_environment(model: str) -> dict[str, str]:
+    """Build Claude Code env, allowing the benchmark to use the shared key."""
+    env = dict(os.environ)
+    api_key = os.getenv("SWE_BENCH_CLAUDE_API_KEY", "placeholder")
+    env["ANTHROPIC_MODEL"] = model if model.endswith("[1m]") else f"{model}[1m]"
+    env["ANTHROPIC_BASE_URL"] = os.getenv("SWE_BENCH_CLAUDE_BASE_URL", "http://127.0.0.1:15721/v1")
+    env["ANTHROPIC_API_KEY"] = api_key
+    env["ANTHROPIC_AUTH_TOKEN"] = api_key
+    return env
+
+
+def preflight_claude_endpoint(model: str) -> None:
+    """Verify Claude's endpoint before any task can be counted."""
+    completed = subprocess.run(
+        ["claude", "-p", "Reply with exactly OK."],
+        env=_claude_environment(model),
+        capture_output=True,
+        text=True,
+        timeout=90,
+    )
+    if completed.returncode != 0 or not completed.stdout.strip():
+        detail = (completed.stderr or completed.stdout or "empty response")[-500:]
+        raise RuntimeError(f"Claude endpoint preflight failed: {detail}")
+
+
+def preflight_swe_agent_environment() -> None:
+    """Verify the isolated SWE-agent interpreter before starting a batch."""
+    completed = subprocess.run(
+        [
+            f"{SWE_AGENT_ENV}/bin/python",
+            "-c",
+            "from swe_agent_local_runner import _load_sweagent_classes; _load_sweagent_classes()",
+        ],
+        cwd=REPO_ROOT,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    if completed.returncode != 0:
+        detail = (completed.stderr or completed.stdout or "unknown import failure")[-1000:]
+        raise RuntimeError(detail)
+
+
 def run_direct(
     task: SWEBenchTask,
     task_output_dir: Path,
@@ -340,16 +445,27 @@ def run_swe_agent(
     duration = time.monotonic() - start
     patch_path = task_output_dir / "swe_agent_run" / "agent.patch"
     patch = patch_path.read_text(encoding="utf-8") if patch_path.exists() else ""
+    runner_error: str | None = None
+    try:
+        runner_payload = json.loads((task_output_dir / "swe_agent.out").read_text(encoding="utf-8"))
+        runner_error = runner_payload.get("error")
+    except (OSError, json.JSONDecodeError, AttributeError):
+        pass
 
     if exit_code != 0:
         return {
             "resolved": False,
             "duration": duration,
-            "error": f"swe-agent exit code {exit_code}",
+            "error": runner_error or f"swe-agent exit code {exit_code}",
             "patch": patch,
         }
     if not patch.strip():
-        return {"resolved": False, "duration": duration, "error": "empty patch", "patch": patch}
+        return {
+            "resolved": False,
+            "duration": duration,
+            "error": runner_error or "empty patch",
+            "patch": patch,
+        }
 
     (task_output_dir / "agent.patch").write_text(patch, encoding="utf-8")
     evaluated = evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval")
@@ -367,6 +483,7 @@ def run_swe_agent(
     "authentication",
     "invalid_api_key",
     "llm error",
+    "dependencies are unavailable",
 )
 
 
@@ -517,8 +634,21 @@ def main() -> int:
         except Exception as exc:
             logger.error("shared direct/SWE-agent endpoint preflight failed: %s", exc)
             return 2
+    if args.mode in ("claude", "all"):
+        try:
+            preflight_claude_endpoint(args.model)
+        except Exception as exc:
+            logger.error("Claude endpoint preflight failed: %s", exc)
+            return 2
+    if args.mode in ("swe-agent", "all"):
+        try:
+            preflight_swe_agent_environment()
+        except Exception as exc:
+            logger.error("SWE-agent environment preflight failed: %s", exc)
+            return 2
 
     for task in tasks:
+        infrastructure_failure = False
         r = by_id[task.id]
         task_output_dir = output_dir / task.id
         task_output_dir.mkdir(parents=True, exist_ok=True)
@@ -599,13 +729,15 @@ def main() -> int:
                     task.id,
                     r.swe_agent_error,
                 )
-                break
+                infrastructure_failure = True
 
         # Save incremental result.
         (task_output_dir / "comparison.json").write_text(
             json.dumps(asdict(r), indent=2, ensure_ascii=False),
             encoding="utf-8",
         )
+        if infrastructure_failure:
+            break
 
     report = {
         "metadata": {

swe_agent_local_runner.py

@@ -23,7 +23,7 @@
 import time
 from dataclasses import dataclass
 from pathlib import Path
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
 from typing import Any
 
 import yaml
@@ -49,6 +49,16 @@ def _load_sweagent_classes():
     try:
         import anyio.to_thread  # noqa: F401
         import together  # type: ignore[import-not-found]  # noqa: F401
+
+        # Agent only uses SWEEnv in annotations; this runner supplies
+        # LocalSWEEnv at runtime. Importing the official SWEEnv would pull in
+        # swebench -> datasets -> pyarrow and add minutes of irrelevant native
+        # import work on Apple Silicon/Rosetta.
+        swe_env_module = "sweagent.environment.swe_env"
+        if swe_env_module not in sys.modules:
+            stub = ModuleType(swe_env_module)
+            stub.SWEEnv = object  # type: ignore[attr-defined]
+            sys.modules[swe_env_module] = stub
         from sweagent.agent.agents import (  # type: ignore[import-not-found]
             Agent,
             AgentArguments,
@@ -88,10 +98,12 @@ def __init__(
         task: Any,
         command_files: list[Path] | None = None,
         timeout: int = 120,
+        command_venv: Path | None = None,
     ) -> None:
         self.workspace = Path(workspace).resolve()
         self.task = task
         self.timeout = timeout
+        self.command_venv = command_venv
         self.communicate_output: str | None = None
         self.returncode: int | None = None
         self.container_obj = SimpleNamespace(id="local")
@@ -176,6 +188,9 @@ def _restart_bash(self) -> None:
     def _start_bash(self) -> None:
         """Start a persistent bash session and source the init script."""
         env = {**os.environ, "ROOT": str(self.workspace)}
+        if self.command_venv is not None:
+            env["VIRTUAL_ENV"] = str(self.command_venv)
+            env["PATH"] = f"{self.command_venv / 'bin'}{os.pathsep}{env.get('PATH', '')}"
         # Use a new session so we can kill the whole process group (bash +
         # any spawned children) when a command times out.
         self._proc = subprocess.Popen(
@@ -507,7 +522,22 @@ def run_swe_agent_local(
         )
     config_file = _materialize_agent_config(Path(config_file).resolve(), output_dir)
 
-    env = LocalSWEEnv(workspace, task, timeout=timeout_per_command)
+    command_venv = output_dir / "command_venv"
+    if command_venv.exists():
+        shutil.rmtree(command_venv)
+    subprocess.run(
+        [sys.executable, "-m", "venv", "--system-site-packages", str(command_venv)],
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    env = LocalSWEEnv(
+        workspace,
+        task,
+        timeout=timeout_per_command,
+        command_venv=command_venv,
+    )
     env.reset()
 
     try:

swe_bench/runner.py

@@ -566,6 +566,11 @@ def _prepare_workspace(self, task: SWEBenchTask, workspace: Path) -> None:
             repo_cache = self.cache_dir / task.repo.replace("/", "__")
             self._ensure_repo_cache(task.repo, repo_cache)
 
+        # Local Docker image builds may derive package versions from Git tags.
+        # Ensure the base commit has enough ancestry before copying the cache;
+        # a depth-1 commit otherwise becomes versions such as ``0.1.dev1``.
+        self._fetch_commit(repo_cache, task.base_commit)
+
         # Copy repo into workspace to avoid mutating the cache.
         if workspace.exists():
             shutil.rmtree(workspace)
@@ -594,16 +599,39 @@ def _fetch_commit(self, repo_dir: Path, commit: str) -> None:
         import random as _random
         import re
 
-        # Check if commit already exists locally before hitting the network.
+        # A present commit is not sufficient: depth-1 history prevents tools
+        # such as setuptools-scm from finding the nearest release tag.
+        commit_exists = False
         try:
             _run_command(
                 ["git", "cat-file", "-t", commit],
                 cwd=repo_dir,
                 timeout=5,
             )
-            return  # commit exists, no need to fetch
+            commit_exists = True
         except Exception:
             pass
+        if commit_exists:
+            try:
+                shallow = (
+                    _run_command(
+                        ["git", "rev-parse", "--is-shallow-repository"],
+                        cwd=repo_dir,
+                        timeout=5,
+                    ).stdout.strip()
+                    == "true"
+                )
+                history_count = int(
+                    _run_command(
+                        ["git", "rev-list", "--count", commit],
+                        cwd=repo_dir,
+                        timeout=10,
+                    ).stdout.strip()
+                )
+                if not shallow or history_count >= 200:
+                    return
+            except Exception:
+                pass
 
         # Build source list: origin → gitee mirror (if URL pattern matches).
         sources: list[str] = ["origin"]
@@ -624,7 +652,7 @@ def _fetch_commit(self, repo_dir: Path, commit: str) -> None:
                 try:
                     timeout = 30 * attempt  # 30s, 60s, 90s
                     _run_command(
-                        ["git", "fetch", "--depth", "1", source, commit],
+                        ["git", "fetch", "--depth", "500", "--tags", source, commit],
                         cwd=repo_dir,
                         timeout=timeout,
                     )