diff --git a/scripts/compare_three_systems.py b/scripts/compare_three_systems.py index 7fef0cc..8b6718a 100644 --- a/scripts/compare_three_systems.py +++ b/scripts/compare_three_systems.py @@ -125,6 +125,8 @@ def prepare_workspace(task: SWEBenchTask, workspace: Path, cache_dir: Path | Non if not repo_cache.exists(): raise FileNotFoundError(f"repo cache not found: {repo_cache}") + _ensure_commit_history(repo_cache, task.base_commit) + if workspace.exists(): shutil.rmtree(workspace) shutil.copytree(repo_cache, workspace) @@ -162,6 +164,59 @@ def prepare_workspace(task: SWEBenchTask, workspace: Path, cache_dir: Path | Non ) +def _ensure_commit_history(repo: Path, commit: str) -> None: + """Fetch enough base-commit ancestry for version derivation in local images.""" + try: + exists = ( + subprocess.run( + ["git", "cat-file", "-t", commit], + cwd=repo, + capture_output=True, + text=True, + timeout=5, + ).returncode + == 0 + ) + shallow = ( + subprocess.run( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=repo, + capture_output=True, + text=True, + timeout=5, + check=True, + ).stdout.strip() + == "true" + ) + count = ( + int( + subprocess.run( + ["git", "rev-list", "--count", commit], + cwd=repo, + capture_output=True, + text=True, + timeout=10, + check=True, + ).stdout.strip() + ) + if exists + else 0 + ) + if exists and (not shallow or count >= 200): + return + except (OSError, subprocess.SubprocessError, ValueError): + pass + + subprocess.run( + ["git", "fetch", "--depth", "500", "--tags", "origin", commit], + cwd=repo, + capture_output=True, + text=True, + timeout=300, + check=True, + ) + + def run_claude( task: SWEBenchTask, task_output_dir: Path, @@ -173,13 +228,20 @@ def run_claude( task_output_dir.mkdir(parents=True, exist_ok=True) prompt = build_goal_description(task) - env = dict(os.environ) + command_venv = task_output_dir / "command_venv" + if command_venv.exists(): + shutil.rmtree(command_venv) + subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(command_venv)], + check=True, + capture_output=True, + text=True, + timeout=120, + ) + env = _claude_environment(model) + env["VIRTUAL_ENV"] = str(command_venv) + env["PATH"] = f"{command_venv / 'bin'}{os.pathsep}{env.get('PATH', '')}" env["CLAUDE_CODE_DEBUG"] = "1" - # Use the same underlying deepseek-v4-flash model via cc-switch. - # cc-switch recognizes the Anthropic-style model id with a [1m] suffix. - env["ANTHROPIC_MODEL"] = model if model.endswith("[1m]") else f"{model}[1m]" - env["ANTHROPIC_BASE_URL"] = "http://127.0.0.1:15721/v1" - env["ANTHROPIC_API_KEY"] = "placeholder" env["ANTHROPIC_MAX_TOKENS"] = "64000" stdout_path = task_output_dir / "claude.out" @@ -235,6 +297,49 @@ def run_claude( return evaluated +def _claude_environment(model: str) -> dict[str, str]: + """Build Claude Code env, allowing the benchmark to use the shared key.""" + env = dict(os.environ) + api_key = os.getenv("SWE_BENCH_CLAUDE_API_KEY", "placeholder") + env["ANTHROPIC_MODEL"] = model if model.endswith("[1m]") else f"{model}[1m]" + env["ANTHROPIC_BASE_URL"] = os.getenv("SWE_BENCH_CLAUDE_BASE_URL", "http://127.0.0.1:15721/v1") + env["ANTHROPIC_API_KEY"] = api_key + env["ANTHROPIC_AUTH_TOKEN"] = api_key + return env + + +def preflight_claude_endpoint(model: str) -> None: + """Verify Claude's endpoint before any task can be counted.""" + completed = subprocess.run( + ["claude", "-p", "Reply with exactly OK."], + env=_claude_environment(model), + capture_output=True, + text=True, + timeout=90, + ) + if completed.returncode != 0 or not completed.stdout.strip(): + detail = (completed.stderr or completed.stdout or "empty response")[-500:] + raise RuntimeError(f"Claude endpoint preflight failed: {detail}") + + +def preflight_swe_agent_environment() -> None: + """Verify the isolated SWE-agent interpreter before starting a batch.""" + completed = subprocess.run( + [ + f"{SWE_AGENT_ENV}/bin/python", + "-c", + "from swe_agent_local_runner import _load_sweagent_classes; _load_sweagent_classes()", + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=120, + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout or "unknown import failure")[-1000:] + raise RuntimeError(detail) + + def run_direct( task: SWEBenchTask, task_output_dir: Path, @@ -340,16 +445,27 @@ def run_swe_agent( duration = time.monotonic() - start patch_path = task_output_dir / "swe_agent_run" / "agent.patch" patch = patch_path.read_text(encoding="utf-8") if patch_path.exists() else "" + runner_error: str | None = None + try: + runner_payload = json.loads((task_output_dir / "swe_agent.out").read_text(encoding="utf-8")) + runner_error = runner_payload.get("error") + except (OSError, json.JSONDecodeError, AttributeError): + pass if exit_code != 0: return { "resolved": False, "duration": duration, - "error": f"swe-agent exit code {exit_code}", + "error": runner_error or f"swe-agent exit code {exit_code}", "patch": patch, } if not patch.strip(): - return {"resolved": False, "duration": duration, "error": "empty patch", "patch": patch} + return { + "resolved": False, + "duration": duration, + "error": runner_error or "empty patch", + "patch": patch, + } (task_output_dir / "agent.patch").write_text(patch, encoding="utf-8") evaluated = evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval") @@ -367,6 +483,7 @@ def run_swe_agent( "authentication", "invalid_api_key", "llm error", + "dependencies are unavailable", ) @@ -517,8 +634,21 @@ def main() -> int: except Exception as exc: logger.error("shared direct/SWE-agent endpoint preflight failed: %s", exc) return 2 + if args.mode in ("claude", "all"): + try: + preflight_claude_endpoint(args.model) + except Exception as exc: + logger.error("Claude endpoint preflight failed: %s", exc) + return 2 + if args.mode in ("swe-agent", "all"): + try: + preflight_swe_agent_environment() + except Exception as exc: + logger.error("SWE-agent environment preflight failed: %s", exc) + return 2 for task in tasks: + infrastructure_failure = False r = by_id[task.id] task_output_dir = output_dir / task.id task_output_dir.mkdir(parents=True, exist_ok=True) @@ -599,13 +729,15 @@ def main() -> int: task.id, r.swe_agent_error, ) - break + infrastructure_failure = True # Save incremental result. (task_output_dir / "comparison.json").write_text( json.dumps(asdict(r), indent=2, ensure_ascii=False), encoding="utf-8", ) + if infrastructure_failure: + break report = { "metadata": { diff --git a/swe_agent_local_runner.py b/swe_agent_local_runner.py index 1d8fc30..40a94dd 100644 --- a/swe_agent_local_runner.py +++ b/swe_agent_local_runner.py @@ -23,7 +23,7 @@ import time from dataclasses import dataclass from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace from typing import Any import yaml @@ -49,6 +49,16 @@ def _load_sweagent_classes(): try: import anyio.to_thread # noqa: F401 import together # type: ignore[import-not-found] # noqa: F401 + + # Agent only uses SWEEnv in annotations; this runner supplies + # LocalSWEEnv at runtime. Importing the official SWEEnv would pull in + # swebench -> datasets -> pyarrow and add minutes of irrelevant native + # import work on Apple Silicon/Rosetta. + swe_env_module = "sweagent.environment.swe_env" + if swe_env_module not in sys.modules: + stub = ModuleType(swe_env_module) + stub.SWEEnv = object # type: ignore[attr-defined] + sys.modules[swe_env_module] = stub from sweagent.agent.agents import ( # type: ignore[import-not-found] Agent, AgentArguments, @@ -88,10 +98,12 @@ def __init__( task: Any, command_files: list[Path] | None = None, timeout: int = 120, + command_venv: Path | None = None, ) -> None: self.workspace = Path(workspace).resolve() self.task = task self.timeout = timeout + self.command_venv = command_venv self.communicate_output: str | None = None self.returncode: int | None = None self.container_obj = SimpleNamespace(id="local") @@ -176,6 +188,9 @@ def _restart_bash(self) -> None: def _start_bash(self) -> None: """Start a persistent bash session and source the init script.""" env = {**os.environ, "ROOT": str(self.workspace)} + if self.command_venv is not None: + env["VIRTUAL_ENV"] = str(self.command_venv) + env["PATH"] = f"{self.command_venv / 'bin'}{os.pathsep}{env.get('PATH', '')}" # Use a new session so we can kill the whole process group (bash + # any spawned children) when a command times out. self._proc = subprocess.Popen( @@ -507,7 +522,22 @@ def run_swe_agent_local( ) config_file = _materialize_agent_config(Path(config_file).resolve(), output_dir) - env = LocalSWEEnv(workspace, task, timeout=timeout_per_command) + command_venv = output_dir / "command_venv" + if command_venv.exists(): + shutil.rmtree(command_venv) + subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(command_venv)], + check=True, + capture_output=True, + text=True, + timeout=120, + ) + env = LocalSWEEnv( + workspace, + task, + timeout=timeout_per_command, + command_venv=command_venv, + ) env.reset() try: diff --git a/swe_bench/runner.py b/swe_bench/runner.py index d78d1c8..c8bfec7 100644 --- a/swe_bench/runner.py +++ b/swe_bench/runner.py @@ -566,6 +566,11 @@ def _prepare_workspace(self, task: SWEBenchTask, workspace: Path) -> None: repo_cache = self.cache_dir / task.repo.replace("/", "__") self._ensure_repo_cache(task.repo, repo_cache) + # Local Docker image builds may derive package versions from Git tags. + # Ensure the base commit has enough ancestry before copying the cache; + # a depth-1 commit otherwise becomes versions such as ``0.1.dev1``. + self._fetch_commit(repo_cache, task.base_commit) + # Copy repo into workspace to avoid mutating the cache. if workspace.exists(): shutil.rmtree(workspace) @@ -594,16 +599,39 @@ def _fetch_commit(self, repo_dir: Path, commit: str) -> None: import random as _random import re - # Check if commit already exists locally before hitting the network. + # A present commit is not sufficient: depth-1 history prevents tools + # such as setuptools-scm from finding the nearest release tag. + commit_exists = False try: _run_command( ["git", "cat-file", "-t", commit], cwd=repo_dir, timeout=5, ) - return # commit exists, no need to fetch + commit_exists = True except Exception: pass + if commit_exists: + try: + shallow = ( + _run_command( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=repo_dir, + timeout=5, + ).stdout.strip() + == "true" + ) + history_count = int( + _run_command( + ["git", "rev-list", "--count", commit], + cwd=repo_dir, + timeout=10, + ).stdout.strip() + ) + if not shallow or history_count >= 200: + return + except Exception: + pass # Build source list: origin → gitee mirror (if URL pattern matches). sources: list[str] = ["origin"] @@ -624,7 +652,7 @@ def _fetch_commit(self, repo_dir: Path, commit: str) -> None: try: timeout = 30 * attempt # 30s, 60s, 90s _run_command( - ["git", "fetch", "--depth", "1", source, commit], + ["git", "fetch", "--depth", "500", "--tags", source, commit], cwd=repo_dir, timeout=timeout, ) diff --git a/tests/test_swe_agent_local_runner.py b/tests/test_swe_agent_local_runner.py index 94a7b74..16dc845 100644 --- a/tests/test_swe_agent_local_runner.py +++ b/tests/test_swe_agent_local_runner.py @@ -6,7 +6,7 @@ import pytest import yaml -from scripts.compare_three_systems import build_goal_description +from scripts.compare_three_systems import _claude_environment, build_goal_description from swe_agent_local_runner import ( LocalSWEEnv, _materialize_agent_config, @@ -46,6 +46,18 @@ def test_local_env_preserves_command_exit_status(tmp_path): env.close() +def test_local_env_uses_per_task_command_venv(tmp_path): + command_venv = tmp_path / "command_venv" + (command_venv / "bin").mkdir(parents=True) + env = LocalSWEEnv(tmp_path, _task(), timeout=5, command_venv=command_venv) + try: + assert env.communicate('printf "$VIRTUAL_ENV"') == str(command_venv) + path = env.communicate('printf "$PATH"') + assert path.split(":")[1] == str(command_venv / "bin") + finally: + env.close() + + def test_inline_state_command_returns_json(tmp_path): config_path = Path("swe_agent_local_config/default_local.yaml") config = yaml.safe_load(config_path.read_text(encoding="utf-8")) @@ -123,6 +135,18 @@ def test_comparison_prompt_does_not_expose_hidden_tests(): assert "public issue description" in prompt +def test_claude_environment_can_use_shared_benchmark_endpoint(monkeypatch): + monkeypatch.setenv("SWE_BENCH_CLAUDE_BASE_URL", "https://api.deepseek.com/anthropic") + monkeypatch.setenv("SWE_BENCH_CLAUDE_API_KEY", "shared-secret") + + env = _claude_environment("deepseek-v4-flash") + + assert env["ANTHROPIC_BASE_URL"] == "https://api.deepseek.com/anthropic" + assert env["ANTHROPIC_API_KEY"] == "shared-secret" + assert env["ANTHROPIC_AUTH_TOKEN"] == "shared-secret" + assert env["ANTHROPIC_MODEL"] == "deepseek-v4-flash[1m]" + + def test_docker_evaluator_does_not_rewrite_official_script_by_default(monkeypatch): monkeypatch.delenv("SWE_BENCH_PATCH_EVAL_ENV", raising=False) evaluator = _docker_evaluator()