From 3767003472ffa2342760e2b5f4b854acbcb15cc6 Mon Sep 17 00:00:00 2001 From: RCSnyder <44173311+RCSnyder@users.noreply.github.com> Date: Sun, 10 May 2026 22:08:41 -0500 Subject: [PATCH] feat(runpod): private-by-default ports, VS Code Ollama bridge, Windows on-ramp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Windows + VS Code + RunPod path to hopper without changing the existing Vast.ai/macOS contract. The pod stays private-by-default; the new bridge lets VS Code Copilot Chat agent mode talk to vLLM over Tailscale by pretending to be an Ollama server. Vendor changes (src/vendors/runpod.sh): * Ports are now derived from runpod_ports(). Default is 22/tcp only; 8000/http requires both RUNPOD_PUBLIC_PORTS_ACK=I_UNDERSTAND_THIS_IS_PUBLIC and a non-empty VLLM_API_KEY so we never accidentally publish an unauthenticated vLLM. RUNPOD_EXPOSE_HTTP=1 / RUNPOD_PORTS opt back in. * record_spend_start + autosupervise_start now fire before run_remote_setup so a setup failure still leaves an active spend-log entry and a running guard. * runpod_restart_vllm re-runs run_remote_setup (re-copies scripts, re-joins Tailscale, restarts vLLM) and calls runpod_ensure_spend_tracking_started so recovery from a half-finished create doesn't silently stop billing tracking. * get_pod_endpoint strips CRLF from POD_IP/POD_SSH_PORT — runpodctl|jq on Git Bash leaves '\r' on parsed values and ssh rejected it as "Bad port". * runpod_status / runpod_logs surface the Ollama bridge when enabled. * Bumped ephemeral container disk to 80GB when no network volume is attached (30GB fills with vLLM install + HF cache for the 27B profile). * Replaced an `rg` shellout in runpod_ssh with grep (not always installed). Shared lib (src/lib/gpu-common.sh): * load_env_file sources via a CRLF-stripped temp copy so Windows-saved .env files don't leak '\r' into ssh -i paths, curl URLs, etc. * run_remote_setup scp's setup-vllm.sh through a CRLF-stripped temp copy too, and also copies vscode-ollama-bridge.py to /workspace when present. * build_remote_env_prefix forwards VLLM_API_KEY and the OLLAMA_SHIM_* vars. * Auto-detect SSH key: prefer ~/.ssh/id_ed25519, ~/.ssh/id_rsa, then ~/.runpod/ssh/RunPod-Key-Go (runpodctl-generated, already registered with RunPod), and warn if none exist. * DEFAULT_VOLUME_CREATE_DC moved to US-KS-2 (EU-SE-1 has been rejecting new volume creation in my account; flip back per .env if you prefer EU). vLLM setup (src/setup-vllm.sh): * Adds --api-key when VLLM_API_KEY is set, with a redacted echo. * Optional Ollama-compatible bridge controlled by OLLAMA_SHIM_ENABLED. The bridge runs in its own venv under /workspace/ollama-shim, started via uvicorn on 11434 (bound to 0.0.0.0 when Tailscale is up, else 127.0.0.1). Separate install/stop/serve/wait functions mirror the vLLM ones; logs go to /workspace/ollama-shim-.log. * uv install runs with UV_NO_CACHE=1 and clears uv/pip caches afterward to keep ephemeral disk usage down. VS Code Ollama bridge (src/vscode-ollama-bridge.py): * Minimal FastAPI app — only the endpoints Copilot Chat's Ollama BYOK provider actually calls: /api/version, /api/tags, /api/show, and a transparent stream-proxy for /v1/chat/completions. /api/show reports capabilities:["tools"] so the model stays in the agent-mode picker, and context_length comes from the vLLM model entry. Reads BACKEND_BASE_URL, BACKEND_API_KEY, REPORTED_VERSION from env. .env.example: * Documents RUNPOD_EXPOSE_HTTP, RUNPOD_PUBLIC_PORTS_ACK, VLLM_API_KEY, and the OLLAMA_SHIM_* knobs with the same private-by-default framing. Docs: * docs/gpu-runbook.md: new sections "RunPod SSH On Git Bash" (CRLF + endpoint-lag quirks, public-port acknowledgement, openssl rand recipe) and "VS Code Ollama Agent Mode" (when to enable the bridge, restart-vllm as the in-place recovery path). Reflowed the failure table. * docs/course/00-vscode-runpod.md: new lesson 0 — full Windows on-ramp with winget installs (Git Bash, jq, Tailscale, OpenSSH, VS Code), runpodctl direct-download, Copilot Chat + Remote-SSH extensions, Git Bash as default VS Code terminal, Tailscale auth-key setup, .env walkthrough, create/teardown flow, and Copilot Chat agent-mode wiring via http://:11434. * README.md + docs/course/README.md link the new lesson without disturbing the existing macOS/Vast path. No vendor contract changes: the dispatcher and KNOWN_COMMANDS are untouched, runpod_* function names are stable, and the spend-log format is unchanged. --- README.md | 20 +-- docs/course/00-vscode-runpod.md | 208 ++++++++++++++++++++++++++++++++ docs/course/README.md | 22 ++-- docs/gpu-runbook.md | 85 +++++++++++-- src/.env.example | 18 +++ src/lib/gpu-common.sh | 50 +++++++- src/setup-vllm.sh | 127 ++++++++++++++++++- src/vendors/runpod.sh | 167 ++++++++++++++++++++++--- src/vscode-ollama-bridge.py | 155 ++++++++++++++++++++++++ 9 files changed, 796 insertions(+), 56 deletions(-) create mode 100644 docs/course/00-vscode-runpod.md create mode 100644 src/vscode-ollama-bridge.py diff --git a/README.md b/README.md index 33832d5..a6cf0ab 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,16 @@ Rent a GPU, run vLLM on it, and stop paying once the daily budget is gone. Works against Vast.ai and RunPod, with Tailscale handling the network so the pod is reachable by hostname from anywhere. -| What you get | How | -| ------------------- | ---------------------------------------------------------------------------- | -| Vendors | Vast.ai, RunPod (one CLI, identical command surface) | -| Inference server | vLLM, OpenAI-compatible at `:8000` — any client that speaks OpenAI works | -| Network | Tailscale hostname (e.g. `vllm-qwen27b-vast:8000`) — no port-forward, no SSH tunnel per session | -| Cost control | Hard `DAILY_BUDGET` ($/day) and optional `DAILY_HOURS` cap with grace countdown before terminate | -| Reproducibility | Per-model + per-GPU profiles in `src/profiles/*.env` | -| Adding a vendor | One file: `src/vendors/.sh` implementing `create`/`ssh`/`terminate`/`status`/`logs` | - -> **New here?** There's a course that walks you from zero to a personal coding agent running on a rented GPU, reachable from your laptop and your phone. Start at [`docs/course/README.md`](docs/course/README.md). +| What you get | How | +| ---------------- | ------------------------------------------------------------------------------------------------ | +| Vendors | Vast.ai, RunPod (one CLI, identical command surface) | +| Inference server | vLLM, OpenAI-compatible at `:8000` — any client that speaks OpenAI works | +| Network | Tailscale hostname (e.g. `vllm-qwen27b-vast:8000`) — no port-forward, no SSH tunnel per session | +| Cost control | Hard `DAILY_BUDGET` ($/day) and optional `DAILY_HOURS` cap with grace countdown before terminate | +| Reproducibility | Per-model + per-GPU profiles in `src/profiles/*.env` | +| Adding a vendor | One file: `src/vendors/.sh` implementing `create`/`ssh`/`terminate`/`status`/`logs` | + +> **New here?** There's a course that walks you from zero to a personal coding agent running on a rented GPU, reachable from your laptop and your phone. Start at [`docs/course/README.md`](docs/course/README.md). On Windows + VS Code + RunPod, take the parallel on-ramp at [`docs/course/00-vscode-runpod.md`](docs/course/00-vscode-runpod.md) instead. ## Quickstart diff --git a/docs/course/00-vscode-runpod.md b/docs/course/00-vscode-runpod.md new file mode 100644 index 0000000..ecb32bc --- /dev/null +++ b/docs/course/00-vscode-runpod.md @@ -0,0 +1,208 @@ +# Lesson 0 — Running Hopper from VS Code on Windows with RunPod + +This is the zero-to-working path if you're on **Windows**, want to drive everything from **VS Code** (including GitHub Copilot Chat in agent mode), and prefer **RunPod** over Vast.ai. + +The rest of the course (lessons 01–06) is macOS + Vast.ai oriented. This lesson is a parallel on-ramp: by the end of it you will have a RunPod pod running vLLM, joined to your tailnet, and reachable from VS Code via the Ollama-compatible bridge. + +If you already use macOS, skip this and go to [the course README](README.md). + +## What you'll have at the end + +- A Windows machine that can run `./src/gpu.sh runpod create` from a Git Bash terminal in VS Code. +- A RunPod pod running vLLM (OpenAI-compatible) on port `8000`, plus the in-pod Ollama bridge on `11434`. +- Both reachable privately over Tailscale by hostname — no public ports, no SSH tunnel for everyday use. +- GitHub Copilot Chat configured to use your pod as an Ollama provider in agent mode. + +## Prerequisites + +- Windows 10 or 11 with admin rights to install software. +- A GitHub account (for VS Code + Copilot Chat). +- A credit card with **$5–$10** for RunPod top-up. +- A Tailscale account (free tier is enough). + +## 1. Install the toolbelt + +You need: Git Bash (shell + ssh + scp + curl), jq, Tailscale for Windows, runpodctl, and VS Code. The easiest path is [winget](https://learn.microsoft.com/windows/package-manager/winget/), bundled with Windows 11 and recent Windows 10. + +Open **PowerShell as Administrator** and run: + +```powershell +winget install --id Git.Git -e +winget install --id stedolan.jq -e +winget install --id tailscale.tailscale -e +winget install --id Microsoft.VisualStudioCode -e +winget install --id Microsoft.OpenSSH.Beta -e # only if `ssh` is missing +``` + +`runpodctl` is not in winget. Install it from the official release into a folder on `PATH`: + +```powershell +$dest = "$env:USERPROFILE\bin" +New-Item -ItemType Directory -Force -Path $dest | Out-Null +Invoke-WebRequest -Uri "https://github.com/runpod/runpodctl/releases/latest/download/runpodctl-windows-amd64.exe" ` + -OutFile "$dest\runpodctl.exe" +# Add to PATH for new shells: +setx PATH "$env:PATH;$dest" +``` + +Close PowerShell. Open a **new** Git Bash terminal (Start menu → "Git Bash") and verify: + +```bash +git --version +bash --version +ssh -V +jq --version +tailscale version +runpodctl version +code --version +``` + +If any command is "not found", fix the install before continuing. PATH changes only apply to **new** terminals — close and reopen if needed. + +> **Why Git Bash, not WSL or PowerShell?** `gpu.sh` is bash and uses POSIX tools (`awk`, `sed`, `scp`, `mktemp`). Git Bash ships all of them and stays out of WSL's filesystem-translation issues. If you already live in WSL, that works too — just clone the repo inside WSL so paths are Linux-native. + +## 2. Open VS Code with the right defaults + +Install these extensions (in VS Code or via the command line): + +```powershell +code --install-extension GitHub.copilot +code --install-extension GitHub.copilot-chat +code --install-extension ms-vscode-remote.remote-ssh +``` + +Set Git Bash as the default integrated terminal so every terminal you open in VS Code is the same shell `gpu.sh` runs in. In VS Code: + +1. `Ctrl+Shift+P` → **Terminal: Select Default Profile** → **Git Bash**. +2. Open a new terminal (`Ctrl+\``) and confirm the prompt looks like `user@host MINGW64 ~`. + +## 3. Set up Tailscale + +1. Sign in to the Tailscale tray app on Windows (`Tailscale → Log in`). Your laptop joins your tailnet. +2. In the [admin console](https://login.tailscale.com/admin/dns), turn on **MagicDNS**. This lets you reach pods by hostname (e.g. `vllm-qwen27b:8000`) instead of IP. +3. Create a reusable auth key at [`/admin/settings/keys`](https://login.tailscale.com/admin/settings/keys): + - **Reusable**: ON + - **Ephemeral**: ON (so destroyed pods leave no stale device entries) + - **Pre-approved**: ON +4. Copy the key. You'll paste it into `src/.env` in step 5. + +## 4. Set up RunPod + +1. Create an account at [runpod.io](https://www.runpod.io/) and add credit ($5–$10 is fine to start). +2. In **Console → Settings → API Keys**, create a key with read/write access. Copy it. +3. In a Git Bash terminal, configure `runpodctl`: + + ```bash + runpodctl config --apiKey + runpodctl pod list # should return an empty list (or your existing pods), not an auth error + ``` + + This writes `~/.runpod/config.toml` and generates an SSH key at `~/.runpod/ssh/RunPod-Key-Go`. Hopper auto-detects that key, so you don't need to copy it anywhere. + +## 5. Clone the repo and configure `.env` + +In Git Bash: + +```bash +mkdir -p ~/code && cd ~/code +git clone https://github.com/ugudlado/hopper.git +cd hopper +cp src/.env.example src/.env +``` + +Open `src/.env` in VS Code (`code src/.env`) and set: + +```env +TAILSCALE_AUTHKEY=tskey-auth-... # from step 3 +RUNPOD_API_KEY=... # from step 4 + +# Daily caps. The supervisor terminates the pod when either is hit. +DAILY_BUDGET=5 +DAILY_HOURS=5 + +# Enable the in-pod Ollama-compatible bridge for VS Code Copilot Chat agent mode. +OLLAMA_SHIM_ENABLED=1 +``` + +Leave `VLLM_API_KEY` and `RUNPOD_EXPOSE_HTTP` empty — Tailscale is the access path, and public RunPod ports are not needed. + +Save the file. `.env` is gitignored; double-check before any `git add`. + +## 6. Create the pod + +Still in Git Bash, from the repo root: + +```bash +PROFILE=qwen3.6-27b-4b ./src/gpu.sh runpod create +``` + +What this does, in order: + +1. Picks an A100 80GB on RunPod, attaches a 60GB network volume named `vllm-cache` (created on first run). +2. SSHs in, copies `setup-vllm.sh` and `vscode-ollama-bridge.py` to the pod. +3. Installs vLLM, joins Tailscale as `vllm-qwen27b`, starts vLLM on `:8000` and the bridge on `:11434`. +4. Records spend start and launches the supervisor in a tmux session named `guard`. + +First boot takes 8–15 minutes (model download + JIT compile). On reruns with the network volume attached, it's 1–2 minutes. + +Watch progress in another Git Bash terminal: + +```bash +PROFILE=qwen3.6-27b-4b ./src/gpu.sh runpod logs +``` + +When it's serving, this works from your Windows machine: + +```bash +curl http://vllm-qwen27b:8000/v1/models +curl http://vllm-qwen27b:11434/api/tags +``` + +If the curl fails: see the [recovery section](#recovery-from-a-broken-create) below. + +## 7. Wire VS Code Copilot Chat to your pod + +Copilot Chat's agent mode supports BYOK via the Ollama API. The in-pod bridge speaks Ollama and proxies to vLLM, so VS Code treats your pod like a local Ollama install. + +1. In VS Code, open the Copilot Chat side panel. +2. Click the **model picker** at the bottom of the chat box → **Manage Models** → **Ollama**. +3. Set the Ollama endpoint to: + + ```text + http://vllm-qwen27b:11434 + ``` + +4. VS Code lists the available models (you'll see whatever `VLLM_MODEL` resolves to in the profile, e.g. `cyankiwi/Qwen3.6-27B-AWQ-INT4`). Tick it. +5. Switch the chat to **Agent** mode and pick that model. + +Sanity check: ask Copilot Chat to "list the files in this repo." It should pick up the workspace and call its filesystem tool against your pod-served model. + +## 8. Tear it down + +When you're done for the day: + +```bash +PROFILE=qwen3.6-27b-4b ./src/gpu.sh runpod terminate +``` + +The network volume survives, so the next `create` reuses the downloaded model. The supervisor would also terminate automatically once `DAILY_BUDGET` or `DAILY_HOURS` is hit. + +## Recovery from a broken create + +If `runpod create` exits after the pod is created but before setup finishes (network blip, SSH timeout, etc.) — the pod is still billing, but vLLM and the bridge are not running. Re-run setup in place instead of recreating: + +```bash +PROFILE=qwen3.6-27b-4b ./src/gpu.sh runpod restart-vllm +``` + +This re-copies the setup script and bridge, re-joins Tailscale, restarts vLLM, and (re)starts the supervisor. See [`gpu-runbook.md`](../gpu-runbook.md) for the underlying behavior and known RunPod-on-Git-Bash quirks (CRLF in `runpodctl` output, SSH endpoint lag, etc.). + +## Where to go next + +You now have the same architecture lessons 01–06 build up to, just from a Windows + VS Code starting point. Skim these if you want the conceptual background: + +- [Lesson 01 — tmux](01-tmux.md): why the agent loop survives terminal closes (works the same in Git Bash). +- [Lesson 02 — Tailscale](02-tailscale.md): how MagicDNS turns `vllm-qwen27b` into a routable hostname. +- [Lesson 04 — `gpu.sh`](04-gpu-sh.md): the dispatcher, profiles, and supervisor in detail. + +The Pi lesson ([05](05-pi.md)) is optional on Windows — Copilot Chat agent mode covers the same "agent talks to vLLM" loop without installing Pi. diff --git a/docs/course/README.md b/docs/course/README.md index d2810b4..3f2df68 100644 --- a/docs/course/README.md +++ b/docs/course/README.md @@ -108,6 +108,8 @@ If `command -v remote-session` prints nothing, add `export PATH="$HOME/.local/bi **Ready?** [**Start with lesson 01 — tmux →**](01-tmux.md) +> **On Windows, or want to drive everything from VS Code with RunPod?** Take the parallel on-ramp first: [**Lesson 0 — Running Hopper from VS Code on Windows with RunPod**](00-vscode-runpod.md). It installs every Windows prerequisite (Git Bash, jq, Tailscale, runpodctl) and ends with Copilot Chat agent mode pointed at your pod. + ## Lessons 1. **[tmux](01-tmux.md)**: Sessions that outlive the terminal you started them in. _This is what makes the device handoff possible._ @@ -121,14 +123,14 @@ If `command -v remote-session` prints nothing, add `export PATH="$HOME/.local/bi **Plan on roughly $1/hr** for a verified A100 SXM4 80GB on Vast with a recent driver (CUDA 12.6+). It can be a bit less or more depending on the day and what's available. With `DAILY_HOURS=5`, that's about $5/day, or ~$100/month if you run the daily envelope every weekday. -| Lesson | Time | GPU cost | Device needed | -| -------------- | --------- | ---------------------------------- | ---------------- | -| 01 — tmux | 15–20 min | $0 | Laptop | -| 02 — Tailscale | 20–30 min | $0 | Laptop + phone | -| 03 — Vast.ai | 20–30 min | $0 until you rent | Laptop | -| 04 — `gpu.sh` | 30–45 min | ~$1 for a create + idle window | Laptop + GPU pod | -| 05 — Pi | 30–45 min | GPU running while you test | Laptop + phone | -| 06 — Capstone | 20–30 min | GPU running during handoff | Laptop + phone | +| Lesson | Time | GPU cost | Device needed | +| -------------- | --------- | ------------------------------ | ---------------- | +| 01 — tmux | 15–20 min | $0 | Laptop | +| 02 — Tailscale | 20–30 min | $0 | Laptop + phone | +| 03 — Vast.ai | 20–30 min | $0 until you rent | Laptop | +| 04 — `gpu.sh` | 30–45 min | ~$1 for a create + idle window | Laptop + GPU pod | +| 05 — Pi | 30–45 min | GPU running while you test | Laptop + phone | +| 06 — Capstone | 20–30 min | GPU running during handoff | Laptop + phone | The main cost risk is not the course itself; it is forgetting to terminate the pod. Keep the supervisor running whenever the GPU is up. @@ -149,5 +151,5 @@ The main cost risk is not the course itself; it is forgetting to terminate the p --- -| | | **[Lesson 01 — tmux →](01-tmux.md)** | -| :--- | :---: | ---: | +| | | **[Lesson 01 — tmux →](01-tmux.md)** | +| :-- | :-: | -----------------------------------: | diff --git a/docs/gpu-runbook.md b/docs/gpu-runbook.md index a977c2d..6e42b26 100644 --- a/docs/gpu-runbook.md +++ b/docs/gpu-runbook.md @@ -14,14 +14,14 @@ The Vast CLI has returned false zero-result searches for `gpu_ram>=N` filters. ` Common Vast create failures: -| Symptom | Likely cause | Fix | -| --- | --- | --- | -| Pod stays in `loading` | Fresh host is pulling a large image | Wait up to 15 minutes before retrying. | -| `manifest unknown` or invalid image | Bad Docker image tag | Use the known-good Vast profile image. | -| SSH permission denied | Key missing from Vast account | Run `vastai show ssh-keys`; register `~/.ssh/id_ed25519.pub` if missing. | -| Instance created but never starts | Vast `create instance --ssh` does not start instances | Harness calls `vastai start instance ` after create. | -| Destroy hangs | Vast prompts interactively | Harness pipes confirmation for destroy. | -| SSH proxy logs remote forwarding errors | Vast platform proxy issue | Retry later or consider a direct-port refactor if it becomes frequent. | +| Symptom | Likely cause | Fix | +| --------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------ | +| Pod stays in `loading` | Fresh host is pulling a large image | Wait up to 15 minutes before retrying. | +| `manifest unknown` or invalid image | Bad Docker image tag | Use the known-good Vast profile image. | +| SSH permission denied | Key missing from Vast account | Run `vastai show ssh-keys`; register `~/.ssh/id_ed25519.pub` if missing. | +| Instance created but never starts | Vast `create instance --ssh` does not start instances | Harness calls `vastai start instance ` after create. | +| Destroy hangs | Vast prompts interactively | Harness pipes confirmation for destroy. | +| SSH proxy logs remote forwarding errors | Vast platform proxy issue | Retry later or consider a direct-port refactor if it becomes frequent. | When a create attempt fails, check the provider state before layering on local fixes: @@ -31,6 +31,75 @@ vastai show instances --raw | jq '.[0] | {id, actual_status, status_msg, ssh_hos ## vLLM Runtime Fixes +## RunPod SSH On Git Bash + +RunPod creates public ingress for pod `ports` entries beyond `22/tcp`, including +`*.proxy.runpod.net` URLs for `*/http` ports. Hopper defaults to `22/tcp` only, +so vLLM is reached via Tailscale or an explicit SSH tunnel. + +Public RunPod ports are blocked unless both conditions are true: + +```env +RUNPOD_PUBLIC_PORTS_ACK=I_UNDERSTAND_THIS_IS_PUBLIC +VLLM_API_KEY= +``` + +Generate the API key from Git Bash with OpenSSL: + +```bash +openssl rand -hex 32 +``` + +With `VLLM_API_KEY` set, `setup-vllm.sh` starts vLLM with `--api-key`, and +clients must send `Authorization: Bearer `. `RUNPOD_EXPOSE_HTTP=1` +and custom `RUNPOD_PORTS` are for short manual tests only. + +On Windows Git Bash, `runpodctl pod get ... | jq -r ...` can leave a trailing +carriage return on the parsed SSH port. That turns `22008` into `$'22008\r'`, +so the harness fails with `Bad port` even though the same endpoint works when +typed manually from `runpodctl` output. `get_pod_endpoint` strips `\r` from the +parsed IP and port before calling `ssh`. + +RunPod can also publish a pod as `RUNNING` before the SSH endpoint is usable. +The harness waits for `runpodctl pod get ` to expose `.ssh.ip` and +`.ssh.port` before connecting. + +## VS Code Ollama Agent Mode + +Hopper's primary contract is still an OpenAI-compatible vLLM endpoint on `:8000`. +For VS Code clients that only speak the Ollama API, enable the in-pod shim: + +```env +OLLAMA_SHIM_ENABLED=1 +OLLAMA_SHIM_VERSION=0.6.4 +OLLAMA_SHIM_PORT=11434 +``` + +This keeps vLLM as the only model server and starts a sibling compatibility +process inside the pod that proxies Ollama requests to `http://127.0.0.1:8000/v1`. +Use Tailscale to reach it privately: + +```text +http://:11434 +``` + +Do not route this through OpenRouter unless you explicitly want a third-party +broker in the path. The intended Hopper pattern is local vLLM + local shim, +both private to the tailnet. + +If a create command exits after the pod is created but before setup completes, +the pod may be reachable by SSH with an empty `/workspace` and no vLLM or +Tailscale process yet. Recover in place instead of recreating: + +```bash +PROFILE=qwen3.6-27b-4b ./src/gpu.sh runpod restart-vllm +``` + +That command re-copies `src/setup-vllm.sh`, joins Tailscale if configured, and +starts vLLM on the existing pod. It also records an active spend-log entry and +starts the budget supervisor if the original create command exited before those +steps ran. + ### Qwen3.x GDN On CUDA < 12.6 Qwen3.5/Qwen3.6 can fail on first inference when flashinfer tries to JIT Gated Delta Net kernels against CUDA 12.4: diff --git a/src/.env.example b/src/.env.example index de5f182..53f3a63 100644 --- a/src/.env.example +++ b/src/.env.example @@ -32,6 +32,24 @@ VAST_API_KEY= # Only needed if you use `gpu.sh runpod ...`. RUNPOD_API_KEY= +# RunPod ports are private-by-default: only 22/tcp is exposed for provider SSH. +# Tailscale is the intended path to vLLM. +RUNPOD_EXPOSE_HTTP=0 +RUNPOD_PUBLIC_PORTS_ACK= + +# Required only if you deliberately expose public RunPod ports. Generate with: +# openssl rand -hex 32 +# When set, vLLM requires clients to send: +# Authorization: Bearer +VLLM_API_KEY= + +# Optional Ollama-compatible facade for VS Code clients that only speak Ollama. +# Hopper still runs vLLM as the primary server; the shim proxies to the local +# vLLM OpenAI endpoint inside the pod. +OLLAMA_SHIM_ENABLED=0 +OLLAMA_SHIM_VERSION=0.6.4 +OLLAMA_SHIM_PORT=11434 + # --------------------------------------------------------------------------- # Daily caps enforced by the budget supervisor that `gpu.sh create` # auto-launches in a tmux session named "guard". Either cap (whichever fires diff --git a/src/lib/gpu-common.sh b/src/lib/gpu-common.sh index 4426955..4e011b0 100644 --- a/src/lib/gpu-common.sh +++ b/src/lib/gpu-common.sh @@ -21,17 +21,24 @@ DEFAULT_RUNPOD_VOLUME_ID="" DEFAULT_DATA_CENTER_IDS="" DEFAULT_VOLUME_NAME="vllm-cache" DEFAULT_VOLUME_SIZE_GB="60" -DEFAULT_VOLUME_CREATE_DC="EU-SE-1" +DEFAULT_VOLUME_CREATE_DC="US-KS-2" DEFAULT_VLLM_MODEL="QuantTrio/Qwen3-Coder-30B-A3B-Instruct-AWQ" DEFAULT_VLLM_MAX_LEN="65536" load_env_file() { local env_path="$1" if [ -f "$env_path" ]; then + # Source via a CRLF-stripped temp copy so values saved with Windows line + # endings don't carry a trailing \r into bash variables (which silently + # breaks ssh -i, curl URLs, file paths, etc.). + local tmp + tmp="$(mktemp 2>/dev/null || echo "${env_path}.tmp.$$")" + sed 's/\r$//' "$env_path" > "$tmp" # shellcheck source=/dev/null set -a - . "$env_path" + . "$tmp" set +a + rm -f "$tmp" fi } @@ -79,6 +86,7 @@ REMOTE_PORT="${REMOTE_PORT:-8000}" VLLM_PORT="${VLLM_PORT:-$REMOTE_PORT}" VLLM_MODEL="${VLLM_MODEL:-$DEFAULT_VLLM_MODEL}" VLLM_MAX_LEN="${VLLM_MAX_LEN:-$DEFAULT_VLLM_MAX_LEN}" +VLLM_API_KEY="${VLLM_API_KEY:-}" TAILSCALE_AUTHKEY="${TAILSCALE_AUTHKEY:-}" TAILSCALE_HOSTNAME="${TAILSCALE_HOSTNAME:-}" TAILSCALE_STATE_FILE="${TAILSCALE_STATE_FILE:-/workspace/tailscale.state}" @@ -124,7 +132,25 @@ pod_access_url() { fi } +# Auto-detect SSH key: explicit SSH_KEY wins; otherwise probe a list of +# common locations and pick the first that exists. runpodctl generates its +# own key at ~/.runpod/ssh/RunPod-Key-Go and syncs the public side to the +# RunPod account, which is the right one to use on a fresh setup. +if [ -z "${SSH_KEY:-}" ]; then + for _candidate in \ + "$HOME/.ssh/id_ed25519" \ + "$HOME/.ssh/id_rsa" \ + "$HOME/.runpod/ssh/RunPod-Key-Go"; do + if [ -f "$_candidate" ]; then + SSH_KEY="$_candidate" + break + fi + done +fi SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519}" +if [ ! -f "$SSH_KEY" ]; then + echo "WARNING: SSH key not found at $SSH_KEY — set SSH_KEY in src/.env" >&2 +fi SSH_OPTS="-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -i $SSH_KEY" RUNPOD_BIN="${RUNPOD_BIN:-}" USE_RUNPODCTL_SSH="${USE_RUNPODCTL_SSH:-0}" @@ -180,7 +206,7 @@ build_remote_env_prefix() { TAILSCALE_HOSTNAME="$(effective_tailnet_hostname)" fi - for var in VLLM_MODEL VLLM_MAX_LEN VLLM_PORT VLLM_GPU_UTIL VLLM_TOOL_PARSER VLLM_REASONING_PARSER VLLM_EXTRA_ARGS TAILSCALE_AUTHKEY TAILSCALE_HOSTNAME TAILSCALE_STATE_FILE; do + for var in VLLM_MODEL VLLM_MAX_LEN VLLM_PORT VLLM_GPU_UTIL VLLM_TOOL_PARSER VLLM_REASONING_PARSER VLLM_EXTRA_ARGS VLLM_API_KEY OLLAMA_SHIM_ENABLED OLLAMA_SHIM_VERSION OLLAMA_SHIM_PORT OLLAMA_SHIM_BIND_HOST TAILSCALE_AUTHKEY TAILSCALE_HOSTNAME TAILSCALE_STATE_FILE; do value="${!var:-}" if [ -n "$value" ]; then prefix+="$var=$(printf '%q' "$value") " @@ -191,14 +217,28 @@ build_remote_env_prefix() { run_remote_setup() { local setup_dest="/workspace/setup-vllm.sh" - local remote_env + local bridge_dest="/workspace/vscode-ollama-bridge.py" + local remote_env setup_tmp if [ -z "$CREATE_RUNPOD_VOLUME_ID" ]; then echo "WARNING: no network volume attached — model will not persist across pod recreations." fi echo "Copying setup script to pod ($setup_dest)..." - scp $SSH_OPTS -P "$POD_SSH_PORT" "$SCRIPT_DIR/setup-vllm.sh" "root@$POD_IP:$setup_dest" + setup_tmp="$(mktemp 2>/dev/null || echo "$SCRIPT_DIR/.setup-vllm.tmp.$$")" + sed 's/\r$//' "$SCRIPT_DIR/setup-vllm.sh" > "$setup_tmp" + if ! scp $SSH_OPTS -P "$POD_SSH_PORT" "$setup_tmp" "root@$POD_IP:$setup_dest"; then + rm -f "$setup_tmp" + return 1 + fi + rm -f "$setup_tmp" + + if [ -f "$SCRIPT_DIR/vscode-ollama-bridge.py" ]; then + echo "Copying Ollama bridge to pod ($bridge_dest)..." + if ! scp $SSH_OPTS -P "$POD_SSH_PORT" "$SCRIPT_DIR/vscode-ollama-bridge.py" "root@$POD_IP:$bridge_dest"; then + return 1 + fi + fi remote_env="$(build_remote_env_prefix)" if direct_access_enabled; then diff --git a/src/setup-vllm.sh b/src/setup-vllm.sh index f4f68ce..b24e8e5 100755 --- a/src/setup-vllm.sh +++ b/src/setup-vllm.sh @@ -11,6 +11,13 @@ VOLUME_MOUNT_PATH="${VOLUME_MOUNT_PATH:-/workspace}" VLLM_GPU_UTIL="${VLLM_GPU_UTIL:-0.9}" VLLM_TOOL_PARSER="${VLLM_TOOL_PARSER:-qwen3_coder}" VLLM_REASONING_PARSER="${VLLM_REASONING_PARSER:-}" +VLLM_API_KEY="${VLLM_API_KEY:-}" +OLLAMA_SHIM_ENABLED="${OLLAMA_SHIM_ENABLED:-0}" +OLLAMA_SHIM_VERSION="${OLLAMA_SHIM_VERSION:-0.6.4}" +OLLAMA_SHIM_PORT="${OLLAMA_SHIM_PORT:-11434}" +OLLAMA_SHIM_BIND_HOST="${OLLAMA_SHIM_BIND_HOST:-127.0.0.1}" +OLLAMA_SHIM_LOG="${OLLAMA_SHIM_LOG:-$VOLUME_MOUNT_PATH/ollama-shim-${TAILSCALE_HOSTNAME}.log}" +OLLAMA_SHIM_DIR="${OLLAMA_SHIM_DIR:-$VOLUME_MOUNT_PATH/ollama-shim}" # Free-form extra flags appended to vllm serve. Used for hardware-specific # workarounds (e.g. --gdn-prefill-backend triton on CUDA<12.6 + Hopper/Ampere). VLLM_EXTRA_ARGS="${VLLM_EXTRA_ARGS:-}" @@ -39,7 +46,8 @@ ensure_cache_dirs() { "$VOLUME_MOUNT_PATH/triton_cache" \ "$VOLUME_MOUNT_PATH/torch_compile_cache" \ "$VOLUME_MOUNT_PATH/vllm_cache" \ - "$VOLUME_MOUNT_PATH/tmp" + "$VOLUME_MOUNT_PATH/tmp" \ + "$VOLUME_MOUNT_PATH/bin" } install_tailscale_if_missing() { @@ -87,6 +95,9 @@ start_tailscale_if_configured() { --ssh VLLM_BIND_HOST="0.0.0.0" + if [ "$OLLAMA_SHIM_BIND_HOST" = "127.0.0.1" ]; then + OLLAMA_SHIM_BIND_HOST="0.0.0.0" + fi echo "Tailscale ready: http://$TAILSCALE_HOSTNAME:$VLLM_PORT" } @@ -109,7 +120,9 @@ install_vllm_if_missing() { echo "Installing uv..." pip install --timeout 60 --retries 5 uv 2>&1 | tail -3 fi - uv pip install --system --break-system-packages --torch-backend=cu128 "vllm==${vllm_version}" 2>&1 | tail -5 + UV_NO_CACHE=1 uv pip install --system --break-system-packages --torch-backend=cu128 "vllm==${vllm_version}" 2>&1 | tail -5 + uv cache clean >/dev/null 2>&1 || true + pip cache purge >/dev/null 2>&1 || true } stop_existing_vllm() { @@ -135,22 +148,83 @@ stop_existing_vllm() { fi } +install_ollama_shim_if_enabled() { + if [ "$OLLAMA_SHIM_ENABLED" != "1" ]; then + return 0 + fi + + local bridge_src="/workspace/vscode-ollama-bridge.py" + if [ ! -f "$bridge_src" ]; then + echo "ERROR: $bridge_src missing — gpu-common.sh should have copied it." + return 1 + fi + + mkdir -p "$OLLAMA_SHIM_DIR" + cp "$bridge_src" "$OLLAMA_SHIM_DIR/bridge.py" + + if [ ! -d "$OLLAMA_SHIM_DIR/.venv" ]; then + python3 -m venv "$OLLAMA_SHIM_DIR/.venv" + fi + + # Minimal deps: bridge.py only uses fastapi, httpx, uvicorn. + if ! "$OLLAMA_SHIM_DIR/.venv/bin/python" -m pip install --upgrade --quiet uv; then + echo "ERROR: Failed to install uv into bridge venv." + return 1 + fi + echo "Installing VS Code Ollama bridge deps via uv..." + "$OLLAMA_SHIM_DIR/.venv/bin/python" -m uv pip install --quiet \ + "fastapi>=0.110" \ + "httpx>=0.25" \ + "uvicorn>=0.23" +} + +stop_existing_ollama_shim() { + # Kill any prior bridge / legacy shim by pattern. + pkill -9 -f 'vscode-ollama-bridge' 2>/dev/null || true + pkill -9 -f 'bridge:app' 2>/dev/null || true + pkill -9 -f 'src.main:app' 2>/dev/null || true + pkill -9 -f 'ollama-shim' 2>/dev/null || true + # Belt-and-suspenders: kill anything bound to the shim port. + # The pipeline can return non-zero under pipefail when nothing matches — + # wrap in `|| true` so set -e doesn't kill the script. + if command -v fuser >/dev/null 2>&1; then + fuser -k -9 "${OLLAMA_SHIM_PORT}/tcp" 2>/dev/null || true + elif command -v ss >/dev/null 2>&1; then + local pids + pids="$(ss -lntp 2>/dev/null | awk -v p=":${OLLAMA_SHIM_PORT} " '$4 ~ p' | grep -oP 'pid=\K[0-9]+' | sort -u || true)" + if [ -n "$pids" ]; then + # shellcheck disable=SC2086 + kill -9 $pids 2>/dev/null || true + fi + fi + sleep 2 +} + serve_model() { echo "Starting vLLM: model=$VLLM_MODEL max_len=$VLLM_MAX_LEN port=$VLLM_PORT" local extra_args=() + local display_args=() if [ -n "$VLLM_TOOL_PARSER" ]; then extra_args+=(--enable-auto-tool-choice --tool-call-parser "$VLLM_TOOL_PARSER") + display_args+=(--enable-auto-tool-choice --tool-call-parser "$VLLM_TOOL_PARSER") fi if [ -n "$VLLM_REASONING_PARSER" ]; then extra_args+=(--enable-reasoning --reasoning-parser "$VLLM_REASONING_PARSER") + display_args+=(--enable-reasoning --reasoning-parser "$VLLM_REASONING_PARSER") fi if [ -n "$VLLM_EXTRA_ARGS" ]; then # Word-split intentional — VLLM_EXTRA_ARGS holds CLI flags. # shellcheck disable=SC2206 extra_args+=($VLLM_EXTRA_ARGS) + # shellcheck disable=SC2206 + display_args+=($VLLM_EXTRA_ARGS) + fi + if [ -n "$VLLM_API_KEY" ]; then + extra_args+=(--api-key "$VLLM_API_KEY") + display_args+=(--api-key '') fi - echo " extra flags: ${extra_args[*]:-none}" + echo " extra flags: ${display_args[*]:-none}" cd /root HF_HUB_DISABLE_XET=1 \ @@ -171,6 +245,26 @@ serve_model() { echo "vLLM started (pid $!). Log: $VLLM_LOG" } +start_ollama_shim() { + if [ "$OLLAMA_SHIM_ENABLED" != "1" ]; then + return 0 + fi + + local shim_python="$OLLAMA_SHIM_DIR/.venv/bin/python" + + echo "Starting VS Code Ollama bridge on $OLLAMA_SHIM_BIND_HOST:$OLLAMA_SHIM_PORT -> http://127.0.0.1:$VLLM_PORT" + BACKEND_BASE_URL="http://127.0.0.1:$VLLM_PORT" \ + BACKEND_API_KEY="${VLLM_API_KEY:-}" \ + REPORTED_VERSION="${OLLAMA_SHIM_VERSION:-0.11.0}" \ + nohup "$shim_python" -m uvicorn bridge:app \ + --app-dir "$OLLAMA_SHIM_DIR" \ + --host "$OLLAMA_SHIM_BIND_HOST" \ + --port "$OLLAMA_SHIM_PORT" \ + > "$OLLAMA_SHIM_LOG" 2>&1 & + disown + echo "Bridge started (pid $!). Log: $OLLAMA_SHIM_LOG" +} + wait_until_serving() { echo "Waiting for vLLM to serve (first start may take ~3-10 min for model download + compile)..." local i max=60 # up to 30 min (60 * 30s) @@ -196,6 +290,29 @@ wait_until_serving() { return 1 } +wait_for_ollama_shim() { + if [ "$OLLAMA_SHIM_ENABLED" != "1" ]; then + return 0 + fi + + echo "Waiting for Ollama bridge to serve on port $OLLAMA_SHIM_PORT..." + local i + for i in $(seq 1 20); do + if curl -fsS --max-time 3 "http://127.0.0.1:$OLLAMA_SHIM_PORT/api/version" >/dev/null 2>&1; then + echo "Ollama bridge serving on port $OLLAMA_SHIM_PORT." + return 0 + fi + if ! pgrep -f 'bridge:app' >/dev/null 2>&1; then + echo "ERROR: Ollama bridge died during startup. Last log lines:" + tail -30 "$OLLAMA_SHIM_LOG" + return 1 + fi + sleep 3 + done + echo "ERROR: Ollama bridge did not start in time. Check $OLLAMA_SHIM_LOG." + return 1 +} + main() { # --bench-only: print hardware info and exit (skip vLLM serve). if [ "${1:-}" = "--bench-only" ]; then @@ -210,9 +327,13 @@ main() { ensure_cache_dirs start_tailscale_if_configured install_vllm_if_missing + install_ollama_shim_if_enabled stop_existing_vllm + stop_existing_ollama_shim serve_model wait_until_serving + start_ollama_shim + wait_for_ollama_shim echo "=== Setup complete ===" } diff --git a/src/vendors/runpod.sh b/src/vendors/runpod.sh index 88f4589..4d02ba8 100644 --- a/src/vendors/runpod.sh +++ b/src/vendors/runpod.sh @@ -154,6 +154,72 @@ runpod_graphql() { -d "$(jq -n --arg q "$query" '{query: $q}')" } +runpod_ports_public_requested() { + local ports="$1" part trimmed + local -a runpod_port_parts + ports="${ports//$'\r'/}" + IFS=',' read -r -a runpod_port_parts <<< "$ports" + for part in "${runpod_port_parts[@]}"; do + trimmed="$(printf '%s' "$part" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -n "$trimmed" ] && [ "$trimmed" != "22/tcp" ]; then + return 0 + fi + done + return 1 +} + +runpod_validate_ports() { + local ports="$1" + ports="${ports//$'\r'/}" + if [ -z "$ports" ]; then + echo "ERROR: RunPod ports resolved to an empty string." >&2 + return 1 + fi + + if runpod_ports_public_requested "$ports"; then + if [ "${RUNPOD_PUBLIC_PORTS_ACK:-}" != "I_UNDERSTAND_THIS_IS_PUBLIC" ]; then + cat >&2 <<'EOF' +ERROR: refusing to create RunPod pod with public ports. + +RunPod ports other than 22/tcp create public proxy/direct ingress. Hopper is +private-by-default and expects vLLM access through Tailscale or an SSH tunnel. + +To intentionally expose a public endpoint, set both: + RUNPOD_PUBLIC_PORTS_ACK=I_UNDERSTAND_THIS_IS_PUBLIC + VLLM_API_KEY= +EOF + return 1 + fi + if [ -z "${VLLM_API_KEY:-}" ]; then + echo "ERROR: public RunPod ports require VLLM_API_KEY so vLLM enforces bearer auth." >&2 + return 1 + fi + fi +} + +runpod_ports() { + local ports + if [ -n "${RUNPOD_PORTS:-}" ]; then + ports="$RUNPOD_PORTS" + elif [ "${RUNPOD_EXPOSE_HTTP:-0}" = "1" ]; then + ports="8000/http,22/tcp" + else + ports="22/tcp" + fi + ports="${ports//$'\r'/}" + runpod_validate_ports "$ports" + printf '%s' "$ports" +} + +runpod_describe_ports() { + local ports="$1" + if runpod_ports_public_requested "$ports"; then + echo "RunPod ports: $ports (PUBLIC; vLLM API key required)." + else + echo "RunPod ports: $ports (private; vLLM via Tailscale or SSH tunnel)." + fi +} + create_spot_pod() { detect_runpod_cli discover_or_create_volume @@ -174,6 +240,13 @@ create_spot_pod() { *) gpu_type_id="$CREATE_GPU_TYPE" ;; esac + local ports_field + if [ -n "${CREATE_RUNPOD_PORTS:-}" ]; then + ports_field="$CREATE_RUNPOD_PORTS" + else + ports_field="$(runpod_ports)" + fi + local query query=$(cat </dev/null) + POD_IP="${POD_IP//$'\r'/}" + POD_SSH_PORT="${POD_SSH_PORT//$'\r'/}" if [ "$POD_IP" = "null" ] || [ -z "$POD_IP" ]; then POD_IP="" POD_SSH_PORT="" @@ -320,6 +397,8 @@ get_pod_endpoint() { fi POD_IP="$(printf "%s\n" "$pod_line" | awk '{print $6}')" POD_SSH_PORT="$(printf "%s\n" "$pod_line" | awk '{print $7}')" + POD_IP="${POD_IP//$'\r'/}" + POD_SSH_PORT="${POD_SSH_PORT//$'\r'/}" fi if [ -z "$POD_IP" ] || [ -z "$POD_SSH_PORT" ]; then @@ -332,13 +411,34 @@ get_pod_endpoint() { # Command entry points: runpod_ # --------------------------------------------------------------------------- +runpod_spend_log_has_active_entry() { + local pod_id="$1" + [ -f "$SPEND_LOG" ] || return 1 + awk -v pod="$pod_id" ' + $2 == pod { active = ($3 != "STOP") } + END { exit(active ? 0 : 1) } + ' "$SPEND_LOG" 2>/dev/null +} + +runpod_ensure_spend_tracking_started() { + if runpod_spend_log_has_active_entry "$POD_ID"; then + return 0 + fi + echo "No active spend entry for $POD_ID; recording spend start now." + record_spend_start +} + runpod_create() { local -a gpu_candidates fallback_gpus local candidate # Branch to spot-create path if SPOT=1 if [ "$SPOT" = "1" ]; then + CREATE_RUNPOD_PORTS="$(runpod_ports)" + runpod_describe_ports "$CREATE_RUNPOD_PORTS" create_spot_pod || exit 1 + record_spend_start + autosupervise_start runpod sleep 10 get_pod_endpoint wait_for_ssh @@ -348,7 +448,6 @@ runpod_create() { else open_tunnel_bg fi - record_spend_start echo "" if direct_access_enabled; then echo "✅ Spot pod ready. vLLM serving $VLLM_MODEL on http://$(effective_tailnet_hostname):$REMOTE_PORT" @@ -362,9 +461,15 @@ runpod_create() { detect_runpod_cli discover_or_create_volume + CREATE_RUNPOD_PORTS="$(runpod_ports)" + runpod_describe_ports "$CREATE_RUNPOD_PORTS" if [ -z "$CREATE_RUNPOD_VOLUME_ID" ] && [ "$VOLUME_NAME" != "none" ]; then echo "No network volume attached — model will re-download on every boot." + if [ "${CREATE_CONTAINER_DISK_GB:-0}" -lt 80 ]; then + CREATE_CONTAINER_DISK_GB=80 + echo "Bumping container disk to ${CREATE_CONTAINER_DISK_GB}GB for ephemeral model cache." + fi fi gpu_candidates=("$CREATE_GPU_TYPE") @@ -396,6 +501,8 @@ runpod_create() { save_pod_id "$POD_ID" echo "Created pod: $POD_ID" + record_spend_start + autosupervise_start runpod sleep 10 get_pod_endpoint @@ -406,12 +513,13 @@ runpod_create() { else open_tunnel_bg fi - record_spend_start - echo "" if direct_access_enabled; then echo "✅ Pod ready. vLLM serving $VLLM_MODEL on http://$(effective_tailnet_hostname):$REMOTE_PORT" echo " Tailscale direct access is active; use '$0 tunnel' only for the legacy fallback." + if [ "${OLLAMA_SHIM_ENABLED:-0}" = "1" ]; then + echo " Ollama shim is active on http://$(effective_tailnet_hostname):${OLLAMA_SHIM_PORT:-11434}" + fi else echo "✅ Pod ready. vLLM serving $VLLM_MODEL on http://localhost:$LOCAL_PORT" fi @@ -419,6 +527,9 @@ runpod_create() { echo "Common commands:" if direct_access_enabled; then echo " curl http://$(effective_tailnet_hostname):$REMOTE_PORT/v1/models" + if [ "${OLLAMA_SHIM_ENABLED:-0}" = "1" ]; then + echo " curl http://$(effective_tailnet_hostname):${OLLAMA_SHIM_PORT:-11434}/api/tags" + fi echo " $0 tunnel # open the legacy SSH tunnel fallback" else echo " curl http://localhost:$LOCAL_PORT/v1/models" @@ -427,15 +538,13 @@ runpod_create() { echo " $0 ssh # shell on pod" echo " $0 restart-vllm # bounce vLLM without recreating pod" echo " $0 terminate # delete pod (volume survives)" - - autosupervise_start runpod } runpod_ssh() { resolve_pod_id detect_runpod_cli - if [ "$USE_RUNPODCTL_SSH" = "1" ] && [ "$RUNPOD_BIN" = "runpodctl" ] && runpodctl ssh --help 2>/dev/null | rg -q "connect"; then + if [ "$USE_RUNPODCTL_SSH" = "1" ] && [ "$RUNPOD_BIN" = "runpodctl" ] && runpodctl ssh --help 2>/dev/null | grep -q "connect"; then runpodctl ssh connect "$POD_ID" return fi @@ -478,7 +587,7 @@ runpod_status() { detect_runpod_cli echo "Pod: $POD_ID" if [ "$RUNPOD_BIN" = "runpodctl" ]; then - "$RUNPOD_BIN" pod get "$POD_ID" 2>&1 | jq '{status: .desiredStatus, uptime_s: .uptimeSeconds, gpu: .machine.gpuId, dc: .machine.dataCenterId, cost: .costPerHr, ssh_ip: .ssh.ip, ssh_port: .ssh.port}' 2>/dev/null || echo "Failed to fetch pod status." + "$RUNPOD_BIN" pod get "$POD_ID" --include-machine 2>&1 | jq '{status: .desiredStatus, uptime_s: .uptimeSeconds, gpu: .machine.gpuId, dc: .machine.dataCenterId, cost: .costPerHr, ssh_ip: .ssh.ip, ssh_port: .ssh.port}' 2>/dev/null || echo "Failed to fetch pod status." else "$RUNPOD_BIN" pod list | awk -v pod="$POD_ID" 'NR==1 || $0 ~ pod' fi @@ -513,6 +622,21 @@ runpod_status() { echo " unreachable on localhost:$LOCAL_PORT" fi fi + + if [ "${OLLAMA_SHIM_ENABLED:-0}" = "1" ]; then + echo "" + if direct_access_enabled; then + echo "Ollama shim (direct tailnet):" + if curl -fsS --max-time 3 "http://$(effective_tailnet_hostname):${OLLAMA_SHIM_PORT:-11434}/api/tags" 2>/dev/null | jq -r '.models[0].name // "ok"' 2>/dev/null | head -1; then + : + else + echo " unreachable on http://$(effective_tailnet_hostname):${OLLAMA_SHIM_PORT:-11434}" + fi + else + echo "Ollama shim:" + echo " not directly exposed without Tailscale; use SSH port-forward if needed" + fi + fi } runpod_health() { @@ -553,13 +677,17 @@ runpod_logs() { detect_runpod_cli get_pod_endpoint wait_for_ssh - local log_file="/workspace/vllm-${CREATE_POD_NAME}.log" + local log_file="/workspace/vllm-$(effective_tailnet_hostname).log" + local shim_log_file="/workspace/ollama-shim-$(effective_tailnet_hostname).log" ssh $SSH_OPTS -p "$POD_SSH_PORT" "root@$POD_IP" " if [ -f $log_file ]; then echo 'Tailing vLLM runtime log: $log_file (Ctrl+C to stop)...' tail -f $log_file + elif [ -f $shim_log_file ]; then + echo 'Tailing Ollama shim log: $shim_log_file (Ctrl+C to stop)...' + tail -f $shim_log_file else - echo 'No vLLM log at $log_file yet. vLLM may still be starting.' + echo 'No vLLM log at $log_file or shim log at $shim_log_file yet. Services may still be starting.' echo '(RunPod runs setup synchronously during create; if create finished cleanly, retry in 30s.)' exit 1 fi @@ -569,14 +697,13 @@ runpod_logs() { runpod_restart_vllm() { resolve_pod_id detect_runpod_cli + runpod_ensure_spend_tracking_started + autosupervise_start runpod get_pod_endpoint wait_for_ssh - local remote_env - remote_env="$(build_remote_env_prefix)" - echo "Restarting vLLM on pod (model=$VLLM_MODEL)..." - ssh $SSH_OPTS -p "$POD_SSH_PORT" "root@$POD_IP" \ - "pkill -9 -f 'vllm serve' 2>/dev/null; sleep 3; ${remote_env}bash /workspace/setup-vllm.sh" - echo "vLLM restarted. Tunnel (if any) should resume automatically once server is up." + echo "Re-running remote vLLM setup on pod (model=$VLLM_MODEL)..." + run_remote_setup + echo "vLLM setup complete. Tunnel (if any) should resume automatically once server is up." } runpod___supervise() { diff --git a/src/vscode-ollama-bridge.py b/src/vscode-ollama-bridge.py new file mode 100644 index 0000000..004cefb --- /dev/null +++ b/src/vscode-ollama-bridge.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Minimal Ollama-compatible bridge for vLLM. + +Implements exactly the endpoints VS Code Copilot Chat's Ollama BYOK provider +calls (see microsoft/vscode-copilot-chat ollamaProvider.ts): + + - GET /api/version required; must report >= 0.6.4 + - GET /api/tags lists models (proxied from vLLM /v1/models) + - POST /api/show per-model metadata incl. capabilities + context + - POST /v1/chat/completions transparent streaming proxy to vLLM + +Configuration via environment: + BACKEND_BASE_URL default http://127.0.0.1:8000 (vLLM OpenAI endpoint) + BACKEND_API_KEY optional bearer token for vLLM (--api-key) + REPORTED_VERSION default 0.11.0 (must be >= 0.6.4) + +Intentionally tiny: no auth, no logging of bodies, no caching beyond what +vLLM already does. The shim is reachable only over the pod's Tailscale +interface — see docs/gpu-runbook.md. +""" +from __future__ import annotations + +import os +import time +from typing import Any + +import httpx +from fastapi import FastAPI, HTTPException, Request, Response +from fastapi.responses import JSONResponse, StreamingResponse + +BACKEND = os.environ.get("BACKEND_BASE_URL", "http://127.0.0.1:8000").rstrip("/") +API_KEY = os.environ.get("BACKEND_API_KEY", "") +VERSION = os.environ.get("REPORTED_VERSION", "0.11.0") + +app = FastAPI(title="hopper-ollama-bridge", version=VERSION) +_client = httpx.AsyncClient(timeout=None) + + +def _auth_headers(extra: dict[str, str] | None = None) -> dict[str, str]: + h = dict(extra or {}) + if API_KEY: + h.setdefault("Authorization", f"Bearer {API_KEY}") + return h + + +def _infer_arch(model_id: str) -> str: + m = model_id.lower() + for k in ("qwen3", "qwen2", "qwen", "llama", "mistral", "deepseek", "gemma", "phi"): + if k in m: + return k + return "llama" + + +async def _list_models() -> list[dict[str, Any]]: + r = await _client.get(f"{BACKEND}/v1/models", headers=_auth_headers()) + r.raise_for_status() + return r.json().get("data", []) + + +@app.get("/api/version") +async def version() -> dict[str, str]: + return {"version": VERSION} + + +@app.get("/api/tags") +async def tags() -> dict[str, Any]: + models = await _list_models() + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + out = [] + for m in models: + arch = _infer_arch(m["id"]) + out.append( + { + "name": m["id"], + "model": m["id"], + "modified_at": now, + "size": 0, + "digest": "", + "details": { + "parent_model": "", + "format": "gguf", + "family": arch, + "families": [arch], + "parameter_size": "", + "quantization_level": "", + }, + } + ) + return {"models": out} + + +@app.post("/api/show") +async def show(req: Request) -> dict[str, Any]: + body = await req.json() + model_id = body.get("model") or body.get("name") + if not model_id: + raise HTTPException(400, "missing 'model'") + models = await _list_models() + match = next((m for m in models if m["id"] == model_id), None) + if match is None: + raise HTTPException(404, f"model {model_id} not found") + arch = _infer_arch(model_id) + ctx = int(match.get("max_model_len") or 32768) + basename = model_id.rsplit("/", 1)[-1] + return { + "template": "", + # "tools" is required for VS Code agent mode to keep the model in the picker. + "capabilities": ["tools"], + "details": {"family": arch}, + "model_info": { + "general.architecture": arch, + "general.basename": basename, + f"{arch}.context_length": ctx, + }, + } + + +@app.post("/v1/chat/completions") +async def chat_completions(req: Request) -> Response: + body = await req.body() + fwd_headers = { + k: v + for k, v in req.headers.items() + if k.lower() not in ("host", "content-length", "connection") + } + if API_KEY and "authorization" not in {k.lower() for k in fwd_headers}: + fwd_headers["Authorization"] = f"Bearer {API_KEY}" + + upstream_req = _client.build_request( + "POST", + f"{BACKEND}/v1/chat/completions", + content=body, + headers=fwd_headers, + ) + upstream = await _client.send(upstream_req, stream=True) + + async def relay(): + try: + async for chunk in upstream.aiter_raw(): + yield chunk + finally: + await upstream.aclose() + + return StreamingResponse( + relay(), + status_code=upstream.status_code, + media_type=upstream.headers.get("content-type", "application/json"), + ) + + +@app.get("/") +async def root() -> JSONResponse: + return JSONResponse( + {"status": "ok", "backend": BACKEND, "version": VERSION, "name": "hopper-ollama-bridge"} + )