From c1e354d5c0d3f1fc66dd6361a0001bfc08549f5b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 31 May 2026 03:24:56 +0000 Subject: [PATCH] Add official Asana MCP via mcp-remote OAuth bridge Reach the official Asana MCP server (remote, OAuth-protected at https://mcp.asana.com/v2/mcp) through the mcp-remote stdio bridge, which performs the OAuth 2.1 authorization-code (PKCE) flow and refreshes the access token automatically. - process_runner: scrape the authorization URL printed to stderr via a background reader into a shared pending_auth_urls registry, and extend the initialize-handshake timeout (MCPPROXY_AUTH_INIT_TIMEOUT, default 300s) so a first-time interactive OAuth flow can complete. The stderr tail buffer still surfaces crash causes on EOF. - frontend: add GET /api/pending-auth and a wizard poller that shows a clickable Authorize link while introspection is blocked on auth. - docker-compose: persist the token cache (mcpproxy-mcp-auth volume at /app/.mcp-auth, kept out of /app/files so tokens aren't exposed via mcpproxy__getfile), set MCP_REMOTE_CONFIG_DIR, and map the OAuth callback port 3334. Dev override bind-mounts ./.mcp-auth (gitignored). - examples/asana.yaml + README: document the provider and the authorize-once-then-silent-refresh flow. - tests: cover auth-URL extraction, stderr capture, and registry clearing. https://claude.ai/code/session_01WBwGMEMH6xsuV3gNVtdLba --- .gitignore | 3 + README.md | 62 +++++++++++++++++++ docker-compose.override.yml | 4 ++ docker-compose.yml | 14 +++++ examples/asana.yaml | 47 ++++++++++++++ frontend/app.py | 33 ++++++++++ process_runner.py | 117 +++++++++++++++++++++++++++++++---- tests/test_process_runner.py | 75 +++++++++++++++++++--- 8 files changed, 336 insertions(+), 19 deletions(-) create mode 100644 examples/asana.yaml diff --git a/.gitignore b/.gitignore index a9256c5..88c2c59 100755 --- a/.gitignore +++ b/.gitignore @@ -35,5 +35,8 @@ tools/ files/ repos/ +# OAuth token cache for mcp-remote bridges (contains live access/refresh tokens) +.mcp-auth/ + # Legacy Playwright MCP output directory (replaced by ./files in docker-compose) .playwright-mcp/ diff --git a/README.md b/README.md index 0f9774c..d32ad2f 100755 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ Each tool **provider** is a single YAML file under `tools/`. The YAML contains: via `npx`, `uvx`, `python -m`, or any installed binary - Or a `package:` + `repository:` pair to clone a git repo, run build commands, and spawn the resulting stdio MCP server — useful for servers distributed only as source +- Or a `package:` block running the [`mcp-remote`](https://www.npmjs.com/package/mcp-remote) + bridge to reach a **remote, OAuth-protected** server (e.g. the official Asana MCP) — the + bridge walks you through the OAuth flow and refreshes the token automatically `server.py` loads every YAML at startup, installs declared `requirements` (pip packages), runs `setup_commands`, then registers each tool automatically — no Python files to @@ -777,6 +780,65 @@ call to it. The process is reused across calls (started lazily on the first tool --- +### Part 3.25 — a remote, OAuth-protected server (e.g. the official Asana MCP) + +Some MCP servers aren't stdio packages at all — they're **remote, OAuth-protected HTTP +endpoints**. The official Asana server is one: it lives at `https://mcp.asana.com/v2/mcp` +(Streamable HTTP) and is reached through an OAuth 2.1 authorization-code (PKCE) flow — there's +no static API key. mcpproxy speaks stdio to its upstreams, so these are bridged with the +community [`mcp-remote`](https://www.npmjs.com/package/mcp-remote) adapter, which is itself just +a `package:` command: + +```yaml +# examples/asana.yaml — copy into your tools/ config dir (or use the wizard) +package: + command: npx -y mcp-remote https://mcp.asana.com/v2/mcp + +tools: + - name: get_me # advertised as asana__get_me; the rest are auto-introspected + description: Return the Asana user that the authorized token belongs to. + input_schema: { type: object, properties: {} } +``` + +**`mcp-remote` performs the OAuth walkthrough and refreshes the token for you** — mcpproxy +itself stays a thin stdio proxy: + +- **First run** (or after the refresh token expires / is revoked): `mcp-remote` prints an + authorization URL and blocks the MCP handshake until you authorize. When you introspect the + command in the **+ New Provider → Package** wizard, mcpproxy scrapes that URL from stderr and + shows a clickable **🔐 Authorize** link (it's also logged as + `authorization required … visit:`). Open it, approve access in Asana, and the localhost + callback (`:3334`) completes the flow — introspection then continues automatically and the + tool list populates. +- **Afterwards** the OAuth token cache is written under `MCP_REMOTE_CONFIG_DIR` and **the access + token is refreshed silently** on every expiry. You don't authorize again until the refresh + token itself lapses. + +#### Persisting the token cache (so you authorize once) + +`docker-compose.yml` wires this up: it sets `MCP_REMOTE_CONFIG_DIR=/app/.mcp-auth`, mounts the +`mcpproxy-mcp-auth` volume there (kept **out** of `/app/files` so tokens are never exposed via +`mcpproxy__getfile`), and maps the OAuth callback port `3334`. Keep that volume and the refresh +token survives restarts. In dev, `docker-compose.override.yml` bind-mounts `./.mcp-auth` +(gitignored). + +#### Headless / one-time bootstrap + +The OAuth redirect targets `localhost:3334`. Either authorize via the wizard link with port +`3334` mapped (the default), **or** run the flow once on the host to pre-populate the cache, +then start the proxy with the same dir mounted: + +```bash +MCP_REMOTE_CONFIG_DIR=./.mcp-auth npx -y mcp-remote https://mcp.asana.com/v2/mcp +# authorize in the browser, then `docker compose up` — tools work with no further prompts +``` + +> **Pin the bridge** for reproducible builds once you've settled on a version, e.g. +> `npx -y mcp-remote@ …`. Add `--debug` to write a detailed auth/refresh log under +> `MCP_REMOTE_CONFIG_DIR`. + +--- + ### Part 3.5 — a repository provider (clone + build + introspect) For MCP servers that are published only as source code (no `npx` / `uvx` / pip distribution), diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 672e9d5..a1398f0 100755 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -20,4 +20,8 @@ services: - ./tools:/app/tools - ./files:/app/files - ./repos:/app/repos + # OAuth token cache for mcp-remote bridges (Asana, …). Bind-mounted so the + # refresh token survives `docker compose down` and you authorize only once. + # Gitignored — never commit it (it contains live OAuth tokens). + - ./.mcp-auth:/app/.mcp-auth - ./.env:/app/.env diff --git a/docker-compose.yml b/docker-compose.yml index eeece99..f69a1ff 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,12 @@ # Playwright browser binaries, …) # mcpproxy-npm /root/.npm — npm/npx package cache # mcpproxy-uv-tools /root/.local/share/uv — uvx per-tool venvs +# mcpproxy-mcp-auth /app/.mcp-auth — OAuth token cache for mcp-remote +# bridges (e.g. the Asana provider). +# PERSIST THIS: it holds the refresh +# token, so you only authorize once. +# Kept out of /app/files so tokens are +# never exposed via mcpproxy__getfile. # # Every cache/artefact volume is optional in spirit — remove the entry and the # container falls back to ephemeral storage on its writable layer (re-clones, @@ -32,6 +38,9 @@ services: ports: - "${MCP_HOST_PORT:-8888}:8888" - "${UI_HOST_PORT:-8889}:8889" + # OAuth callback for mcp-remote bridges (e.g. Asana). Only needed while + # authorizing in a browser the first time; harmless to leave mapped. + - "${MCP_OAUTH_CALLBACK_PORT:-3334}:3334" env_file: - ./.env environment: @@ -39,6 +48,9 @@ services: MCP_ENV_FILE: "/app/.env" MCPPROXY_FILES_DIR: "/app/files" MCPPROXY_REPOS_DIR: "/app/repos" + # Where mcp-remote caches OAuth tokens (access + refresh). Persisted via + # the mcpproxy-mcp-auth volume so you authorize once and refresh silently. + MCP_REMOTE_CONFIG_DIR: "/app/.mcp-auth" volumes: - mcpproxy-tools:/app/tools - mcpproxy-files:/app/files @@ -46,6 +58,7 @@ services: - mcpproxy-cache:/root/.cache - mcpproxy-npm:/root/.npm - mcpproxy-uv-tools:/root/.local/share/uv + - mcpproxy-mcp-auth:/app/.mcp-auth - ./.env:/app/.env volumes: @@ -55,3 +68,4 @@ volumes: mcpproxy-cache: mcpproxy-npm: mcpproxy-uv-tools: + mcpproxy-mcp-auth: diff --git a/examples/asana.yaml b/examples/asana.yaml new file mode 100644 index 0000000..161d2ee --- /dev/null +++ b/examples/asana.yaml @@ -0,0 +1,47 @@ +documentation: | + # Asana (official remote MCP, OAuth) + + Bridges the official Asana MCP server — a **remote, OAuth-protected** server at + `https://mcp.asana.com/v2/mcp` (Streamable HTTP) — into mcpproxy via the + community [`mcp-remote`](https://www.npmjs.com/package/mcp-remote) stdio bridge. + There is no static API key: access is granted through an OAuth 2.1 + authorization-code (PKCE) flow. + + ## How auth works + - **First run (or after the refresh token expires / is revoked):** `mcp-remote` + prints an authorization URL to stderr and blocks the MCP handshake until you + authorize. The provider wizard surfaces this as a clickable **Authorize** + link (it also appears in the server log: `authorization required … visit:`). + Open it, approve access in Asana, and the localhost callback completes the + flow automatically. + - **Afterwards (steady state):** the OAuth token cache is written under + `MCP_REMOTE_CONFIG_DIR` (mounted as a persistent volume — see + docker-compose.yml). `mcp-remote` then **refreshes the access token silently** + on its own. No further interaction is needed across restarts until the + refresh token itself expires. + + ## One-time bootstrap (headless / Docker) + The OAuth callback listens on `localhost:3334`. Either: + 1. Map port `3334` out of the container (done in docker-compose.yml) and click + the Authorize link from the wizard, **or** + 2. Run the flow once with host networking to populate the token cache, e.g. + `MCP_REMOTE_CONFIG_DIR=./files/.mcp-auth npx -y mcp-remote https://mcp.asana.com/v2/mcp`, + then start the proxy with that same volume mounted. + + ## Tools + The full Asana tool set (tasks, projects, search, comments, workspaces, …) is + auto-populated when the wizard introspects the command after you authorize. + The single declaration below (`get_me`) is a safe, zero-argument starter you + can call to confirm the connection. + +package: + command: npx -y mcp-remote https://mcp.asana.com/v2/mcp + +tools: + # Auto-populated by the wizard's introspection step once authorized. + # `get_me` returns the authenticated user — handy for verifying the connection. + - name: get_me # advertised to the LLM as asana__get_me + description: Return the Asana user that the authorized token belongs to. + input_schema: + type: object + properties: {} diff --git a/frontend/app.py b/frontend/app.py index a5a68af..a12450f 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -560,6 +560,24 @@ async def introspect_package(request: Request) -> dict: traceback.print_exc() return {"ok": False, "error": str(exc), "tools": [], "package_manager": pm} + @app.get("/api/pending-auth") + async def pending_auth(command: str = "") -> dict: + """Return the OAuth authorization URL a spawn is currently waiting on. + + Remote OAuth-protected MCP servers reached via the `mcp-remote` bridge + print an authorization URL to stderr and block the MCP handshake until + the user authorizes in a browser. `process_runner` scrapes that URL; + the wizard polls this endpoint (while an introspect call is blocked) to + show a clickable "Authorize" link. Once a valid token cache exists the + bridge refreshes silently and this stays empty. + + With no `command`, returns every pending URL keyed by spawn command. + """ + from process_runner import pending_auth_urls + if command: + return {"ok": True, "auth_url": pending_auth_urls.get(command.strip())} + return {"ok": True, "pending": dict(pending_auth_urls)} + # ── Repository clone-and-build ─────────────────────────────────────────── @app.post("/api/clone-and-build") @@ -2297,6 +2315,19 @@ async def index(): const requirements = _wzGetListValues('wz-pkg-reqs-container'); const setup_commands = _wzGetListValues('wz-pkg-cmds-container'); el.innerHTML = 'Introspecting — this may take a moment on first use…'; + // Remote OAuth servers (bridged via mcp-remote) block introspection until the + // user authorizes in a browser. Poll for the authorization URL meanwhile and + // surface it as a clickable link so the wizard can walk the user through it. + const authPoll = setInterval(async () => { + try { + const a = await api('GET', '/api/pending-auth?command=' + encodeURIComponent(cmd)); + if (a && a.auth_url) { + el.innerHTML = `
🔐 Authorization required — ` + + `click here to authorize, ` + + `then complete the browser flow. Introspection continues automatically once you finish.
`; + } + } catch {} + }, 1500); try { const r = await api('POST', '/api/introspect', {command: cmd, requirements, setup_commands}); if (!r.ok) throw new Error(r.error || 'Introspection failed'); @@ -2309,6 +2340,8 @@ async def index(): } catch(e) { wzIntrospectedTools = []; el.innerHTML = `
⚠ Introspection failed (${esc(e.message)}). Continuing without auto-detected tools — add them manually in the editor.
`; + } finally { + clearInterval(authPoll); } } diff --git a/process_runner.py b/process_runner.py index 6d51098..ff3ad3c 100644 --- a/process_runner.py +++ b/process_runner.py @@ -18,10 +18,53 @@ import asyncio import json import os +import re import shlex import traceback from typing import Any +# --------------------------------------------------------------------------- +# OAuth-bridge (mcp-remote) support +# --------------------------------------------------------------------------- +# +# Remote, OAuth-protected MCP servers (e.g. the official Asana server at +# https://mcp.asana.com/v2/mcp) are reached through the community `mcp-remote` +# bridge, spawned exactly like any other stdio package provider. On first run — +# or whenever the cached refresh token has expired or been revoked — mcp-remote +# prints an authorization URL to *stderr* and blocks the MCP `initialize` +# handshake until the user completes the browser OAuth flow. +# +# We scrape that URL out of stderr so the UI can surface a clickable +# "Authorize" link, and we give the handshake a longer, configurable timeout +# so a human has time to finish authorizing. Once a valid token cache exists +# mcp-remote refreshes silently and none of this is exercised. + +# How long (seconds) to wait for the `initialize` response. Generous by +# default so a first-time interactive OAuth flow can complete; override with +# MCPPROXY_AUTH_INIT_TIMEOUT. +AUTH_INIT_TIMEOUT = float(os.environ.get("MCPPROXY_AUTH_INIT_TIMEOUT", "300")) + +# Latest pending authorization URL per spawn command, populated from stderr. +# The UI (same process — the frontend runs as a daemon thread inside the MCP +# server) polls this so it can show the link while a spawn is blocked on auth. +pending_auth_urls: dict[str, str] = {} + +_URL_RE = re.compile(r"https?://[^\s'\"<>]+") +# Lines that hint mcp-remote (or a similar bridge) is asking the user to +# authorize. Matched case-insensitively against each stderr line. +_AUTH_HINT_RE = re.compile( + r"authoriz|oauth|visit (?:this|the following)|open (?:this|the following)", + re.IGNORECASE, +) + + +def _extract_auth_url(line: str) -> str | None: + """Return an authorization URL from *line* if it looks like an auth prompt.""" + if not _AUTH_HINT_RE.search(line): + return None + m = _URL_RE.search(line) + return m.group(0) if m else None + class ProcessSession: """A long-lived connection to a single stdio MCP server process.""" @@ -39,6 +82,13 @@ def __init__( self._proc: asyncio.subprocess.Process | None = None self._lock = asyncio.Lock() self._next_id = 0 + # stderr is consumed by a background reader (see _consume_stderr) so we + # can scrape OAuth authorization URLs in real time; the reader keeps a + # bounded tail buffer that _drain_stderr_tail reports on failure. + self._stderr_tail: list[str] = [] + self._stderr_task: asyncio.Task | None = None + # Authorization URL most recently printed by the subprocess, if any. + self.pending_auth_url: str | None = None # ── internal ────────────────────────────────────────────────────────────── @@ -64,17 +114,49 @@ async def _recv(self, timeout: float = 30.0) -> dict[str, Any]: raise EOFError(f"MCP process closed stdout{suffix}") return json.loads(line) - async def _drain_stderr_tail(self, max_bytes: int = 4096) -> str: - """Return up to ``max_bytes`` of buffered stderr from the subprocess.""" - if not self._proc or not self._proc.stderr: - return "" + async def _consume_stderr(self) -> None: + """Continuously read subprocess stderr. + + Keeps a bounded tail (for crash diagnostics) and scrapes any OAuth + authorization URL so the UI can surface a clickable "Authorize" link + while the spawn is blocked on the user completing the browser flow. + """ + assert self._proc and self._proc.stderr try: - data = await asyncio.wait_for( - self._proc.stderr.read(max_bytes), timeout=2.0 - ) - except (asyncio.TimeoutError, Exception): - return "" - return data.decode(errors="replace").strip() + while True: + raw = await self._proc.stderr.readline() + if not raw: + break + line = raw.decode(errors="replace").rstrip("\n") + self._stderr_tail.append(line) + if len(self._stderr_tail) > 50: + del self._stderr_tail[:-50] + url = _extract_auth_url(line) + if url: + self.pending_auth_url = url + pending_auth_urls[self.command] = url + print( + f"[mcpproxy] authorization required for " + f"'{self.command}' — visit: {url}", + flush=True, + ) + except Exception: + traceback.print_exc() + + def _start_stderr_reader(self) -> None: + if self._stderr_task is None or self._stderr_task.done(): + self._stderr_task = asyncio.ensure_future(self._consume_stderr()) + + def _clear_pending_auth(self) -> None: + self.pending_auth_url = None + pending_auth_urls.pop(self.command, None) + + async def _drain_stderr_tail(self, max_bytes: int = 4096) -> str: + """Return the buffered tail of subprocess stderr (best-effort).""" + # Give the background reader a moment to flush any final lines. + await asyncio.sleep(0.1) + text = "\n".join(self._stderr_tail).strip() + return text[-max_bytes:] async def _start(self) -> None: env = self._build_env() @@ -86,6 +168,10 @@ async def _start(self) -> None: cwd=self.cwd, env=env, ) + # Begin scraping stderr immediately so an OAuth authorization URL is + # captured even though the initialize response below blocks until the + # user finishes authorizing. + self._start_stderr_reader() # initialize handshake rid = self._new_id() await self._send({ @@ -96,7 +182,12 @@ async def _start(self) -> None: "clientInfo": {"name": "mcpproxy", "version": "1.0"}, }, }) - await self._recv(timeout=60) # initialize response + # A generous timeout: an OAuth bridge (mcp-remote) holds the handshake + # open until the interactive browser authorization completes. With a + # valid cached token this returns immediately. + await self._recv(timeout=AUTH_INIT_TIMEOUT) # initialize response + # Handshake completed → any pending authorization is resolved. + self._clear_pending_auth() # notifications/initialized (no response expected) await self._send({"jsonrpc": "2.0", "method": "notifications/initialized"}) @@ -177,6 +268,10 @@ async def call_tool(self, tool_name: str, arguments: dict[str, Any]) -> Any: return {"ok": True, "result": parts[0] if len(parts) == 1 else parts} async def close(self) -> None: + self._clear_pending_auth() + if self._stderr_task is not None: + self._stderr_task.cancel() + self._stderr_task = None if self._proc: try: self._proc.stdin.close() # type: ignore[union-attr] diff --git a/tests/test_process_runner.py b/tests/test_process_runner.py index efb0859..4bb509d 100644 --- a/tests/test_process_runner.py +++ b/tests/test_process_runner.py @@ -81,26 +81,85 @@ def test_no_env_keys_skips_file_read(self, tmp_path, monkeypatch): class TestIntrospectStderrCapture: """When the subprocess crashes during the handshake, the error message - should include stderr so the user can see the cause.""" + should include stderr so the user can see the cause. stderr is consumed by + a background reader into ``_stderr_tail``; ``_drain_stderr_tail`` reports it.""" @pytest.mark.asyncio async def test_eof_error_includes_stderr_tail(self): class FakeStdout: async def readline(self): return b"" - class FakeStderr: - def __init__(self, data): - self._data = data - async def read(self, n): - d, self._data = self._data[:n], self._data[n:] - return d session = process_runner.ProcessSession("does-not-matter") + # Simulate what the background reader would have captured. + session._stderr_tail = ["Environment validation failed: KEY: Required"] class _Proc: stdout = FakeStdout() - stderr = FakeStderr(b"Environment validation failed: KEY: Required\n") + stderr = None session._proc = _Proc() with pytest.raises(EOFError) as exc_info: await session._recv(timeout=1.0) assert "Environment validation failed" in str(exc_info.value) + + +class TestAuthUrlExtraction: + """mcp-remote prints an OAuth authorization URL to stderr; we scrape it so + the UI can offer a clickable Authorize link.""" + + def test_extracts_url_from_authorize_prompt(self): + line = "Please authorize this client by visiting: https://app.asana.com/-/oauth_authorize?client_id=123" + url = process_runner._extract_auth_url(line) + assert url == "https://app.asana.com/-/oauth_authorize?client_id=123" + + def test_extracts_url_from_open_this_url_prompt(self): + line = "If your browser does not open, open this URL: https://example.com/oauth?x=1" + assert process_runner._extract_auth_url(line) == "https://example.com/oauth?x=1" + + def test_ignores_unrelated_lines_with_urls(self): + # A URL with no authorization hint must not be treated as an auth prompt. + assert process_runner._extract_auth_url("Fetching https://mcp.asana.com/v2/mcp") is None + + def test_ignores_hint_without_url(self): + assert process_runner._extract_auth_url("authorization pending…") is None + + +class TestConsumeStderr: + def teardown_method(self): + process_runner.pending_auth_urls.clear() + + @pytest.mark.asyncio + async def test_consume_stderr_captures_auth_url_and_tail(self): + lines = [ + b"booting mcp-remote\n", + b"Please authorize by visiting: https://app.asana.com/-/oauth_authorize?c=1\n", + b"", # EOF + ] + + class FakeStderr: + async def readline(self): + return lines.pop(0) if lines else b"" + + cmd = "npx -y mcp-remote https://mcp.asana.com/v2/mcp" + session = process_runner.ProcessSession(cmd) + class _Proc: + stderr = FakeStderr() + session._proc = _Proc() + + await session._consume_stderr() + + assert session.pending_auth_url == "https://app.asana.com/-/oauth_authorize?c=1" + # Exposed in the shared registry the UI polls, keyed by command. + assert process_runner.pending_auth_urls[cmd] == "https://app.asana.com/-/oauth_authorize?c=1" + # Tail retains the captured lines for crash diagnostics. + assert "booting mcp-remote" in "\n".join(session._stderr_tail) + + @pytest.mark.asyncio + async def test_clear_pending_auth_removes_registry_entry(self): + cmd = "some-cmd" + session = process_runner.ProcessSession(cmd) + session.pending_auth_url = "https://x/oauth" + process_runner.pending_auth_urls[cmd] = "https://x/oauth" + session._clear_pending_auth() + assert session.pending_auth_url is None + assert cmd not in process_runner.pending_auth_urls