diff --git a/Dockerfile b/Dockerfile index 9d098b8..48010c8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir uv ENV PATH="/root/.local/bin:$PATH" -COPY server.py config.py process_runner.py builtin_tools.py tool_registry.py rest_provider.py oauth_bootstrap.py ./ +COPY server.py config.py process_runner.py builtin_tools.py tool_registry.py rest_provider.py oauth_bootstrap.py catalog.py ./ COPY frontend/ ./frontend/ COPY handlers/ ./handlers/ diff --git a/README.md b/README.md index 9f4913d..cc044a7 100755 --- a/README.md +++ b/README.md @@ -107,18 +107,45 @@ Open **`http://localhost:8889`** in your browser after starting the server. ### New Provider wizard -Click **+ New Provider** and choose a provider type: +Click **+ New Provider** and choose a provider type. Each mode card carries a short +*"Best for…"* hint with concrete examples so it's clear which one fits your situation: -| Type | Description | -|---|---| -| **Python code** | Write `async def` functions; the UI lists the ones it finds as you type. Each becomes a tool entry. | -| **Package** | Enter any command that launches a stdio MCP server (`npx`, `uvx`, `python -m`, or an installed binary). When you click **Next**, mcpproxy auto-introspects the command and pre-populates the tool list; if introspection fails you can still proceed and add tools by hand. | -| **Repository** | Provide a git URL and a list of build commands. mcpproxy clones the repo, runs the build commands, then introspects the resulting stdio MCP server. The URL and build commands are persisted in YAML so the repo can be re-cloned and re-built automatically on every container restart. | -| **REST / OAuth API** | Point at a REST API: a base URL plus an OpenAPI spec (imported into tools automatically) or hand-entered endpoints, with optional OAuth. Each endpoint becomes an MCP tool. See [REST / OAuth providers](#rest--oauth-providers). | +| Type | Best for | Description | +|---|---|---| +| **Remote MCP Server** | Hosted SaaS tools that already speak MCP (Asana, Linear, Notion, GitHub) where you just have a URL | Paste a remote, OAuth-protected MCP server URL; mcpproxy bridges it with `npx -y mcp-remote `, introspects its tools, and handles auth automatically. | +| **Package** | Published MCP servers you install and run locally (Playwright, filesystem, Slack) | Enter any command that launches a stdio MCP server (`npx`, `uvx`, `python -m`, or an installed binary). When you click **Next**, mcpproxy auto-introspects the command and pre-populates the tool list; if introspection fails you can still proceed and add tools by hand. | +| **Repository** | MCP servers distributed as source you build yourself | Provide a git URL and a list of build commands. mcpproxy clones the repo, runs the build commands, then introspects the resulting stdio MCP server. The URL and build commands are persisted in YAML so the repo can be re-cloned and re-built automatically on every container restart. | +| **REST / OAuth API** | Any plain web API with no prebuilt MCP server (Stripe, OpenWeather, internal services) | Point at a REST API: a base URL plus an OpenAPI spec (imported into tools automatically) or hand-entered endpoints, with optional OAuth. Each endpoint becomes an MCP tool. See [REST / OAuth providers](#rest--oauth-providers). | +| **Python code** | Quick custom logic, glue, or calculations you write inline | Write `async def` functions; the UI lists the ones it finds as you type. Each becomes a tool entry. | After the provider step, the wizard shows a **Secrets** step: any `secrets.env` entries in the provider are listed, and you can fill in their values to save them directly to `.env`. +### Browse providers catalog + +The **πŸ—‚ Browse** button (next to **+ New Provider**) opens a searchable catalog of known +MCP servers and REST/OpenAPI APIs. Pick one and click **Configure β†’** β€” it opens the New +Provider wizard with the right mode selected and the URL or OpenAPI spec pre-filled, then +the usual introspection flow runs as if you'd typed it by hand. + +The catalog is **hybrid**: + +- A **curated list** is bundled in the repo at `frontend/catalog.json` β€” the offline-safe + default. Add or edit entries there (each is either a `mcp_remote` entry with a `url`, or + a `rest_openapi` entry with an `openapi_url`). +- Ticking **Probe live registries** also queries external sources β€” the official + [MCP registry](https://registry.modelcontextprotocol.io), [Smithery](https://smithery.ai) + (needs `SMITHERY_API_KEY`), and [APIs.guru](https://apis.guru) for OpenAPI specs. Sources + are fetched concurrently with per-source error isolation and a short cache, so one slow or + unavailable registry never blocks the others or the curated list. + +`/api/catalog` only ever contacts that fixed set of registry hosts (it takes no caller-supplied +URL), and the server never fetches a catalog entry's own URL β€” that only happens through the +existing, already-guarded wizard introspection. Knobs: `MCPPROXY_CATALOG_LIVE` (set `0` to +disable live probing entirely), `MCPPROXY_CATALOG_TTL` (cache seconds, default `900`), +`MCPPROXY_CATALOG_TIMEOUT` (per-request seconds, default `8`), `MCPPROXY_CATALOG_MAX_PER_SOURCE` +(entry cap per live source, default `150`). + ### Secrets manager The **πŸ”‘ Secrets** button (also available in the wizard's final step) reads all `secrets.env` diff --git a/catalog.py b/catalog.py new file mode 100644 index 0000000..af4cb5a --- /dev/null +++ b/catalog.py @@ -0,0 +1,284 @@ +"""Provider catalog β€” curated + optional live registry aggregation. + +Powers the web UI's "Browse providers" modal. A *catalog entry* describes a +known provider the user can configure with one click; the UI deep-links the +entry into the existing "New Provider" wizard with its fields pre-filled. + +Two entry kinds, discriminated by ``kind``: + + * ``mcp_remote`` β€” a remote MCP server reachable at ``url`` (bridged with + ``npx -y mcp-remote ``). + * ``rest_openapi`` β€” a REST API described by an OpenAPI/Swagger spec at + ``openapi_url`` (optionally with a ``base_url`` and an + ``auth_hint``). + +Data sources are hybrid: + + * a curated JSON file bundled in the repo (``frontend/catalog.json``) β€” the + offline-safe default; and + * optional live probing of external registries (MCP registry, Smithery, + APIs.guru), enabled per-request and behind the ``MCPPROXY_CATALOG_LIVE`` + gate. Live sources are fetched concurrently with per-source error + isolation and a short-lived cache, so one slow/erroring registry never + blocks the others or the curated list. + +This module is framework-free (no FastAPI imports) so it stays unit-testable. +""" +from __future__ import annotations + +import asyncio +import json +import os +import time +from pathlib import Path +from typing import Any, Awaitable, Callable + +import httpx + +# Curated catalog ships next to the UI app so the existing ``COPY frontend/`` +# Docker step picks it up. +CURATED_PATH = Path(__file__).parent / "frontend" / "catalog.json" + +# Master switch for live registry probing. Set MCPPROXY_CATALOG_LIVE=0 to +# disable all outbound probes in locked-down deployments (mirrors the +# MCPPROXY_WEB_TERMINAL gate). Default on. +CATALOG_LIVE = os.environ.get("MCPPROXY_CATALOG_LIVE", "1") not in ("0", "false", "False", "") + +# Per-source cache lifetime and outbound timeout (seconds). +CATALOG_TTL = float(os.environ.get("MCPPROXY_CATALOG_TTL", "900")) +CATALOG_TIMEOUT = float(os.environ.get("MCPPROXY_CATALOG_TIMEOUT", "8")) + +# Upper bound on how many entries a single live source may contribute, so a +# huge registry (APIs.guru lists thousands) can't flood the UI. +CATALOG_MAX_PER_SOURCE = int(os.environ.get("MCPPROXY_CATALOG_MAX_PER_SOURCE", "150")) + +_USER_AGENT = "mcpproxy-catalog/1.0 (+https://github.com/BillJr99/mcpproxy)" + +# source -> (fetched_at, entries) +_cache: dict[str, tuple[float, list[dict[str, Any]]]] = {} + + +# --------------------------------------------------------------------------- +# Normalization helpers +# --------------------------------------------------------------------------- + +def _slugify(value: str) -> str: + out = "".join(c if c.isalnum() else "-" for c in (value or "").lower()) + while "--" in out: + out = out.replace("--", "-") + return out.strip("-") or "provider" + + +def _http_url(value: Any) -> str | None: + """Return ``value`` only if it is a plain http(s) URL, else ``None``.""" + if isinstance(value, str) and value.startswith(("http://", "https://")): + return value + return None + + +def _normalize_entry(raw: dict[str, Any], source: str) -> dict[str, Any] | None: + """Coerce a raw dict into a valid catalog entry, or ``None`` if unusable. + + Enforces that URLs are http(s) only and that the entry carries the field + its ``kind`` requires. Untrusted registry text (name/description) is kept + as-is here; the browser ``esc()``s it at render time. + """ + kind = raw.get("kind") + name = (raw.get("name") or "").strip() + if not name or kind not in ("mcp_remote", "rest_openapi"): + return None + + entry: dict[str, Any] = { + "id": raw.get("id") or _slugify(name), + "kind": kind, + "name": name, + "description": (raw.get("description") or "").strip(), + "categories": [c for c in (raw.get("categories") or []) if isinstance(c, str)], + "homepage": _http_url(raw.get("homepage")), + "source": source, + } + + if kind == "mcp_remote": + url = _http_url(raw.get("url")) + if not url: + return None + entry["url"] = url + else: # rest_openapi + openapi_url = _http_url(raw.get("openapi_url")) + if not openapi_url: + return None + entry["openapi_url"] = openapi_url + entry["base_url"] = _http_url(raw.get("base_url")) + if raw.get("auth_hint"): + entry["auth_hint"] = str(raw["auth_hint"]) + + return entry + + +def _dedupe(entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Drop duplicate entries by (kind, url|openapi_url); first seen wins. + + ``build_catalog`` always prepends curated entries, so curated wins ties. + """ + seen: set[tuple[str, str]] = set() + out: list[dict[str, Any]] = [] + for e in entries: + key = (e["kind"], e.get("url") or e.get("openapi_url") or e["id"]) + if key in seen: + continue + seen.add(key) + out.append(e) + return out + + +# --------------------------------------------------------------------------- +# Curated source +# --------------------------------------------------------------------------- + +def load_curated() -> list[dict[str, Any]]: + """Load and normalize the bundled curated catalog. Never raises.""" + try: + data = json.loads(CURATED_PATH.read_text(encoding="utf-8")) + except Exception: + return [] + entries = [] + for raw in (data.get("entries") or []): + norm = _normalize_entry(raw, raw.get("source") or "curated") + if norm: + entries.append(norm) + return entries + + +# --------------------------------------------------------------------------- +# Live registry adapters +# --------------------------------------------------------------------------- + +async def _fetch_mcp_registry(client: httpx.AsyncClient) -> list[dict[str, Any]]: + """Official MCP registry β€” servers exposing a remote endpoint.""" + resp = await client.get("https://registry.modelcontextprotocol.io/v0/servers") + resp.raise_for_status() + out: list[dict[str, Any]] = [] + for srv in (resp.json().get("servers") or []): + remotes = srv.get("remotes") or [] + url = next((_http_url(r.get("url")) for r in remotes if _http_url(r.get("url"))), None) + if not url: + continue # only servers we can bridge by URL belong in the catalog + out.append({ + "kind": "mcp_remote", + "name": srv.get("title") or srv.get("name") or url, + "description": srv.get("description") or "", + "homepage": (srv.get("repository") or {}).get("url") or srv.get("websiteUrl"), + "url": url, + }) + return out + + +async def _fetch_smithery(client: httpx.AsyncClient) -> list[dict[str, Any]]: + """Smithery registry β€” requires a SMITHERY_API_KEY to query.""" + api_key = os.environ.get("SMITHERY_API_KEY") + if not api_key: + raise RuntimeError("SMITHERY_API_KEY not set") + resp = await client.get( + "https://registry.smithery.ai/servers", + params={"pageSize": CATALOG_MAX_PER_SOURCE}, + headers={"Authorization": f"Bearer {api_key}"}, + ) + resp.raise_for_status() + out: list[dict[str, Any]] = [] + for srv in (resp.json().get("servers") or []): + url = _http_url(srv.get("deploymentUrl") or srv.get("url")) + if not url: + continue + out.append({ + "kind": "mcp_remote", + "name": srv.get("displayName") or srv.get("qualifiedName") or url, + "description": srv.get("description") or "", + "homepage": srv.get("homepage"), + "url": url, + }) + return out + + +async def _fetch_apis_guru(client: httpx.AsyncClient) -> list[dict[str, Any]]: + """APIs.guru directory β€” preferred OpenAPI/Swagger spec per API.""" + resp = await client.get("https://api.apis.guru/v2/list.json") + resp.raise_for_status() + out: list[dict[str, Any]] = [] + for api in resp.json().values(): + versions = api.get("versions") or {} + ver = versions.get(api.get("preferred")) or next(iter(versions.values()), None) + if not ver: + continue + spec_url = _http_url(ver.get("swaggerUrl") or ver.get("openapiUrl")) + if not spec_url: + continue + info = ver.get("info") or {} + out.append({ + "kind": "rest_openapi", + "name": info.get("title") or spec_url, + "description": (info.get("description") or "").strip()[:300], + "categories": info.get("x-apisguru-categories") or [], + "homepage": info.get("contact", {}).get("url") if isinstance(info.get("contact"), dict) else None, + "openapi_url": spec_url, + }) + return out + + +SOURCES: dict[str, Callable[[httpx.AsyncClient], Awaitable[list[dict[str, Any]]]]] = { + "mcp_registry": _fetch_mcp_registry, + "smithery": _fetch_smithery, + "apis_guru": _fetch_apis_guru, +} + + +# --------------------------------------------------------------------------- +# Cache + orchestration +# --------------------------------------------------------------------------- + +async def _cached_fetch(source: str, client: httpx.AsyncClient) -> list[dict[str, Any]]: + now = time.time() + hit = _cache.get(source) + if hit and now - hit[0] < CATALOG_TTL: + return hit[1] + raw_entries = await SOURCES[source](client) + entries = [] + for raw in raw_entries[:CATALOG_MAX_PER_SOURCE]: + norm = _normalize_entry(raw, source) + if norm: + entries.append(norm) + _cache[source] = (now, entries) + return entries + + +async def build_catalog( + live: bool = False, sources: list[str] | None = None +) -> dict[str, Any]: + """Merge curated entries with (optionally) live registry entries. + + Always returns the curated list even if every live source errors. Live + probing is skipped entirely unless ``live`` is true *and* the + ``MCPPROXY_CATALOG_LIVE`` gate is on. Failing sources are reported in the + ``errors`` map rather than raising. + """ + entries = list(load_curated()) # curated first so it wins de-dupe ties + errors: dict[str, str] = {} + live = bool(live) and CATALOG_LIVE + + if live: + wanted = [s for s in (sources or list(SOURCES)) if s in SOURCES] + async with httpx.AsyncClient( + timeout=CATALOG_TIMEOUT, + follow_redirects=True, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) as client: + results = await asyncio.gather( + *(_cached_fetch(s, client) for s in wanted), + return_exceptions=True, + ) + for source, res in zip(wanted, results): + if isinstance(res, Exception): + errors[source] = str(res) or res.__class__.__name__ + else: + entries.extend(res) + + return {"entries": _dedupe(entries), "errors": errors, "live": live} diff --git a/frontend/app.py b/frontend/app.py index bfeb4bb..0820319 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -862,6 +862,20 @@ async def introspect_openapi_endpoint(request: Request) -> dict: traceback.print_exc() return {"ok": False, "error": str(exc), "endpoints": [], "tools": []} + @app.get("/api/catalog") + async def get_catalog(live: bool = False, source: str | None = None) -> dict: + """Browseable provider catalog β€” curated entries plus optional live registries. + + ``?live=true`` enables probing external registries (MCP registry, + Smithery, APIs.guru); ``?source=mcp_registry,apis_guru`` narrows which + ones. The endpoint takes no caller-supplied URL β€” it only ever hits a + fixed allowlist of registry hosts β€” and always returns the curated list + even if every live source errors (failures land in ``errors``). + """ + import catalog + sources = [s.strip() for s in source.split(",") if s.strip()] if source else None + return await catalog.build_catalog(live=live, sources=sources) + @app.post("/api/rest-authorize") async def rest_authorize(request: Request) -> dict: """Begin an authorization_code flow for a saved REST provider. @@ -1517,6 +1531,7 @@ async def index(): .empty-state{color:var(--muted);text-align:center;padding:30px;font-size:.9em} .wizard-choice{cursor:pointer;transition:border-color .15s,background .15s;border:1px solid var(--border)!important} .wizard-choice:hover,.wizard-choice.selected{border-color:var(--accent)!important;background:#252535!important} +.wizard-choice .best-for{display:block;margin-top:8px;font-size:.78em;font-style:italic;color:var(--accent);opacity:.85} .wizard-step{display:none}.wizard-step.active{display:block} .secret-set{border-left:3px solid var(--green)!important} .secret-unset{border-left:3px solid var(--yellow)!important} @@ -1546,6 +1561,8 @@ async def index(): ⚑ mcpproxy
+
Remote MCP Server
Bridge a remote, OAuth-protected MCP server β€” just paste its URL (e.g. the official Asana server). Tools & auth are handled automatically. + Best for hosted SaaS tools that already speak MCP β€” e.g. Asana, Linear, Notion, GitHub β€” where you just have a URL. @@ -1843,6 +1861,7 @@ async def index():
πŸ“¦
Package
Run an existing MCP server via npx, uvx, python -m, or any command β€” or bridge a remote server with npx -y mcp-remote <url>. Tools are auto-detected. + Best for published MCP servers you install and run locally β€” e.g. Playwright, filesystem, Slack, Puppeteer via npx/uvx/pip. @@ -1852,6 +1871,7 @@ async def index():
πŸ“‚
Repository
Clone a git repo, run build commands, then introspect & spawn the resulting stdio MCP server + Best for MCP servers distributed as source you build yourself before running. @@ -1861,6 +1881,7 @@ async def index():
πŸ”Œ
REST / OAuth API
Point at a REST API (base URL + endpoints, or an OpenAPI spec) with optional OAuth β€” each endpoint becomes an MCP tool + Best for any plain web API that has no prebuilt MCP server β€” e.g. Stripe, OpenWeather, or an internal service β€” described by a base URL or OpenAPI spec. @@ -1870,6 +1891,7 @@ async def index():
🐍
Python Code
Write async def functions β€” each one becomes an MCP tool + Best for quick custom logic, glue, or calculations you write inline β€” no external server needed. @@ -2180,6 +2202,36 @@ async def index(): + + + @@ -2193,6 +2245,7 @@ async def index(): let currentProvider = null; // the structured JSON object being edited let codeEditor = null; // CodeMirror instance for the code block let secretsModal = null, wizModal = null, termModal = null; +let catalogModal = null, catalogEntries = []; let filesModal = null, ttModal = null; let filesRoot = 'tools', filesPath = '', filesRoots = ['tools', 'files', 'repos']; let ttTools = [], ttSelected = null; // tool tester: /v1/tools entries + selected name @@ -3488,11 +3541,113 @@ async def index(): document.getElementById('wz-repo-cmd').value = 'node build/index.js'; } -function wzSelectType(type) { +function wzSelectType(type, prefill) { wzType = type; document.querySelectorAll('.wizard-choice').forEach(el => el.classList.remove('selected')); - event.currentTarget.classList.add('selected'); - setTimeout(() => wzShowStep(type), 120); + // Highlight the clicked card when invoked from an onclick handler; when called + // programmatically (e.g. from the catalog) there is no matching card event. + const ct = window.event && window.event.currentTarget; + if (ct && ct.classList && ct.classList.contains('wizard-choice')) ct.classList.add('selected'); + setTimeout(() => { + wzShowStep(type); + if (typeof prefill === 'function') prefill(); + }, 120); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Browse providers catalog +// ───────────────────────────────────────────────────────────────────────────── +function _slugify(s) { + return (String(s||'').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')) || 'provider'; +} + +function openCatalog() { + catalogModal = catalogModal || new bootstrap.Modal(document.getElementById('catalog-modal')); + catalogModal.show(); + loadCatalog(document.getElementById('catalog-live').checked); +} + +async function loadCatalog(live) { + const list = document.getElementById('catalog-list'); + list.innerHTML = '
Loading…
'; + try { + const data = await api('GET', '/api/catalog?live=' + (live ? 'true' : 'false')); + catalogEntries = data.entries || []; + renderCatalog(); + const errs = Object.keys(data.errors || {}); + if (errs.length) toast('Some registries were unavailable: ' + errs.join(', '), false); + } catch (e) { + list.innerHTML = '
' + esc(e.message) + '
'; + } +} + +function catalogSearch() { renderCatalog(); } + +function catalogToggleLive(cb) { loadCatalog(cb.checked); } + +function renderCatalog() { + const q = (document.getElementById('catalog-search').value || '').toLowerCase().trim(); + const kind = document.getElementById('catalog-kind').value; + const list = document.getElementById('catalog-list'); + const rows = catalogEntries.filter(e => { + if (kind && e.kind !== kind) return false; + if (!q) return true; + return (e.name + ' ' + (e.description || '') + ' ' + (e.categories || []).join(' ')).toLowerCase().includes(q); + }); + if (!rows.length) { list.innerHTML = '
No matching providers.
'; return; } + list.innerHTML = rows.map((e, i) => { + const badge = e.kind === 'mcp_remote' + ? 'MCP server' + : 'REST / OpenAPI'; + const src = e.source && e.source !== 'curated' + ? '' + esc(e.source) + '' : ''; + const home = e.homepage + ? ' β†—' : ''; + return '
' + + '
' + + '
' + + '' + esc(e.name) + '' + badge + src + home + '
' + + '
' + esc(e.description || '') + '
' + + '
' + + '' + + '
'; + }).join(''); + // Stash the filtered rows so the Configure buttons resolve by index. + list._rows = rows; +} + +function catalogConfigure(idx) { + const entry = (document.getElementById('catalog-list')._rows || [])[idx]; + if (!entry) return; + const el = document.getElementById('catalog-modal'); + // Wait for the catalog modal to finish closing before opening the wizard so + // Bootstrap doesn't leave a stale backdrop behind. + el.addEventListener('hidden.bs.modal', () => _wizardFromEntry(entry), { once: true }); + catalogModal.hide(); +} + +function _wizardFromEntry(entry) { + openWizard(); + if (entry.kind === 'mcp_remote') { + wzSelectType('remote', () => { + document.getElementById('wz-remote-name').value = entry.id || _slugify(entry.name); + document.getElementById('wz-remote-url').value = entry.url || ''; + }); + } else if (entry.kind === 'rest_openapi') { + wzSelectType('rest', () => { + document.getElementById('wz-rest-name').value = entry.id || _slugify(entry.name); + document.getElementById('wz-rest-base-url').value = entry.base_url || ''; + document.getElementById('wz-rest-openapi').value = entry.openapi_url || ''; + if (entry.auth_hint) { + const sel = document.getElementById('wz-rest-auth-type'); + if (sel && [...sel.options].some(o => o.value === entry.auth_hint)) { + sel.value = entry.auth_hint; + wzRestAuthChanged(); + } + } + wzRestTab('openapi'); + }); + } } // ── REST wizard helpers ────────────────────────────────────────────────────── diff --git a/frontend/catalog.json b/frontend/catalog.json new file mode 100644 index 0000000..f367c2a --- /dev/null +++ b/frontend/catalog.json @@ -0,0 +1,101 @@ +{ + "version": 1, + "entries": [ + { + "id": "asana", + "kind": "mcp_remote", + "name": "Asana", + "description": "Official Asana MCP server β€” manage tasks, projects, and workspaces. OAuth-protected.", + "categories": ["productivity"], + "homepage": "https://developers.asana.com/docs/using-asanas-model-control-protocol-mcp-server", + "url": "https://mcp.asana.com/v2/mcp" + }, + { + "id": "linear", + "kind": "mcp_remote", + "name": "Linear", + "description": "Official Linear MCP server β€” issues, projects, and cycles. OAuth-protected.", + "categories": ["productivity", "developer"], + "homepage": "https://linear.app/docs/mcp", + "url": "https://mcp.linear.app/sse" + }, + { + "id": "notion", + "kind": "mcp_remote", + "name": "Notion", + "description": "Official Notion MCP server β€” search, read, and update pages and databases. OAuth-protected.", + "categories": ["productivity", "knowledge"], + "homepage": "https://developers.notion.com/docs/mcp", + "url": "https://mcp.notion.com/mcp" + }, + { + "id": "sentry", + "kind": "mcp_remote", + "name": "Sentry", + "description": "Official Sentry MCP server β€” query issues, events, and releases. OAuth-protected.", + "categories": ["developer", "observability"], + "homepage": "https://docs.sentry.io/product/sentry-mcp/", + "url": "https://mcp.sentry.dev/mcp" + }, + { + "id": "stripe-mcp", + "kind": "mcp_remote", + "name": "Stripe", + "description": "Official Stripe MCP server β€” payments, customers, and balances. OAuth-protected.", + "categories": ["payments", "developer"], + "homepage": "https://docs.stripe.com/mcp", + "url": "https://mcp.stripe.com" + }, + { + "id": "deepwiki", + "kind": "mcp_remote", + "name": "DeepWiki", + "description": "Ask questions about any public GitHub repository's documentation. No auth.", + "categories": ["developer", "knowledge"], + "homepage": "https://deepwiki.com", + "url": "https://mcp.deepwiki.com/mcp" + }, + { + "id": "huggingface", + "kind": "mcp_remote", + "name": "Hugging Face", + "description": "Official Hugging Face MCP server β€” search models, datasets, and Spaces.", + "categories": ["ai", "developer"], + "homepage": "https://huggingface.co/settings/mcp", + "url": "https://huggingface.co/mcp" + }, + { + "id": "petstore", + "kind": "rest_openapi", + "name": "Swagger Petstore", + "description": "The canonical OpenAPI demo API β€” a great way to try the REST wizard end to end.", + "categories": ["demo", "developer"], + "homepage": "https://petstore3.swagger.io", + "openapi_url": "https://petstore3.swagger.io/api/v3/openapi.json", + "base_url": "https://petstore3.swagger.io/api/v3", + "auth_hint": "none" + }, + { + "id": "stripe-rest", + "kind": "rest_openapi", + "name": "Stripe REST API", + "description": "The full Stripe REST API as OpenAPI β€” payments, customers, subscriptions. Uses a bearer secret key.", + "categories": ["payments", "developer"], + "homepage": "https://stripe.com/docs/api", + "openapi_url": "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json", + "base_url": "https://api.stripe.com", + "auth_hint": "bearer" + }, + { + "id": "apis-guru", + "kind": "rest_openapi", + "name": "APIs.guru Directory", + "description": "The APIs.guru directory API itself β€” list and look up thousands of public OpenAPI specs. No auth.", + "categories": ["developer", "directory"], + "homepage": "https://apis.guru", + "openapi_url": "https://api.apis.guru/v2/openapi.json", + "base_url": "https://api.apis.guru/v2", + "auth_hint": "none" + } + ] +} diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..b57c960 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,127 @@ +"""Unit tests for catalog β€” curated loading, normalization, and the live merge. + +Live registry calls are faked with a stub httpx client so no network is +touched; tests focus on error isolation, de-dupe, and the offline default. +""" +import asyncio + +import pytest + +import catalog + + +def _run(coro): + return asyncio.run(coro) + + +# --------------------------------------------------------------------------- +# Normalization +# --------------------------------------------------------------------------- + +def test_normalize_requires_kind_name_and_url(): + assert catalog._normalize_entry({"name": "x"}, "s") is None # no kind + assert catalog._normalize_entry({"kind": "mcp_remote", "name": ""}, "s") is None + # mcp_remote without a usable url is dropped + assert catalog._normalize_entry({"kind": "mcp_remote", "name": "x"}, "s") is None + # non-http url is rejected + assert catalog._normalize_entry( + {"kind": "mcp_remote", "name": "x", "url": "ftp://h/y"}, "s" + ) is None + + +def test_normalize_valid_entries_and_slug(): + remote = catalog._normalize_entry( + {"kind": "mcp_remote", "name": "My Server", "url": "https://h/mcp"}, "src" + ) + assert remote["id"] == "my-server" + assert remote["source"] == "src" + assert remote["url"] == "https://h/mcp" + + rest = catalog._normalize_entry( + {"kind": "rest_openapi", "name": "API", "openapi_url": "https://h/o.json", + "base_url": "https://h", "auth_hint": "bearer"}, "src" + ) + assert rest["openapi_url"] == "https://h/o.json" + assert rest["base_url"] == "https://h" + assert rest["auth_hint"] == "bearer" + + +# --------------------------------------------------------------------------- +# Curated + offline build +# --------------------------------------------------------------------------- + +def test_curated_loads_and_is_valid(): + entries = catalog.load_curated() + assert entries, "bundled catalog.json should contain entries" + for e in entries: + assert e["kind"] in ("mcp_remote", "rest_openapi") + assert e["name"] and e["id"] + + +def test_build_offline_returns_curated_only(): + data = _run(catalog.build_catalog(live=False)) + assert data["live"] is False + assert data["errors"] == {} + assert len(data["entries"]) == len(catalog.load_curated()) + + +# --------------------------------------------------------------------------- +# Live merge: error isolation + de-dupe +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def _reset_cache(): + catalog._cache.clear() + yield + catalog._cache.clear() + + +def test_live_isolates_failing_sources(monkeypatch): + async def good(client): + return [{"kind": "mcp_remote", "name": "Live One", "url": "https://live/one"}] + + async def bad(client): + raise RuntimeError("boom") + + monkeypatch.setattr(catalog, "CATALOG_LIVE", True) + monkeypatch.setattr(catalog, "SOURCES", {"good": good, "bad": bad}) + + data = _run(catalog.build_catalog(live=True)) + assert "bad" in data["errors"] and "boom" in data["errors"]["bad"] + # the good source still contributed despite the bad one raising + names = [e["name"] for e in data["entries"]] + assert "Live One" in names + # curated entries are still present + assert len(data["entries"]) >= len(catalog.load_curated()) + 1 + + +def test_live_dedupe_prefers_curated(monkeypatch): + curated = catalog.load_curated() + dup = next(e for e in curated if e["kind"] == "mcp_remote") + + async def dupe_source(client): + return [{"kind": "mcp_remote", "name": "DIFFERENT NAME", "url": dup["url"]}] + + monkeypatch.setattr(catalog, "CATALOG_LIVE", True) + monkeypatch.setattr(catalog, "SOURCES", {"dupe": dupe_source}) + + data = _run(catalog.build_catalog(live=True)) + matches = [e for e in data["entries"] if e.get("url") == dup["url"]] + assert len(matches) == 1 + assert matches[0]["name"] == dup["name"] # curated won + + +def test_live_gate_disables_probing(monkeypatch): + called = {"n": 0} + + async def src(client): + called["n"] += 1 + return [] + + monkeypatch.setattr(catalog, "CATALOG_LIVE", False) # global gate off + monkeypatch.setattr(catalog, "SOURCES", {"src": src}) + + data = _run(catalog.build_catalog(live=True)) + assert data["live"] is False + assert called["n"] == 0 + assert len(data["entries"]) == len(catalog.load_curated()) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 2cee9d6..8f41a2f 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -1693,6 +1693,31 @@ def test_index_contains_files_and_tooltester_ui(self, client): "filesUpload", "ttInvoke"): assert needle in html, needle + def test_index_contains_catalog_ui(self, client): + html = client.get("/").text + for needle in ("catalog-modal", "openCatalog()", "catalogConfigure", "best-for"): + assert needle in html, needle + + +class TestCatalogAPI: + def test_catalog_offline_returns_curated(self, client): + data = client.get("/api/catalog").json() + assert data["live"] is False + assert data["errors"] == {} + assert data["entries"], "curated catalog should not be empty" + for e in data["entries"]: + assert e["kind"] in ("mcp_remote", "rest_openapi") + assert e["name"] and e["id"] and e["source"] + + def test_catalog_live_flag_without_probing_is_safe(self, client, monkeypatch): + # Gate live probing off so the endpoint never touches the network even + # when ?live=true is passed; it must still return the curated list. + import catalog + monkeypatch.setattr(catalog, "CATALOG_LIVE", False) + data = client.get("/api/catalog?live=true").json() + assert data["live"] is False + assert data["entries"] + # --------------------------------------------------------------------------- # OAuth bootstrap (provider-declared oauth: block)