From 9e5d8164c8c74aa3f78fd7ca31a17692273e288f Mon Sep 17 00:00:00 2001 From: Praveen Sampath Date: Fri, 24 Jul 2026 11:46:43 +0000 Subject: [PATCH 01/19] feat(integrations): add direct remote MCP support - add admin-managed remote MCP sources, probing, credentials, and OAuth setup - reconcile MCP catalogs into manifests and route tools, resources, and prompts through the gateway - prevent sync and slug collisions while adding SSRF protections and focused tests --- connectors/atlassian/src/sync.rs | 3 +- .../atlassian/tests/common/mock_atlassian.rs | 4 +- connectors/atlassian/tests/common/mod.rs | 9 +- connectors/google/src/connector.rs | 4 +- connectors/google/src/sync.rs | 25 +- sdk/rust/src/connector.rs | 9 +- sdk/rust/src/lib.rs | 6 +- sdk/rust/src/server.rs | 14 +- services/ai/agents/executor.py | 10 +- services/ai/db/models.py | 2 + services/ai/routers/chat.py | 7 +- .../tests/unit/test_mcp_capability_handler.py | 236 ++- .../unit/test_remote_mcp_connector_actions.py | 224 +++ services/ai/tools/connector_handler.py | 78 +- services/ai/tools/mcp_capability_handler.py | 225 ++- services/connector-manager/src/handlers.rs | 354 +++- services/connector-manager/src/lib.rs | 20 + services/connector-manager/src/models.rs | 6 +- .../src/remote_mcp/gateway.rs | 1618 +++++++++++++++++ .../connector-manager/src/remote_mcp/mod.rs | 3 + .../connector-manager/src/remote_mcp/oauth.rs | 336 ++++ .../src/remote_mcp/registry.rs | 349 ++++ services/connector-manager/src/scheduler.rs | 64 +- .../connector-manager/src/source_cleanup.rs | 6 +- .../connector-manager/src/sync_manager.rs | 27 +- .../tests/common/mock_connector.rs | 6 +- .../connector-manager/tests/common/mod.rs | 14 +- .../migrations/106_add_remote_mcp_sources.sql | 43 + services/searcher/src/search.rs | 4 +- services/searcher/src/search_repository.rs | 4 +- services/searcher/src/suggested_questions.rs | 11 +- .../db/repositories/service_credentials.rs | 16 + shared/src/db/repositories/source.rs | 22 +- shared/src/models.rs | 134 +- web/src/lib/server/db/schema.ts | 1 + web/src/lib/server/mcp/client.test.ts | 69 + web/src/lib/server/mcp/client.ts | 515 ++++++ web/src/lib/server/mcp/probe.test.ts | 121 ++ web/src/lib/server/oauth/connectorOAuth.ts | 130 +- web/src/lib/types.test.ts | 14 + web/src/lib/types.ts | 10 + .../(admin)/admin/settings/+layout.svelte | 21 +- .../settings/integrations/+layout.server.ts | 1 + .../settings/integrations/+page.server.ts | 15 +- .../admin/settings/integrations/+page.svelte | 9 +- .../admin/settings/mcp/+page.server.ts | 7 + .../(admin)/admin/settings/mcp/+page.svelte | 193 ++ .../settings/mcp/[sourceId]/+page.server.ts | 81 + .../settings/mcp/[sourceId]/+page.svelte | 240 +++ .../mcp/[sourceId]/page.server.test.ts | 77 + .../settings/integrations/+page.server.ts | 13 +- web/src/routes/api/oauth/callback/+server.ts | 47 +- web/src/routes/api/oauth/start/+server.ts | 5 +- web/src/routes/api/remote-mcp/+server.ts | 213 +++ .../api/remote-mcp/[sourceId]/+server.ts | 272 +++ .../api/remote-mcp/[sourceId]/server.test.ts | 83 + web/src/routes/api/remote-mcp/test/+server.ts | 30 + .../routes/api/service-credentials/+server.ts | 6 +- web/src/routes/api/sources/+server.ts | 67 +- .../api/sources/[sourceId]/sync/+server.ts | 10 + 60 files changed, 5907 insertions(+), 236 deletions(-) create mode 100644 services/ai/tests/unit/test_remote_mcp_connector_actions.py create mode 100644 services/connector-manager/src/remote_mcp/gateway.rs create mode 100644 services/connector-manager/src/remote_mcp/mod.rs create mode 100644 services/connector-manager/src/remote_mcp/oauth.rs create mode 100644 services/connector-manager/src/remote_mcp/registry.rs create mode 100644 services/migrations/106_add_remote_mcp_sources.sql create mode 100644 web/src/lib/server/mcp/client.test.ts create mode 100644 web/src/lib/server/mcp/client.ts create mode 100644 web/src/lib/server/mcp/probe.test.ts create mode 100644 web/src/lib/types.test.ts create mode 100644 web/src/routes/(admin)/admin/settings/mcp/+page.server.ts create mode 100644 web/src/routes/(admin)/admin/settings/mcp/+page.svelte create mode 100644 web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.server.ts create mode 100644 web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.svelte create mode 100644 web/src/routes/(admin)/admin/settings/mcp/[sourceId]/page.server.test.ts create mode 100644 web/src/routes/api/remote-mcp/+server.ts create mode 100644 web/src/routes/api/remote-mcp/[sourceId]/+server.ts create mode 100644 web/src/routes/api/remote-mcp/[sourceId]/server.test.ts create mode 100644 web/src/routes/api/remote-mcp/test/+server.ts diff --git a/connectors/atlassian/src/sync.rs b/connectors/atlassian/src/sync.rs index b353b356d..364fd3939 100644 --- a/connectors/atlassian/src/sync.rs +++ b/connectors/atlassian/src/sync.rs @@ -101,7 +101,8 @@ impl SyncManager { return Err(anyhow!("Source is not active: {}", source_id)); } - let source_type = source.source_type; + let source_type = SourceType::try_from(source.source_type.as_str()) + .map_err(|e| anyhow!("Invalid source type for Atlassian connector: {}", e))?; if source_type != SourceType::Confluence && source_type != SourceType::Jira { return Err(anyhow!( "Invalid source type for Atlassian connector: {:?}", diff --git a/connectors/atlassian/tests/common/mock_atlassian.rs b/connectors/atlassian/tests/common/mock_atlassian.rs index 06947650d..484a3e703 100644 --- a/connectors/atlassian/tests/common/mock_atlassian.rs +++ b/connectors/atlassian/tests/common/mock_atlassian.rs @@ -5,8 +5,6 @@ use std::collections::HashMap; use std::pin::Pin; use std::sync::Mutex; -use omni_atlassian_connector::AtlassianApi; -use omni_atlassian_connector::AtlassianCredentials; use omni_atlassian_connector::client::{OrgGroupInfo, PageReadRestrictions}; use omni_atlassian_connector::models::{ ConfluenceCqlPage, ConfluencePage, ConfluenceSpace, ConfluenceSpacePermission, JiraField, @@ -14,6 +12,8 @@ use omni_atlassian_connector::models::{ JiraProjectIssueSecuritySchemeResponse, JiraProjectRolesResponse, JiraRoleActorsResponse, JiraSearchResponse, JiraSecurityLevelMember, }; +use omni_atlassian_connector::AtlassianApi; +use omni_atlassian_connector::AtlassianCredentials; #[derive(Debug, Clone)] pub struct MethodCall { diff --git a/connectors/atlassian/tests/common/mod.rs b/connectors/atlassian/tests/common/mod.rs index 05e54973c..a6cfd4427 100644 --- a/connectors/atlassian/tests/common/mod.rs +++ b/connectors/atlassian/tests/common/mod.rs @@ -7,7 +7,8 @@ use omni_connector_sdk::SdkClient; use redis::AsyncCommands; use shared::db::repositories::service_credentials::ServiceCredentialsRepo; use shared::models::{ - AuthType, ConnectorManifest, ServiceCredential, ServiceProvider, SourceType, SyncType, + AuthType, ConnectorManifest, IntegrationType, ServiceCredential, ServiceProvider, SourceType, + SyncType, }; use shared::storage::postgres::PostgresStorage; use shared::test_environment::TestEnvironment; @@ -81,7 +82,11 @@ pub async fn setup_test_fixture(source_type: SourceType) -> Result sync_modes: vec![SyncType::Full], connector_id: "atlassian".to_string(), connector_url: "http://127.0.0.1:1".to_string(), - source_types: vec![SourceType::Confluence, SourceType::Jira], + integration_type: IntegrationType::Connector, + source_types: vec![ + SourceType::Confluence.to_string(), + SourceType::Jira.to_string(), + ], description: None, actions: vec![], search_operators: vec![], diff --git a/connectors/google/src/connector.rs b/connectors/google/src/connector.rs index 829cea6ed..4c90eec03 100644 --- a/connectors/google/src/connector.rs +++ b/connectors/google/src/connector.rs @@ -1197,7 +1197,9 @@ impl Connector for GoogleConnector { } } _ => { - create_service_auth(creds, source.source_type).map_err(|e| { + let native_source_type = SourceType::try_from(source.source_type.as_str()) + .map_err(SyncRequestValidationError::BadRequest)?; + create_service_auth(creds, native_source_type).map_err(|e| { SyncRequestValidationError::BadRequest(format!( "Invalid Google service-account credentials: {}", e diff --git a/connectors/google/src/sync.rs b/connectors/google/src/sync.rs index 026a1e722..33cd93c01 100644 --- a/connectors/google/src/sync.rs +++ b/connectors/google/src/sync.rs @@ -991,7 +991,10 @@ impl SyncManager { // mid-sync. let existing_state = existing_state.unwrap_or_default(); - let result = match source.source_type { + let native_source_type = SourceType::try_from(source.source_type.as_str()) + .map_err(|e| anyhow!("Unsupported source type: {}", e))?; + + let result = match native_source_type { SourceType::GoogleDrive => { self.sync_drive_source_internal( source, @@ -1027,7 +1030,7 @@ impl SyncManager { _ => Err(anyhow!("Unsupported source type: {:?}", source.source_type)), }; - if result.is_ok() && source.source_type == SourceType::GoogleDrive { + if result.is_ok() && native_source_type == SourceType::GoogleDrive { self.ensure_webhook_registered(source_id).await; } @@ -2289,7 +2292,10 @@ impl SyncManager { ctx: &SyncContext, ) -> Result { let sync_run_id = ctx.sync_run_id(); - let service_auth = Arc::new(self.create_auth(service_creds, source.source_type).await?); + + let native_source_type = SourceType::try_from(source.source_type.as_str()) + .map_err(|e| anyhow!("Unsupported source type: {}", e))?; + let service_auth = Arc::new(self.create_auth(service_creds, native_source_type).await?); // Parse folder-path filters. Malformed config is a sync error. let parsed_filters = match crate::models::parse_folder_path_filters(&source.config) { @@ -2612,7 +2618,9 @@ impl SyncManager { ) -> Result { let sync_run_id = ctx.sync_run_id(); - let service_auth = Arc::new(self.create_auth(service_creds, source.source_type).await?); + let native_source_type = SourceType::try_from(source.source_type.as_str()) + .map_err(|e| anyhow!("Unsupported source type: {}", e))?; + let service_auth = Arc::new(self.create_auth(service_creds, native_source_type).await?); let (_drive_cutoff_date, gmail_cutoff_date) = self.get_cutoff_date()?; info!("Using Gmail cutoff date: {}", gmail_cutoff_date); @@ -4911,7 +4919,14 @@ impl SyncManager { service_creds: &ServiceCredential, ctx: &SyncContext, ) -> HashSet { - let service_auth = match self.create_auth(service_creds, source.source_type).await { + let native_source_type = match SourceType::try_from(source.source_type.as_str()) { + Ok(source_type) => source_type, + Err(e) => { + warn!("Skipping group sync for unsupported source type: {}", e); + return HashSet::new(); + } + }; + let service_auth = match self.create_auth(service_creds, native_source_type).await { Ok(auth) => auth, Err(e) => { warn!("Failed to create auth for group sync: {}", e); diff --git a/sdk/rust/src/connector.rs b/sdk/rust/src/connector.rs index 2d6c19046..dccd4b711 100644 --- a/sdk/rust/src/connector.rs +++ b/sdk/rust/src/connector.rs @@ -13,7 +13,7 @@ use serde::de::DeserializeOwned; use serde::Serialize; use serde_json::Value as JsonValue; use shared::models::{ - ActionDefinition, ConnectorManifest, ConnectorSkillDefinition, SearchOperator, + ActionDefinition, ConnectorManifest, ConnectorSkillDefinition, IntegrationType, SearchOperator, ServiceCredential, Source, SourceType, SyncType, }; @@ -165,7 +165,12 @@ pub trait Connector: Send + Sync + 'static { sync_modes: self.sync_modes(), connector_id: self.name().to_string(), connector_url, - source_types: self.source_types(), + integration_type: IntegrationType::Connector, + source_types: self + .source_types() + .into_iter() + .map(|source_type| source_type.to_string()) + .collect(), description: self.description(), actions: self.actions(), search_operators: self.search_operators(), diff --git a/sdk/rust/src/lib.rs b/sdk/rust/src/lib.rs index 3d7436096..be9f1eb63 100644 --- a/sdk/rust/src/lib.rs +++ b/sdk/rust/src/lib.rs @@ -20,9 +20,9 @@ pub use server::{create_router, serve, serve_with_config, serve_with_extra_route pub use shared::models::DocumentAttributes; pub use shared::models::{ ActionDefinition, ActionMode, AuthType, ConnectorEvent, ConnectorManifest, - ConnectorSkillDefinition, DocumentMetadata, DocumentPermissions, McpPromptDefinition, - McpResourceDefinition, SearchOperator, ServiceCredential, ServiceProvider, Source, SourceType, - SyncRun, SyncStatus, SyncType, + ConnectorSkillDefinition, DocumentMetadata, DocumentPermissions, IntegrationType, + McpPromptDefinition, McpResourceDefinition, SearchOperator, ServiceCredential, ServiceProvider, + Source, SourceType, SyncRun, SyncStatus, SyncType, }; pub use shared::rate_limiter::{RateLimiter, RetryableError}; pub use shared::telemetry; diff --git a/sdk/rust/src/server.rs b/sdk/rust/src/server.rs index 23ecb0aa6..25121d482 100644 --- a/sdk/rust/src/server.rs +++ b/sdk/rust/src/server.rs @@ -18,7 +18,7 @@ use axum::{ use dashmap::mapref::entry::Entry; use dashmap::DashMap; use serde::de::DeserializeOwned; -use shared::models::{ConnectorSkillDefinition, SyncSlotClass, SyncType}; +use shared::models::{ConnectorSkillDefinition, SourceType, SyncSlotClass, SyncType}; use shared::telemetry; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; @@ -482,11 +482,21 @@ where .register_sync(&sync_run_id, request.sync_mode) .await; + let native_source_type = SourceType::try_from(source.source_type.as_str()).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + Json(SyncResponse::error(format!( + "source {} is not a native connector source: {}", + source_id, e + ))), + ) + })?; + let ctx = SyncContext::new_with_resume( state.sdk_client.clone(), sync_run_id.clone(), source_id.clone(), - source.source_type, + native_source_type, request.sync_mode, request.is_resume, cancelled, diff --git a/services/ai/agents/executor.py b/services/ai/agents/executor.py index 7fec185e5..1f70dc072 100644 --- a/services/ai/agents/executor.py +++ b/services/ai/agents/executor.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import cast -import httpx from anthropic.types import ( BashCodeExecutionToolResultBlockParam, CodeExecutionToolResultBlockParam, @@ -68,7 +67,7 @@ ConnectorToolHandler, SourceFilter, ToolsetSummary, - sources_from_sync_overview_response, + fetch_active_sources_from_connector_manager, ) from tools.email_handler import EmailToolHandler from tools.mcp_capability_handler import McpCapabilityHandler @@ -125,12 +124,9 @@ async def _resolve_llm_provider(state: AppState, agent: Agent) -> ResolvedModel: async def _fetch_sources() -> list[Source] | None: - """Fetch all sources from the connector manager.""" + """Fetch active sources, including remote MCP rows, from connector manager.""" try: - async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.get(f"{CONNECTOR_MANAGER_URL.rstrip('/')}/sources") - resp.raise_for_status() - return sources_from_sync_overview_response(resp.json()) + return await fetch_active_sources_from_connector_manager(CONNECTOR_MANAGER_URL) except Exception as e: logger.warning(f"Failed to fetch sources: {e}") return None diff --git a/services/ai/db/models.py b/services/ai/db/models.py index cd790a8c9..5e33a5367 100644 --- a/services/ai/db/models.py +++ b/services/ai/db/models.py @@ -415,6 +415,7 @@ class Source: source_type: str is_active: bool is_deleted: bool + integration_type: str = "connector" @classmethod def from_row(cls, row: Mapping[str, object]) -> "Source": @@ -423,6 +424,7 @@ def from_row(cls, row: Mapping[str, object]) -> "Source": name=cast(str, row["name"]), source_type=cast(str, row["source_type"]), is_active=cast(bool, row["is_active"]), + integration_type=cast(str, row.get("integration_type", "connector")), is_deleted=cast(bool, row["is_deleted"]), ) diff --git a/services/ai/routers/chat.py b/services/ai/routers/chat.py index 312357573..606228e4a 100644 --- a/services/ai/routers/chat.py +++ b/services/ai/routers/chat.py @@ -102,7 +102,7 @@ from tools.connector_handler import ( SearchOperator, ToolsetSummary, - sources_from_sync_overview_response, + fetch_active_sources_from_connector_manager, ) from tools.mcp_capability_handler import McpCapabilityHandler from tools.meta_handler import MetaToolHandler, OnLoad @@ -233,10 +233,7 @@ def _loaded_source_ids( async def _fetch_sources_from_connector_manager() -> list[Source] | None: try: - async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.get(f"{CONNECTOR_MANAGER_URL.rstrip('/')}/sources") - resp.raise_for_status() - return sources_from_sync_overview_response(resp.json()) + return await fetch_active_sources_from_connector_manager(CONNECTOR_MANAGER_URL) except Exception as e: logger.warning(f"Failed to fetch sources from connector manager: {e}") return None diff --git a/services/ai/tests/unit/test_mcp_capability_handler.py b/services/ai/tests/unit/test_mcp_capability_handler.py index c70ef7222..7778c6bd9 100644 --- a/services/ai/tests/unit/test_mcp_capability_handler.py +++ b/services/ai/tests/unit/test_mcp_capability_handler.py @@ -7,7 +7,7 @@ from httpx import Response from db.models import Source -from tools.mcp_capability_handler import McpCapabilityHandler +from tools.mcp_capability_handler import McpCapabilityHandler, McpResourceDefinition from tools.registry import ToolContext from tools.searcher_client import CapabilitySearchResponse, CapabilitySearchResult @@ -41,11 +41,18 @@ async def search_capabilities(self, request): return CapabilitySearchResponse(results=results[: request.limit]) -def _source(source_id: str, source_type: str = "docs", *, active: bool = True) -> Source: +def _source( + source_id: str, + source_type: str = "docs", + *, + active: bool = True, + integration_type: str = "connector", +) -> Source: return Source( id=source_id, name=f"{source_type} source", source_type=source_type, + integration_type=integration_type, is_active=active, is_deleted=False, ) @@ -55,6 +62,30 @@ def _ctx() -> ToolContext: return ToolContext(chat_id="chat-1", user_id="user-1") +def test_mcp_capability_412_returns_oauth_required_result() -> None: + handler = McpCapabilityHandler(connector_manager_url="http://cm.test") + record = handler._resource_record( + _source("src-1", integration_type="remote_mcp"), + "docs", + "Docs", + McpResourceDefinition(uri_template="docs://guide", name="Guide"), + ) + response = Response( + 412, + json={ + "provider": "remote_mcp", + "oauth_start_url": "/api/oauth/start?source_id=src-1", + }, + ) + + result = handler._oauth_required_result(response, record) + + assert result is not None + assert result.oauth_required is not None + assert result.oauth_required.source_id == "src-1" + assert result.oauth_required.provider == "remote_mcp" + + def _manifest() -> dict: return { "mcp_enabled": True, @@ -106,13 +137,19 @@ async def test_publishes_resource_and_prompt_capabilities() -> None: ) ) handler = McpCapabilityHandler( - "http://cm.test", searcher_client=searcher, prefetched_sources=[_source("src-1")] + "http://cm.test", + searcher_client=searcher, + prefetched_sources=[_source("src-1")], ) await handler.publish_capabilities() - assert len(searcher.upserts) == 1 - capabilities = searcher.upserts[0].capabilities + assert len(searcher.upserts) == 2 + assert {request.capability_type for request in searcher.upserts} == { + "resource", + "prompt", + } + capabilities = [cap for request in searcher.upserts for cap in request.capabilities] assert {cap.capability_type for cap in capabilities} == {"resource", "prompt"} resource_caps = [cap for cap in capabilities if cap.capability_type == "resource"] prompt_caps = [cap for cap in capabilities if cap.capability_type == "prompt"] @@ -127,15 +164,57 @@ async def test_publishes_resource_and_prompt_capabilities() -> None: assert prompt.data["arguments"][0]["required"] is True +@pytest.mark.asyncio +@respx.mock +async def test_empty_remote_mcp_catalog_syncs_empty_capability_publishers() -> None: + searcher = _FakeSearcherClient() + respx.get("http://cm.test/connectors").mock( + return_value=Response( + 200, + json=[ + { + "source_type": "docs", + "healthy": True, + "manifest": { + "integration_type": "remote_mcp", + "mcp_enabled": True, + "resources": [], + "prompts": [], + }, + } + ], + ) + ) + handler = McpCapabilityHandler( + "http://cm.test", + searcher_client=searcher, + prefetched_sources=[_source("src-remote", integration_type="remote_mcp")], + ) + + await handler.publish_capabilities() + + assert len(searcher.upserts) == 2 + assert {(req.publisher_id, req.capability_type) for req in searcher.upserts} == { + ("src-remote", "resource"), + ("src-remote", "prompt"), + } + assert all(req.capabilities == [] for req in searcher.upserts) + + @pytest.mark.asyncio @respx.mock async def test_search_uses_allowed_ids_and_source_ids() -> None: searcher = _FakeSearcherClient() respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) ) handler = McpCapabilityHandler( - "http://cm.test", searcher_client=searcher, prefetched_sources=[_source("src-1")] + "http://cm.test", + searcher_client=searcher, + prefetched_sources=[_source("src-1")], ) await handler.refresh() resource_id = next(iter(handler._resources)) @@ -165,7 +244,9 @@ async def test_search_uses_allowed_ids_and_source_ids() -> None: ), ] - resource_result = await handler.execute("resource_search", {"query": "guide"}, _ctx()) + resource_result = await handler.execute( + "resource_search", {"query": "guide"}, _ctx() + ) prompt_result = await handler.execute("prompt_search", {"query": "debug"}, _ctx()) assert resource_id in resource_result.content[0]["text"] @@ -181,15 +262,22 @@ async def test_search_uses_allowed_ids_and_source_ids() -> None: @respx.mock async def test_load_resource_requires_uri_for_templates() -> None: respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) + ) + handler = McpCapabilityHandler( + "http://cm.test", prefetched_sources=[_source("src-1")] ) - handler = McpCapabilityHandler("http://cm.test", prefetched_sources=[_source("src-1")]) await handler.refresh() template_id = next( record.id for record in handler._resources.values() if record.requires_uri ) - result = await handler.execute("load_resource", {"resource_id": template_id}, _ctx()) + result = await handler.execute( + "load_resource", {"resource_id": template_id}, _ctx() + ) assert result.is_error assert "provide a concrete uri" in result.content[0]["text"] @@ -199,15 +287,28 @@ async def test_load_resource_requires_uri_for_templates() -> None: @respx.mock async def test_load_resource_applies_line_range() -> None: respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) ) respx.post("http://cm.test/resource").mock( return_value=Response( 200, - json={"contents": [{"uri": "docs://guide", "text": "one\ntwo\nthree\nfour", "mime_type": "text/plain"}]}, + json={ + "contents": [ + { + "uri": "docs://guide", + "text": "one\ntwo\nthree\nfour", + "mime_type": "text/plain", + } + ] + }, ) ) - handler = McpCapabilityHandler("http://cm.test", prefetched_sources=[_source("src-1")]) + handler = McpCapabilityHandler( + "http://cm.test", prefetched_sources=[_source("src-1")] + ) await handler.refresh() guide_id = next( record.id for record in handler._resources.values() if record.name == "Guide" @@ -228,15 +329,24 @@ async def test_load_resource_applies_line_range() -> None: @pytest.mark.asyncio @respx.mock -async def test_load_resource_large_content_returns_preview_and_reload_instruction() -> None: +async def test_load_resource_large_content_returns_preview_and_reload_instruction() -> ( + None +): respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) ) large_text = "\n".join(f"line {idx} " + "x" * 200 for idx in range(1, 220)) respx.post("http://cm.test/resource").mock( - return_value=Response(200, json={"contents": [{"uri": "docs://guide", "text": large_text}]}) + return_value=Response( + 200, json={"contents": [{"uri": "docs://guide", "text": large_text}]} + ) + ) + handler = McpCapabilityHandler( + "http://cm.test", prefetched_sources=[_source("src-1")] ) - handler = McpCapabilityHandler("http://cm.test", prefetched_sources=[_source("src-1")]) await handler.refresh() guide_id = next( record.id for record in handler._resources.values() if record.name == "Guide" @@ -255,9 +365,14 @@ async def test_load_resource_large_content_returns_preview_and_reload_instructio @respx.mock async def test_load_prompt_validates_required_arguments() -> None: respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) + ) + handler = McpCapabilityHandler( + "http://cm.test", prefetched_sources=[_source("src-1")] ) - handler = McpCapabilityHandler("http://cm.test", prefetched_sources=[_source("src-1")]) await handler.refresh() prompt_id = next(iter(handler._prompts)) @@ -271,7 +386,10 @@ async def test_load_prompt_validates_required_arguments() -> None: @respx.mock async def test_load_prompt_returns_structured_template_tool_result() -> None: respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) ) prompt_route = respx.post("http://cm.test/prompt").mock( return_value=Response( @@ -279,14 +397,22 @@ async def test_load_prompt_returns_structured_template_tool_result() -> None: json={ "description": "Debug prompt", "messages": [ - {"role": "user", "content": {"type": "text", "text": "I'm seeing this error:"}}, + { + "role": "user", + "content": {"type": "text", "text": "I'm seeing this error:"}, + }, {"role": "user", "content": {"type": "text", "text": "boom"}}, - {"role": "assistant", "content": {"type": "text", "text": "I'll help debug."}}, + { + "role": "assistant", + "content": {"type": "text", "text": "I'll help debug."}, + }, ], }, ) ) - handler = McpCapabilityHandler("http://cm.test", prefetched_sources=[_source("src-1")]) + handler = McpCapabilityHandler( + "http://cm.test", prefetched_sources=[_source("src-1")] + ) await handler.refresh() prompt_id = next(iter(handler._prompts)) @@ -299,13 +425,22 @@ async def test_load_prompt_returns_structured_template_tool_result() -> None: assert not result.is_error assert prompt_route.calls[0].request.content body = json.loads(prompt_route.calls[0].request.content) - assert body == {"source_id": "src-1", "name": "debug_error", "arguments": {"error": "boom"}} + assert body == { + "source_id": "src-1", + "user_id": "user-1", + "name": "debug_error", + "arguments": {"error": "boom"}, + } text = result.content[0]["text"] assert "not actual user/assistant chat history" in text assert "```json" in text data = json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0]) assert data["prompt_id"] == prompt_id - assert [message["role"] for message in data["messages"]] == ["user", "user", "assistant"] + assert [message["role"] for message in data["messages"]] == [ + "user", + "user", + "assistant", + ] assert data["messages"][0]["content"]["text"] == "I'm seeing this error:" @@ -313,7 +448,10 @@ async def test_load_prompt_returns_structured_template_tool_result() -> None: @respx.mock async def test_source_filter_limits_records_to_readable_sources() -> None: respx.get("http://cm.test/connectors").mock( - return_value=Response(200, json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}]) + return_value=Response( + 200, + json=[{"source_type": "docs", "healthy": True, "manifest": _manifest()}], + ) ) handler = McpCapabilityHandler( "http://cm.test", @@ -326,3 +464,47 @@ async def test_source_filter_limits_records_to_readable_sources() -> None: assert handler.has_capabilities() assert {record.source_id for record in handler._resources.values()} == {"src-read"} assert {record.source_id for record in handler._prompts.values()} == {"src-read"} + + +@pytest.mark.asyncio +@respx.mock +async def test_remote_mcp_capabilities_match_by_integration_type_and_source_type() -> ( + None +): + manifest = _manifest() | {"integration_type": "remote_mcp"} + respx.get("http://cm.test/connectors").mock( + return_value=Response( + 200, + json=[ + {"source_type": "docs", "healthy": True, "manifest": manifest}, + { + "source_type": "docs", + "healthy": True, + "manifest": _manifest() | {"integration_type": "connector"}, + }, + ], + ) + ) + handler = McpCapabilityHandler( + "http://cm.test", + prefetched_sources=[ + _source("src-native", integration_type="connector"), + _source("src-remote", integration_type="remote_mcp"), + ], + ) + + await handler.refresh() + + assert handler.has_capabilities() + assert len(handler._resources) == 4 + assert {record.source_id for record in handler._resources.values()} == { + "src-native", + "src-remote", + } + remote_records = [ + record + for record in handler._resources.values() + if record.source_id == "src-remote" + ] + assert remote_records + assert all(record.source_type == "docs" for record in remote_records) diff --git a/services/ai/tests/unit/test_remote_mcp_connector_actions.py b/services/ai/tests/unit/test_remote_mcp_connector_actions.py new file mode 100644 index 000000000..2a78c2787 --- /dev/null +++ b/services/ai/tests/unit/test_remote_mcp_connector_actions.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import importlib + +import pytest +import respx +from httpx import Response + +from db.models import Source +from tools.connector_handler import ( + ConnectorToolHandler, + fetch_active_sources_from_connector_manager, +) + + +def _source(source_id: str, integration_type: str) -> Source: + return Source( + id=source_id, + name=f"{integration_type} docs", + source_type="docs", + integration_type=integration_type, + is_active=True, + is_deleted=False, + ) + + +@pytest.mark.asyncio +@respx.mock +async def test_actions_match_source_by_integration_type_and_source_type() -> None: + respx.get("http://cm.test/connectors").mock( + return_value=Response( + 200, + json=[ + { + "source_type": "docs", + "healthy": True, + "manifest": { + "integration_type": "connector", + "display_name": "Native Docs", + "actions": [ + { + "name": "search", + "description": "Native search", + "mode": "read", + } + ], + }, + }, + { + "source_type": "docs", + "healthy": True, + "manifest": { + "integration_type": "remote_mcp", + "display_name": "Remote Docs", + "actions": [ + { + "name": "search", + "description": "Remote MCP search", + "mode": "read", + } + ], + }, + }, + ], + ) + ) + handler = ConnectorToolHandler( + connector_manager_url="http://cm.test", + user_id="user-1", + prefetched_sources=[ + _source("src-native", "connector"), + _source("src-remote", "remote_mcp"), + ], + ) + + await handler._ensure_initialized() + + assert len(handler.actions) == 2 + assert {action.source_id for action in handler.actions.values()} == { + "src-native", + "src-remote", + } + remote_action = next( + action + for action in handler.actions.values() + if action.source_id == "src-remote" + ) + assert remote_action.integration_type == "remote_mcp" + assert remote_action.description == "Remote MCP search" + + +@pytest.mark.asyncio +@respx.mock +async def test_actions_fetch_active_sources_endpoint_including_remote_mcp_rows() -> None: + respx.get("http://cm.test/connectors").mock( + return_value=Response( + 200, + json=[ + { + "source_type": "docs", + "healthy": True, + "manifest": { + "integration_type": "remote_mcp", + "display_name": "Remote Docs", + "actions": [ + { + "name": "search", + "description": "Remote MCP search", + "mode": "read", + } + ], + }, + } + ], + ) + ) + active_sources = respx.get("http://cm.test/sources/active").mock( + return_value=Response( + 200, + json=[ + { + "source": { + "id": "src-remote", + "name": "Remote docs", + "source_type": "docs", + "integration_type": "remote_mcp", + "is_active": True, + "is_deleted": False, + }, + "sync_runs": [], + "health": "healthy", + } + ], + ) + ) + legacy_sources = respx.get("http://cm.test/sources").mock( + return_value=Response(500, json={"error": "sync endpoint should not be used"}) + ) + + handler = ConnectorToolHandler( + connector_manager_url="http://cm.test", + user_id="user-1", + ) + + await handler._ensure_initialized() + + assert active_sources.called + assert not legacy_sources.called + assert len(handler.actions) == 1 + assert next(iter(handler.actions.values())).source_id == "src-remote" + + +@pytest.mark.asyncio +@respx.mock +async def test_active_source_prefetch_helper_uses_active_endpoint_for_remote_mcp_rows() -> None: + active_sources = respx.get("http://cm.test/sources/active").mock( + return_value=Response( + 200, + json=[ + { + "source": { + "id": "src-remote", + "name": "Remote docs", + "source_type": "docs", + "integration_type": "remote_mcp", + "is_active": True, + "is_deleted": False, + }, + "sync_runs": [], + "health": "healthy", + } + ], + ) + ) + legacy_sources = respx.get("http://cm.test/sources").mock( + return_value=Response(500, json={"error": "sync-only endpoint should not be used"}) + ) + + sources = await fetch_active_sources_from_connector_manager("http://cm.test") + + assert active_sources.called + assert not legacy_sources.called + assert [source.id for source in sources] == ["src-remote"] + assert sources[0].integration_type == "remote_mcp" + + +@pytest.mark.asyncio +@respx.mock +async def test_chat_and_agent_prefetch_helpers_use_active_sources_endpoint(monkeypatch) -> None: + chat_module = importlib.import_module("routers.chat") + executor_module = importlib.import_module("agents.executor") + monkeypatch.setattr(chat_module, "CONNECTOR_MANAGER_URL", "http://cm.test") + monkeypatch.setattr(executor_module, "CONNECTOR_MANAGER_URL", "http://cm.test") + + active_sources = respx.get("http://cm.test/sources/active").mock( + return_value=Response( + 200, + json=[ + { + "source": { + "id": "src-remote", + "name": "Remote docs", + "source_type": "docs", + "integration_type": "remote_mcp", + "is_active": True, + "is_deleted": False, + }, + "sync_runs": [], + "health": "healthy", + } + ], + ) + ) + legacy_sources = respx.get("http://cm.test/sources").mock( + return_value=Response(500, json={"error": "sync-only endpoint should not be used"}) + ) + + chat_sources = await chat_module._fetch_sources_from_connector_manager() + agent_sources = await executor_module._fetch_sources() + + assert active_sources.call_count == 2 + assert not legacy_sources.called + assert [source.id for source in chat_sources or []] == ["src-remote"] + assert [source.id for source in agent_sources or []] == ["src-remote"] diff --git a/services/ai/tools/connector_handler.py b/services/ai/tools/connector_handler.py index f74aa9949..c3dfc402d 100644 --- a/services/ai/tools/connector_handler.py +++ b/services/ai/tools/connector_handler.py @@ -51,6 +51,23 @@ def sources_from_sync_overview_response(payload: object) -> list[Source]: return sources +async def fetch_active_sources_from_connector_manager( + connector_manager_url: str, + timeout: float = 10.0, +) -> list[Source]: + """Fetch active source rows, including remote MCP rows. + + Connector-manager's legacy /sources endpoint is sync-overview oriented and + excludes remote MCP sources because they never sync. Tool prefetch paths must + use /sources/active so action/resource manifests can join active rows by + (integration_type, source_type) without re-enabling sync. + """ + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(f"{connector_manager_url.rstrip('/')}/sources/active") + resp.raise_for_status() + return sources_from_sync_overview_response(resp.json()) + + class ToolsetSummary(TypedDict): source_id: str source_type: str @@ -83,6 +100,7 @@ class ConnectorAction: mode: SourceMode admin_only: bool = False hidden: bool = False + integration_type: str = "connector" class ConnectorToolHandler: @@ -170,30 +188,38 @@ async def _fetch_actions(self) -> list[ConnectorAction]: if self._prefetched_sources is not None: sources = self._prefetched_sources else: - sources_resp = await client.get( - f"{self._connector_manager_url}/sources" + sources = await fetch_active_sources_from_connector_manager( + self._connector_manager_url ) - sources_resp.raise_for_status() - sources = sources_from_sync_overview_response(sources_resp.json()) except Exception as e: logger.error(f"Failed to fetch connector info: {e}") return [] - # Build a mapping from source_type to list of active sources - source_by_type: dict[str, list[Source]] = {} + # Build a mapping from (integration_type, source_type) to active sources. + source_by_identity: dict[tuple[str, str], list[Source]] = {} for source in sources: if source.is_active and not source.is_deleted: - source_by_type.setdefault(source.source_type, []).append(source) + source_by_identity.setdefault( + (source.integration_type, source.source_type), [] + ).append(source) # Extract search operators from connector manifests search_operators: list[SearchOperator] = [] for connector in connectors: source_type = connector.get("source_type", "") manifest = connector.get("manifest") + integration_type = ( + manifest.get("integration_type", "connector") + if manifest + else "connector" + ) if not manifest or not connector.get("healthy"): continue + if (integration_type, source_type) not in source_by_identity: + continue + display_name = manifest.get("display_name", source_type) for op in manifest.get("search_operators", []): operator = op.get("operator") @@ -217,6 +243,11 @@ async def _fetch_actions(self) -> list[ConnectorAction]: for connector in connectors: source_type = connector.get("source_type", "") manifest = connector.get("manifest") + integration_type = ( + manifest.get("integration_type", "connector") + if manifest + else "connector" + ) if not manifest or not connector.get("healthy"): continue @@ -224,8 +255,10 @@ async def _fetch_actions(self) -> list[ConnectorAction]: action_source_types = action_def.get("source_types") or [] if action_source_types and source_type not in action_source_types: continue - # Find matching active sources for this connector type - for source in source_by_type.get(source_type, []): + # Find matching active sources for this integration/source type. + for source in source_by_identity.get( + (integration_type, source_type), [] + ): actions.append( ConnectorAction( source_id=source.id, @@ -239,6 +272,7 @@ async def _fetch_actions(self) -> list[ConnectorAction]: mode=action_def.get("mode", "write"), admin_only=action_def.get("admin_only", False), hidden=action_def.get("hidden", False), + integration_type=integration_type, ) ) @@ -384,6 +418,15 @@ async def check_oauth_required( if user_credential is not None: return None + source_row = await conn.fetchrow( + """ + SELECT integration_type, config + FROM sources + WHERE id = $1 + LIMIT 1 + """, + action.source_id, + ) org_credential = await conn.fetchrow( """ SELECT provider @@ -394,10 +437,21 @@ async def check_oauth_required( action.source_id, ) - if org_credential is None: - return None + if source_row and source_row["integration_type"] == "remote_mcp": + config = source_row["config"] or {} + if isinstance(config, str): + try: + config = json.loads(config) + except json.JSONDecodeError: + config = {} + if not isinstance(config, dict) or config.get("auth_type") != "oauth": + return None + provider = "remote_mcp" + else: + if org_credential is None: + return None + provider = org_credential["provider"] - provider = org_credential["provider"] if not provider: return None diff --git a/services/ai/tools/mcp_capability_handler.py b/services/ai/tools/mcp_capability_handler.py index c25d2bc95..fc9e2325b 100644 --- a/services/ai/tools/mcp_capability_handler.py +++ b/services/ai/tools/mcp_capability_handler.py @@ -16,6 +16,7 @@ from db.models import Source from tools.connector_handler import SourceFilter, sources_from_sync_overview_response +from tools.omni_tool_result import OAuthRequiredPayload, encode_oauth_required from tools.registry import ToolContext, ToolResult from tools.searcher_client import ( CapabilitiesSyncRequest, @@ -73,6 +74,7 @@ class McpPromptDefinition(BaseModel): class McpConnectorManifest(BaseModel): model_config = ConfigDict(extra="ignore") + integration_type: str = "connector" mcp_enabled: bool = False resources: list[McpResourceDefinition] = Field(default_factory=list) prompts: list[McpPromptDefinition] = Field(default_factory=list) @@ -135,6 +137,7 @@ def __init__( self._source_filter = source_filter self._resources: dict[str, McpResourceRecord] = {} self._prompts: dict[str, McpPromptRecord] = {} + self._publisher_source_ids: set[str] = set() self._initialized = False async def refresh(self) -> None: @@ -155,7 +158,7 @@ async def refresh(self) -> None: sources = self._prefetched_sources else: sources_resp = await client.get( - f"{self._connector_manager_url}/sources" + f"{self._connector_manager_url}/sources/active" ) sources_resp.raise_for_status() sources = sources_from_sync_overview_response(sources_resp.json()) @@ -164,16 +167,19 @@ async def refresh(self) -> None: self._initialized = True return - active_sources_by_type: dict[str, list[Source]] = {} + active_sources_by_identity: dict[tuple[str, str], list[Source]] = {} for source in sources: if not source.is_active or source.is_deleted: continue if not self._source_allows_read(source.id): continue - active_sources_by_type.setdefault(source.source_type, []).append(source) + active_sources_by_identity.setdefault( + (source.integration_type, source.source_type), [] + ).append(source) resources: dict[str, McpResourceRecord] = {} prompts: dict[str, McpPromptRecord] = {} + publisher_source_ids: set[str] = set() for connector in connectors: if not connector.healthy or connector.manifest is None: @@ -183,7 +189,12 @@ async def refresh(self) -> None: if not manifest.mcp_enabled: continue - matching_sources = active_sources_by_type.get(source_type, []) + matching_sources = active_sources_by_identity.get( + (manifest.integration_type, source_type), [] + ) + for source in matching_sources: + publisher_source_ids.add(source.id) + if not matching_sources: continue @@ -203,6 +214,7 @@ async def refresh(self) -> None: self._resources = resources self._prompts = prompts + self._publisher_source_ids = publisher_source_ids self._initialized = True def has_capabilities(self) -> bool: @@ -328,13 +340,15 @@ async def execute( if tool_name == "resource_search": return await self._resource_search(tool_input) if tool_name == "load_resource": - return await self._load_resource(tool_input) + return await self._load_resource(tool_input, context) if tool_name == "prompt_search": return await self._prompt_search(tool_input) if tool_name == "load_prompt": - return await self._load_prompt(tool_input) + return await self._load_prompt(tool_input, context) return ToolResult( - content=[{"type": "text", "text": f"Unknown MCP capability tool: {tool_name}"}], + content=[ + {"type": "text", "text": f"Unknown MCP capability tool: {tool_name}"} + ], is_error=True, ) @@ -344,12 +358,11 @@ async def publish_capabilities(self) -> None: return capabilities = self._capabilities() - if not capabilities: - return - publish_key = ( id(self._searcher_client), - self._capability_fingerprint(capabilities), + self._capability_fingerprint(capabilities) + + "|publishers=" + + ",".join(sorted(self._publisher_source_ids)), ) if publish_key in self._published_capability_keys: return @@ -362,17 +375,18 @@ async def publish_capabilities(self) -> None: for capability in capabilities: if not capability.source_id: continue - grouped.setdefault((capability.source_id, capability.capability_type), []).append( - capability - ) - for (publisher_id, capability_type), group in grouped.items(): - await self._searcher_client.sync_capabilities( - CapabilitiesSyncRequest( - publisher_id=publisher_id, - capability_type=capability_type, - capabilities=group, + grouped.setdefault( + (capability.source_id, capability.capability_type), [] + ).append(capability) + for publisher_id in self._publisher_source_ids: + for capability_type in ("resource", "prompt"): + await self._searcher_client.sync_capabilities( + CapabilitiesSyncRequest( + publisher_id=publisher_id, + capability_type=capability_type, + capabilities=grouped.get((publisher_id, capability_type), []), + ) ) - ) except Exception as e: logger.warning(f"Failed to publish MCP capabilities: {e}") return @@ -387,24 +401,39 @@ async def _resource_search(self, tool_input: dict) -> ToolResult: ) if not _TOKEN_RE.findall(query.lower()): return ToolResult( - content=[{"type": "text", "text": f"No searchable tokens in query: {query!r}"}], + content=[ + { + "type": "text", + "text": f"No searchable tokens in query: {query!r}", + } + ], is_error=True, ) limit = self._parse_limit(tool_input.get("limit")) matches = await self._search_capabilities("resource", query, limit) if not matches: - return ToolResult(content=[{"type": "text", "text": f"No MCP resources matched {query!r}."}]) + return ToolResult( + content=[ + {"type": "text", "text": f"No MCP resources matched {query!r}."} + ] + ) lines = [f"Found {len(matches)} MCP resource(s) matching {query!r}:"] for record in matches: assert isinstance(record, McpResourceRecord) desc = f" — {record.description}" if record.description else "" - uri_note = " (URI template; pass concrete uri to load_resource)" if record.requires_uri else "" + uri_note = ( + " (URI template; pass concrete uri to load_resource)" + if record.requires_uri + else "" + ) mime = f" · {record.mime_type}" if record.mime_type else "" lines.append( f"- {record.id}: {record.name}{desc} [{record.source_name}/{record.source_type}] uri_template={record.uri_template!r}{mime}{uri_note}" ) - lines.append("Call load_resource with the exact resource_id to read a resource.") + lines.append( + "Call load_resource with the exact resource_id to read a resource." + ) return ToolResult(content=[{"type": "text", "text": "\n".join(lines)}]) async def _prompt_search(self, tool_input: dict) -> ToolResult: @@ -416,13 +445,20 @@ async def _prompt_search(self, tool_input: dict) -> ToolResult: ) if not _TOKEN_RE.findall(query.lower()): return ToolResult( - content=[{"type": "text", "text": f"No searchable tokens in query: {query!r}"}], + content=[ + { + "type": "text", + "text": f"No searchable tokens in query: {query!r}", + } + ], is_error=True, ) limit = self._parse_limit(tool_input.get("limit")) matches = await self._search_capabilities("prompt", query, limit) if not matches: - return ToolResult(content=[{"type": "text", "text": f"No MCP prompts matched {query!r}."}]) + return ToolResult( + content=[{"type": "text", "text": f"No MCP prompts matched {query!r}."}] + ) lines = [f"Found {len(matches)} MCP prompt(s) matching {query!r}:"] for record in matches: @@ -433,7 +469,9 @@ async def _prompt_search(self, tool_input: dict) -> ToolResult: lines.append( f"- {record.id}: {record.name}{desc} [{record.source_name}/{record.source_type}]{arg_note}" ) - lines.append("Call load_prompt with the exact prompt_id and any required arguments.") + lines.append( + "Call load_prompt with the exact prompt_id and any required arguments." + ) return ToolResult(content=[{"type": "text", "text": "\n".join(lines)}]) async def _search_capabilities( @@ -465,17 +503,26 @@ async def _search_capabilities( matches.append(record) return matches - async def _load_resource(self, tool_input: dict) -> ToolResult: + async def _load_resource( + self, tool_input: dict, context: ToolContext + ) -> ToolResult: resource_id = (tool_input.get("resource_id") or "").strip() if not resource_id: return ToolResult( - content=[{"type": "text", "text": "Missing required parameter: resource_id"}], + content=[ + {"type": "text", "text": "Missing required parameter: resource_id"} + ], is_error=True, ) record = self._resources.get(resource_id) if record is None: return ToolResult( - content=[{"type": "text", "text": f"Unknown or inaccessible MCP resource: {resource_id}"}], + content=[ + { + "type": "text", + "text": f"Unknown or inaccessible MCP resource: {resource_id}", + } + ], is_error=True, ) @@ -499,20 +546,34 @@ async def _load_resource(self, tool_input: dict) -> ToolResult: tool_input.get("start_line"), tool_input.get("end_line") ) if line_error: - return ToolResult(content=[{"type": "text", "text": line_error}], is_error=True) + return ToolResult( + content=[{"type": "text", "text": line_error}], is_error=True + ) try: async with httpx.AsyncClient(timeout=60.0) as client: response = await client.post( f"{self._connector_manager_url}/resource", - json={"source_id": record.source_id, "uri": read_uri}, + json={ + "source_id": record.source_id, + "user_id": context.user_id, + "uri": read_uri, + }, ) + oauth_required = self._oauth_required_result(response, record) + if oauth_required is not None: + return oauth_required response.raise_for_status() payload = response.json() except Exception as e: logger.warning(f"Failed to load MCP resource {resource_id}: {e}") return ToolResult( - content=[{"type": "text", "text": f"Failed to load MCP resource: {resource_id}"}], + content=[ + { + "type": "text", + "text": f"Failed to load MCP resource: {resource_id}", + } + ], is_error=True, ) @@ -527,17 +588,24 @@ async def _load_resource(self, tool_input: dict) -> ToolResult: ] ) - async def _load_prompt(self, tool_input: dict) -> ToolResult: + async def _load_prompt(self, tool_input: dict, context: ToolContext) -> ToolResult: prompt_id = (tool_input.get("prompt_id") or "").strip() if not prompt_id: return ToolResult( - content=[{"type": "text", "text": "Missing required parameter: prompt_id"}], + content=[ + {"type": "text", "text": "Missing required parameter: prompt_id"} + ], is_error=True, ) record = self._prompts.get(prompt_id) if record is None: return ToolResult( - content=[{"type": "text", "text": f"Unknown or inaccessible MCP prompt: {prompt_id}"}], + content=[ + { + "type": "text", + "text": f"Unknown or inaccessible MCP prompt: {prompt_id}", + } + ], is_error=True, ) @@ -545,13 +613,20 @@ async def _load_prompt(self, tool_input: dict) -> ToolResult: arguments = raw_arguments if isinstance(raw_arguments, dict) else {} if raw_arguments is not None and not isinstance(raw_arguments, dict): return ToolResult( - content=[{"type": "text", "text": "Prompt arguments must be an object."}], + content=[ + {"type": "text", "text": "Prompt arguments must be an object."} + ], is_error=True, ) missing = self._missing_required_arguments(record.arguments, arguments) if missing: return ToolResult( - content=[{"type": "text", "text": f"Missing required prompt argument(s): {', '.join(missing)}"}], + content=[ + { + "type": "text", + "text": f"Missing required prompt argument(s): {', '.join(missing)}", + } + ], is_error=True, ) @@ -561,16 +636,22 @@ async def _load_prompt(self, tool_input: dict) -> ToolResult: f"{self._connector_manager_url}/prompt", json={ "source_id": record.source_id, + "user_id": context.user_id, "name": record.name, "arguments": arguments or None, }, ) + oauth_required = self._oauth_required_result(response, record) + if oauth_required is not None: + return oauth_required response.raise_for_status() payload = response.json() except Exception as e: logger.warning(f"Failed to load MCP prompt {prompt_id}: {e}") return ToolResult( - content=[{"type": "text", "text": f"Failed to load MCP prompt: {prompt_id}"}], + content=[ + {"type": "text", "text": f"Failed to load MCP prompt: {prompt_id}"} + ], is_error=True, ) @@ -583,6 +664,43 @@ async def _load_prompt(self, tool_input: dict) -> ToolResult: ] ) + def _oauth_required_result( + self, response: httpx.Response, record: McpResourceRecord | McpPromptRecord + ) -> ToolResult | None: + if response.status_code != 412: + return None + body = response.json() + provider = body.get("provider") + oauth_start_url = body.get("oauth_start_url") + if not provider or not oauth_start_url: + logger.error( + "connector-manager 412 missing provider/oauth_start_url for MCP capability; body=%s", + body, + ) + return ToolResult( + content=[ + { + "type": "text", + "text": ( + "This MCP capability requires authorization, but the OAuth " + "start URL was not provided by connector-manager." + ), + } + ], + is_error=True, + ) + payload = OAuthRequiredPayload( + source_id=record.source_id, + source_type=record.source_type, + provider=provider, + oauth_start_url=oauth_start_url, + ) + return ToolResult( + content=[encode_oauth_required(payload)], + is_error=False, + oauth_required=payload, + ) + def _capabilities(self) -> list[CapabilityUpsert]: capabilities: list[CapabilityUpsert] = [] for record in self._resources.values(): @@ -675,7 +793,10 @@ def _prompt_record( def _source_allows_read(self, source_id: str) -> bool: if self._source_filter is None: return True - return source_id in self._source_filter and "read" in self._source_filter[source_id] + return ( + source_id in self._source_filter + and "read" in self._source_filter[source_id] + ) @staticmethod def _short_hash(value: str) -> str: @@ -764,8 +885,14 @@ def _format_resource_result( header.append(f"description: {record.description}") sections: list[str] = ["\n".join(header)] - text_items = [item for item in contents if isinstance(item, dict) and isinstance(item.get("text"), str)] - blob_items = [item for item in contents if isinstance(item, dict) and "blob" in item] + text_items = [ + item + for item in contents + if isinstance(item, dict) and isinstance(item.get("text"), str) + ] + blob_items = [ + item for item in contents if isinstance(item, dict) and "blob" in item + ] if not text_items: if blob_items: @@ -780,7 +907,11 @@ def _format_resource_result( for index, item in enumerate(text_items, start=1): text = item["text"] item_uri = item.get("uri") if isinstance(item.get("uri"), str) else read_uri - mime_type = item.get("mime_type") if isinstance(item.get("mime_type"), str) else record.mime_type + mime_type = ( + item.get("mime_type") + if isinstance(item.get("mime_type"), str) + else record.mime_type + ) total_bytes = len(text.encode("utf-8")) lines = text.split("\n") total_lines = len(lines) @@ -840,7 +971,11 @@ def _format_prompt_result(self, record: McpPromptRecord, payload: Any) -> str: messages: list[Any] = [] if isinstance(payload, dict): raw_description = payload.get("description") - description = raw_description if isinstance(raw_description, str) else record.description + description = ( + raw_description + if isinstance(raw_description, str) + else record.description + ) raw_messages = payload.get("messages") messages = raw_messages if isinstance(raw_messages, list) else [] diff --git a/services/connector-manager/src/handlers.rs b/services/connector-manager/src/handlers.rs index 754e76ff4..ed7aace26 100644 --- a/services/connector-manager/src/handlers.rs +++ b/services/connector-manager/src/handlers.rs @@ -24,8 +24,8 @@ use serde_json::{json, Value}; use shared::clients::docling::{DoclingClient, DoclingError}; use shared::db::repositories::{ConfigurationRepository, SyncRunRepository}; use shared::models::{ - ActionMode, ConnectorManifest, GlobalConfiguration, SearchOperator, ServiceCredential, - ServiceProvider, Source, SourceType, SyncRun, SyncType, + ActionMode, ConnectorManifest, GlobalConfiguration, IntegrationType, SearchOperator, + ServiceCredential, ServiceProvider, Source, SourceType, SyncRun, SyncType, }; use shared::queue::EventQueue; use shared::utils; @@ -183,6 +183,10 @@ pub async fn list_schedules( .find_active_sources() .await .map_err(|e| ApiError::Internal(e.to_string()))?; + let sources: Vec = sources + .into_iter() + .filter(source_supports_data_sync) + .collect(); let source_ids: Vec = sources.iter().map(|s| s.id.clone()).collect(); let latest_runs = sync_run_repo @@ -210,10 +214,7 @@ pub async fn list_schedules( ScheduleInfo { source_id: source.id, source_name: source.name, - source_type: serde_json::to_value(&source.source_type) - .ok() - .and_then(|v| v.as_str().map(String::from)) - .unwrap_or_default(), + source_type: source.source_type, sync_interval_seconds: source.sync_interval_seconds, next_sync_at: next_sync_at.map(|t| t.to_string()), last_sync_at: last_sync_at.map(|t| t.to_string()), @@ -239,7 +240,28 @@ pub async fn list_sources( .await .map_err(|e| ApiError::Internal(e.to_string()))?; - Ok(Json(build_source_sync_overviews(&state, sources).await?)) + let syncable_sources = sources + .into_iter() + .filter(source_supports_data_sync) + .collect(); + Ok(Json( + build_source_sync_overviews(&state, syncable_sources).await?, + )) +} + +pub async fn list_active_sources_for_capabilities( + State(state): State, +) -> Result>, ApiError> { + let source_repo = SourceRepository::new(state.db_pool.pool()); + let sources = source_repo + .find_active_sources() + .await + .map_err(|e| ApiError::Internal(e.to_string()))? + .into_iter() + .filter(|source| !source.is_deleted) + .collect(); + + Ok(Json(build_source_identity_overviews(sources))) } pub async fn get_source( @@ -252,6 +274,7 @@ pub async fn get_source( .await .map_err(|e| ApiError::Internal(e.to_string()))? .filter(|source| !source.is_deleted) + .filter(source_supports_data_sync) .ok_or_else(|| ApiError::NotFound(format!("Source not found: {}", source_id)))?; let mut overviews = build_source_sync_overviews(&state, vec![source]).await?; @@ -262,6 +285,25 @@ pub async fn get_source( Ok(Json(overview)) } +fn source_supports_data_sync(source: &Source) -> bool { + source.integration_type == IntegrationType::Connector +} + +fn build_source_identity_overviews(sources: Vec) -> Vec { + sources + .into_iter() + .map(|source| SourceSyncOverview { + sync_runs: Vec::new(), + source: Source { + connector_state: None, + checkpoint: None, + ..source + }, + health: SourceHealth::Healthy, + }) + .collect() +} + async fn build_source_sync_overviews( state: &AppState, sources: Vec, @@ -334,7 +376,9 @@ pub async fn list_connectors( for manifest in manifests { let url = manifest.connector_url.clone(); - let healthy = if !url.is_empty() { + let healthy = if remote_mcp_in_process_manifest_is_healthy(&manifest) { + true + } else if !url.is_empty() { client.health_check(&url).await } else { false @@ -353,6 +397,10 @@ pub async fn list_connectors( Ok(Json(connectors)) } +fn remote_mcp_in_process_manifest_is_healthy(manifest: &ConnectorManifest) -> bool { + manifest.integration_type == IntegrationType::RemoteMcp && manifest.connector_url.is_empty() +} + pub async fn execute_action( State(state): State, _headers: HeaderMap, @@ -386,10 +434,11 @@ pub async fn execute_action( ApiError::BadRequest("user_id is required in transient mode".to_string()) })?; + // Look up the connector manifest by source_type. let manifest = manifests .iter() - .find(|m| m.source_types.contains(&source_type)) + .find(|m| m.source_types.contains(&source_type.to_string())) .ok_or_else(|| { ApiError::NotFound(format!( "Connector not registered for type: {:?}", @@ -424,6 +473,11 @@ pub async fn execute_action( ))); } + // Resolve user/admin from user_id (required in transient mode). + let user_id = request.user_id.as_ref().ok_or_else(|| { + ApiError::BadRequest("user_id is required in transient mode".to_string()) + })?; + let user_repo = UserRepository::new(state.db_pool.pool()); let user = user_repo .find_by_id(user_id.clone()) @@ -483,11 +537,41 @@ pub async fn execute_action( .map_err(|e| ApiError::Internal(e.to_string()))? .ok_or_else(|| ApiError::NotFound(format!("Source not found: {source_id}")))?; - source_type = db_source.source_type; + source_type = SourceType::try_from(db_source.source_type.as_str()) + .map_err(|e| ApiError::Internal(format!("Invalid source type: {}", e)))?; + + if db_source.integration_type == IntegrationType::RemoteMcp { + if !db_source.is_active || db_source.is_deleted { + return Err(ApiError::BadRequest(format!( + "Remote MCP source is inactive or deleted: {}", + source_id + ))); + } + let result = state + .remote_mcp_gateway + .execute_action( + &db_source, + &request.action, + request.params.clone(), + request.user_id.as_deref(), + ) + .await + .map_err(remote_mcp_gateway_error_to_api_error)?; + let body = + serde_json::to_vec(&result).map_err(|e| ApiError::Internal(e.to_string()))?; + return axum::response::Response::builder() + .status(StatusCode::OK) + .header( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ) + .body(axum::body::Body::from(body)) + .map_err(|e| ApiError::Internal(e.to_string())); + } let manifest = manifests .iter() - .find(|m| m.source_types.contains(&source_type)) + .find(|m| m.source_types.contains(&source_type.to_string())) .ok_or_else(|| { ApiError::NotFound(format!( "Connector not registered for type: {:?}", @@ -540,7 +624,7 @@ pub async fn execute_action( { CredentialResolution::Resolved(c) => c, CredentialResolution::NeedsUserAuth { provider } => { - return Ok(needs_user_auth_response(&source_id, source_type, provider)?); + return Ok(needs_user_auth_response(&source_id, source_type.to_string(), provider)?); } CredentialResolution::NoCredentials => { return Err(ApiError::NotFound(format!( @@ -840,14 +924,58 @@ async fn resolve_credentials( struct NeedsUserAuthResponse { error: &'static str, source_id: String, - source_type: SourceType, + source_type: String, provider: ServiceProvider, oauth_start_url: String, } +fn remote_mcp_gateway_error_to_api_error( + err: crate::remote_mcp::gateway::GatewayError, +) -> ApiError { + match err { + crate::remote_mcp::gateway::GatewayError::NeedsUserAuth { + source_id, + source_type, + provider, + } => ApiError::PreconditionFailedJson(json!({ + "error": "needs_user_auth", + "source_id": source_id, + "source_type": source_type, + "provider": provider, + "oauth_start_url": format!("/api/oauth/start?source_id={}", source_id), + })), + crate::remote_mcp::gateway::GatewayError::MissingCredentials(source_id) => { + ApiError::NotFound(format!("Credentials not found for source: {source_id}")) + } + crate::remote_mcp::gateway::GatewayError::SourceInactive(source_id) => { + ApiError::BadRequest(format!( + "Remote MCP source is inactive or deleted: {source_id}" + )) + } + crate::remote_mcp::gateway::GatewayError::SourceNotFound(source_id) => { + ApiError::NotFound(format!("Source not found: {source_id}")) + } + crate::remote_mcp::gateway::GatewayError::NotRemoteMcp(source_id) => { + ApiError::BadRequest(format!("Source is not remote MCP: {source_id}")) + } + crate::remote_mcp::gateway::GatewayError::InvalidConfig { source_id, message } => { + ApiError::BadRequest(format!( + "Invalid remote MCP config for {source_id}: {message}" + )) + } + crate::remote_mcp::gateway::GatewayError::UnsupportedAuthType { + source_id, + auth_type, + } => ApiError::BadRequest(format!( + "Unsupported remote MCP auth type for {source_id}: {auth_type:?}" + )), + other => ApiError::Internal(other.to_string()), + } +} + fn needs_user_auth_response( source_id: &str, - source_type: SourceType, + source_type: String, provider: ServiceProvider, ) -> Result { let body = NeedsUserAuthResponse { @@ -895,7 +1023,12 @@ pub async fn list_actions( if (manifest.read_only || source_read_only) && action.mode == ActionMode::Write { continue; } - if !action.source_types.is_empty() && !action.source_types.contains(source_type) { + if !action.source_types.is_empty() + && !action + .source_types + .iter() + .any(|action_source_type| action_source_type.as_str() == source_type) + { continue; } if action.hidden { @@ -983,7 +1116,43 @@ pub async fn read_resource( .map_err(|e| ApiError::Internal(e.to_string()))? .ok_or_else(|| ApiError::NotFound(format!("Source not found: {}", request.source_id)))?; - let connector_url = get_connector_url_for_source(&state.redis_client, source.source_type) + if source.integration_type == IntegrationType::RemoteMcp { + if !source.is_active || source.is_deleted { + return Err(ApiError::BadRequest(format!( + "Remote MCP source is inactive or deleted: {}", + source.id + ))); + } + let manifests = get_registered_manifests(&state.redis_client).await; + let manifest = manifests.iter().find(|m| { + m.integration_type == source.integration_type + && m.source_types.contains(&source.source_type) + }); + if !manifest + .map(|m| { + m.resources.iter().any(|resource| { + request.uri == resource.uri_template + || request + .uri + .starts_with(resource.uri_template.trim_end_matches('*')) + }) + }) + .unwrap_or(false) + { + return Err(ApiError::NotFound(format!( + "Resource not advertised: {}", + request.uri + ))); + } + let result = state + .remote_mcp_gateway + .read_resource(&source, &request.uri, request.user_id.as_deref()) + .await + .map_err(remote_mcp_gateway_error_to_api_error)?; + return Ok(Json(result)); + } + + let connector_url = get_connector_url_for_source(&state.redis_client, &source.source_type) .await .ok_or_else(|| { ApiError::NotFound(format!( @@ -1035,7 +1204,41 @@ pub async fn get_prompt( .map_err(|e| ApiError::Internal(e.to_string()))? .ok_or_else(|| ApiError::NotFound(format!("Source not found: {}", request.source_id)))?; - let connector_url = get_connector_url_for_source(&state.redis_client, source.source_type) + if source.integration_type == IntegrationType::RemoteMcp { + if !source.is_active || source.is_deleted { + return Err(ApiError::BadRequest(format!( + "Remote MCP source is inactive or deleted: {}", + source.id + ))); + } + let manifests = get_registered_manifests(&state.redis_client).await; + let manifest = manifests.iter().find(|m| { + m.integration_type == source.integration_type + && m.source_types.contains(&source.source_type) + }); + if !manifest + .map(|m| m.prompts.iter().any(|prompt| prompt.name == request.name)) + .unwrap_or(false) + { + return Err(ApiError::NotFound(format!( + "Prompt not advertised: {}", + request.name + ))); + } + let result = state + .remote_mcp_gateway + .get_prompt( + &source, + &request.name, + request.arguments.clone(), + request.user_id.as_deref(), + ) + .await + .map_err(remote_mcp_gateway_error_to_api_error)?; + return Ok(Json(result)); + } + + let connector_url = get_connector_url_for_source(&state.redis_client, &source.source_type) .await .ok_or_else(|| { ApiError::NotFound(format!( @@ -1092,7 +1295,29 @@ pub async fn oauth_credential_ready( .map_err(|e| ApiError::Internal(e.to_string()))? .ok_or_else(|| ApiError::NotFound(format!("Source not found: {}", request.source_id)))?; - let connector_url = get_connector_url_for_source(&state.redis_client, source.source_type) + if source.integration_type == IntegrationType::RemoteMcp { + if request.user_id.is_none() { + match state.remote_mcp_gateway.refresh_catalog(&source.id).await { + Ok(manifest) => { + return Ok(Json(json!({ + "status": "completed", + "catalog_updated": true, + "connector_id": manifest.connector_id, + }))); + } + Err(err) => { + warn!(source_id = %source.id, error = %err, "remote MCP OAuth credential-ready catalog refresh failed"); + return Err(remote_mcp_gateway_error_to_api_error(err)); + } + } + } + return Ok(Json(json!({ + "status": "delivered", + "catalog_updated": false, + }))); + } + + let connector_url = get_connector_url_for_source(&state.redis_client, &source.source_type) .await .ok_or_else(|| { ApiError::NotFound(format!( @@ -1202,12 +1427,16 @@ pub async fn list_skills( for manifest in manifests { for skill in &manifest.skills { - let source_types = if skill.source_types.is_empty() { - &manifest.source_types + let source_types: Vec = if skill.source_types.is_empty() { + manifest.source_types.clone() } else { - &skill.source_types + skill + .source_types + .iter() + .map(|source_type| source_type.to_string()) + .collect() }; - for source_type in source_types { + for source_type in &source_types { let matching_sources: Vec<_> = sources .iter() .filter(|source| source.source_type == *source_type) @@ -1328,6 +1557,9 @@ pub enum ApiError { #[error("Payload too large: {0}")] PayloadTooLarge(String), + #[error("Precondition failed")] + PreconditionFailedJson(Value), + #[error("Too many requests: {message} (retry after {retry_after_secs}s)")] TooManyRequests { message: String, @@ -1350,6 +1582,9 @@ impl From for ApiError { SyncError::SourceInactive(id) => { ApiError::BadRequest(format!("Source is inactive: {}", id)) } + SyncError::SourceDoesNotSync(id) => { + ApiError::BadRequest(format!("Source does not support data sync: {}", id)) + } SyncError::SyncAlreadyRunning(id) => { ApiError::Conflict(format!("Sync already running for source: {}", id)) } @@ -1399,6 +1634,9 @@ impl IntoResponse for ApiError { ApiError::Conflict(msg) => (StatusCode::CONFLICT, msg.clone()), ApiError::Internal(msg) => (StatusCode::INTERNAL_SERVER_ERROR, msg.clone()), ApiError::PayloadTooLarge(msg) => (StatusCode::PAYLOAD_TOO_LARGE, msg.clone()), + ApiError::PreconditionFailedJson(body) => { + return (StatusCode::PRECONDITION_FAILED, Json(body.clone())).into_response(); + } ApiError::TooManyRequests { .. } => unreachable!(), }; @@ -1558,9 +1796,7 @@ pub async fn sdk_register( .map_err(|e| ApiError::Internal(format!("Failed to store registration: {}", e)))?; // Aggregate search operators from all registered connectors - let keys: Vec = redis::cmd("KEYS") - .arg("connector:manifest:*") - .query_async(&mut conn) + let keys = scan_redis_keys(&mut conn, "connector:manifest:*") .await .unwrap_or_default(); @@ -1667,6 +1903,30 @@ pub async fn sdk_register( })) } +async fn scan_redis_keys( + conn: &mut redis::aio::MultiplexedConnection, + pattern: &str, +) -> redis::RedisResult> { + let mut cursor: u64 = 0; + let mut keys = Vec::new(); + loop { + let (next_cursor, batch): (u64, Vec) = redis::cmd("SCAN") + .cursor_arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(100) + .query_async(conn) + .await?; + keys.extend(batch); + if next_cursor == 0 { + break; + } + cursor = next_cursor; + } + Ok(keys) +} + /// Scan Redis for all registered connector manifests. pub async fn get_registered_manifests(redis_client: &redis::Client) -> Vec { let mut conn = match redis_client.get_multiplexed_async_connection().await { @@ -1677,9 +1937,7 @@ pub async fn get_registered_manifests(redis_client: &redis::Client) -> Vec = redis::cmd("KEYS") - .arg("connector:manifest:*") - .query_async(&mut conn) + let keys = scan_redis_keys(&mut conn, "connector:manifest:*") .await .unwrap_or_default(); @@ -1697,11 +1955,15 @@ pub async fn get_registered_manifests(redis_client: &redis::Client) -> Vec Option { let manifests = get_registered_manifests(redis_client).await; for manifest in manifests { - if manifest.source_types.contains(&source_type) { + if manifest + .source_types + .iter() + .any(|manifest_source_type| manifest_source_type == source_type) + { return Some(manifest.connector_url); } } @@ -1712,10 +1974,14 @@ pub async fn get_connector_url_for_source( /// Returns an empty vec when no connector is registered for the source_type. pub async fn get_sync_modes_for_source( redis_client: &redis::Client, - source_type: SourceType, + source_type: &str, ) -> Vec { for manifest in get_registered_manifests(redis_client).await { - if manifest.source_types.contains(&source_type) { + if manifest + .source_types + .iter() + .any(|manifest_source_type| manifest_source_type == source_type) + { return manifest.sync_modes; } } @@ -2577,12 +2843,19 @@ pub async fn sdk_get_source_sync_config( .map(|c| c.credentials) .unwrap_or_else(|| serde_json::json!({})); + let source_type = SourceType::try_from(source.source_type.as_str()).map_err(|e| { + ApiError::BadRequest(format!( + "Source {} cannot be served to a native connector SDK: {}", + source.id, e + )) + })?; + Ok(Json(SdkSourceSyncConfigResponse { config: source.config, credentials, connector_state: source.connector_state, checkpoint: source.checkpoint, - source_type: source.source_type, + source_type, user_filter_mode: source.user_filter_mode, user_whitelist: source.user_whitelist, user_blacklist: source.user_blacklist, @@ -2819,7 +3092,8 @@ mod tests { sync_modes: vec![SyncType::Full], connector_id: "test-connector".to_string(), connector_url: "http://test-connector:4000".to_string(), - source_types: vec![SourceType::Notion], + integration_type: shared::models::IntegrationType::Connector, + source_types: vec![SourceType::Notion.to_string()], description: None, actions: vec![shared::models::ActionDefinition { name: "export_data_source_csv".to_string(), @@ -2843,6 +3117,18 @@ mod tests { } } + #[test] + fn remote_mcp_in_process_manifest_is_healthy_without_connector_url() { + let mut manifest = manifest_with_action_schema(json!({})); + manifest.integration_type = IntegrationType::RemoteMcp; + manifest.connector_url = String::new(); + + assert!(remote_mcp_in_process_manifest_is_healthy(&manifest)); + + manifest.integration_type = IntegrationType::Connector; + assert!(!remote_mcp_in_process_manifest_is_healthy(&manifest)); + } + #[test] fn test_validate_action_schema_accepts_provider_safe_object_schema() { let manifest = manifest_with_action_schema(json!({ diff --git a/services/connector-manager/src/lib.rs b/services/connector-manager/src/lib.rs index ccb0d9f18..0205edb15 100644 --- a/services/connector-manager/src/lib.rs +++ b/services/connector-manager/src/lib.rs @@ -2,6 +2,7 @@ pub mod config; pub mod connector_client; pub mod handlers; pub mod models; +pub mod remote_mcp; pub mod scheduler; pub mod source_cleanup; pub mod sync_circuit_breaker; @@ -16,6 +17,7 @@ use axum::{ }; use config::ConnectorManagerConfig; use redis::Client as RedisClient; +use remote_mcp::{gateway::RemoteMcpGateway, registry}; use shared::{ telemetry::{self, TelemetryConfig}, DatabasePool, ObjectStorage, @@ -34,6 +36,7 @@ pub struct AppState { pub redis_client: RedisClient, pub config: ConnectorManagerConfig, pub sync_manager: Arc, + pub remote_mcp_gateway: Arc, pub content_storage: Arc, pub extraction_semaphore: Arc, } @@ -48,6 +51,10 @@ pub fn create_app(state: AppState) -> Router { .route("/sync/:id/progress", get(handlers::get_sync_progress)) .route("/schedules", get(handlers::list_schedules)) .route("/sources", get(handlers::list_sources)) + .route( + "/sources/active", + get(handlers::list_active_sources_for_capabilities), + ) .route("/sources/:source_id", get(handlers::get_source)) .route("/connectors", get(handlers::list_connectors)) .route("/action", post(handlers::execute_action)) @@ -155,12 +162,17 @@ pub async fn run_server() -> AnyhowResult<()> { config.clone(), redis_client.clone(), )); + let remote_mcp_gateway = Arc::new(RemoteMcpGateway::new( + db_pool.clone(), + redis_client.clone(), + )?); let app_state = AppState { db_pool: db_pool.clone(), redis_client: redis_client.clone(), config: config.clone(), sync_manager: sync_manager.clone(), + remote_mcp_gateway: remote_mcp_gateway.clone(), content_storage, extraction_semaphore: Arc::new(Semaphore::new(config.extraction_concurrency)), }; @@ -171,6 +183,14 @@ pub async fn run_server() -> AnyhowResult<()> { warn!("Startup sync reconciliation failed: {}", e); } + registry::startup_register_remote_mcp_sources((*remote_mcp_gateway).clone(), db_pool.clone()) + .await; + tokio::spawn(registry::run_remote_mcp_registry_loop( + (*remote_mcp_gateway).clone(), + db_pool.clone(), + )); + info!("Remote MCP registry started"); + tokio::spawn(scheduler::Scheduler::run( db_pool.pool().clone(), redis_client, diff --git a/services/connector-manager/src/models.rs b/services/connector-manager/src/models.rs index 0f4cac67d..775c7f332 100644 --- a/services/connector-manager/src/models.rs +++ b/services/connector-manager/src/models.rs @@ -57,7 +57,7 @@ pub struct ScheduleInfo { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConnectorInfo { - pub source_type: SourceType, + pub source_type: String, pub url: String, pub healthy: bool, #[serde(skip_serializing_if = "Option::is_none")] @@ -281,12 +281,16 @@ pub struct SdkWebhookResponse { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExecuteResourceRequest { pub source_id: String, + #[serde(default)] + pub user_id: Option, pub uri: String, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExecutePromptRequest { pub source_id: String, + #[serde(default)] + pub user_id: Option, pub name: String, #[serde(default)] pub arguments: Option, diff --git a/services/connector-manager/src/remote_mcp/gateway.rs b/services/connector-manager/src/remote_mcp/gateway.rs new file mode 100644 index 000000000..62e078402 --- /dev/null +++ b/services/connector-manager/src/remote_mcp/gateway.rs @@ -0,0 +1,1618 @@ +use crate::models::ConnectorManifest; +use crate::remote_mcp::oauth::{parse_oauth_config, usable_oauth_credential, OAuthError}; +use futures::StreamExt; +use redis::AsyncCommands; +use reqwest::{Client, Url}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value as JsonValue}; +use shared::db::repositories::ServiceCredentialsRepo; +use shared::models::{ + ActionDefinition, ActionMode, AuthType, IntegrationType, McpPromptArgument, + McpPromptDefinition, McpResourceDefinition, ServiceCredential, ServiceProvider, Source, +}; +use shared::{traits::Repository, DatabasePool}; +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; +use thiserror::Error; +use tokio::sync::Mutex; +use tracing::warn; + +pub const REMOTE_MCP_CONNECTOR_ID_PREFIX: &str = "remote_mcp:"; +pub const REMOTE_MCP_MANIFEST_TTL_SECONDS: u64 = 300; +const REMOTE_MCP_CONNECT_TIMEOUT_SECONDS: u64 = 5; +const REMOTE_MCP_READ_TIMEOUT_SECONDS: u64 = 20; +const REMOTE_MCP_OVERALL_TIMEOUT_SECONDS: u64 = 30; +const MAX_MCP_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +const MAX_MCP_CATALOG_ITEMS: usize = 200; +const MAX_MCP_SCHEMA_BYTES: usize = 64 * 1024; +const MAX_MCP_STRING_BYTES: usize = 8 * 1024; + +static REMOTE_MCP_OAUTH_REFRESH_LOCKS: OnceLock>>>> = + OnceLock::new(); + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RemoteMcpConfig { + pub endpoint_url: String, + #[serde(default)] + pub auth_type: Option, + #[serde(default = "default_write_tools_enabled")] + pub write_tools_enabled: bool, +} + +fn default_write_tools_enabled() -> bool { + true +} + +#[derive(Debug, Error)] +pub enum GatewayError { + #[error("source not found: {0}")] + SourceNotFound(String), + #[error("source {0} is not a remote MCP source")] + NotRemoteMcp(String), + #[error("invalid remote MCP config for source {source_id}: {message}")] + InvalidConfig { source_id: String, message: String }, + #[error("unsupported remote MCP auth type for source {source_id}: {auth_type:?}")] + UnsupportedAuthType { + source_id: String, + auth_type: AuthType, + }, + #[error("remote MCP endpoint resolves to a disallowed address: {0}")] + DisallowedAddress(String), + #[error("database error: {0}")] + Database(#[from] shared::db::error::DatabaseError), + #[error("redis error: {0}")] + Redis(#[from] redis::RedisError), + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + #[error("serialization error: {0}")] + Serde(#[from] serde_json::Error), + #[error("remote MCP protocol error: {0}")] + Protocol(String), + #[error("remote MCP source {source_id} requires per-user OAuth credentials")] + NeedsUserAuth { + source_id: String, + source_type: String, + provider: ServiceProvider, + }, + #[error("missing runtime credentials for remote MCP source {0}")] + MissingCredentials(String), + #[error("remote MCP source is inactive or deleted: {0}")] + SourceInactive(String), +} + +#[derive(Clone)] +pub struct RemoteMcpGateway { + db_pool: DatabasePool, + redis_client: redis::Client, + http_client: Client, +} + +impl RemoteMcpGateway { + pub fn new(db_pool: DatabasePool, redis_client: redis::Client) -> Result { + let http_client = remote_mcp_client_builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; + Ok(Self { + db_pool, + redis_client, + http_client, + }) + } + + pub(crate) fn redis_client_for_registry(&self) -> redis::Client { + self.redis_client.clone() + } + + pub async fn discover_and_register( + &self, + source_id: &str, + ) -> Result { + let source = self.load_source(source_id).await?; + let config = parse_remote_mcp_config(&source)?; + let manifest = self.discover_manifest(&source, &config).await?; + self.publish_manifest(&manifest).await?; + Ok(manifest) + } + + pub async fn refresh_catalog( + &self, + source_id: &str, + ) -> Result { + self.discover_and_register(source_id).await + } + + pub async fn execute_action( + &self, + source: &Source, + action: &str, + params: JsonValue, + user_id: Option<&str>, + ) -> Result { + ensure_active_remote_mcp_source(source)?; + let config = parse_remote_mcp_config(source)?; + let headers = self.runtime_auth_headers(source, &config, user_id).await?; + let session_id = self + .initialize_session(&config.endpoint_url, &headers) + .await?; + let response = self + .json_rpc( + &config.endpoint_url, + session_id.as_deref(), + &headers, + json!({ + "jsonrpc": "2.0", + "id": 10, + "method": "tools/call", + "params": { + "name": action, + "arguments": params, + }, + }), + ) + .await?; + Ok(response + .body + .get("result") + .cloned() + .unwrap_or(response.body)) + } + + pub async fn read_resource( + &self, + source: &Source, + uri: &str, + user_id: Option<&str>, + ) -> Result { + ensure_active_remote_mcp_source(source)?; + let config = parse_remote_mcp_config(source)?; + let headers = self.runtime_auth_headers(source, &config, user_id).await?; + let session_id = self + .initialize_session(&config.endpoint_url, &headers) + .await?; + let response = self + .json_rpc( + &config.endpoint_url, + session_id.as_deref(), + &headers, + json!({ + "jsonrpc": "2.0", + "id": 11, + "method": "resources/read", + "params": { "uri": uri }, + }), + ) + .await?; + Ok(response + .body + .get("result") + .cloned() + .unwrap_or(response.body)) + } + + pub async fn get_prompt( + &self, + source: &Source, + name: &str, + arguments: Option, + user_id: Option<&str>, + ) -> Result { + ensure_active_remote_mcp_source(source)?; + let config = parse_remote_mcp_config(source)?; + let headers = self.runtime_auth_headers(source, &config, user_id).await?; + let session_id = self + .initialize_session(&config.endpoint_url, &headers) + .await?; + let response = self + .json_rpc( + &config.endpoint_url, + session_id.as_deref(), + &headers, + json!({ + "jsonrpc": "2.0", + "id": 12, + "method": "prompts/get", + "params": { + "name": name, + "arguments": arguments.unwrap_or_else(|| json!({})), + }, + }), + ) + .await?; + Ok(response + .body + .get("result") + .cloned() + .unwrap_or(response.body)) + } + + pub async fn remove_manifest_for_source_type( + &self, + source_type: &str, + ) -> Result<(), GatewayError> { + let key = manifest_key(source_type); + let mut conn = self.redis_client.get_multiplexed_async_connection().await?; + let _: () = conn.del(key).await?; + Ok(()) + } + + async fn load_source(&self, source_id: &str) -> Result { + let repo = shared::db::repositories::SourceRepository::new(self.db_pool.pool()); + repo.find_by_id(source_id.to_string()) + .await? + .ok_or_else(|| GatewayError::SourceNotFound(source_id.to_string())) + } + + async fn discover_manifest( + &self, + source: &Source, + config: &RemoteMcpConfig, + ) -> Result { + validate_endpoint_for_gateway(&config.endpoint_url).await?; + let mut session_id = None; + let headers = self.discovery_auth_headers(source, config).await?; + + let initialize = self + .json_rpc( + &config.endpoint_url, + session_id.as_deref(), + &headers, + json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { "name": "omni-connector-manager", "version": env!("CARGO_PKG_VERSION") } + } + }), + ) + .await?; + session_id = initialize.session_id; + let server_info = initialize + .body + .pointer("/result/serverInfo") + .cloned() + .unwrap_or(json!({})); + let version = server_info + .get("version") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let _ = self + .json_rpc( + &config.endpoint_url, + session_id.as_deref(), + &headers, + json!({"jsonrpc":"2.0","method":"notifications/initialized","params":{}}), + ) + .await; + + let actions = self + .discover_actions( + &config.endpoint_url, + session_id.as_deref(), + &headers, + config, + ) + .await + .unwrap_or_else(|e| { + warn!(source_id = %source.id, error = %e, "remote MCP tools/list failed"); + Vec::new() + }); + let resources = self + .discover_resources(&config.endpoint_url, session_id.as_deref(), &headers) + .await + .unwrap_or_else(|e| { + warn!(source_id = %source.id, error = %e, "remote MCP resources discovery failed"); + Vec::new() + }); + let prompts = self + .discover_prompts(&config.endpoint_url, session_id.as_deref(), &headers) + .await + .unwrap_or_else(|e| { + warn!(source_id = %source.id, error = %e, "remote MCP prompts/list failed"); + Vec::new() + }); + + let oauth = if config.auth_type == Some(AuthType::OAuth) { + self.discover_oauth_metadata(&config.endpoint_url, &source.source_type) + .await + .unwrap_or_else(|e| { + warn!(source_id = %source.id, error = %e, "remote MCP OAuth metadata discovery failed"); + None + }) + } else { + None + }; + + Ok(build_manifest( + source, config, version, actions, resources, prompts, oauth, + )) + } + + async fn publish_manifest(&self, manifest: &ConnectorManifest) -> Result<(), GatewayError> { + let manifest_json = serde_json::to_string(manifest)?; + let mut conn = self.redis_client.get_multiplexed_async_connection().await?; + let _: () = conn + .set_ex( + manifest_key(&manifest.source_types[0]), + manifest_json, + REMOTE_MCP_MANIFEST_TTL_SECONDS, + ) + .await?; + Ok(()) + } + + async fn initialize_session( + &self, + endpoint_url: &str, + headers: &[(String, String)], + ) -> Result, GatewayError> { + let initialize = self + .json_rpc( + endpoint_url, + None, + headers, + json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { "name": "omni-connector-manager", "version": env!("CARGO_PKG_VERSION") } + } + }), + ) + .await?; + let session_id = initialize.session_id; + let _ = self + .json_rpc( + endpoint_url, + session_id.as_deref(), + headers, + json!({"jsonrpc":"2.0","method":"notifications/initialized","params":{}}), + ) + .await; + Ok(session_id) + } + + async fn discovery_auth_headers( + &self, + source: &Source, + config: &RemoteMcpConfig, + ) -> Result, GatewayError> { + match config.auth_type { + None => Ok(Vec::new()), + Some(AuthType::BearerToken) => { + let repo = + ServiceCredentialsRepo::new(self.db_pool.pool().clone()).map_err(|e| { + GatewayError::Protocol(format!( + "failed to initialize credential repository: {e}" + )) + })?; + let credential = repo.find_org_credential(&source.id).await.map_err(|e| { + GatewayError::Protocol(format!("failed to load bearer credential: {e}")) + })?; + let token = credential + .and_then(|c| { + c.credentials + .get("token") + .and_then(|v| v.as_str()) + .map(str::to_owned) + }) + .ok_or_else(|| { + GatewayError::Protocol("missing bearer token credential".to_string()) + })?; + Ok(vec![( + "authorization".to_string(), + format!("Bearer {token}"), + )]) + } + Some(AuthType::OAuth) => { + let repo = + ServiceCredentialsRepo::new(self.db_pool.pool().clone()).map_err(|e| { + GatewayError::Protocol(format!( + "failed to initialize credential repository: {e}" + )) + })?; + let credential = repo.find_org_credential(&source.id).await.map_err(|e| { + GatewayError::Protocol(format!( + "failed to load OAuth bootstrap credential: {e}" + )) + })?; + let credential = match credential { + Some(credential) => { + let oauth_value = self + .discover_oauth_metadata(&config.endpoint_url, &source.source_type) + .await? + .ok_or_else(|| { + GatewayError::Protocol( + "missing OAuth metadata for bootstrap credential refresh" + .to_string(), + ) + })?; + let oauth = parse_oauth_config(&oauth_value).map_err(|e| { + GatewayError::Protocol(format!("invalid OAuth metadata: {e}")) + })?; + Some( + self.usable_oauth_credential_serialized(source, credential, &oauth) + .await?, + ) + } + None => None, + }; + let token = credential + .and_then(|c| { + c.credentials + .get("access_token") + .and_then(|v| v.as_str()) + .map(str::to_owned) + }) + .ok_or_else(|| { + GatewayError::Protocol("missing OAuth bootstrap access_token".to_string()) + })?; + Ok(vec![( + "authorization".to_string(), + format!("Bearer {token}"), + )]) + } + Some(auth_type) => Err(GatewayError::UnsupportedAuthType { + source_id: source.id.clone(), + auth_type, + }), + } + } + + async fn runtime_auth_headers( + &self, + source: &Source, + config: &RemoteMcpConfig, + user_id: Option<&str>, + ) -> Result, GatewayError> { + let repo = match config.auth_type { + None => return Ok(Vec::new()), + Some(AuthType::BearerToken | AuthType::OAuth) => { + ServiceCredentialsRepo::new(self.db_pool.pool().clone()).map_err(|e| { + GatewayError::Protocol(format!( + "failed to initialize credential repository: {e}" + )) + })? + } + Some(auth_type) => { + return Err(GatewayError::UnsupportedAuthType { + source_id: source.id.clone(), + auth_type, + }); + } + }; + + let org_credential = if config.auth_type == Some(AuthType::BearerToken) { + repo.find_org_credential(&source.id).await.map_err(|e| { + GatewayError::Protocol(format!("failed to load org credential: {e}")) + })? + } else { + None + }; + let mut user_credential = match (config.auth_type, user_id) { + (Some(AuthType::OAuth), Some(uid)) => repo + .find_user_credential(&source.id, uid) + .await + .map_err(|e| { + GatewayError::Protocol(format!("failed to load user credential: {e}")) + })?, + _ => None, + }; + if config.auth_type == Some(AuthType::OAuth) { + if let Some(credential) = user_credential.take() { + let oauth_value = self + .discover_oauth_metadata(&config.endpoint_url, &source.source_type) + .await? + .ok_or_else(|| { + GatewayError::Protocol( + "missing OAuth metadata for credential refresh".to_string(), + ) + })?; + let oauth = parse_oauth_config(&oauth_value) + .map_err(|e| GatewayError::Protocol(format!("invalid OAuth metadata: {e}")))?; + user_credential = Some( + self.usable_oauth_credential_serialized(source, credential, &oauth) + .await?, + ); + } + } + runtime_auth_headers_from_credentials( + config, + &source.id, + &source.source_type, + user_id, + org_credential.as_ref(), + user_credential.as_ref(), + ) + } + + async fn usable_oauth_credential_serialized( + &self, + source: &Source, + credential: ServiceCredential, + oauth: &crate::remote_mcp::oauth::RemoteMcpOAuthConfig, + ) -> Result { + let key = format!( + "{}:{}", + source.id, + credential.user_id.as_deref().unwrap_or("__org__") + ); + let locks = REMOTE_MCP_OAUTH_REFRESH_LOCKS.get_or_init(|| Mutex::new(HashMap::new())); + let lock = { + let mut guard = locks.lock().await; + guard + .entry(key) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + }; + let _guard = lock.lock().await; + + let repo = ServiceCredentialsRepo::new(self.db_pool.pool().clone()).map_err(|e| { + GatewayError::Protocol(format!("failed to create credential repo: {e}")) + })?; + let credential = repo + .find_by_id(&credential.id) + .await + .map_err(|e| GatewayError::Protocol(format!("failed to reload credential: {e}")))? + .ok_or_else(|| GatewayError::MissingCredentials(source.id.clone()))?; + + usable_oauth_credential(&self.db_pool, &self.http_client, source, credential, oauth) + .await + .map_err(|e| oauth_error_to_gateway_error(source, e)) + } + + async fn discover_oauth_metadata( + &self, + endpoint_url: &str, + source_type: &str, + ) -> Result, GatewayError> { + let endpoint = Url::parse(endpoint_url) + .map_err(|e| GatewayError::Protocol(format!("invalid endpoint URL: {e}")))?; + let mut candidates = Vec::new(); + if endpoint.path() != "/" { + candidates.push(format!( + "{}/.well-known/oauth-protected-resource{}", + endpoint.origin().ascii_serialization(), + endpoint.path() + )); + } + candidates.push(format!( + "{}/.well-known/oauth-protected-resource", + endpoint.origin().ascii_serialization() + )); + + for candidate in candidates { + let response = pinned_http_client_for_url(&candidate) + .await? + .get(&candidate) + .header("accept", "application/json") + .send() + .await + .map_err(|e| GatewayError::Protocol(format!("OAuth metadata fetch failed: {e}")))?; + if !response.status().is_success() { + continue; + } + let prm_text = read_limited_response_text(response).await?; + let prm: JsonValue = serde_json::from_str(&prm_text).map_err(|e| { + GatewayError::Protocol(format!("OAuth metadata JSON parse failed: {e}")) + })?; + let auth_server = prm + .get("authorization_servers") + .and_then(|v| v.as_array()) + .and_then(|servers| servers.iter().find_map(|v| v.as_str())) + .ok_or_else(|| { + GatewayError::Protocol( + "OAuth protected resource metadata has no authorization server".to_string(), + ) + })?; + let auth_url = Url::parse(auth_server).map_err(|e| { + GatewayError::Protocol(format!("invalid authorization server URL: {e}")) + })?; + let auth_metadata_url = format!( + "{}/.well-known/oauth-authorization-server{}", + auth_url.origin().ascii_serialization(), + if auth_url.path() == "/" { + "" + } else { + auth_url.path() + } + ); + let as_response = pinned_http_client_for_url(&auth_metadata_url) + .await? + .get(&auth_metadata_url) + .header("accept", "application/json") + .send() + .await + .map_err(|e| { + GatewayError::Protocol(format!("authorization metadata fetch failed: {e}")) + })?; + if !as_response.status().is_success() { + continue; + } + let as_text = read_limited_response_text(as_response).await?; + let as_metadata: JsonValue = serde_json::from_str(&as_text).map_err(|e| { + GatewayError::Protocol(format!("authorization metadata JSON parse failed: {e}")) + })?; + let auth_endpoint = as_metadata + .get("authorization_endpoint") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + GatewayError::Protocol("missing authorization_endpoint".to_string()) + })?; + validate_endpoint_for_gateway(auth_endpoint).await?; + let token_endpoint = as_metadata + .get("token_endpoint") + .and_then(|v| v.as_str()) + .ok_or_else(|| GatewayError::Protocol("missing token_endpoint".to_string()))?; + validate_endpoint_for_gateway(token_endpoint).await?; + let userinfo_endpoint = match as_metadata + .get("userinfo_endpoint") + .and_then(|v| v.as_str()) + { + Some(value) => { + validate_endpoint_for_gateway(value).await?; + value.to_string() + } + None => endpoint.origin().ascii_serialization(), + }; + let registration_endpoint = match as_metadata + .get("registration_endpoint") + .and_then(|v| v.as_str()) + { + Some(value) => { + validate_endpoint_for_gateway(value).await?; + Some(value.to_string()) + } + None => None, + }; + let resource = match prm.get("resource").and_then(|v| v.as_str()) { + Some(value) => { + validate_endpoint_for_gateway(value).await?; + value.to_string() + } + None => endpoint_url.to_string(), + }; + let scopes = as_metadata + .get("scopes_supported") + .and_then(|v| v.as_array()) + .map(|values| values.iter().filter_map(|v| v.as_str()).collect::>()) + .unwrap_or_default(); + let token_auth_method = as_metadata + .get("token_endpoint_auth_methods_supported") + .and_then(|v| v.as_array()) + .and_then(|values| { + values + .iter() + .find_map(|v| (v.as_str() == Some("none")).then_some("none")) + }) + .unwrap_or("client_secret_post"); + return Ok(Some(json!({ + "provider": format!("{REMOTE_MCP_CONNECTOR_ID_PREFIX}{source_type}"), + "credential_provider": "remote_mcp", + "auth_endpoint": auth_endpoint, + "token_endpoint": token_endpoint, + "userinfo_endpoint": userinfo_endpoint, + "userinfo_email_field": "email", + "identity_scopes": [], + "scopes": { source_type: { "read": scopes, "write": scopes } }, + "extra_auth_params": {}, + "scope_separator": " ", + "registration_endpoint": registration_endpoint, + "token_endpoint_auth_method": token_auth_method, + "resource": resource, + "protected_resource_metadata_url": candidate, + "authorization_server_metadata_url": auth_metadata_url, + }))); + } + Ok(None) + } + + async fn json_rpc( + &self, + endpoint_url: &str, + session_id: Option<&str>, + headers: &[(String, String)], + body: JsonValue, + ) -> Result { + let http_client = pinned_http_client_for_url(endpoint_url).await?; + let mut request = http_client + .post(endpoint_url) + .header("accept", "application/json, text/event-stream") + .header("content-type", "application/json"); + if let Some(session_id) = session_id { + request = request.header("mcp-session-id", session_id); + } + for (name, value) in headers { + request = request.header(name, value); + } + let response = request.json(&body).send().await?; + let next_session_id = response + .headers() + .get("mcp-session-id") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned) + .or_else(|| session_id.map(str::to_owned)); + if !response.status().is_success() { + return Err(GatewayError::Protocol(format!( + "MCP HTTP request failed with status {}", + response.status() + ))); + } + let content_type = response + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + let text = read_limited_response_text(response).await?; + let body = parse_mcp_response_body(&text, &content_type)?; + if let Some(error) = body.get("error") { + return Err(GatewayError::Protocol(format!( + "MCP JSON-RPC error: {error}" + ))); + } + Ok(JsonRpcResponse { + body, + session_id: next_session_id, + }) + } + + async fn discover_actions( + &self, + endpoint_url: &str, + session_id: Option<&str>, + headers: &[(String, String)], + config: &RemoteMcpConfig, + ) -> Result, GatewayError> { + let response = self + .json_rpc( + endpoint_url, + session_id, + headers, + json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}), + ) + .await?; + let tools = response + .body + .pointer("/result/tools") + .and_then(|v| v.as_array()); + Ok(tools + .into_iter() + .flatten() + .take(MAX_MCP_CATALOG_ITEMS) + .filter_map(|tool| action_from_tool(tool, config.write_tools_enabled)) + .collect()) + } + + async fn discover_resources( + &self, + endpoint_url: &str, + session_id: Option<&str>, + headers: &[(String, String)], + ) -> Result, GatewayError> { + let mut resources = Vec::new(); + let listed = self + .json_rpc( + endpoint_url, + session_id, + headers, + json!({"jsonrpc":"2.0","id":3,"method":"resources/list","params":{}}), + ) + .await?; + if let Some(items) = listed + .body + .pointer("/result/resources") + .and_then(|v| v.as_array()) + { + resources.extend( + items + .iter() + .take(MAX_MCP_CATALOG_ITEMS.saturating_sub(resources.len())) + .filter_map(resource_from_value), + ); + } + if let Ok(templates) = self + .json_rpc( + endpoint_url, + session_id, + headers, + json!({"jsonrpc":"2.0","id":4,"method":"resources/templates/list","params":{}}), + ) + .await + { + if let Some(items) = templates + .body + .pointer("/result/resourceTemplates") + .and_then(|v| v.as_array()) + { + resources.extend( + items + .iter() + .take(MAX_MCP_CATALOG_ITEMS.saturating_sub(resources.len())) + .filter_map(resource_from_value), + ); + } + } + Ok(resources) + } + + async fn discover_prompts( + &self, + endpoint_url: &str, + session_id: Option<&str>, + headers: &[(String, String)], + ) -> Result, GatewayError> { + let response = self + .json_rpc( + endpoint_url, + session_id, + headers, + json!({"jsonrpc":"2.0","id":5,"method":"prompts/list","params":{}}), + ) + .await?; + let prompts = response + .body + .pointer("/result/prompts") + .and_then(|v| v.as_array()); + Ok(prompts + .into_iter() + .flatten() + .take(MAX_MCP_CATALOG_ITEMS) + .filter_map(prompt_from_value) + .collect()) + } +} + +struct JsonRpcResponse { + body: JsonValue, + session_id: Option, +} + +pub fn parse_remote_mcp_config(source: &Source) -> Result { + if source.integration_type != IntegrationType::RemoteMcp { + return Err(GatewayError::NotRemoteMcp(source.id.clone())); + } + let config: RemoteMcpConfig = + serde_json::from_value(source.config.clone()).map_err(|e| GatewayError::InvalidConfig { + source_id: source.id.clone(), + message: e.to_string(), + })?; + Url::parse(&config.endpoint_url) + .map_err(|e| GatewayError::InvalidConfig { + source_id: source.id.clone(), + message: e.to_string(), + }) + .and_then(|url| { + if url.scheme() != "https" && url.scheme() != "http" { + Err(GatewayError::InvalidConfig { + source_id: source.id.clone(), + message: "endpoint_url must use http or https".to_string(), + }) + } else if url.username() != "" || url.password().is_some() || url.fragment().is_some() { + Err(GatewayError::InvalidConfig { + source_id: source.id.clone(), + message: "endpoint_url must not include credentials or fragments".to_string(), + }) + } else { + Ok(()) + } + })?; + if let Some(auth_type) = config.auth_type { + if !matches!(auth_type, AuthType::BearerToken | AuthType::OAuth) { + return Err(GatewayError::UnsupportedAuthType { + source_id: source.id.clone(), + auth_type, + }); + } + } + Ok(config) +} + +pub fn manifest_key(source_type: &str) -> String { + format!("connector:manifest:{REMOTE_MCP_CONNECTOR_ID_PREFIX}{source_type}") +} + +fn authorization_header(token: &str) -> Vec<(String, String)> { + vec![("authorization".to_string(), format!("Bearer {token}"))] +} + +pub fn runtime_auth_headers_from_credentials( + config: &RemoteMcpConfig, + source_id: &str, + source_type: &str, + user_id: Option<&str>, + org_credential: Option<&ServiceCredential>, + user_credential: Option<&ServiceCredential>, +) -> Result, GatewayError> { + match config.auth_type { + None => Ok(Vec::new()), + Some(AuthType::BearerToken) => { + let token = org_credential + .and_then(|credential| credential.credentials.get("token")) + .and_then(|value| value.as_str()) + .ok_or_else(|| GatewayError::MissingCredentials(source_id.to_string()))?; + Ok(authorization_header(token)) + } + Some(AuthType::OAuth) => { + let Some(_uid) = user_id else { + return Err(GatewayError::MissingCredentials(source_id.to_string())); + }; + let token = user_credential + .and_then(|credential| credential.credentials.get("access_token")) + .and_then(|value| value.as_str()) + .ok_or_else(|| GatewayError::NeedsUserAuth { + source_id: source_id.to_string(), + source_type: source_type.to_string(), + provider: ServiceProvider::RemoteMcp, + })?; + Ok(authorization_header(token)) + } + Some(auth_type) => Err(GatewayError::UnsupportedAuthType { + source_id: source_id.to_string(), + auth_type, + }), + } +} + +pub fn build_manifest( + source: &Source, + config: &RemoteMcpConfig, + version: String, + actions: Vec, + resources: Vec, + prompts: Vec, + oauth: Option, +) -> ConnectorManifest { + ConnectorManifest { + name: source.source_type.clone(), + display_name: source.name.clone(), + version, + sync_modes: Vec::new(), + connector_id: format!("{REMOTE_MCP_CONNECTOR_ID_PREFIX}{}", source.source_type), + connector_url: String::new(), + integration_type: IntegrationType::RemoteMcp, + source_types: vec![source.source_type.clone()], + description: None, + actions, + search_operators: Vec::new(), + read_only: !config.write_tools_enabled, + extra_schema: None, + attributes_schema: None, + mcp_enabled: true, + mcp_catalog_loaded: true, + resources, + prompts, + skills: Vec::new(), + oauth: oauth.or_else(|| { + config.auth_type.filter(|a| *a == AuthType::OAuth).map(|_| { + json!({ + "provider": format!("{REMOTE_MCP_CONNECTOR_ID_PREFIX}{}", source.source_type), + "credential_provider": "remote_mcp", + }) + }) + }), + } +} + +fn action_from_tool(tool: &JsonValue, write_tools_enabled: bool) -> Option { + if serde_json::to_vec(tool).ok()?.len() > MAX_MCP_SCHEMA_BYTES { + return None; + } + let name = bounded_string(tool.get("name")?.as_str()?)?; + let read_only = tool + .pointer("/annotations/readOnlyHint") + .or_else(|| tool.pointer("/annotations/read_only_hint")) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let mode = if read_only { + ActionMode::Read + } else { + ActionMode::Write + }; + if mode == ActionMode::Write && !write_tools_enabled { + return None; + } + Some(ActionDefinition { + name, + description: bounded_string( + tool.get("description") + .and_then(|v| v.as_str()) + .unwrap_or_default(), + ) + .unwrap_or_default(), + input_schema: bounded_schema( + tool.get("inputSchema") + .or_else(|| tool.get("input_schema")) + .cloned() + .unwrap_or_else(|| json!({"type":"object","properties":{}})), + )?, + mode, + source_types: Vec::new(), + admin_only: false, + hidden: false, + }) +} + +fn resource_from_value(value: &JsonValue) -> Option { + if serde_json::to_vec(value).ok()?.len() > MAX_MCP_SCHEMA_BYTES { + return None; + } + let uri_template = bounded_string( + value + .get("uriTemplate") + .or_else(|| value.get("uri_template")) + .or_else(|| value.get("uri"))? + .as_str()?, + )?; + Some(McpResourceDefinition { + uri_template, + name: bounded_string(value.get("name")?.as_str()?)?, + description: value + .get("description") + .and_then(|v| v.as_str()) + .and_then(bounded_string), + mime_type: value + .get("mimeType") + .or_else(|| value.get("mime_type")) + .and_then(|v| v.as_str()) + .and_then(bounded_string), + }) +} + +fn prompt_from_value(value: &JsonValue) -> Option { + if serde_json::to_vec(value).ok()?.len() > MAX_MCP_SCHEMA_BYTES { + return None; + } + let arguments = value + .get("arguments") + .and_then(|v| v.as_array()) + .map(|args| { + args.iter() + .take(MAX_MCP_CATALOG_ITEMS) + .filter_map(|arg| { + Some(McpPromptArgument { + name: bounded_string(arg.get("name")?.as_str()?)?, + description: arg + .get("description") + .and_then(|v| v.as_str()) + .and_then(bounded_string), + required: arg + .get("required") + .and_then(|v| v.as_bool()) + .unwrap_or(false), + }) + }) + .collect() + }) + .unwrap_or_default(); + Some(McpPromptDefinition { + name: bounded_string(value.get("name")?.as_str()?)?, + description: value + .get("description") + .and_then(|v| v.as_str()) + .and_then(bounded_string), + arguments, + }) +} + +fn bounded_string(value: &str) -> Option { + (value.len() <= MAX_MCP_STRING_BYTES).then(|| value.to_string()) +} + +fn bounded_schema(value: JsonValue) -> Option { + (serde_json::to_vec(&value).ok()?.len() <= MAX_MCP_SCHEMA_BYTES).then_some(value) +} + +pub(crate) async fn read_limited_response_text( + response: reqwest::Response, +) -> Result { + if let Some(length) = response.content_length() { + if length > MAX_MCP_RESPONSE_BYTES as u64 { + return Err(GatewayError::Protocol(format!( + "MCP response exceeded {} bytes", + MAX_MCP_RESPONSE_BYTES + ))); + } + } + let mut body = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if body.len().saturating_add(chunk.len()) > MAX_MCP_RESPONSE_BYTES { + return Err(GatewayError::Protocol(format!( + "MCP response exceeded {} bytes", + MAX_MCP_RESPONSE_BYTES + ))); + } + body.extend_from_slice(&chunk); + } + String::from_utf8(body) + .map_err(|e| GatewayError::Protocol(format!("MCP response was not UTF-8: {e}"))) +} + +fn oauth_error_to_gateway_error(source: &Source, error: OAuthError) -> GatewayError { + match error { + OAuthError::ReconnectRequired => GatewayError::NeedsUserAuth { + source_id: source.id.clone(), + source_type: source.source_type.clone(), + provider: ServiceProvider::RemoteMcp, + }, + other => GatewayError::Protocol(format!("OAuth refresh failed: {other}")), + } +} + +fn parse_mcp_response_body(text: &str, content_type: &str) -> Result { + if content_type.contains("text/event-stream") { + let event_data = text + .lines() + .filter_map(|line| line.strip_prefix("data:")) + .map(str::trim) + .find(|line| !line.is_empty() && *line != "[DONE]") + .ok_or_else(|| GatewayError::Protocol("empty MCP event stream".to_string()))?; + Ok(serde_json::from_str(event_data)?) + } else if text.trim().is_empty() { + Ok(json!({})) + } else { + Ok(serde_json::from_str(text)?) + } +} + +pub(crate) async fn validate_endpoint_for_gateway(endpoint_url: &str) -> Result<(), GatewayError> { + let _ = validated_remote_mcp_addrs(endpoint_url).await?; + Ok(()) +} + +async fn validated_remote_mcp_addrs( + endpoint_url: &str, +) -> Result<(Url, String, Vec), GatewayError> { + let url = Url::parse(endpoint_url) + .map_err(|e| GatewayError::Protocol(format!("invalid endpoint URL: {e}")))?; + if url.scheme() != "https" && url.scheme() != "http" { + return Err(GatewayError::Protocol( + "endpoint URL must use http or https".to_string(), + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(GatewayError::Protocol( + "endpoint URL must not include credentials".to_string(), + )); + } + if url.fragment().is_some() { + return Err(GatewayError::Protocol( + "endpoint URL must not include a fragment".to_string(), + )); + } + let host = url + .host_str() + .ok_or_else(|| GatewayError::Protocol("endpoint URL has no host".to_string()))? + .to_string(); + let addrs: Vec = + tokio::net::lookup_host((host.as_str(), url.port_or_known_default().unwrap_or(443))) + .await + .map_err(|e| GatewayError::Protocol(format!("endpoint DNS lookup failed: {e}")))? + .collect(); + if addrs.is_empty() { + return Err(GatewayError::Protocol( + "endpoint host did not resolve".to_string(), + )); + } + for addr in &addrs { + let ip = addr.ip(); + if is_disallowed_ip(ip) { + return Err(GatewayError::DisallowedAddress(ip.to_string())); + } + } + Ok((url, host, addrs)) +} + +fn remote_mcp_client_builder() -> reqwest::ClientBuilder { + Client::builder() + .connect_timeout(Duration::from_secs(REMOTE_MCP_CONNECT_TIMEOUT_SECONDS)) + .read_timeout(Duration::from_secs(REMOTE_MCP_READ_TIMEOUT_SECONDS)) + .timeout(Duration::from_secs(REMOTE_MCP_OVERALL_TIMEOUT_SECONDS)) +} + +pub(crate) async fn pinned_http_client_for_url(endpoint_url: &str) -> Result { + let (_url, host, addrs) = validated_remote_mcp_addrs(endpoint_url).await?; + remote_mcp_client_builder() + .redirect(reqwest::redirect::Policy::none()) + .resolve_to_addrs(&host, &addrs) + .build() + .map_err(GatewayError::Http) +} + +fn ensure_active_remote_mcp_source(source: &Source) -> Result<(), GatewayError> { + if source.integration_type != IntegrationType::RemoteMcp { + return Err(GatewayError::NotRemoteMcp(source.id.clone())); + } + if !source.is_active || source.is_deleted { + return Err(GatewayError::SourceInactive(source.id.clone())); + } + Ok(()) +} + +fn is_disallowed_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => is_disallowed_ipv4(ip), + IpAddr::V6(ip) => { + if let Some(mapped) = ip.to_ipv4_mapped() { + return is_disallowed_ipv4(mapped); + } + ip.is_loopback() + || ip.is_multicast() + || ip.is_unspecified() + || ((ip.segments()[0] & 0xfe00) == 0xfc00) + || ((ip.segments()[0] & 0xffc0) == 0xfe80) + || ip.segments()[0] == 0x2002 + || (ip.segments()[0] == 0x2001 && ip.segments()[1] == 0x0db8) + } + } +} + +fn is_disallowed_ipv4(ip: std::net::Ipv4Addr) -> bool { + ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_multicast() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.octets()[0] == 0 + || ip.octets()[0] >= 224 + || (ip.octets()[0] == 100 && (64..=127).contains(&ip.octets()[1])) + || (ip.octets()[0] == 192 && ip.octets()[1] == 0 && ip.octets()[2] == 0) + || (ip.octets()[0] == 192 && ip.octets()[1] == 88 && ip.octets()[2] == 99) + || (ip.octets()[0] == 198 && (18..=19).contains(&ip.octets()[1])) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use shared::models::{SourceScope, UserFilterMode}; + use time::OffsetDateTime; + + fn credential( + user_id: Option<&str>, + auth_type: AuthType, + credentials: JsonValue, + ) -> ServiceCredential { + ServiceCredential { + id: format!("cred_{}", user_id.unwrap_or("org")), + source_id: "src_1".to_string(), + user_id: user_id.map(str::to_owned), + provider: ServiceProvider::RemoteMcp, + auth_type, + principal_email: None, + credentials, + config: json!({}), + expires_at: None, + last_validated_at: None, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + } + } + + fn source(config: JsonValue) -> Source { + Source { + id: "src_1".to_string(), + name: "Acme MCP".to_string(), + source_type: "acme".to_string(), + integration_type: IntegrationType::RemoteMcp, + config, + is_active: true, + is_deleted: false, + scope: SourceScope::Org, + user_filter_mode: UserFilterMode::All, + user_whitelist: None, + user_blacklist: None, + connector_state: None, + checkpoint: None, + sync_interval_seconds: None, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + created_by: "admin".to_string(), + } + } + + #[test] + fn oversized_catalog_entries_are_ignored() { + let oversized = "x".repeat(MAX_MCP_SCHEMA_BYTES + 1); + assert!(action_from_tool( + &json!({"name":"huge","inputSchema":{"description": oversized}}), + true + ) + .is_none()); + assert!(resource_from_value( + &json!({"name":"huge","uri":"file://huge","description": oversized}) + ) + .is_none()); + assert!(prompt_from_value(&json!({"name":"huge","description": oversized})).is_none()); + } + + #[test] + fn oauth_reconnect_maps_to_needs_user_auth() { + let source = source(json!({"endpoint_url":"https://mcp.example.com/mcp"})); + let err = oauth_error_to_gateway_error(&source, OAuthError::ReconnectRequired); + assert!(matches!( + err, + GatewayError::NeedsUserAuth { source_id, source_type, provider } + if source_id == "src_1" && source_type == "acme" && provider == ServiceProvider::RemoteMcp + )); + } + + #[test] + fn runtime_dispatch_rejects_inactive_or_deleted_remote_mcp_sources() { + let mut inactive = source(json!({"endpoint_url":"https://mcp.example.com/mcp"})); + inactive.is_active = false; + assert!(matches!( + ensure_active_remote_mcp_source(&inactive), + Err(GatewayError::SourceInactive(id)) if id == "src_1" + )); + + let mut deleted = source(json!({"endpoint_url":"https://mcp.example.com/mcp"})); + deleted.is_deleted = true; + assert!(matches!( + ensure_active_remote_mcp_source(&deleted), + Err(GatewayError::SourceInactive(id)) if id == "src_1" + )); + } + + #[test] + fn parses_remote_mcp_config_and_rejects_unsupported_auth() { + let valid = parse_remote_mcp_config(&source(json!({ + "endpoint_url": "https://mcp.example.com/mcp", + "auth_type": "bearer_token", + "write_tools_enabled": false + }))) + .unwrap(); + assert_eq!(valid.auth_type, Some(AuthType::BearerToken)); + assert!(!valid.write_tools_enabled); + + let err = parse_remote_mcp_config(&source(json!({ + "endpoint_url": "https://mcp.example.com/mcp", + "auth_type": "api_key" + }))) + .unwrap_err(); + assert!(matches!(err, GatewayError::UnsupportedAuthType { .. })); + } + + #[test] + fn manifest_uses_normal_source_type_identity_and_no_sync_modes() { + let cfg = RemoteMcpConfig { + endpoint_url: "https://mcp.example.com/mcp".to_string(), + auth_type: None, + write_tools_enabled: true, + }; + let manifest = build_manifest( + &source(json!({})), + &cfg, + "1.2.3".to_string(), + vec![], + vec![], + vec![], + None, + ); + assert_eq!(manifest.connector_id, "remote_mcp:acme"); + assert_eq!(manifest.integration_type, IntegrationType::RemoteMcp); + assert_eq!(manifest.source_types, vec!["acme".to_string()]); + assert!(manifest.sync_modes.is_empty()); + assert!(manifest.connector_url.is_empty()); + } + + #[test] + fn manifest_preserves_normalized_remote_mcp_oauth_config() { + let cfg = RemoteMcpConfig { + endpoint_url: "https://mcp.example.com/mcp".to_string(), + auth_type: Some(AuthType::OAuth), + write_tools_enabled: true, + }; + let oauth = json!({ + "provider": "remote_mcp:acme", + "credential_provider": "remote_mcp", + "auth_endpoint": "https://auth.example.com/authorize", + "token_endpoint": "https://auth.example.com/token", + "resource": "https://mcp.example.com/mcp" + }); + let manifest = build_manifest( + &source(json!({})), + &cfg, + "1.2.3".to_string(), + vec![], + vec![], + vec![], + Some(oauth), + ); + assert_eq!( + manifest + .oauth + .as_ref() + .and_then(|v| v.get("provider")) + .and_then(|v| v.as_str()), + Some("remote_mcp:acme") + ); + assert_eq!( + manifest + .oauth + .as_ref() + .and_then(|v| v.get("credential_provider")) + .and_then(|v| v.as_str()), + Some("remote_mcp") + ); + } + + #[test] + fn write_tools_can_be_omitted_from_manifest_actions() { + let read = json!({"name":"read","description":"r","annotations":{"readOnlyHint":true}}); + let write = json!({"name":"write","description":"w","annotations":{"readOnlyHint":false}}); + assert!(action_from_tool(&read, false).is_some()); + assert!(action_from_tool(&write, false).is_none()); + } + + #[test] + fn runtime_auth_allows_public_without_credentials() { + let cfg = RemoteMcpConfig { + endpoint_url: "https://mcp.example.com/mcp".to_string(), + auth_type: None, + write_tools_enabled: true, + }; + let headers = runtime_auth_headers_from_credentials( + &cfg, + "src_1", + "acme", + Some("user_1"), + None, + None, + ) + .unwrap(); + assert!(headers.is_empty()); + } + + #[test] + fn runtime_auth_uses_org_bearer_token_for_shared_bearer_sources() { + let cfg = RemoteMcpConfig { + endpoint_url: "https://mcp.example.com/mcp".to_string(), + auth_type: Some(AuthType::BearerToken), + write_tools_enabled: true, + }; + let org = credential(None, AuthType::BearerToken, json!({"token":"org-token"})); + let headers = runtime_auth_headers_from_credentials( + &cfg, + "src_1", + "acme", + Some("user_1"), + Some(&org), + None, + ) + .unwrap(); + assert_eq!( + headers, + vec![("authorization".to_string(), "Bearer org-token".to_string())] + ); + } + + #[test] + fn runtime_oauth_requires_exact_user_credential_without_org_fallback() { + let cfg = RemoteMcpConfig { + endpoint_url: "https://mcp.example.com/mcp".to_string(), + auth_type: Some(AuthType::OAuth), + write_tools_enabled: true, + }; + let org = credential( + None, + AuthType::OAuth, + json!({"access_token":"bootstrap-token"}), + ); + let err = runtime_auth_headers_from_credentials( + &cfg, + "src_1", + "acme", + Some("user_1"), + Some(&org), + None, + ) + .unwrap_err(); + assert!(matches!(err, GatewayError::NeedsUserAuth { .. })); + + let user = credential( + Some("user_1"), + AuthType::OAuth, + json!({"access_token":"user-token"}), + ); + let headers = runtime_auth_headers_from_credentials( + &cfg, + "src_1", + "acme", + Some("user_1"), + Some(&org), + Some(&user), + ) + .unwrap(); + assert_eq!( + headers, + vec![("authorization".to_string(), "Bearer user-token".to_string())] + ); + } + + #[test] + fn parses_sse_json_rpc_response() { + let parsed = parse_mcp_response_body( + "event: message\ndata: {\"jsonrpc\":\"2.0\",\"result\":{}}\n\n", + "text/event-stream", + ) + .unwrap(); + assert!(parsed.get("result").is_some()); + } + + #[test] + fn gateway_ssrf_filter_blocks_reserved_ipv6_and_mapped_ipv4_ranges() { + for ip in [ + "::1", + "fc00::1", + "fe80::1", + "2001:db8::1", + "2002:c000:0201::1", + "::ffff:127.0.0.1", + "::ffff:169.254.169.254", + ] { + assert!( + is_disallowed_ip(ip.parse().unwrap()), + "expected {ip} to be blocked" + ); + } + assert!(!is_disallowed_ip("2001:4860:4860::8888".parse().unwrap())); + } + + #[test] + fn gateway_ssrf_filter_blocks_additional_reserved_ipv4_ranges() { + for ip in [ + "0.0.0.0", + "100.64.0.1", + "192.0.0.8", + "192.88.99.1", + "198.18.0.1", + "203.0.113.10", + ] { + assert!( + is_disallowed_ip(ip.parse().unwrap()), + "expected {ip} to be blocked" + ); + } + assert!(!is_disallowed_ip("8.8.8.8".parse().unwrap())); + } + + #[test] + fn resource_and_prompt_catalog_entries_accept_mcp_wire_naming() { + let resource = resource_from_value(&json!({ + "uriTemplate": "repo://files/{path}", + "name": "Repository File", + "description": "Read a file", + "mimeType": "text/plain" + })) + .unwrap(); + assert_eq!(resource.uri_template, "repo://files/{path}"); + assert_eq!(resource.mime_type.as_deref(), Some("text/plain")); + + let prompt = prompt_from_value(&json!({ + "name": "review_change", + "description": "Review a change", + "arguments": [ + {"name": "diff", "description": "Patch", "required": true}, + {"name": "tone"} + ] + })) + .unwrap(); + assert_eq!(prompt.arguments.len(), 2); + assert!(prompt.arguments[0].required); + assert!(!prompt.arguments[1].required); + } +} diff --git a/services/connector-manager/src/remote_mcp/mod.rs b/services/connector-manager/src/remote_mcp/mod.rs new file mode 100644 index 000000000..f281aa2dc --- /dev/null +++ b/services/connector-manager/src/remote_mcp/mod.rs @@ -0,0 +1,3 @@ +pub mod gateway; +pub mod oauth; +pub mod registry; diff --git a/services/connector-manager/src/remote_mcp/oauth.rs b/services/connector-manager/src/remote_mcp/oauth.rs new file mode 100644 index 000000000..c4f9cf0d7 --- /dev/null +++ b/services/connector-manager/src/remote_mcp/oauth.rs @@ -0,0 +1,336 @@ +use crate::remote_mcp::gateway::{ + pinned_http_client_for_url, read_limited_response_text, validate_endpoint_for_gateway, +}; +use reqwest::Client; +use serde::Deserialize; +use serde_json::Value as JsonValue; +use shared::db::repositories::{ConnectorConfigRepository, ServiceCredentialsRepo}; +use shared::models::{ServiceCredential, Source}; +use shared::DatabasePool; +use std::time::Duration; +use thiserror::Error; +use time::OffsetDateTime; + +const REFRESH_SKEW_SECONDS: i64 = 300; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TokenEndpointAuthMethod { + ClientSecretPost, + ClientSecretBasic, + None, +} + +impl TokenEndpointAuthMethod { + fn parse(value: Option<&str>) -> Self { + match value { + Some("client_secret_basic") => Self::ClientSecretBasic, + Some("none") => Self::None, + _ => Self::ClientSecretPost, + } + } +} + +#[derive(Debug, Clone)] +pub struct RemoteMcpOAuthConfig { + pub provider: String, + pub credential_provider: String, + pub token_endpoint: String, + pub token_endpoint_auth_method: TokenEndpointAuthMethod, + pub resource: Option, +} + +#[derive(Debug, Error)] +pub enum OAuthError { + #[error("missing OAuth field: {0}")] + MissingField(&'static str), + #[error("credential requires OAuth reconnect")] + ReconnectRequired, + #[error("OAuth refresh failed: {0}")] + RefreshFailed(String), + #[error("repository error: {0}")] + Repository(String), +} + +pub fn parse_oauth_config(value: &JsonValue) -> Result { + let provider = value + .get("provider") + .and_then(|v| v.as_str()) + .ok_or(OAuthError::MissingField("provider"))? + .to_string(); + let token_endpoint = value + .get("token_endpoint") + .and_then(|v| v.as_str()) + .ok_or(OAuthError::MissingField("token_endpoint"))? + .to_string(); + let credential_provider = value + .get("credential_provider") + .and_then(|v| v.as_str()) + .unwrap_or("remote_mcp") + .to_string(); + let token_endpoint_auth_method = TokenEndpointAuthMethod::parse( + value + .get("token_endpoint_auth_method") + .and_then(|v| v.as_str()), + ); + let resource = value + .get("resource") + .and_then(|v| v.as_str()) + .map(str::to_owned); + + Ok(RemoteMcpOAuthConfig { + provider, + credential_provider, + token_endpoint, + token_endpoint_auth_method, + resource, + }) +} + +pub fn credential_needs_refresh(credential: &ServiceCredential, now: OffsetDateTime) -> bool { + match credential.expires_at { + Some(expires_at) => expires_at <= now + time::Duration::seconds(REFRESH_SKEW_SECONDS), + None => false, + } +} + +#[derive(Debug, Deserialize)] +struct RefreshResponse { + access_token: String, + refresh_token: Option, + token_type: Option, + expires_in: Option, +} + +pub async fn usable_oauth_credential( + db_pool: &DatabasePool, + _http_client: &Client, + source: &Source, + mut credential: ServiceCredential, + oauth: &RemoteMcpOAuthConfig, +) -> Result { + if !credential_needs_refresh(&credential, OffsetDateTime::now_utc()) { + return Ok(credential); + } + + let refresh_token = credential + .credentials + .get("refresh_token") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .ok_or(OAuthError::ReconnectRequired)? + .to_string(); + + let config_repo = ConnectorConfigRepository::new(db_pool.pool().clone()); + let connector_config = config_repo + .get_by_provider(&oauth.provider) + .await + .map_err(|e| OAuthError::Repository(e.to_string()))? + .map(|row| row.config) + .unwrap_or_else(|| serde_json::json!({})); + + let client_id = string_from(&connector_config, "oauth_client_id") + .or_else(|| string_from(&credential.credentials, "client_id")) + .ok_or(OAuthError::MissingField("oauth_client_id"))?; + let client_secret = string_from(&connector_config, "oauth_client_secret") + .or_else(|| string_from(&credential.credentials, "client_secret")); + let token_endpoint = string_from(&connector_config, "oauth_token_endpoint") + .or_else(|| string_from(&credential.credentials, "token_uri")) + .unwrap_or_else(|| oauth.token_endpoint.clone()); + let token_method = TokenEndpointAuthMethod::parse( + string_from(&connector_config, "oauth_token_endpoint_auth_method").as_deref(), + ); + let token_method = if matches!(token_method, TokenEndpointAuthMethod::ClientSecretPost) + && connector_config + .get("oauth_token_endpoint_auth_method") + .is_none() + { + oauth.token_endpoint_auth_method.clone() + } else { + token_method + }; + + let mut params = vec![ + ("grant_type".to_string(), "refresh_token".to_string()), + ("refresh_token".to_string(), refresh_token), + ]; + let mut basic_auth: Option<(String, String)> = None; + match token_method { + TokenEndpointAuthMethod::None => { + params.push(("client_id".to_string(), client_id)); + } + TokenEndpointAuthMethod::ClientSecretPost => { + params.push(("client_id".to_string(), client_id)); + if let Some(secret) = client_secret { + params.push(("client_secret".to_string(), secret)); + } + } + TokenEndpointAuthMethod::ClientSecretBasic => { + let secret = client_secret.ok_or(OAuthError::MissingField("oauth_client_secret"))?; + basic_auth = Some((client_id, secret)); + } + } + if let Some(resource) = &oauth.resource { + validate_endpoint_for_gateway(resource) + .await + .map_err(|e| OAuthError::RefreshFailed(e.to_string()))?; + params.push(("resource".to_string(), resource.clone())); + } + + let pinned_client = pinned_http_client_for_url(&token_endpoint) + .await + .map_err(|e| OAuthError::RefreshFailed(e.to_string()))?; + let mut request = pinned_client + .post(&token_endpoint) + .timeout(Duration::from_secs(20)) + .header("accept", "application/json"); + if let Some((client_id, client_secret)) = basic_auth { + request = request.basic_auth(client_id, Some(client_secret)); + } + + let response = request + .form(¶ms) + .send() + .await + .map_err(|e| OAuthError::RefreshFailed(e.to_string()))?; + let status = response.status(); + let body = read_limited_response_text(response) + .await + .map_err(|e| OAuthError::RefreshFailed(e.to_string()))?; + if !status.is_success() { + if is_reconnect_required_refresh_failure(status.as_u16(), &body) { + return Err(OAuthError::ReconnectRequired); + } + return Err(OAuthError::RefreshFailed(format!( + "token endpoint returned HTTP {status}" + ))); + } + let refreshed: RefreshResponse = + serde_json::from_str(&body).map_err(|e| OAuthError::RefreshFailed(e.to_string()))?; + + credential.credentials["access_token"] = JsonValue::String(refreshed.access_token); + if let Some(refresh_token) = refreshed.refresh_token { + credential.credentials["refresh_token"] = JsonValue::String(refresh_token); + } + credential.credentials["token_type"] = + JsonValue::String(refreshed.token_type.unwrap_or_else(|| "Bearer".to_string())); + credential.credentials["token_uri"] = JsonValue::String(token_endpoint); + if let Some(expires_in) = refreshed.expires_in { + credential.expires_at = + Some(OffsetDateTime::now_utc() + time::Duration::seconds(expires_in)); + } + + let repo = ServiceCredentialsRepo::new(db_pool.pool().clone()) + .map_err(|e| OAuthError::Repository(e.to_string()))?; + repo.update_credentials(&credential) + .await + .map_err(|e| OAuthError::Repository(e.to_string()))?; + + tracing::debug!(source_id = %source.id, provider = %oauth.provider, credential_provider = %oauth.credential_provider, "refreshed remote MCP OAuth credential"); + Ok(credential) +} + +fn is_reconnect_required_refresh_failure(status: u16, body: &str) -> bool { + if !(400..500).contains(&status) + || status == 408 + || status == 409 + || status == 425 + || status == 429 + { + return false; + } + let Ok(value) = serde_json::from_str::(body) else { + return true; + }; + let error = value + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + matches!( + error, + "invalid_grant" | "invalid_client" | "unauthorized_client" | "access_denied" + ) +} + +fn string_from(value: &JsonValue, key: &str) -> Option { + value.get(key).and_then(|v| v.as_str()).map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use shared::models::{AuthType, ServiceProvider}; + + fn credential(expires_at: Option) -> ServiceCredential { + ServiceCredential { + id: "cred_1".to_string(), + source_id: "src_1".to_string(), + user_id: Some("user_1".to_string()), + provider: ServiceProvider::RemoteMcp, + auth_type: AuthType::OAuth, + principal_email: None, + credentials: json!({"access_token":"old"}), + config: json!({}), + expires_at, + last_validated_at: None, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + } + } + + #[test] + fn parses_normalized_remote_mcp_oauth_config() { + let cfg = parse_oauth_config(&json!({ + "provider": "remote_mcp:acme", + "credential_provider": "remote_mcp", + "token_endpoint": "https://auth.example.com/token", + "token_endpoint_auth_method": "none", + "resource": "https://mcp.example.com/mcp" + })) + .unwrap(); + assert_eq!(cfg.provider, "remote_mcp:acme"); + assert_eq!(cfg.credential_provider, "remote_mcp"); + assert_eq!( + cfg.token_endpoint_auth_method, + TokenEndpointAuthMethod::None + ); + } + + #[test] + fn detects_expiring_credentials_with_skew() { + assert!(!credential_needs_refresh( + &credential(None), + OffsetDateTime::now_utc() + )); + assert!(credential_needs_refresh( + &credential(Some( + OffsetDateTime::now_utc() + time::Duration::seconds(60) + )), + OffsetDateTime::now_utc(), + )); + assert!(!credential_needs_refresh( + &credential(Some(OffsetDateTime::now_utc() + time::Duration::hours(1))), + OffsetDateTime::now_utc(), + )); + } + + #[test] + fn refresh_failure_classification_distinguishes_reconnect_from_transient() { + assert!(is_reconnect_required_refresh_failure( + 400, + r#"{"error":"invalid_grant"}"#, + )); + assert!(is_reconnect_required_refresh_failure( + 401, + r#"{"error":"invalid_client"}"#, + )); + assert!(!is_reconnect_required_refresh_failure( + 429, + r#"{"error":"rate_limited"}"#, + )); + assert!(!is_reconnect_required_refresh_failure( + 500, + r#"{"error":"server_error"}"#, + )); + } +} diff --git a/services/connector-manager/src/remote_mcp/registry.rs b/services/connector-manager/src/remote_mcp/registry.rs new file mode 100644 index 000000000..f29e0bb6f --- /dev/null +++ b/services/connector-manager/src/remote_mcp/registry.rs @@ -0,0 +1,349 @@ +use crate::remote_mcp::gateway::{GatewayError, RemoteMcpGateway, REMOTE_MCP_CONNECTOR_ID_PREFIX}; +use futures::{stream, StreamExt}; +use redis::AsyncCommands; +use serde_json::json; +use shared::db::repositories::SourceRepository; +use shared::models::{IntegrationType, Source}; +use shared::DatabasePool; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::time::Duration; +use tracing::{info, warn}; + +const MAX_MCP_STARTUP_DISCOVERY: usize = 4; +const REMOTE_MCP_RECONCILE_INTERVAL_SECONDS: u64 = 240; + +pub async fn startup_register_remote_mcp_sources(gateway: RemoteMcpGateway, db_pool: DatabasePool) { + match active_usable_remote_mcp_sources(&db_pool).await { + Ok(sources) => { + stream::iter(sources) + .for_each_concurrent(MAX_MCP_STARTUP_DISCOVERY, |source| { + let gateway = gateway.clone(); + async move { + if let Err(error) = gateway.discover_and_register(&source.id).await { + warn!(source_id = %source.id, error = %redact_gateway_error(&error), "Remote MCP startup registration failed"); + } + } + }) + .await; + } + Err(error) => warn!(error = %error, "Remote MCP startup source query failed"), + } +} + +pub async fn run_remote_mcp_registry_loop(gateway: RemoteMcpGateway, db_pool: DatabasePool) { + let mut interval = + tokio::time::interval(Duration::from_secs(REMOTE_MCP_RECONCILE_INTERVAL_SECONDS)); + loop { + interval.tick().await; + if let Err(error) = reconcile_remote_mcp_sources(&gateway, &db_pool).await { + warn!(error = %redact_gateway_error(&error), "Remote MCP registry reconciliation failed"); + } + } +} + +pub async fn reconcile_remote_mcp_sources( + gateway: &RemoteMcpGateway, + db_pool: &DatabasePool, +) -> Result<(), GatewayError> { + let usable_sources = active_usable_remote_mcp_sources(db_pool).await?; + let active_source_types: HashSet = usable_sources + .iter() + .map(|source| source.source_type.clone()) + .collect(); + let active_source_ids: HashSet = usable_sources + .iter() + .map(|source| source.id.clone()) + .collect(); + + for source in usable_sources { + if let Err(error) = gateway.refresh_catalog(&source.id).await { + warn!(source_id = %source.id, source_type = %source.source_type, error = %redact_gateway_error(&error), "Remote MCP catalog refresh failed; existing Redis manifest left untouched"); + } + } + + remove_stale_remote_mcp_manifests(gateway, &active_source_types).await?; + remove_stale_remote_mcp_capabilities(db_pool, &active_source_ids).await?; + Ok(()) +} + +async fn active_usable_remote_mcp_sources( + db_pool: &DatabasePool, +) -> Result, GatewayError> { + let repo = SourceRepository::new(db_pool.pool()); + let sources = repo.find_active_sources().await?; + Ok(reject_conflicting_remote_mcp_sources(sources)) +} + +fn reject_conflicting_remote_mcp_sources(sources: Vec) -> Vec { + let native_source_types: HashSet = sources + .iter() + .filter(|source| source.integration_type == IntegrationType::Connector) + .map(|source| source.source_type.clone()) + .collect(); + let remote_sources = sources + .into_iter() + .filter(|source| source.integration_type == IntegrationType::RemoteMcp); + + let mut by_slug: HashMap> = HashMap::new(); + for source in remote_sources { + by_slug + .entry(source.source_type.clone()) + .or_default() + .push(source); + } + + let mut usable = Vec::new(); + for (source_type, mut sources) in by_slug { + if native_source_types.contains(&source_type) { + let ids: Vec = sources.iter().map(|source| source.id.clone()).collect(); + warn!(source_type = %source_type, source_ids = ?ids, "Native/MCP slug conflict detected; skipping remote MCP rows"); + continue; + } + if sources.len() > 1 { + let ids: Vec = sources.iter().map(|source| source.id.clone()).collect(); + warn!(source_type = %source_type, source_ids = ?ids, "Remote MCP slug conflict detected; skipping conflicted rows"); + continue; + } + if let Some(source) = sources.pop() { + usable.push(source); + } + } + usable +} + +async fn remove_stale_remote_mcp_manifests( + gateway: &RemoteMcpGateway, + active_source_types: &HashSet, +) -> Result<(), GatewayError> { + let redis_client = gateway_redis_client(gateway); + let mut conn = redis_client.get_multiplexed_async_connection().await?; + let keys = scan_redis_keys( + &mut conn, + &format!("connector:manifest:{REMOTE_MCP_CONNECTOR_ID_PREFIX}*"), + ) + .await?; + for key in keys { + if let Some(source_type) = source_type_from_manifest_key(&key) { + if !active_source_types.contains(source_type) { + let _: () = conn.del(&key).await?; + info!(source_type = %source_type, "Removed stale remote MCP manifest from Redis"); + } + } + } + Ok(()) +} + +async fn remove_stale_remote_mcp_capabilities( + db_pool: &DatabasePool, + active_source_ids: &HashSet, +) -> Result<(), GatewayError> { + let all_remote_source_ids: Vec = sqlx::query_scalar( + r#" + SELECT id + FROM sources + WHERE integration_type = 'remote_mcp' + "#, + ) + .fetch_all(db_pool.pool()) + .await + .map_err(shared::db::error::DatabaseError::from)?; + + let stale_source_ids = stale_remote_mcp_source_ids(&all_remote_source_ids, active_source_ids); + if stale_source_ids.is_empty() { + return Ok(()); + } + + publish_empty_remote_mcp_capability_catalogs(&stale_source_ids).await; + Ok(()) +} + +async fn publish_empty_remote_mcp_capability_catalogs(source_ids: &[String]) { + let Ok(searcher_url) = env::var("SEARCHER_URL") else { + warn!( + source_ids = ?source_ids, + "Stale remote MCP capability publishers detected but SEARCHER_URL is unset; AI/searcher capability sync will prune them on next publication." + ); + return; + }; + let client = reqwest::Client::new(); + let endpoint = format!("{}/capabilities/sync", searcher_url.trim_end_matches('/')); + for source_id in source_ids { + for capability_type in ["resource", "prompt"] { + let response = client + .post(&endpoint) + .timeout(Duration::from_secs(10)) + .json(&json!({ + "publisher_id": source_id, + "capability_type": capability_type, + "capabilities": [], + })) + .send() + .await; + match response { + Ok(resp) if resp.status().is_success() => info!( + source_id = %source_id, + capability_type, + "Pruned stale remote MCP capabilities through searcher sync path" + ), + Ok(resp) => warn!( + source_id = %source_id, + capability_type, + status = %resp.status(), + "Searcher rejected stale remote MCP capability prune" + ), + Err(error) => warn!( + source_id = %source_id, + capability_type, + error = %error, + "Failed to prune stale remote MCP capabilities through searcher sync path" + ), + } + } + } +} + +fn stale_remote_mcp_source_ids( + all_remote_source_ids: &[String], + active_source_ids: &HashSet, +) -> Vec { + all_remote_source_ids + .iter() + .filter(|source_id| !active_source_ids.contains(source_id.as_str())) + .cloned() + .collect() +} + +async fn scan_redis_keys( + conn: &mut redis::aio::MultiplexedConnection, + pattern: &str, +) -> redis::RedisResult> { + let mut cursor: u64 = 0; + let mut keys = Vec::new(); + loop { + let (next_cursor, batch): (u64, Vec) = redis::cmd("SCAN") + .cursor_arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(100) + .query_async(conn) + .await?; + keys.extend(batch); + if next_cursor == 0 { + break; + } + cursor = next_cursor; + } + Ok(keys) +} + +fn gateway_redis_client(gateway: &RemoteMcpGateway) -> redis::Client { + // Keep Redis ownership inside the gateway API while allowing registry + // cleanup to use the same configured client without adding global state. + gateway.redis_client_for_registry() +} + +pub fn source_type_from_manifest_key(key: &str) -> Option<&str> { + key.strip_prefix(&format!( + "connector:manifest:{REMOTE_MCP_CONNECTOR_ID_PREFIX}" + )) + .filter(|source_type| !source_type.is_empty()) +} + +fn redact_gateway_error(error: &GatewayError) -> String { + match error { + GatewayError::Protocol(message) => format!("protocol error: {}", redact_secretish(message)), + other => other.to_string(), + } +} + +fn redact_secretish(value: &str) -> String { + value + .replace("Bearer ", "Bearer [redacted]") + .replace("access_token", "[redacted_token]") + .replace("token", "[redacted_token]") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use shared::models::{SourceScope, UserFilterMode}; + use time::OffsetDateTime; + + fn source(id: &str, slug: &str) -> Source { + Source { + id: id.to_string(), + name: slug.to_string(), + source_type: slug.to_string(), + integration_type: IntegrationType::RemoteMcp, + config: json!({"endpoint_url":"https://example.com/mcp"}), + is_active: true, + is_deleted: false, + scope: SourceScope::Org, + user_filter_mode: UserFilterMode::All, + user_whitelist: None, + user_blacklist: None, + connector_state: None, + checkpoint: None, + sync_interval_seconds: None, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + created_by: "admin".to_string(), + } + } + + #[test] + fn parses_remote_mcp_manifest_keys() { + assert_eq!( + source_type_from_manifest_key("connector:manifest:remote_mcp:acme"), + Some("acme") + ); + assert_eq!( + source_type_from_manifest_key("connector:manifest:slack"), + None + ); + } + + #[test] + fn conflicting_remote_mcp_slugs_are_skipped() { + let usable = reject_conflicting_remote_mcp_sources(vec![ + source("one", "acme"), + source("two", "acme"), + source("three", "beta"), + ]); + assert_eq!(usable.len(), 1); + assert_eq!(usable[0].source_type, "beta"); + } + + #[test] + fn native_slug_conflicts_skip_remote_mcp_rows() { + let mut native = source("native", "acme"); + native.integration_type = IntegrationType::Connector; + let usable = reject_conflicting_remote_mcp_sources(vec![native, source("remote", "acme")]); + assert!(usable.is_empty()); + } + + #[test] + fn manifest_key_matches_registry_key_parser() { + let key = crate::remote_mcp::gateway::manifest_key("acme"); + assert_eq!(source_type_from_manifest_key(&key), Some("acme")); + } + + #[test] + fn stale_remote_mcp_source_ids_exclude_active_ids() { + let active_source_ids = HashSet::from(["active".to_string()]); + assert_eq!( + stale_remote_mcp_source_ids( + &[ + "active".to_string(), + "deleted".to_string(), + "inactive".to_string() + ], + &active_source_ids, + ), + vec!["deleted".to_string(), "inactive".to_string()] + ); + } +} diff --git a/services/connector-manager/src/scheduler.rs b/services/connector-manager/src/scheduler.rs index 5b96a644c..864f6db90 100644 --- a/services/connector-manager/src/scheduler.rs +++ b/services/connector-manager/src/scheduler.rs @@ -192,7 +192,17 @@ impl Scheduler { let now = OffsetDateTime::now_utc(); for source in active_sources { - let modes = get_sync_modes_for_source(&self.redis_client, source.source_type).await; + if !is_syncable_source(&source) { + debug!( + source_id = %source.id, + source_type = %source.source_type, + integration_type = ?source.integration_type, + "Skipping realtime supervisor for non-syncable source" + ); + continue; + } + + let modes = get_sync_modes_for_source(&self.redis_client, &source.source_type).await; if !modes.contains(&SyncType::Realtime) { continue; } @@ -336,6 +346,7 @@ impl Scheduler { .find_active_sources() .await .map_err(|e| SchedulerError::DatabaseError(e.to_string()))?; + let sources: Vec = sources.into_iter().filter(is_syncable_source).collect(); let source_ids: Vec = sources.iter().map(|source| source.id.clone()).collect(); let recent_run_limit = self .config @@ -385,7 +396,7 @@ impl Scheduler { } let sync_type = pick_scheduled_sync_type( - &get_sync_modes_for_source(&self.redis_client, source.source_type).await, + &get_sync_modes_for_source(&self.redis_client, &source.source_type).await, ); match self @@ -422,6 +433,10 @@ impl Scheduler { } } +fn is_syncable_source(source: &Source) -> bool { + source.integration_type == shared::models::IntegrationType::Connector +} + fn sources_due_for_sync( sources: Vec, sync_runs: Vec, @@ -442,6 +457,14 @@ fn sources_due_for_sync( let mut due_sources: Vec<(Source, Option)> = sources .into_iter() .filter_map(|source| { + if !is_syncable_source(&source) { + info!( + "Skipping scheduled sync for source {} ({:?}): integration type {:?} does not support data sync", + source.id, source.source_type, source.integration_type + ); + return None; + } + let sync_interval_seconds = match source.sync_interval_seconds { Some(interval) => interval, None => { @@ -623,14 +646,15 @@ struct SlotHealth { mod tests { use super::*; use serde_json::json; - use shared::models::{SourceScope, SourceType, UserFilterMode}; + use shared::models::{IntegrationType, SourceScope, SourceType, UserFilterMode}; fn source(id: &str, interval_seconds: Option) -> Source { let now = OffsetDateTime::now_utc(); Source { id: id.to_string(), name: format!("source-{id}"), - source_type: SourceType::LocalFiles, + source_type: SourceType::LocalFiles.to_string(), + integration_type: IntegrationType::Connector, config: json!({}), is_active: true, is_deleted: false, @@ -647,6 +671,13 @@ mod tests { } } + fn remote_mcp_source(id: &str, interval_seconds: Option) -> Source { + Source { + integration_type: IntegrationType::RemoteMcp, + ..source(id, interval_seconds) + } + } + fn sync_run( id: &str, source_id: &str, @@ -740,6 +771,31 @@ mod tests { assert!(active_backoff(&health, now, 30, 3600).is_none()); } + #[test] + fn remote_mcp_sources_are_not_syncable() { + assert!(is_syncable_source(&source("native", Some(60)))); + assert!(!is_syncable_source(&remote_mcp_source("mcp", Some(60)))); + } + + #[test] + fn due_sources_exclude_remote_mcp_sources() { + let now = OffsetDateTime::now_utc(); + let due = sources_due_for_sync( + vec![ + source("native", Some(60)), + remote_mcp_source("mcp", Some(60)), + ], + vec![], + now, + 10, + 30, + 3600, + ); + + assert_eq!(due.len(), 1); + assert_eq!(due[0].id, "native"); + } + #[test] fn due_sources_include_elapsed_successes() { let now = OffsetDateTime::now_utc(); diff --git a/services/connector-manager/src/source_cleanup.rs b/services/connector-manager/src/source_cleanup.rs index 504cabeb9..1f7fb6e9d 100644 --- a/services/connector-manager/src/source_cleanup.rs +++ b/services/connector-manager/src/source_cleanup.rs @@ -59,7 +59,11 @@ async fn cleanup_source(pool: &PgPool, source_id: &str) -> Result<(), sqlx::Erro return Ok(()); } - // No documents left — safe to delete the source row (cascades to sync_runs, etc.) + // No documents left. Do not write searcher-owned agent_capabilities from + // connector-manager; stale source-scoped capabilities are pruned by the + // existing AI/searcher capability sync path. + + // Safe to delete the source row (cascades to sync_runs, credentials, etc.) sqlx::query("DELETE FROM sources WHERE id = $1") .bind(source_id) .execute(pool) diff --git a/services/connector-manager/src/sync_manager.rs b/services/connector-manager/src/sync_manager.rs index ae6250750..723735010 100644 --- a/services/connector-manager/src/sync_manager.rs +++ b/services/connector-manager/src/sync_manager.rs @@ -6,7 +6,7 @@ use dashmap::DashMap; use redis::Client as RedisClient; use shared::db::error::DatabaseError; use shared::db::repositories::SyncRunRepository; -use shared::models::{SourceType, SyncSlotClass, SyncStatus, SyncType}; +use shared::models::{IntegrationType, SyncSlotClass, SyncStatus, SyncType}; use shared::{DatabasePool, Repository, SourceRepository}; use sqlx::PgPool; use std::collections::HashMap; @@ -91,8 +91,12 @@ impl SyncManager { return Err(SyncError::SourceInactive(source_id.to_string())); } + if source.integration_type == IntegrationType::RemoteMcp { + return Err(SyncError::SourceDoesNotSync(source_id.to_string())); + } + // Get connector URL from registry - let connector_url = get_connector_url_for_source(&self.redis_client, source.source_type) + let connector_url = get_connector_url_for_source(&self.redis_client, &source.source_type) .await .ok_or_else(|| { SyncError::ConnectorNotConfigured(format!("{:?}", source.source_type)) @@ -214,7 +218,7 @@ impl SyncManager { .map_err(|e| SyncError::DatabaseError(e.to_string()))? .ok_or_else(|| SyncError::SourceNotFound(sync_run.source_id.clone()))?; - let connector_url = get_connector_url_for_source(&self.redis_client, source.source_type) + let connector_url = get_connector_url_for_source(&self.redis_client, &source.source_type) .await .ok_or_else(|| { SyncError::ConnectorNotConfigured(format!("{:?}", source.source_type)) @@ -327,7 +331,7 @@ impl SyncManager { let connector_url = match get_connector_url_for_source( &self.redis_client, - source.source_type, + &source.source_type, ) .await { @@ -454,7 +458,7 @@ impl SyncManager { }; let connector_url = - match get_connector_url_for_source(&self.redis_client, source.source_type).await { + match get_connector_url_for_source(&self.redis_client, &source.source_type).await { Some(url) => url, None => { // Connector is still unregistered. Attempt was counted @@ -558,9 +562,9 @@ impl SyncManager { return Ok(vec![]); } - let source_type_map: HashMap = inactive_sources + let source_type_map: HashMap = inactive_sources .iter() - .map(|s| (s.id.clone(), s.source_type)) + .map(|s| (s.id.clone(), s.source_type.clone())) .collect(); let source_ids: Vec = inactive_sources.into_iter().map(|s| s.id).collect(); @@ -579,7 +583,7 @@ impl SyncManager { "Cancelling sync {} for inactive/deleted source {}", run.id, run.source_id ); - if let Some(&source_type) = source_type_map.get(&run.source_id) { + if let Some(source_type) = source_type_map.get(&run.source_id) { if let Some(connector_url) = get_connector_url_for_source(&self.redis_client, source_type).await { @@ -613,7 +617,7 @@ impl SyncManager { let timeout_minutes = self.config.stale_sync_timeout_minutes as i64; let cutoff = OffsetDateTime::now_utc() - time::Duration::minutes(timeout_minutes); - let stale_syncs: Vec<(String, String, SourceType)> = sqlx::query_as( + let stale_syncs: Vec<(String, String, String)> = sqlx::query_as( r#" SELECT sr.id, sr.source_id, s.source_type FROM sync_runs sr @@ -638,7 +642,7 @@ impl SyncManager { ); if let Some(connector_url) = - get_connector_url_for_source(&self.redis_client, source_type).await + get_connector_url_for_source(&self.redis_client, &source_type).await { if let Err(e) = self .connector_client @@ -704,6 +708,9 @@ pub enum SyncError { #[error("Source is inactive: {0}")] SourceInactive(String), + #[error("Source does not support data sync: {0}")] + SourceDoesNotSync(String), + #[error("Sync already running for source: {0}")] SyncAlreadyRunning(String), diff --git a/services/connector-manager/tests/common/mock_connector.rs b/services/connector-manager/tests/common/mock_connector.rs index f9e4ab3e0..b5fb942bf 100644 --- a/services/connector-manager/tests/common/mock_connector.rs +++ b/services/connector-manager/tests/common/mock_connector.rs @@ -1,17 +1,17 @@ use axum::{ - Router, extract::{Path, State}, http::StatusCode, response::Json, routing::{get, post}, + Router, }; use serde::{Deserialize, Serialize}; -use serde_json::{Value as JsonValue, json}; +use serde_json::{json, Value as JsonValue}; use std::collections::HashSet; use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; use tokio::sync::oneshot; -use tokio::time::{Duration, sleep}; +use tokio::time::{sleep, Duration}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RecordedSyncRequest { diff --git a/services/connector-manager/tests/common/mod.rs b/services/connector-manager/tests/common/mod.rs index d70aa8891..66cd417ea 100644 --- a/services/connector-manager/tests/common/mod.rs +++ b/services/connector-manager/tests/common/mod.rs @@ -3,10 +3,11 @@ pub mod mock_connector; use anyhow::Result; use mock_connector::MockConnector; use omni_connector_manager::{ - config::ConnectorManagerConfig, create_app, sync_manager::SyncManager, AppState, + config::ConnectorManagerConfig, create_app, remote_mcp::gateway::RemoteMcpGateway, + sync_manager::SyncManager, AppState, }; use redis::{AsyncCommands, Client as RedisClient}; -use shared::models::{ConnectorManifest, SourceType, SyncType}; +use shared::models::{ConnectorManifest, IntegrationType, SourceType, SyncType}; use shared::storage::postgres::PostgresStorage; use shared::test_environment::TestEnvironment; use shared::ObjectStorage; @@ -62,7 +63,8 @@ pub async fn setup_test_fixture() -> Result { sync_modes: vec![SyncType::Full], connector_id: "filesystem".to_string(), connector_url: mock_connector.base_url.clone(), - source_types: vec![SourceType::LocalFiles], + integration_type: IntegrationType::Connector, + source_types: vec![SourceType::LocalFiles.to_string()], description: None, actions: vec![], search_operators: vec![], @@ -91,12 +93,18 @@ pub async fn setup_test_fixture() -> Result { redis_client.clone(), )); + let remote_mcp_gateway = Arc::new(RemoteMcpGateway::new( + test_env.db_pool.clone(), + redis_client.clone(), + )?); + let app_state = AppState { db_pool: test_env.db_pool.clone(), redis_client, extraction_semaphore: Arc::new(Semaphore::new(config.extraction_concurrency)), config, sync_manager, + remote_mcp_gateway, content_storage, }; diff --git a/services/migrations/106_add_remote_mcp_sources.sql b/services/migrations/106_add_remote_mcp_sources.sql new file mode 100644 index 000000000..8c63b5a55 --- /dev/null +++ b/services/migrations/106_add_remote_mcp_sources.sql @@ -0,0 +1,43 @@ +-- Add support for UI-created remote MCP source rows without overloading +-- source_type. source_type remains the app/provider slug; integration_type +-- distinguishes native connector-backed rows from remote MCP rows. + +-- 1. Integration discriminator on sources. Existing rows are native connectors. +ALTER TABLE sources ADD COLUMN integration_type TEXT NOT NULL DEFAULT 'connector'; +ALTER TABLE sources ADD CONSTRAINT sources_integration_type_check + CHECK (integration_type IN ('connector', 'remote_mcp')); + +CREATE INDEX idx_sources_integration_type ON sources (integration_type); + +-- 2. Add the generic credential provider used for credentials belonging to +-- UI-created remote MCP integrations. Existing auth_type values are reused. +ALTER TABLE service_credentials DROP CONSTRAINT IF EXISTS service_credentials_provider_check; +ALTER TABLE service_credentials ADD CONSTRAINT service_credentials_provider_check +CHECK (provider IN ( + 'google', 'slack', 'atlassian', 'github', 'notion', 'fireflies', + 'hubspot', 'microsoft', 'imap', 'clickup', 'linear', 'paperless_ngx', + 'nextcloud', 'google_ads', 'darwinbox', + 'remote_mcp' +)); + +-- 3. Keep source_type's existing "app name" meaning. Native connector rows +-- remain constrained to known SourceType values, while remote MCP rows may use +-- a validated app/server slug supplied during setup. +ALTER TABLE sources DROP CONSTRAINT IF EXISTS sources_source_type_check; +ALTER TABLE sources ADD CONSTRAINT sources_source_type_check CHECK ( + (integration_type = 'connector' AND source_type IN ( + 'google_drive', 'gmail', 'google_chat', 'confluence', 'jira', 'slack', + 'notion', 'web', 'github', 'local_files', 'file_system', 'fireflies', + 'hubspot', 'one_drive', 'share_point', 'outlook', 'outlook_calendar', + 'imap', 'clickup', 'linear', 'ms_teams', 'paperless_ngx', 'nextcloud', + 'google_ads', 'darwinbox' + )) + OR + (integration_type = 'remote_mcp' AND source_type ~ '^[a-z][a-z0-9_]{1,49}$') +); + +-- At most one non-deleted remote MCP source may claim an app slug. Native/native +-- duplicates remain legal for providers that support multiple source rows. +CREATE UNIQUE INDEX sources_remote_mcp_source_type_uniq + ON sources (source_type) + WHERE integration_type = 'remote_mcp' AND is_deleted = false; diff --git a/services/searcher/src/search.rs b/services/searcher/src/search.rs index dc3ff73e6..732a6712c 100644 --- a/services/searcher/src/search.rs +++ b/services/searcher/src/search.rs @@ -6,19 +6,19 @@ use crate::query_parser; use crate::search_repository::SearchDocumentRepository; use anyhow::Result; use redis::{AsyncCommands, Client as RedisClient}; -use shared::SourceType; use shared::db::repositories::{ DocumentRepository, EmbeddingRepository, GroupRepository, PersonRepository, SourceRepository, }; use shared::models::{ChunkResult, Document, Facet, FacetValue}; use shared::utils::safe_str_slice; +use shared::SourceType; use shared::{ AIClient, DatabasePool, ObjectStorage, Repository, SearcherConfig, StorageFactory, UserRepository, }; use std::cmp::Ordering; -use std::collections::HashMap; use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; use std::hash::{Hash, Hasher}; use std::sync::Arc; use std::time::{Duration, Instant}; diff --git a/services/searcher/src/search_repository.rs b/services/searcher/src/search_repository.rs index 747802cb9..b7ef94582 100644 --- a/services/searcher/src/search_repository.rs +++ b/services/searcher/src/search_repository.rs @@ -1,12 +1,12 @@ use pgvector::Vector; use serde_json::Value as JsonValue; use shared::{ - SourceType, db::error::DatabaseError, db::repositories::document, models::{AttributeFilter, ChunkResult, DateFilter, Document, Facet, FacetValue}, + SourceType, }; -use sqlx::{FromRow, PgPool, Row, postgres::PgRow}; +use sqlx::{postgres::PgRow, FromRow, PgPool, Row}; use std::collections::{HashMap, HashSet}; use tracing::debug; diff --git a/services/searcher/src/suggested_questions.rs b/services/searcher/src/suggested_questions.rs index 4d667d46a..54f88b9d5 100644 --- a/services/searcher/src/suggested_questions.rs +++ b/services/searcher/src/suggested_questions.rs @@ -1,6 +1,6 @@ use crate::models::{SuggestedQuestion, SuggestedQuestionsResponse}; use crate::{Result as SearcherResult, SearcherError}; -use anyhow::{Context, Result, anyhow}; +use anyhow::{anyhow, Context, Result}; use dashmap::DashSet; use futures_util::StreamExt; use redis::AsyncCommands; @@ -318,7 +318,10 @@ impl SuggestedQuestionsGenerator { ); } Err(e) => { - warn!("Failed to generate suggestion for document {}: {}", doc.id, e); + warn!( + "Failed to generate suggestion for document {}: {}", + doc.id, e + ); } } @@ -414,7 +417,9 @@ impl SuggestedQuestionsGenerator { "Document {} deemed unsuitable for suggestion (model returned SKIP)", document_id ); - return Err(anyhow!("Document unsuitable for suggestion generation (SKIP)")); + return Err(anyhow!( + "Document unsuitable for suggestion generation (SKIP)" + )); } if expect_question_mark && !output.contains('?') { diff --git a/shared/src/db/repositories/service_credentials.rs b/shared/src/db/repositories/service_credentials.rs index c90c58ecf..21c010acc 100644 --- a/shared/src/db/repositories/service_credentials.rs +++ b/shared/src/db/repositories/service_credentials.rs @@ -20,6 +20,22 @@ impl ServiceCredentialsRepo { }) } + /// Fetch a credential row by id. + pub async fn find_by_id(&self, id: &str) -> Result> { + let mut creds = sqlx::query_as::<_, ServiceCredential>( + "SELECT * FROM service_credentials WHERE id = $1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + + if let Some(ref mut creds) = creds { + self.decrypt_credentials_in_place(creds)?; + } + + Ok(creds) + } + /// Fetch the org-wide credential row for a source (`user_id IS NULL`). pub async fn find_org_credential(&self, source_id: &str) -> Result> { let mut creds = sqlx::query_as::<_, ServiceCredential>( diff --git a/shared/src/db/repositories/source.rs b/shared/src/db/repositories/source.rs index d7cd571b6..20b49d5fb 100644 --- a/shared/src/db/repositories/source.rs +++ b/shared/src/db/repositories/source.rs @@ -16,7 +16,7 @@ impl SourceRepository { pub async fn find_by_type(&self, source_type: &str) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -34,7 +34,7 @@ impl SourceRepository { pub async fn find_all_sources(&self) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -54,7 +54,7 @@ impl SourceRepository { pub async fn find_all_sources_without_state(&self) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, NULL::jsonb AS connector_state, NULL::jsonb AS checkpoint, sync_interval_seconds, created_at, updated_at, created_by @@ -72,7 +72,7 @@ impl SourceRepository { pub async fn find_active_sources(&self) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -89,7 +89,7 @@ impl SourceRepository { pub async fn find_inactive(&self) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -133,7 +133,7 @@ impl SourceRepository { ) -> Result, DatabaseError> { let mut query_builder = sqlx::QueryBuilder::new( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -145,7 +145,7 @@ impl SourceRepository { query_builder.push(" AND source_type IN ("); let mut separated = query_builder.separated(", "); for source_type in source_types { - separated.push_bind(source_type); + separated.push_bind(source_type.to_string()); } query_builder.push(")"); } @@ -214,7 +214,7 @@ impl Repository for SourceRepository { async fn find_by_id(&self, id: String) -> Result, DatabaseError> { let source = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -231,7 +231,7 @@ impl Repository for SourceRepository { async fn find_all(&self, limit: i64, offset: i64) -> Result, DatabaseError> { let sources = sqlx::query_as::<_, Source>( r#" - SELECT id, name, source_type, config, is_active, is_deleted, scope, + SELECT id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by FROM sources @@ -253,7 +253,7 @@ impl Repository for SourceRepository { r#" INSERT INTO sources (id, name, source_type, config, is_active, created_by) VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id, name, source_type, config, is_active, is_deleted, scope, + RETURNING id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by "#, @@ -282,7 +282,7 @@ impl Repository for SourceRepository { UPDATE sources SET name = $2, source_type = $3, config = $4, is_active = $5, updated_at = CURRENT_TIMESTAMP WHERE id = $1 - RETURNING id, name, source_type, config, is_active, is_deleted, scope, + RETURNING id, name, source_type, integration_type, config, is_active, is_deleted, scope, user_filter_mode, user_whitelist, user_blacklist, connector_state, checkpoint, sync_interval_seconds, created_at, updated_at, created_by "#, diff --git a/shared/src/models.rs b/shared/src/models.rs index 13f754086..020cbeed7 100644 --- a/shared/src/models.rs +++ b/shared/src/models.rs @@ -68,7 +68,10 @@ pub enum SourceScope { pub struct Source { pub id: String, pub name: String, - pub source_type: SourceType, + pub source_type: String, + #[serde(default)] + #[sqlx(default)] + pub integration_type: IntegrationType, pub config: JsonValue, pub is_active: bool, pub is_deleted: bool, @@ -88,6 +91,16 @@ pub struct Source { } impl Source { + pub fn native_source_type(&self) -> Result { + if self.integration_type != IntegrationType::Connector { + return Err(format!( + "source {} has integration_type {:?}, not connector", + self.id, self.integration_type + )); + } + SourceType::try_from(self.source_type.as_str()) + } + pub fn get_user_whitelist(&self) -> Vec { self.user_whitelist .as_ref() @@ -194,6 +207,101 @@ pub enum SourceType { Darwinbox, } +impl SourceType { + pub fn as_str(&self) -> &'static str { + match self { + SourceType::GoogleDrive => "google_drive", + SourceType::Gmail => "gmail", + SourceType::GoogleChat => "google_chat", + SourceType::Confluence => "confluence", + SourceType::Jira => "jira", + SourceType::Slack => "slack", + SourceType::Github => "github", + SourceType::LocalFiles => "local_files", + SourceType::FileSystem => "file_system", + SourceType::Web => "web", + SourceType::Notion => "notion", + SourceType::Hubspot => "hubspot", + SourceType::OneDrive => "one_drive", + SourceType::SharePoint => "share_point", + SourceType::Outlook => "outlook", + SourceType::OutlookCalendar => "outlook_calendar", + SourceType::MsTeams => "ms_teams", + SourceType::Fireflies => "fireflies", + SourceType::Imap => "imap", + SourceType::Clickup => "clickup", + SourceType::Linear => "linear", + SourceType::PaperlessNgx => "paperless_ngx", + SourceType::Nextcloud => "nextcloud", + SourceType::GoogleAds => "google_ads", + SourceType::Darwinbox => "darwinbox", + } + } +} + +impl std::fmt::Display for SourceType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for SourceType { + type Err = String; + + fn from_str(value: &str) -> Result { + SourceType::try_from(value) + } +} + +impl TryFrom<&str> for SourceType { + type Error = String; + + fn try_from(value: &str) -> Result { + match value { + "google_drive" => Ok(SourceType::GoogleDrive), + "gmail" => Ok(SourceType::Gmail), + "google_chat" => Ok(SourceType::GoogleChat), + "confluence" => Ok(SourceType::Confluence), + "jira" => Ok(SourceType::Jira), + "slack" => Ok(SourceType::Slack), + "github" => Ok(SourceType::Github), + "local_files" => Ok(SourceType::LocalFiles), + "file_system" => Ok(SourceType::FileSystem), + "web" => Ok(SourceType::Web), + "notion" => Ok(SourceType::Notion), + "hubspot" => Ok(SourceType::Hubspot), + "one_drive" => Ok(SourceType::OneDrive), + "share_point" => Ok(SourceType::SharePoint), + "outlook" => Ok(SourceType::Outlook), + "outlook_calendar" => Ok(SourceType::OutlookCalendar), + "ms_teams" => Ok(SourceType::MsTeams), + "fireflies" => Ok(SourceType::Fireflies), + "imap" => Ok(SourceType::Imap), + "clickup" => Ok(SourceType::Clickup), + "linear" => Ok(SourceType::Linear), + "paperless_ngx" => Ok(SourceType::PaperlessNgx), + "nextcloud" => Ok(SourceType::Nextcloud), + "google_ads" => Ok(SourceType::GoogleAds), + "darwinbox" => Ok(SourceType::Darwinbox), + other => Err(format!("unknown source type: {other}")), + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, sqlx::Type, PartialEq, Eq, Hash)] +#[sqlx(type_name = "text", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum IntegrationType { + Connector, + RemoteMcp, +} + +impl Default for IntegrationType { + fn default() -> Self { + IntegrationType::Connector + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, sqlx::Type, PartialEq)] #[sqlx(type_name = "varchar", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] @@ -217,6 +325,9 @@ pub enum ServiceProvider { #[serde(rename = "google_ads")] GoogleAds, Darwinbox, + #[sqlx(rename = "remote_mcp")] + #[serde(rename = "remote_mcp")] + RemoteMcp, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, sqlx::Type, PartialEq)] @@ -821,7 +932,9 @@ pub struct ConnectorManifest { pub connector_id: String, pub connector_url: String, #[serde(default)] - pub source_types: Vec, + pub integration_type: IntegrationType, + #[serde(default)] + pub source_types: Vec, #[serde(default)] pub description: Option, #[serde(default)] @@ -1318,7 +1431,8 @@ mod tests { Source { id: "src-1".to_string(), name: "Test".to_string(), - source_type: SourceType::Web, + source_type: SourceType::Web.to_string(), + integration_type: IntegrationType::Connector, config: json!({}), is_active: true, is_deleted: false, @@ -1335,6 +1449,20 @@ mod tests { } } + #[test] + fn integration_type_uses_text_sqlx_type_and_snake_case_json() { + use sqlx::{Postgres, Type, TypeInfo}; + + assert_eq!( + >::type_info().name(), + "text" + ); + assert_eq!( + serde_json::to_string(&IntegrationType::RemoteMcp).unwrap(), + "\"remote_mcp\"" + ); + } + #[test] fn test_user_configuration_normalizes_timezone_aliases() { let configuration = UserConfiguration::from_rows(vec![( diff --git a/web/src/lib/server/db/schema.ts b/web/src/lib/server/db/schema.ts index 4e4b62809..126f2815c 100644 --- a/web/src/lib/server/db/schema.ts +++ b/web/src/lib/server/db/schema.ts @@ -37,6 +37,7 @@ export const sources = pgTable('sources', { id: text('id').primaryKey(), name: text('name').notNull(), sourceType: text('source_type').notNull(), + integrationType: text('integration_type').notNull().default('connector'), config: jsonb('config').notNull().default({}), isActive: boolean('is_active').notNull().default(true), isDeleted: boolean('is_deleted').notNull().default(false), diff --git a/web/src/lib/server/mcp/client.test.ts b/web/src/lib/server/mcp/client.test.ts new file mode 100644 index 000000000..0db93d598 --- /dev/null +++ b/web/src/lib/server/mcp/client.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { AuthType } from '$lib/types' +import { + isRemoteMcpBlockedIp, + remoteMcpConfigFromInput, + validateRemoteMcpSlug, + validateRemoteMcpUrl, + validateRemoteMcpUrlForCredentialUse, +} from './client' + +describe('remote MCP client validation', () => { + it('blocks private and reserved addresses used in SSRF attempts', () => { + expect(isRemoteMcpBlockedIp('127.0.0.1')).toBe(true) + expect(isRemoteMcpBlockedIp('10.1.2.3')).toBe(true) + expect(isRemoteMcpBlockedIp('172.16.0.1')).toBe(true) + expect(isRemoteMcpBlockedIp('192.168.1.10')).toBe(true) + expect(isRemoteMcpBlockedIp('169.254.169.254')).toBe(true) + expect(isRemoteMcpBlockedIp('198.18.0.1')).toBe(true) + expect(isRemoteMcpBlockedIp('203.0.113.10')).toBe(true) + expect(isRemoteMcpBlockedIp('::1')).toBe(true) + expect(isRemoteMcpBlockedIp('fc00::1')).toBe(true) + expect(isRemoteMcpBlockedIp('fe80::1')).toBe(true) + expect(isRemoteMcpBlockedIp('::ffff:127.0.0.1')).toBe(true) + }) + + it('allows public routable addresses', () => { + expect(isRemoteMcpBlockedIp('8.8.8.8')).toBe(false) + expect(isRemoteMcpBlockedIp('1.1.1.1')).toBe(false) + expect(isRemoteMcpBlockedIp('2001:4860:4860::8888')).toBe(false) + }) + + it('rejects URL credentials, fragments, and unsupported schemes', () => { + expect(() => validateRemoteMcpUrl('file:///tmp/server')).toThrow() + expect(() => validateRemoteMcpUrl('https://user:pass@example.com/mcp')).toThrow() + expect(() => validateRemoteMcpUrl('https://example.com/mcp#fragment')).toThrow() + }) + + it('parses allowed config using existing AuthType values', () => { + expect( + remoteMcpConfigFromInput({ + endpointUrl: 'https://example.com/mcp', + authType: AuthType.BEARER_TOKEN, + writeToolsEnabled: false, + }), + ).toEqual({ + endpoint_url: 'https://example.com/mcp', + auth_type: AuthType.BEARER_TOKEN, + write_tools_enabled: false, + }) + expect(() => + remoteMcpConfigFromInput({ + endpointUrl: 'https://example.com/mcp', + authType: 'api_key', + }), + ).toThrow() + }) + + it('rejects OAuth metadata-derived credential URLs resolving to blocked addresses', async () => { + await expect( + validateRemoteMcpUrlForCredentialUse('http://127.0.0.1/token'), + ).rejects.toThrow() + }) + + it('validates immutable source slug shape', () => { + expect(validateRemoteMcpSlug('github_mcp')).toBe('github_mcp') + expect(() => validateRemoteMcpSlug('Remote-MCP')).toThrow() + expect(() => validateRemoteMcpSlug('1github')).toThrow() + }) +}) diff --git a/web/src/lib/server/mcp/client.ts b/web/src/lib/server/mcp/client.ts new file mode 100644 index 000000000..64cceebfc --- /dev/null +++ b/web/src/lib/server/mcp/client.ts @@ -0,0 +1,515 @@ +import { error } from '@sveltejs/kit' +import { lookup } from 'node:dns/promises' +import net from 'node:net' +import { Agent } from 'undici' +import { AuthType } from '$lib/types' + +export interface RemoteMcpConfig { + endpoint_url: string + auth_type?: AuthType.BEARER_TOKEN | AuthType.OAUTH | null + write_tools_enabled: boolean +} + +export interface RemoteMcpProbeOptions { + endpointUrl: string + authType?: AuthType.BEARER_TOKEN | AuthType.OAUTH | null + bearerToken?: string | null +} + +export interface RemoteMcpProbeResult { + ok: boolean + serverName: string | null + serverVersion: string | null + toolCount: number | null + resourceCount: number | null + oauth: Record | null + error?: string +} + +const MAX_RESPONSE_BYTES = 1024 * 1024 +const REMOTE_MCP_HTTP_TIMEOUT_MS = 20_000 +const MCP_PROTOCOL_VERSION = '2024-11-05' + +export function isRemoteMcpBlockedIp(address: string): boolean { + const family = net.isIP(address) + if (family === 4) { + const parts = address.split('.').map((p) => Number.parseInt(p, 10)) + const [a, b, c] = parts + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 0 && c === 0) || + (a === 192 && b === 0 && c === 2) || + (a === 192 && b === 88 && c === 99) || + (a === 192 && b === 168) || + (a === 198 && (b === 18 || b === 19)) || + (a === 198 && b === 51 && c === 100) || + (a === 203 && b === 0 && c === 113) || + a >= 224 + ) + } + if (family === 6) { + const normalized = address.toLowerCase() + const mappedV4 = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1] + if (mappedV4) return isRemoteMcpBlockedIp(mappedV4) + return ( + normalized === '::' || + normalized === '::1' || + normalized.startsWith('fc') || + normalized.startsWith('fd') || + normalized.startsWith('fe80:') || + normalized.startsWith('ff') || + normalized.startsWith('2001:db8:') || + normalized.startsWith('2002:') + ) + } + return true +} + +export function validateRemoteMcpUrl(endpointUrl: string): URL { + let url: URL + try { + url = new URL(endpointUrl) + } catch { + throw error(400, 'Invalid endpoint URL') + } + if (!['https:', 'http:'].includes(url.protocol)) { + throw error(400, 'Endpoint URL must use http or https') + } + if (url.username || url.password) { + throw error(400, 'Endpoint URL must not include credentials') + } + if (url.hash) { + throw error(400, 'Endpoint URL must not include a fragment') + } + if (!url.hostname) { + throw error(400, 'Endpoint URL must include a host') + } + return url +} + +type PinnedAddress = { address: string; family: 4 | 6 } + +async function resolveAllowedRemoteMcpAddresses(url: URL): Promise { + const records = await lookup(url.hostname, { all: true, verbatim: true }) + if (records.length === 0) throw error(400, 'Endpoint host did not resolve') + if (records.some((record) => isRemoteMcpBlockedIp(record.address))) { + throw error(400, 'Endpoint resolves to a disallowed network address') + } + return records.map((record) => ({ address: record.address, family: record.family as 4 | 6 })) +} + +export async function assertRemoteMcpDestinationAllowed(url: URL): Promise { + await resolveAllowedRemoteMcpAddresses(url) +} + +export async function validateRemoteMcpUrlForCredentialUse(endpointUrl: string): Promise { + const url = validateRemoteMcpUrl(endpointUrl) + await assertRemoteMcpDestinationAllowed(url) + return url.toString() +} + +export async function fetchWithPinnedRemoteMcpDns( + url: URL, + init: RequestInit = {}, +): Promise { + const addresses = await resolveAllowedRemoteMcpAddresses(url) + let next = 0 + const dispatcher = new Agent({ + connect: { + lookup(_hostname: string, _options: unknown, callback: (...args: unknown[]) => void) { + const record = addresses[next++ % addresses.length] + callback(null, record.address, record.family) + }, + }, + } as any) + const timeoutSignal = AbortSignal.timeout(REMOTE_MCP_HTTP_TIMEOUT_MS) + const signal = init.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal + try { + const response = await fetch(url.toString(), { + ...init, + signal, + redirect: 'manual', + dispatcher, + } as RequestInit & { dispatcher: unknown }) + const body = await readRemoteMcpLimitedBytes(response) + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }) + } finally { + await dispatcher.close().catch(() => undefined) + } +} + +function parseSlugCandidate(name: string | null): string | null { + if (!name) return null + const slug = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + .slice(0, 50) + return /^[a-z][a-z0-9_]{1,49}$/.test(slug) ? slug : null +} + +export function validateRemoteMcpSlug(sourceType: string): string { + if (!/^[a-z][a-z0-9_]{1,49}$/.test(sourceType)) { + throw error(400, 'sourceType must match ^[a-z][a-z0-9_]{1,49}$') + } + return sourceType +} + +async function readRemoteMcpLimitedBytes(response: Response): Promise { + const reader = response.body?.getReader() + if (!reader) return new Uint8Array() + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > MAX_RESPONSE_BYTES) throw new Error('MCP response too large') + chunks.push(value) + } + return Buffer.concat(chunks) +} + +export async function readRemoteMcpLimitedJson(response: Response): Promise { + const body = new TextDecoder().decode(await readRemoteMcpLimitedBytes(response)) + if (!body.trim()) return null + const eventData = body + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trim()) + .find((line) => line && line !== '[DONE]') + return JSON.parse(eventData ?? body) +} + +function resourceMetadataUrlFromChallenge(response: Response): string | null { + const challenge = response.headers.get('www-authenticate') + if (!challenge) return null + return /resource_metadata="([^"]+)"/.exec(challenge)?.[1] ?? null +} + +function oauthFromChallenge(response: Response): Record | null { + const resourceMetadata = resourceMetadataUrlFromChallenge(response) + if (!resourceMetadata) return null + return { + protected_resource_metadata_url: resourceMetadata, + } +} + +async function mcpPost( + endpointUrl: string, + headers: HeadersInit, + sessionId: string | null, + body: Record, +): Promise<{ response: Response; json: any; sessionId: string | null }> { + const url = validateRemoteMcpUrl(endpointUrl) + + const requestHeaders: Record = { + accept: 'application/json, text/event-stream', + 'content-type': 'application/json', + ...Object.fromEntries(new Headers(headers).entries()), + } + if (sessionId) requestHeaders['mcp-session-id'] = sessionId + + const response = await fetchWithPinnedRemoteMcpDns(url, { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(body), + }) + const nextSessionId = response.headers.get('mcp-session-id') ?? sessionId + const json = response.ok ? await readRemoteMcpLimitedJson(response) : null + return { response, json, sessionId: nextSessionId } +} + +function remoteMcpProbeErrorMessage(err: unknown): string { + if (err instanceof Error) return err.message + if (err && typeof err === 'object') { + const body = 'body' in err ? (err as { body?: unknown }).body : undefined + if (body && typeof body === 'object' && 'message' in body) { + const message = (body as { message?: unknown }).message + if (typeof message === 'string') return message + } + if ('message' in err && typeof (err as { message?: unknown }).message === 'string') { + return (err as { message: string }).message + } + } + return 'Remote MCP probe failed' +} + +export async function probeRemoteMcpServer( + options: RemoteMcpProbeOptions, +): Promise { + const url = validateRemoteMcpUrl(options.endpointUrl) + await assertRemoteMcpDestinationAllowed(url) + + const authHeaders: Record = {} + if (options.authType === AuthType.BEARER_TOKEN && options.bearerToken) { + authHeaders.authorization = `Bearer ${options.bearerToken}` + } + + try { + const init = await mcpPost(options.endpointUrl, authHeaders, null, { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: MCP_PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: 'omni-web', version: '1.0.0' }, + }, + }) + + if (init.response.status === 401 || init.response.status === 403) { + return { + ok: options.authType === AuthType.OAUTH, + serverName: null, + serverVersion: null, + toolCount: null, + resourceCount: null, + oauth: oauthFromChallenge(init.response), + suggestedSourceType: null, + error: options.authType === AuthType.OAUTH ? undefined : 'Authorization required', + } + } + if (!init.response.ok) { + return { + ok: false, + serverName: null, + serverVersion: null, + toolCount: null, + resourceCount: null, + oauth: null, + suggestedSourceType: null, + error: `MCP initialize failed with HTTP ${init.response.status}`, + } + } + + const serverInfo = init.json?.result?.serverInfo ?? {} + const sessionId = init.sessionId + await mcpPost(options.endpointUrl, authHeaders, sessionId, { + jsonrpc: '2.0', + method: 'notifications/initialized', + params: {}, + }).catch(() => null) + + const tools = await mcpPost(options.endpointUrl, authHeaders, sessionId, { + jsonrpc: '2.0', + id: 2, + method: 'tools/list', + params: {}, + }).catch(() => null) + const resources = await mcpPost(options.endpointUrl, authHeaders, sessionId, { + jsonrpc: '2.0', + id: 3, + method: 'resources/list', + params: {}, + }).catch(() => null) + const templates = await mcpPost(options.endpointUrl, authHeaders, sessionId, { + jsonrpc: '2.0', + id: 4, + method: 'resources/templates/list', + params: {}, + }).catch(() => null) + + const serverName = typeof serverInfo.name === 'string' ? serverInfo.name : null + const serverVersion = typeof serverInfo.version === 'string' ? serverInfo.version : null + const toolCount = Array.isArray(tools?.json?.result?.tools) + ? tools.json.result.tools.length + : 0 + const resourceCount = + (Array.isArray(resources?.json?.result?.resources) + ? resources.json.result.resources.length + : 0) + + (Array.isArray(templates?.json?.result?.resourceTemplates) + ? templates.json.result.resourceTemplates.length + : 0) + + return { + ok: true, + serverName, + serverVersion, + toolCount, + resourceCount, + oauth: null, + suggestedSourceType: parseSlugCandidate(serverName), + } + } catch (err) { + return { + ok: false, + serverName: null, + serverVersion: null, + toolCount: null, + resourceCount: null, + oauth: null, + suggestedSourceType: null, + error: remoteMcpProbeErrorMessage(err), + } + } +} + +function wellKnownProtectedResourceUrls(endpointUrl: string): string[] { + const endpoint = new URL(endpointUrl) + const urls = [`${endpoint.origin}/.well-known/oauth-protected-resource`] + if (endpoint.pathname && endpoint.pathname !== '/') { + urls.unshift(`${endpoint.origin}/.well-known/oauth-protected-resource${endpoint.pathname}`) + } + return [...new Set(urls)] +} + +async function fetchAllowedJson(url: string): Promise { + const parsed = validateRemoteMcpUrl(url) + const response = await fetchWithPinnedRemoteMcpDns(parsed, { + method: 'GET', + headers: { accept: 'application/json' }, + }) + if (!response.ok) return null + return readRemoteMcpLimitedJson(response) +} + +async function protectedResourceMetadataUrl(endpointUrl: string): Promise { + const init = await mcpPost(endpointUrl, {}, null, { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: MCP_PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: 'omni-web', version: '1.0.0' }, + }, + }).catch(() => null) + const challenged = init ? resourceMetadataUrlFromChallenge(init.response) : null + if (challenged) return challenged + + for (const candidate of wellKnownProtectedResourceUrls(endpointUrl)) { + if (await fetchAllowedJson(candidate)) return candidate + } + return null +} + +function authServerMetadataUrls(issuerOrMetadata: string): string[] { + const url = validateRemoteMcpUrl(issuerOrMetadata) + if (url.pathname.includes('/.well-known/oauth-authorization-server')) { + return [url.toString()] + } + return [ + `${url.origin}/.well-known/oauth-authorization-server${url.pathname === '/' ? '' : url.pathname}`, + `${url.origin}/.well-known/openid-configuration${url.pathname === '/' ? '' : url.pathname}`, + ] +} + +export async function discoverRemoteMcpOAuthConfig(args: { + endpointUrl: string + sourceType: string +}): Promise | null> { + const endpoint = validateRemoteMcpUrl(args.endpointUrl) + await assertRemoteMcpDestinationAllowed(endpoint) + + const prmUrl = await protectedResourceMetadataUrl(endpoint.toString()) + if (!prmUrl) return null + const prm = await fetchAllowedJson(prmUrl) + if (!prm || typeof prm !== 'object') return null + + const authServers = Array.isArray(prm.authorization_servers) + ? prm.authorization_servers.filter( + (value: unknown): value is string => typeof value === 'string', + ) + : [] + if (authServers.length === 0) return null + + for (const authServer of authServers) { + for (const metadataUrl of authServerMetadataUrls(authServer)) { + const asMetadata = await fetchAllowedJson(metadataUrl) + if (!asMetadata || typeof asMetadata !== 'object') continue + if ( + typeof asMetadata.authorization_endpoint !== 'string' || + typeof asMetadata.token_endpoint !== 'string' + ) { + continue + } + const authEndpoint = await validateRemoteMcpUrlForCredentialUse( + asMetadata.authorization_endpoint, + ).catch(() => null) + const tokenEndpoint = await validateRemoteMcpUrlForCredentialUse( + asMetadata.token_endpoint, + ).catch(() => null) + if (!authEndpoint || !tokenEndpoint) continue + const userinfoEndpoint = + typeof asMetadata.userinfo_endpoint === 'string' + ? await validateRemoteMcpUrlForCredentialUse( + asMetadata.userinfo_endpoint, + ).catch(() => null) + : endpoint.origin + const registrationEndpoint = + typeof asMetadata.registration_endpoint === 'string' + ? await validateRemoteMcpUrlForCredentialUse( + asMetadata.registration_endpoint, + ).catch(() => null) + : null + const resource = + typeof prm.resource === 'string' + ? await validateRemoteMcpUrlForCredentialUse(prm.resource).catch(() => null) + : endpoint.toString() + if ( + !userinfoEndpoint || + (asMetadata.registration_endpoint && !registrationEndpoint) || + !resource + ) { + continue + } + const scopes = Array.isArray(asMetadata.scopes_supported) + ? asMetadata.scopes_supported.filter( + (value: unknown): value is string => typeof value === 'string', + ) + : [] + const tokenAuthMethods = Array.isArray(asMetadata.token_endpoint_auth_methods_supported) + ? asMetadata.token_endpoint_auth_methods_supported + : [] + return { + provider: `remote_mcp:${args.sourceType}`, + credential_provider: 'remote_mcp', + auth_endpoint: authEndpoint, + token_endpoint: tokenEndpoint, + userinfo_endpoint: userinfoEndpoint, + userinfo_email_field: 'email', + identity_scopes: [], + scopes: { [args.sourceType]: { read: scopes, write: scopes } }, + extra_auth_params: {}, + scope_separator: ' ', + registration_endpoint: registrationEndpoint, + token_endpoint_auth_method: tokenAuthMethods.includes('none') + ? 'none' + : 'client_secret_post', + resource, + protected_resource_metadata_url: prmUrl, + authorization_server_metadata_url: metadataUrl, + } + } + } + + return null +} + +export function remoteMcpConfigFromInput(input: { + endpointUrl: string + authType?: string | null + writeToolsEnabled?: boolean +}): RemoteMcpConfig { + const url = validateRemoteMcpUrl(input.endpointUrl) + const authType = input.authType ?? null + if (authType !== null && authType !== AuthType.BEARER_TOKEN && authType !== AuthType.OAUTH) { + throw error(400, 'authType must be null, bearer_token, or oauth') + } + return { + endpoint_url: url.toString(), + auth_type: authType, + write_tools_enabled: input.writeToolsEnabled ?? true, + } +} diff --git a/web/src/lib/server/mcp/probe.test.ts b/web/src/lib/server/mcp/probe.test.ts new file mode 100644 index 000000000..a1b7626c5 --- /dev/null +++ b/web/src/lib/server/mcp/probe.test.ts @@ -0,0 +1,121 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { AuthType } from '$lib/types' + +const { lookupMock, fetchMock } = vi.hoisted(() => ({ + lookupMock: vi.fn(), + fetchMock: vi.fn(), +})) + +vi.mock('node:dns/promises', () => ({ + lookup: lookupMock, +})) + +function jsonRpcResponse(body: unknown, headers?: Record) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json', ...headers }, + }) +} + +describe('remote MCP probe network behavior', () => { + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + lookupMock.mockResolvedValue([{ address: '8.8.8.8', family: 4 }]) + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + it('uses bearer auth, disables redirects, and re-checks DNS before each MCP request', async () => { + const { probeRemoteMcpServer } = await import('./client') + fetchMock + .mockResolvedValueOnce( + jsonRpcResponse( + { + jsonrpc: '2.0', + result: { serverInfo: { name: 'Acme Docs', version: '1.0.0' } }, + }, + { 'mcp-session-id': 'session-1' }, + ), + ) + .mockResolvedValueOnce(jsonRpcResponse({ jsonrpc: '2.0', result: {} })) + .mockResolvedValueOnce( + jsonRpcResponse({ jsonrpc: '2.0', result: { tools: [{ name: 'search' }] } }), + ) + .mockResolvedValueOnce( + jsonRpcResponse({ + jsonrpc: '2.0', + result: { resources: [{ uri: 'docs://guide' }] }, + }), + ) + .mockResolvedValueOnce( + jsonRpcResponse({ jsonrpc: '2.0', result: { resourceTemplates: [] } }), + ) + + const result = await probeRemoteMcpServer({ + endpointUrl: 'https://mcp.example.com/mcp', + authType: AuthType.BEARER_TOKEN, + bearerToken: 'secret-token', + }) + + expect(result).toMatchObject({ + ok: true, + serverName: 'Acme Docs', + serverVersion: '1.0.0', + toolCount: 1, + resourceCount: 1, + suggestedSourceType: 'acme_docs', + }) + expect(lookupMock).toHaveBeenCalledTimes(6) + expect(fetchMock).toHaveBeenCalledWith( + 'https://mcp.example.com/mcp', + expect.objectContaining({ + method: 'POST', + redirect: 'manual', + signal: expect.any(AbortSignal), + headers: expect.objectContaining({ + authorization: 'Bearer secret-token', + 'mcp-session-id': 'session-1', + }), + }), + ) + }) + + it('treats OAuth authorization challenges as a successful setup probe with normalized metadata', async () => { + const { probeRemoteMcpServer } = await import('./client') + fetchMock.mockResolvedValueOnce( + new Response('', { + status: 401, + headers: { + 'www-authenticate': + 'Bearer resource_metadata="https://mcp.example.com/.well-known/oauth-protected-resource"', + }, + }), + ) + + const result = await probeRemoteMcpServer({ + endpointUrl: 'https://mcp.example.com/mcp', + authType: AuthType.OAUTH, + }) + + expect(result.ok).toBe(true) + expect(result.oauth).toEqual({ + protected_resource_metadata_url: + 'https://mcp.example.com/.well-known/oauth-protected-resource', + }) + expect(result.error).toBeUndefined() + }) + + it('fails closed when DNS rebinding changes the resolved address before an MCP request', async () => { + const { probeRemoteMcpServer } = await import('./client') + lookupMock + .mockResolvedValueOnce([{ address: '8.8.8.8', family: 4 }]) + .mockResolvedValueOnce([{ address: '127.0.0.1', family: 4 }]) + + const result = await probeRemoteMcpServer({ endpointUrl: 'https://mcp.example.com/mcp' }) + + expect(result.ok).toBe(false) + expect(result.error).toContain('Endpoint resolves to a disallowed network address') + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/lib/server/oauth/connectorOAuth.ts b/web/src/lib/server/oauth/connectorOAuth.ts index 2aa09b882..f3806eeda 100644 --- a/web/src/lib/server/oauth/connectorOAuth.ts +++ b/web/src/lib/server/oauth/connectorOAuth.ts @@ -1,13 +1,23 @@ import { createHash, randomBytes } from 'crypto' import { app, getConfig } from '../config' import { getConnectorConfig, upsertConnectorConfig } from '../db/connector-configs' +import type { Source } from '../db/schema' import { createLogger } from '../logger' +import { + discoverRemoteMcpOAuthConfig, + fetchWithPinnedRemoteMcpDns, + readRemoteMcpLimitedJson, + validateRemoteMcpUrl, + validateRemoteMcpUrlForCredentialUse, +} from '../mcp/client' import { OAuthStateManager } from './state' import type { OAuthError, OAuthTokens } from './types' +import { IntegrationType } from '$lib/types' export type OAuthTokenEndpointAuthMethod = 'client_secret_post' | 'client_secret_basic' | 'none' const logger = createLogger('connector-oauth') +const OAUTH_RESPONSE_MAX_BYTES = 1024 * 1024 function isOAuthTokenEndpointAuthMethod(value: unknown): value is OAuthTokenEndpointAuthMethod { return value === 'client_secret_post' || value === 'client_secret_basic' || value === 'none' @@ -30,6 +40,9 @@ export interface OAuthManifestConfig { registration_endpoint?: string | null token_endpoint_auth_method?: OAuthTokenEndpointAuthMethod resource?: string | null + credential_provider?: string | null + protected_resource_metadata_url?: string | null + authorization_server_metadata_url?: string | null } /// What flow we're driving — encoded into the OAuth state so the single @@ -88,6 +101,23 @@ export async function getOAuthManifestForSourceType( return entry?.manifest?.oauth ?? null } +export async function getOAuthConfigForSource( + source: Pick, +): Promise { + if (source.integrationType !== IntegrationType.REMOTE_MCP) { + return getOAuthManifestForSourceType(source.sourceType) + } + + const provider = `remote_mcp:${source.sourceType}` + const manifestConfig = await getOAuthManifestForSourceType(source.sourceType) + if (manifestConfig?.provider === provider) return manifestConfig + + return (await discoverRemoteMcpOAuthConfig({ + endpointUrl: String((source.config as Record)?.endpoint_url ?? ''), + sourceType: source.sourceType, + })) as OAuthManifestConfig | null +} + interface ClientCreds { clientId: string clientSecret?: string @@ -130,6 +160,69 @@ async function loadClientCreds( return null } +function isRemoteMcpProvider(provider: string): boolean { + return provider.startsWith('remote_mcp:') +} + +async function remoteMcpCredentialFetch( + provider: string, + endpoint: string, + init: RequestInit, +): Promise { + if (!isRemoteMcpProvider(provider)) { + return fetch(endpoint, { + ...init, + signal: init.signal ?? AbortSignal.timeout(20_000), + }) + } + const validated = await validateRemoteMcpUrlForCredentialUse(endpoint) + return fetchWithPinnedRemoteMcpDns(validateRemoteMcpUrl(validated), init) +} + +async function readLimitedResponseText(response: Response): Promise { + const reader = response.body?.getReader() + if (!reader) return '' + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > OAUTH_RESPONSE_MAX_BYTES) throw new Error('OAuth response too large') + chunks.push(value) + } + return new TextDecoder().decode(Buffer.concat(chunks)) +} + +async function readCredentialJson(provider: string, response: Response): Promise { + if (isRemoteMcpProvider(provider)) return readRemoteMcpLimitedJson(response) + const text = await readLimitedResponseText(response) + return text.trim() ? JSON.parse(text) : {} +} + +async function readCredentialText(_provider: string, response: Response): Promise { + return readLimitedResponseText(response) +} + +async function validateRemoteMcpOAuthConfigUrls(config: OAuthManifestConfig): Promise { + if (!isRemoteMcpProvider(config.provider)) return + await validateRemoteMcpUrlForCredentialUse(config.auth_endpoint) + await validateRemoteMcpUrlForCredentialUse(config.token_endpoint) + await validateRemoteMcpUrlForCredentialUse(config.userinfo_endpoint) + if (config.registration_endpoint) { + await validateRemoteMcpUrlForCredentialUse(config.registration_endpoint) + } + if (config.resource) { + await validateRemoteMcpUrlForCredentialUse(config.resource) + } + if (config.protected_resource_metadata_url) { + await validateRemoteMcpUrlForCredentialUse(config.protected_resource_metadata_url) + } + if (config.authorization_server_metadata_url) { + await validateRemoteMcpUrlForCredentialUse(config.authorization_server_metadata_url) + } +} + async function dynamicallyRegisterClient( provider: string, config: OAuthManifestConfig, @@ -141,7 +234,8 @@ async function dynamicallyRegisterClient( ) let response: Response try { - response = await fetch(config.registration_endpoint!, { + await validateRemoteMcpOAuthConfigUrls(config) + response = await remoteMcpCredentialFetch(provider, config.registration_endpoint!, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -159,7 +253,9 @@ async function dynamicallyRegisterClient( } catch { return null } - const data = (await response.json().catch(() => ({}))) as { client_id?: string } + const data = (await readCredentialJson(provider, response).catch(() => ({}))) as { + client_id?: string + } if (!response.ok || !data.client_id) return null const stored = { @@ -281,10 +377,13 @@ export async function generateAuthUrl(args: { export async function generateAuthUrlForOrgSource(args: { sourceId: string sourceType: string + source?: Pick userId: string returnTo?: string }): Promise<{ url: string; requiredScopes: string[] }> { - const manifestConfig = await getOAuthManifestForSourceType(args.sourceType) + const manifestConfig = args.source + ? await getOAuthConfigForSource(args.source) + : await getOAuthManifestForSourceType(args.sourceType) if (!manifestConfig) { throw new Error(`No OAuth manifest for source_type=${args.sourceType}`) } @@ -323,13 +422,16 @@ export async function generateAuthUrlForOrgSource(args: { async function generateAuthUrlForExistingSourceUserFlow(args: { sourceId: string sourceType: string + source?: Pick userId: string returnTo?: string approvalId?: string approvalChatId?: string mode: 'read' | 'write' }): Promise<{ url: string; requiredScopes: string[] }> { - const manifestConfig = await getOAuthManifestForSourceType(args.sourceType) + const manifestConfig = args.source + ? await getOAuthConfigForSource(args.source) + : await getOAuthManifestForSourceType(args.sourceType) if (!manifestConfig) { throw new Error(`No OAuth manifest for source_type=${args.sourceType}`) } @@ -342,7 +444,7 @@ async function generateAuthUrlForExistingSourceUserFlow(args: { const writeScopes = manifestConfig.scopes[args.sourceType]?.write ?? [] const actionScopes = args.mode === 'write' ? [...new Set([...readScopes, ...writeScopes])] : readScopes - if (actionScopes.length === 0) { + if (actionScopes.length === 0 && args.source?.integrationType !== IntegrationType.REMOTE_MCP) { throw new Error(`No ${args.mode} action scopes declared for source_type=${args.sourceType}`) } @@ -388,6 +490,7 @@ async function generateAuthUrlForExistingSourceUserFlow(args: { export async function generateAuthUrlForUserRead(args: { sourceId: string sourceType: string + source?: Pick userId: string returnTo?: string }): Promise<{ url: string; requiredScopes: string[] }> { @@ -399,6 +502,7 @@ export async function generateAuthUrlForUserRead(args: { export async function generateAuthUrlForUserWrite(args: { sourceId: string sourceType: string + source?: Pick userId: string returnTo?: string approvalId?: string @@ -505,7 +609,11 @@ export async function exchangeCodeAndIdentify( tokenParams.set('resource', config.resource) } + await validateRemoteMcpOAuthConfigUrls(config) const tokenEndpoint = creds.tokenEndpoint ?? config.token_endpoint + if (isRemoteMcpProvider(config.provider)) { + await validateRemoteMcpUrlForCredentialUse(tokenEndpoint) + } logger.info('Starting connector OAuth token exchange', { provider: config.provider, flow: state.metadata.flow.type, @@ -517,12 +625,14 @@ export async function exchangeCodeAndIdentify( resource: config.resource ?? null, requestedScopes: state.metadata.requiredScopes, }) - const tokenResp = await fetch(tokenEndpoint, { + const tokenResp = await remoteMcpCredentialFetch(config.provider, tokenEndpoint, { method: 'POST', headers: tokenHeaders, body: tokenParams.toString(), }) - const tokenData = (await tokenResp.json().catch(() => ({}))) as OAuthTokens | OAuthError + const tokenData = (await readCredentialJson(config.provider, tokenResp).catch(() => ({}))) as + | OAuthTokens + | OAuthError if (!tokenResp.ok) { const err = tokenData as OAuthError logger.warn('Connector OAuth token exchange failed', { @@ -556,14 +666,14 @@ export async function exchangeCodeAndIdentify( return { tokens, state, config, principalEmail: principalEmailOverride, clientCreds: creds } } - const userinfoResp = await fetch(config.userinfo_endpoint, { + const userinfoResp = await remoteMcpCredentialFetch(config.provider, config.userinfo_endpoint, { headers: { Authorization: `Bearer ${tokens.access_token}`, Accept: 'application/json', }, }) if (!userinfoResp.ok) { - const body = await userinfoResp.text().catch(() => '') + const body = await readCredentialText(config.provider, userinfoResp).catch(() => '') logger.warn('Connector OAuth userinfo fetch failed', { provider: config.provider, flow: state.metadata.flow.type, @@ -573,7 +683,7 @@ export async function exchangeCodeAndIdentify( }) throw new Error(`Failed to fetch userinfo: ${userinfoResp.status}`) } - const profile = (await userinfoResp.json()) as unknown + const profile = (await readCredentialJson(config.provider, userinfoResp)) as unknown const email = extractEmailFromUserinfo(profile, config.userinfo_email_field) if (!email) { logger.warn('Connector OAuth userinfo response missing email field', { diff --git a/web/src/lib/types.test.ts b/web/src/lib/types.test.ts new file mode 100644 index 000000000..9353de6cb --- /dev/null +++ b/web/src/lib/types.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'vitest' +import { IntegrationType, supportsDataSync } from './types' + +describe('supportsDataSync', () => { + it('allows native connector sources and legacy missing integration types', () => { + expect(supportsDataSync(IntegrationType.CONNECTOR)).toBe(true) + expect(supportsDataSync(undefined)).toBe(true) + expect(supportsDataSync(null)).toBe(true) + }) + + it('rejects remote MCP sources', () => { + expect(supportsDataSync(IntegrationType.REMOTE_MCP)).toBe(false) + }) +}) diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 2d75d4376..013c3ce2b 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -1,3 +1,12 @@ +export enum IntegrationType { + CONNECTOR = 'connector', + REMOTE_MCP = 'remote_mcp', +} + +export function supportsDataSync(integrationType?: string | null): boolean { + return (integrationType ?? IntegrationType.CONNECTOR) === IntegrationType.CONNECTOR +} + export enum SourceType { GOOGLE_DRIVE = 'google_drive', GMAIL = 'gmail', @@ -41,6 +50,7 @@ export enum ServiceProvider { NEXTCLOUD = 'nextcloud', GOOGLE_ADS = 'google_ads', DARWINBOX = 'darwinbox', + REMOTE_MCP = 'remote_mcp', } export enum AuthType { diff --git a/web/src/routes/(admin)/admin/settings/+layout.svelte b/web/src/routes/(admin)/admin/settings/+layout.svelte index 33c245ee4..d7485eb57 100644 --- a/web/src/routes/(admin)/admin/settings/+layout.svelte +++ b/web/src/routes/(admin)/admin/settings/+layout.svelte @@ -16,6 +16,7 @@ FileText, Brain, Globe, + Server, } from '@lucide/svelte' import Button from '$lib/components/ui/button/button.svelte' import SidebarUserMenu from '$lib/components/sidebar-user-menu.svelte' @@ -195,6 +196,21 @@ {/if} + + + {#snippet child({ props })} + + + MCP + + {/snippet} + + @@ -209,8 +225,9 @@
-
- +
+ Admin Settings
diff --git a/web/src/routes/(admin)/admin/settings/integrations/+layout.server.ts b/web/src/routes/(admin)/admin/settings/integrations/+layout.server.ts index bf51ab251..0cdfee762 100644 --- a/web/src/routes/(admin)/admin/settings/integrations/+layout.server.ts +++ b/web/src/routes/(admin)/admin/settings/integrations/+layout.server.ts @@ -13,6 +13,7 @@ function mapSource(source: Record): Source { id: source.id as string, name: source.name as string, sourceType: source.source_type as string, + integrationType: (source.integration_type as string | undefined) ?? 'connector', config: source.config, isActive: source.is_active as boolean, isDeleted: source.is_deleted as boolean, diff --git a/web/src/routes/(admin)/admin/settings/integrations/+page.server.ts b/web/src/routes/(admin)/admin/settings/integrations/+page.server.ts index 4ee605ddd..baaa74a4d 100644 --- a/web/src/routes/(admin)/admin/settings/integrations/+page.server.ts +++ b/web/src/routes/(admin)/admin/settings/integrations/+page.server.ts @@ -10,6 +10,7 @@ import { type OAuthManifestConfig, } from '$lib/server/oauth/connectorOAuth' import type { SyncRun } from '$lib/server/db/schema' +import { IntegrationType, supportsDataSync } from '$lib/types' import type { PageServerLoad } from './$types' const CONNECTOR_DISPLAY_ORDER: string[] = [ @@ -52,7 +53,10 @@ interface ConnectorInfo { connector_id?: string display_name?: string description?: string + integration_type?: string source_types?: string[] + actions?: unknown[] + resources?: unknown[] oauth?: OAuthManifestConfig | null } } @@ -102,9 +106,11 @@ function mapSyncRun(run: Record): SyncRun { export const load: PageServerLoad = async ({ locals }) => { requireAdmin(locals) - const connectedSources = await sourcesRepository.getOrgWide() - const sourceIds = connectedSources.map((s) => s.id) - const latestSyncRuns = await sourcesRepository.getLatestSyncRunsForSourceIds(sourceIds) + const orgSources = await sourcesRepository.getOrgWide() + const connectedSources = orgSources.filter((source) => supportsDataSync(source.integrationType)) + const connectedSourceIds = connectedSources.map((source) => source.id) + const connectedSourceIdSet = new Set(connectedSourceIds) + const latestSyncRuns = await sourcesRepository.getLatestSyncRunsForSourceIds(connectedSourceIds) const savedOAuthConfigs = await getAllConnectorConfigsPublic() const savedOAuthConfigByProvider = new Map(savedOAuthConfigs.map((row) => [row.provider, row])) @@ -128,6 +134,7 @@ export const load: PageServerLoad = async ({ locals }) => { if (sourcesResponse.ok) { const overviews = (await sourcesResponse.json()) as ConnectorManagerSourceOverview[] for (const overview of overviews) { + if (!connectedSourceIdSet.has(overview.source.id)) continue sourceHealth.set(overview.source.id, overview.health) if (overview.sync_runs[0]) { latestSyncRuns.set(overview.source.id, mapSyncRun(overview.sync_runs[0])) @@ -147,6 +154,7 @@ export const load: PageServerLoad = async ({ locals }) => { const oauthManifestByProvider = new Map() for (const connector of connectors) { + if (connector.manifest?.integration_type === IntegrationType.REMOTE_MCP) continue const connectorId = connector.manifest?.connector_id ?? connector.source_type if (!integrationMap.has(connectorId)) { integrationMap.set(connectorId, { @@ -184,6 +192,7 @@ export const load: PageServerLoad = async ({ locals }) => { oauthProviders = Array.from(sourceTypesByOAuthProvider.keys()) .filter( (provider) => + !provider.startsWith('remote_mcp:') && !isAutoManagedOAuthProvider(oauthManifestByProvider.get(provider)), ) .map((provider) => { diff --git a/web/src/routes/(admin)/admin/settings/integrations/+page.svelte b/web/src/routes/(admin)/admin/settings/integrations/+page.svelte index 2699a401d..be8197b46 100644 --- a/web/src/routes/(admin)/admin/settings/integrations/+page.svelte +++ b/web/src/routes/(admin)/admin/settings/integrations/+page.svelte @@ -41,6 +41,7 @@ HardDrive, KeyRound, Mail, + Plus, } from '@lucide/svelte' import { toast } from 'svelte-sonner' import GoogleWorkspaceSetup from '$lib/components/google-workspace-setup.svelte' @@ -62,7 +63,7 @@ import DarwinboxConnectorSetup from '$lib/components/darwinbox-connector-setup.svelte' import OAuthClientConfigDialog from '$lib/components/oauth-integrations/oauth-client-config-dialog.svelte' import { Badge } from '$lib/components/ui/badge' - import { SourceType } from '$lib/types' + import { AuthType, SourceType } from '$lib/types' import { formatDate, getSourceNoun, getStatusColor } from '$lib/utils/sources' import { invalidateAll } from '$app/navigation' import { page } from '$app/state' @@ -248,6 +249,12 @@ const slug = sourceTypeSlug[sourceType] ?? sourceType return `/admin/settings/integrations/${slug}/${sourceId}` } + + function remoteMcpAuthLabel(authType: string | null) { + if (authType === AuthType.BEARER_TOKEN) return 'Shared bearer' + if (authType === AuthType.OAUTH) return 'Per-user OAuth' + return 'Public' + } diff --git a/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts b/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts new file mode 100644 index 000000000..5338b269a --- /dev/null +++ b/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts @@ -0,0 +1,7 @@ +import { requireAdmin } from '$lib/server/authHelpers' +import type { PageServerLoad } from './$types' + +export const load: PageServerLoad = async ({ locals }) => { + requireAdmin(locals) + return {} +} diff --git a/web/src/routes/(admin)/admin/settings/mcp/+page.svelte b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte new file mode 100644 index 000000000..ac79c5d08 --- /dev/null +++ b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte @@ -0,0 +1,193 @@ + + +Add Remote MCP Server + +
+
+
+

Add remote MCP server

+

+ Configure a remote Streamable HTTP MCP endpoint. Catalogs are discovered at runtime; + secrets are write-only. +

+
+ + + + Connection + Only remote HTTP(S) MCP endpoints are supported. + + +
+ + +
+
+ + +
+
+ + +

+ Immutable after creation. Use lowercase letters, numbers, and underscores. +

+
+
+ + +
+ {#if authType === AuthType.BEARER_TOKEN} +
+ + +

+ Stored encrypted and never returned by the API. +

+
+ {/if} + {#if authType === AuthType.OAUTH} +

+ After creation, configure this server's OAuth client and authorize an admin + bootstrap credential on the edit page. +

+ {/if} + + +
+ + +
+ + {#if probe?.ok} +
+
+ Connected + {#if probe.serverName}{probe.serverName}{/if} + {#if probe.serverVersion}v{probe.serverVersion}{/if} +
+
+ {probe.toolCount ?? 0} tools · {probe.resourceCount ?? 0} resources +
+
+ {/if} + + +
+
diff --git a/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.server.ts b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.server.ts new file mode 100644 index 000000000..8eae33831 --- /dev/null +++ b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.server.ts @@ -0,0 +1,81 @@ +import { error } from '@sveltejs/kit' +import { requireAdmin } from '$lib/server/authHelpers' +import { getConfig } from '$lib/server/config' +import { getConnectorConfigPublic } from '$lib/server/db/connector-configs' +import { + isClientConfigComplete, + tokenEndpointAuthMethodForConfig, +} from '$lib/server/oauth/connectorOAuth' +import { IntegrationType } from '$lib/types' +import type { PageServerLoad } from './$types' + +type RemoteMcpSourceResponse = { + id: string + name: string + sourceType: string + authType: string | null + config: Record + isActive: boolean +} + +type ConnectorInfo = { + source_type: string + manifest?: { + integration_type?: string + actions?: unknown[] + resources?: unknown[] + oauth?: Record | null + } | null +} + +export const load: PageServerLoad = async ({ locals, fetch, params }) => { + requireAdmin(locals) + + const sourceResponse = await fetch(`/api/remote-mcp/${params.sourceId}`) + if (!sourceResponse.ok) { + throw error(sourceResponse.status, 'Remote MCP source not found') + } + const source = (await sourceResponse.json()) as RemoteMcpSourceResponse + + let manifestAvailable = false + let toolCount = 0 + let resourceCount = 0 + let oauthProvider = `remote_mcp:${source.sourceType}` + let oauthManifest: Record | null = null + + try { + const response = await fetch(`${getConfig().services.connectorManagerUrl}/connectors`) + if (response.ok) { + const connectors = (await response.json()) as ConnectorInfo[] + const entry = connectors.find( + (connector) => + connector.source_type === source.sourceType && + connector.manifest?.integration_type === IntegrationType.REMOTE_MCP, + ) + const manifest = entry?.manifest + manifestAvailable = Boolean(manifest) + toolCount = manifest?.actions?.length ?? 0 + resourceCount = manifest?.resources?.length ?? 0 + oauthManifest = manifest?.oauth ?? null + if (typeof oauthManifest?.provider === 'string') oauthProvider = oauthManifest.provider + } + } catch (err) { + locals.logger.warn('Failed to fetch remote MCP manifest status', err) + } + + const oauthClient = await getConnectorConfigPublic(oauthProvider) + const tokenEndpointAuthMethod = tokenEndpointAuthMethodForConfig( + oauthClient?.config, + oauthManifest as any, + ) + + return { + source, + manifest: { available: manifestAvailable, toolCount, resourceCount }, + oauth: { + provider: oauthProvider, + configured: isClientConfigComplete(oauthClient?.config, tokenEndpointAuthMethod), + config: oauthClient?.config ?? {}, + }, + } +} diff --git a/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.svelte b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.svelte new file mode 100644 index 000000000..49348a27a --- /dev/null +++ b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/+page.svelte @@ -0,0 +1,240 @@ + + +{source.name} - Remote MCP + +
+
+
+

Edit remote MCP server

+

+ {source.name} · {source.sourceType} +

+
+ + + + Runtime catalog + Catalog data is discovered by Connector Manager and stored only in Redis. + + + {data.manifest.available ? 'Available' : 'Unavailable'} + {data.manifest.toolCount} tools · {data.manifest.resourceCount} resources + {#if probeSummary}Latest probe: {probeSummary}{/if} + + + + + + + Connection + Source slug is immutable. Secrets are write-only and never displayed. + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {#if authType === AuthType.BEARER_TOKEN} +
+ + +
+ {/if} + +
+ + + +
+ + + + {#if authType === AuthType.OAUTH} + + + OAuth + OAuth clients for remote MCP are scoped to this source: {data.oauth.provider}. + + +
+ {data.oauth.configured + ? 'Client configured' + : 'Client not configured'} + Users authorize individually before invoking protected tools. +
+
+ + +
+ + + {/if} +
+
+ + (oauthDialogOpen = false)} + onCancel={() => (oauthDialogOpen = false)} /> diff --git a/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/page.server.test.ts b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/page.server.test.ts new file mode 100644 index 000000000..fe40ac59e --- /dev/null +++ b/web/src/routes/(admin)/admin/settings/mcp/[sourceId]/page.server.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from 'vitest' +import { IntegrationType } from '$lib/types' +import { load } from './+page.server' + +vi.mock('$lib/server/authHelpers', () => ({ + requireAdmin: vi.fn(), +})) + +vi.mock('$lib/server/config', () => ({ + getConfig: () => ({ services: { connectorManagerUrl: 'http://connector-manager.test' } }), +})) + +vi.mock('$lib/server/db/connector-configs', () => ({ + getConnectorConfigPublic: vi.fn(async (provider: string) => ({ + provider, + config: { oauth_client_id: 'client-id' }, + })), +})) + +describe('remote MCP admin edit page load', () => { + it('uses the remote-MCP manifest identity when a native connector shares the same source_type', async () => { + const fetchMock = vi.fn(async (url: string) => { + if (url === '/api/remote-mcp/src-remote') { + return new Response( + JSON.stringify({ + id: 'src-remote', + name: 'Remote Docs', + sourceType: 'docs', + authType: 'oauth', + config: { endpoint_url: 'https://mcp.example.com/mcp' }, + isActive: true, + }), + { status: 200 }, + ) + } + if (url === 'http://connector-manager.test/connectors') { + return new Response( + JSON.stringify([ + { + source_type: 'docs', + manifest: { + integration_type: IntegrationType.CONNECTOR, + actions: [{ name: 'native_action' }], + resources: [{ name: 'native_resource' }], + oauth: { provider: 'native-docs' }, + }, + }, + { + source_type: 'docs', + manifest: { + integration_type: IntegrationType.REMOTE_MCP, + actions: [{ name: 'remote_tool' }, { name: 'remote_write' }], + resources: [{ name: 'remote_resource' }], + oauth: { + provider: 'remote_mcp:docs', + token_endpoint_auth_method: 'none', + }, + }, + }, + ]), + { status: 200 }, + ) + } + throw new Error(`unexpected fetch ${url}`) + }) + + const result = (await load({ + locals: { user: { id: 'admin', role: 'admin' }, logger: { warn: vi.fn() } }, + params: { sourceId: 'src-remote' }, + fetch: fetchMock, + } as never)) as any + + expect(result.manifest).toEqual({ available: true, toolCount: 2, resourceCount: 1 }) + expect(result.oauth.provider).toBe('remote_mcp:docs') + expect(result.oauth.configured).toBe(true) + }) +}) diff --git a/web/src/routes/(app)/settings/integrations/+page.server.ts b/web/src/routes/(app)/settings/integrations/+page.server.ts index dd6c9dce0..e1527eb28 100644 --- a/web/src/routes/(app)/settings/integrations/+page.server.ts +++ b/web/src/routes/(app)/settings/integrations/+page.server.ts @@ -5,6 +5,7 @@ import { sources, documents } from '$lib/server/db/schema' import { eq, and, count, inArray } from 'drizzle-orm' import { updateSourceById } from '$lib/server/db/sources' import { sourcesRepository } from '$lib/server/repositories/sources' +import { supportsDataSync } from '$lib/types' import type { PageServerLoad, Actions } from './$types' export const load: PageServerLoad = async ({ locals }) => { @@ -14,9 +15,11 @@ export const load: PageServerLoad = async ({ locals }) => { const googleConnectorConfig = await getConnectorConfigPublic('google') - const userSources = await sourcesRepository.getByUserId(locals.user.id) + const userSources = (await sourcesRepository.getByUserId(locals.user.id)).filter((source) => + supportsDataSync(source.integrationType), + ) - // Load sync status and document counts for personal sources owned by this user + // Load sync status and document counts for personal syncable sources owned by this user const userSourceIds = userSources.map((s) => s.id) const latestSyncRuns = await sourcesRepository.getLatestSyncRunsForSourceIds(userSourceIds) @@ -67,6 +70,9 @@ export const actions: Actions = { if (!source) { return fail(403, { error: 'Source not found or not owned by you' }) } + if (!supportsDataSync(source.integrationType)) { + return fail(400, { error: 'Source does not support data sync' }) + } await updateSourceById(sourceId, { isActive: false }) }, @@ -92,6 +98,9 @@ export const actions: Actions = { if (!source) { return fail(403, { error: 'Source not found or not owned by you' }) } + if (!supportsDataSync(source.integrationType)) { + return fail(400, { error: 'Source does not support data sync' }) + } await updateSourceById(sourceId, { isActive: true }) }, diff --git a/web/src/routes/api/oauth/callback/+server.ts b/web/src/routes/api/oauth/callback/+server.ts index ea102c1be..c13b7716c 100644 --- a/web/src/routes/api/oauth/callback/+server.ts +++ b/web/src/routes/api/oauth/callback/+server.ts @@ -3,6 +3,7 @@ import type { RequestHandler } from './$types' import { db } from '$lib/server/db' import { sources } from '$lib/server/db/schema' import { ulid } from 'ulid' +import { eq } from 'drizzle-orm' import { exchangeCodeAndIdentify } from '$lib/server/oauth/connectorOAuth' import { OAuthStateManager } from '$lib/server/oauth/state' import { serviceCredentialsRepository } from '$lib/server/repositories/service-credentials' @@ -12,7 +13,7 @@ import { getConfig } from '$lib/server/config' import { getSourceById, getSourcesByType } from '$lib/server/db/sources' import { toolApprovalRepository } from '$lib/server/db/tool-approvals' import { getSourceDisplayName } from '$lib/utils/icons' -import { SourceType } from '$lib/types' +import { IntegrationType, SourceType } from '$lib/types' function isSafeLocalPath(value: string): boolean { return value.startsWith('/') && !value.startsWith('//') @@ -53,10 +54,12 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { } let failureReturnTo: string | null = null + let pendingProvider: string | null = null try { const pendingState = await OAuthStateManager.getState(stateToken) if (pendingState?.user_id === user.id) { failureReturnTo = returnToFromStateMetadata(pendingState.metadata) + pendingProvider = pendingState.metadata?.provider ?? null } } catch (err) { logger.warn('Failed to read OAuth state for failure redirect', { err: String(err) }) @@ -67,6 +70,9 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { exchange = await exchangeCodeAndIdentify(code, stateToken, { principalEmailOverrides: { clickup: user.email, + ...(pendingProvider?.startsWith('remote_mcp:') + ? { [pendingProvider]: user.email } + : {}), }, }) } catch (err) { @@ -78,6 +84,7 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { } const { tokens, state, principalEmail, config, clientCreds } = exchange + const credentialProvider = config.credential_provider ?? config.provider if (state.user_id !== user.id) { throw error(403, 'OAuth state does not match the signed-in user') @@ -117,7 +124,10 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { token_uri: clientCreds.tokenEndpoint ?? config.token_endpoint, }) - const notifyOAuthCredentialReady = async (sourceId: string, userId?: string) => { + const notifyOAuthCredentialReady = async ( + sourceId: string, + userId?: string, + ): Promise | null> => { try { const cmUrl = getConfig().services.connectorManagerUrl const resp = await fetch(`${cmUrl}/oauth/credential-ready`, { @@ -130,18 +140,22 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { flow: flow.type === 'user_write' ? 'user_write' : 'user_read', }), }) + const body = (await resp.json().catch(() => null)) as Record | null if (!resp.ok) { logger.warn('OAuth credential-ready notification failed', { sourceId, status: resp.status, - body: await resp.text(), + body, }) + return null } + return body } catch (err) { logger.warn('OAuth credential-ready notification failed', { sourceId, err: String(err), }) + return null } } @@ -158,7 +172,7 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { await serviceCredentialsRepository.create({ sourceId: flow.sourceId, - provider: config.provider, + provider: credentialProvider, authType: 'oauth', principalEmail, credentials: { @@ -169,7 +183,24 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { expiresAt, }) - await notifyOAuthCredentialReady(flow.sourceId) + const credentialReady = await notifyOAuthCredentialReady(flow.sourceId) + + if (source.integrationType === IntegrationType.REMOTE_MCP) { + if (credentialReady?.status !== 'completed') { + throw redirect( + 302, + withErrorParam( + flow.returnTo ?? '/admin/settings/integrations', + 'oauth_catalog_refresh_failed', + ), + ) + } + await db + .update(sources) + .set({ isActive: true, updatedAt: new Date() }) + .where(eq(sources.id, flow.sourceId)) + throw redirect(302, flow.returnTo ?? '/admin/settings/integrations?success=connected') + } try { await fetch(`/api/sources/${flow.sourceId}/sync`, { @@ -194,7 +225,7 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { await serviceCredentialsRepository.createForUser({ sourceId: flow.sourceId, userId: user.id, - provider: config.provider, + provider: credentialProvider, authType: 'oauth', principalEmail, credentials: { @@ -248,7 +279,7 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { await serviceCredentialsRepository.createForUser({ sourceId: existing.id, userId: user.id, - provider: config.provider, + provider: credentialProvider, authType: 'oauth', principalEmail, credentials: { @@ -277,7 +308,7 @@ export const GET: RequestHandler = async ({ url, locals, fetch }) => { await serviceCredentialsRepository.createForUser({ sourceId: newSource.id, userId: user.id, - provider: config.provider, + provider: credentialProvider, authType: 'oauth', principalEmail, credentials: credentialsWithRefreshFallback({}), diff --git a/web/src/routes/api/oauth/start/+server.ts b/web/src/routes/api/oauth/start/+server.ts index dbf5fc536..dae29fbe7 100644 --- a/web/src/routes/api/oauth/start/+server.ts +++ b/web/src/routes/api/oauth/start/+server.ts @@ -8,6 +8,7 @@ import { generateAuthUrlForUserRead, generateAuthUrlForUserWrite, isProviderConfigured, + getOAuthConfigForSource, getOAuthManifestForSourceType, } from '$lib/server/oauth/connectorOAuth' @@ -54,7 +55,7 @@ export const GET: RequestHandler = async ({ url, locals }) => { throw error(400, `Unsupported source scope: ${source.scope}`) } - const config = await getOAuthManifestForSourceType(source.sourceType) + const config = await getOAuthConfigForSource(source) if (!config) { throw error(501, `OAuth is not implemented for source_type=${source.sourceType} yet.`) } @@ -88,6 +89,7 @@ export const GET: RequestHandler = async ({ url, locals }) => { sourceId, sourceType: source.sourceType, userId: locals.user.id, + source, returnTo, }) throw redirect(302, authUrl) @@ -98,6 +100,7 @@ export const GET: RequestHandler = async ({ url, locals }) => { const { url: authUrl } = await generator({ sourceId, sourceType: source.sourceType, + source, userId: locals.user.id, returnTo, approvalId, diff --git a/web/src/routes/api/remote-mcp/+server.ts b/web/src/routes/api/remote-mcp/+server.ts new file mode 100644 index 000000000..c3e15c04a --- /dev/null +++ b/web/src/routes/api/remote-mcp/+server.ts @@ -0,0 +1,213 @@ +import { json, error } from '@sveltejs/kit' +import type { RequestHandler } from './$types' +import { and, eq, inArray, isNull, ne, sql } from 'drizzle-orm' +import { ulid } from 'ulid' +import { db } from '$lib/server/db' +import { serviceCredentials, sources, type Source } from '$lib/server/db/schema' +import { encryptConfig } from '$lib/server/crypto/encryption' +import { AuthType, IntegrationType, ServiceProvider } from '$lib/types' +import { + probeRemoteMcpServer, + remoteMcpConfigFromInput, + validateRemoteMcpSlug, + type RemoteMcpConfig, +} from '$lib/server/mcp/client' + +type RemoteMcpSourceResponse = { + id: string + name: string + sourceType: string + integrationType: string + scope: string + config: unknown + isActive: boolean + isDeleted: boolean + syncIntervalSeconds: number | null + createdAt: Date + updatedAt: Date + authType: string | null + isConnected: boolean +} + +function remoteMcpAuthType(source: Source): string | null { + const config = source.config as Partial + return config.auth_type ?? null +} + +function sanitizeRemoteMcpSource( + source: Source, + hasOrgCredential: boolean, +): RemoteMcpSourceResponse { + const authType = remoteMcpAuthType(source) + return { + id: source.id, + name: source.name, + sourceType: source.sourceType, + integrationType: source.integrationType, + scope: source.scope, + config: source.config, + isActive: source.isActive, + isDeleted: source.isDeleted, + syncIntervalSeconds: source.syncIntervalSeconds, + createdAt: source.createdAt, + updatedAt: source.updatedAt, + authType, + isConnected: authType === AuthType.BEARER_TOKEN ? hasOrgCredential : source.isActive, + } +} + +async function lockSourceSlug(tx: any, sourceType: string): Promise { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${`source_slug:${sourceType}`}))`) +} + +async function assertNoActiveSlugCollision( + tx: any, + sourceType: string, + excludingSourceId?: string, +): Promise { + const rows = await tx + .select({ id: sources.id, integrationType: sources.integrationType }) + .from(sources) + .where( + and( + eq(sources.sourceType, sourceType), + eq(sources.isActive, true), + eq(sources.isDeleted, false), + excludingSourceId ? ne(sources.id, excludingSourceId) : undefined, + ), + ) + .limit(1) + + if (rows.length > 0) { + const conflict = rows[0] + const kind = + conflict.integrationType === IntegrationType.REMOTE_MCP + ? 'remote MCP' + : 'native connector' + throw error(409, `An active ${kind} source already uses sourceType ${sourceType}`) + } +} + +async function upsertRemoteMcpBearerCredential( + tx: any, + sourceId: string, + bearerToken: string, +): Promise { + await tx + .delete(serviceCredentials) + .where( + and( + eq(serviceCredentials.sourceId, sourceId), + eq(serviceCredentials.provider, ServiceProvider.REMOTE_MCP), + isNull(serviceCredentials.userId), + ), + ) + + await tx.insert(serviceCredentials).values({ + id: ulid(), + sourceId, + userId: null, + provider: ServiceProvider.REMOTE_MCP, + authType: AuthType.BEARER_TOKEN, + principalEmail: null, + credentials: encryptConfig({ token: bearerToken }), + config: {}, + }) +} + +export const GET: RequestHandler = async ({ locals }) => { + if (!locals.user) throw error(401, 'Unauthorized') + if (locals.user.role !== 'admin') throw error(403, 'Admin access required') + + const remoteSources = await db + .select() + .from(sources) + .where( + and( + eq(sources.integrationType, IntegrationType.REMOTE_MCP), + eq(sources.isDeleted, false), + ), + ) + + const sourceIds = remoteSources.map((source) => source.id) + const orgCredentials = + sourceIds.length > 0 + ? await db + .select({ sourceId: serviceCredentials.sourceId }) + .from(serviceCredentials) + .where( + and( + inArray(serviceCredentials.sourceId, sourceIds), + eq(serviceCredentials.provider, ServiceProvider.REMOTE_MCP), + isNull(serviceCredentials.userId), + ), + ) + : [] + const credentialSourceIds = new Set(orgCredentials.map((credential) => credential.sourceId)) + + return json( + remoteSources.map((source) => + sanitizeRemoteMcpSource(source, credentialSourceIds.has(source.id)), + ), + ) +} + +export const POST: RequestHandler = async ({ request, locals }) => { + if (!locals.user) throw error(401, 'Unauthorized') + const user = locals.user + if (user.role !== 'admin') throw error(403, 'Admin access required') + + const body = await request.json() + const name = typeof body.name === 'string' ? body.name.trim() : '' + const sourceType = validateRemoteMcpSlug(String(body.sourceType ?? body.source_type ?? '')) + const config = remoteMcpConfigFromInput({ + endpointUrl: String(body.endpointUrl ?? body.endpoint_url ?? ''), + authType: body.authType ?? body.auth_type ?? null, + writeToolsEnabled: body.writeToolsEnabled ?? body.write_tools_enabled, + }) + const rawBearerToken = body.bearerToken ?? body.bearer_token + const bearerToken = typeof rawBearerToken === 'string' ? rawBearerToken : null + + if (!name) throw error(400, 'name is required') + if (config.auth_type === AuthType.BEARER_TOKEN && !bearerToken) { + throw error(400, 'bearerToken is required for bearer auth') + } + + const probe = await probeRemoteMcpServer({ + endpointUrl: config.endpoint_url, + authType: config.auth_type, + bearerToken, + }) + if (!probe.ok) throw error(400, probe.error ?? 'Remote MCP probe failed') + + const [created] = await db.transaction(async (tx) => { + await lockSourceSlug(tx, sourceType) + await assertNoActiveSlugCollision(tx, sourceType) + + const inserted = await tx + .insert(sources) + .values({ + id: ulid(), + name, + sourceType, + integrationType: IntegrationType.REMOTE_MCP, + scope: 'org', + config, + createdBy: user.id, + isActive: config.auth_type !== AuthType.OAUTH, + syncIntervalSeconds: null, + }) + .returning() + + if (config.auth_type === AuthType.BEARER_TOKEN && bearerToken) { + await upsertRemoteMcpBearerCredential(tx, inserted[0].id, bearerToken) + } + + return inserted + }) + + return json( + { ...sanitizeRemoteMcpSource(created, config.auth_type === AuthType.BEARER_TOKEN), probe }, + { status: 201 }, + ) +} diff --git a/web/src/routes/api/remote-mcp/[sourceId]/+server.ts b/web/src/routes/api/remote-mcp/[sourceId]/+server.ts new file mode 100644 index 000000000..f1d16b5e1 --- /dev/null +++ b/web/src/routes/api/remote-mcp/[sourceId]/+server.ts @@ -0,0 +1,272 @@ +import { json, error } from '@sveltejs/kit' +import type { RequestHandler } from './$types' +import { and, eq, isNull, sql } from 'drizzle-orm' +import { ulid } from 'ulid' +import { getConfig } from '$lib/server/config' +import { db } from '$lib/server/db' +import { serviceCredentials, sources, type Source } from '$lib/server/db/schema' +import { decryptConfig, encryptConfig } from '$lib/server/crypto/encryption' +import { AuthType, IntegrationType, ServiceProvider } from '$lib/types' +import { + probeRemoteMcpServer, + remoteMcpConfigFromInput, + type RemoteMcpConfig, +} from '$lib/server/mcp/client' + +function remoteMcpAuthType(source: Source): string | null { + const config = source.config as Partial + return config.auth_type ?? null +} + +function sanitizeRemoteMcpSource(source: Source, hasOrgCredential: boolean) { + const authType = remoteMcpAuthType(source) + return { + id: source.id, + name: source.name, + sourceType: source.sourceType, + integrationType: source.integrationType, + scope: source.scope, + config: source.config, + isActive: source.isActive, + isDeleted: source.isDeleted, + syncIntervalSeconds: source.syncIntervalSeconds, + createdAt: source.createdAt, + updatedAt: source.updatedAt, + authType, + isConnected: authType === AuthType.BEARER_TOKEN ? hasOrgCredential : source.isActive, + } +} + +async function loadRemoteMcpSource(sourceId: string): Promise { + const source = await db.query.sources.findFirst({ where: eq(sources.id, sourceId) }) + if (!source || source.isDeleted || source.integrationType !== IntegrationType.REMOTE_MCP) { + throw error(404, 'Remote MCP source not found') + } + return source +} + +async function getOrgRemoteMcpCredential(sourceId: string) { + return await db.query.serviceCredentials.findFirst({ + where: and( + eq(serviceCredentials.sourceId, sourceId), + eq(serviceCredentials.provider, ServiceProvider.REMOTE_MCP), + isNull(serviceCredentials.userId), + ), + }) +} + +function bearerTokenFromCredential( + credential: { credentials: unknown } | undefined, +): string | null { + if (!credential) return null + const decrypted = decryptConfig(credential.credentials) + const token = decrypted.token + return typeof token === 'string' ? token : null +} + +export function remoteMcpPutTransition( + existingIsActive: boolean, + previousConfig: Partial, + nextConfig: RemoteMcpConfig, +): { + shouldBeActive: boolean + shouldDeleteCredentials: boolean + oauthBootstrapRequired: boolean +} { + const previousAuthType = previousConfig.auth_type ?? null + const nextAuthType = nextConfig.auth_type ?? null + const authTypeChanged = previousAuthType !== nextAuthType + const endpointChanged = previousConfig.endpoint_url !== nextConfig.endpoint_url + const oauthBootstrapRequired = + nextAuthType === AuthType.OAUTH && (authTypeChanged || endpointChanged) + + return { + shouldBeActive: + nextAuthType === AuthType.OAUTH ? existingIsActive && !oauthBootstrapRequired : true, + shouldDeleteCredentials: + authTypeChanged || + (previousAuthType === AuthType.OAUTH && + nextAuthType === AuthType.OAUTH && + endpointChanged), + oauthBootstrapRequired, + } +} + +async function pruneRemoteMcpCapabilities(sourceId: string, fetchFn: typeof fetch): Promise { + const searcherUrl = getConfig().services.searcherUrl + await Promise.all( + ['resource', 'prompt'].map((capabilityType) => + fetchFn(`${searcherUrl}/capabilities/sync`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + publisher_id: sourceId, + capability_type: capabilityType, + capabilities: [], + }), + }).catch(() => undefined), + ), + ) +} + +async function upsertRemoteMcpBearerCredential( + tx: any, + sourceId: string, + bearerToken: string, +): Promise { + await tx + .delete(serviceCredentials) + .where( + and( + eq(serviceCredentials.sourceId, sourceId), + eq(serviceCredentials.provider, ServiceProvider.REMOTE_MCP), + isNull(serviceCredentials.userId), + ), + ) + + await tx.insert(serviceCredentials).values({ + id: ulid(), + sourceId, + userId: null, + provider: ServiceProvider.REMOTE_MCP, + authType: AuthType.BEARER_TOKEN, + principalEmail: null, + credentials: encryptConfig({ token: bearerToken }), + config: {}, + }) +} + +export const GET: RequestHandler = async ({ params, locals }) => { + if (!locals.user) throw error(401, 'Unauthorized') + if (locals.user.role !== 'admin') throw error(403, 'Admin access required') + + const source = await loadRemoteMcpSource(params.sourceId) + const credential = await getOrgRemoteMcpCredential(source.id) + return json(sanitizeRemoteMcpSource(source, Boolean(credential))) +} + +export const PUT: RequestHandler = async ({ params, request, locals, fetch }) => { + if (!locals.user) throw error(401, 'Unauthorized') + if (locals.user.role !== 'admin') throw error(403, 'Admin access required') + + const existing = await loadRemoteMcpSource(params.sourceId) + const existingConfig = existing.config as Partial + const body = await request.json() + + const requestedSourceType = body.sourceType ?? body.source_type + if (requestedSourceType !== undefined && requestedSourceType !== existing.sourceType) { + throw error(400, 'Remote MCP sourceType is immutable') + } + + const name = typeof body.name === 'string' ? body.name.trim() : existing.name + if (!name) throw error(400, 'name is required') + + const config = remoteMcpConfigFromInput({ + endpointUrl: String( + body.endpointUrl ?? body.endpoint_url ?? existingConfig.endpoint_url ?? '', + ), + authType: body.authType ?? body.auth_type ?? existingConfig.auth_type ?? null, + writeToolsEnabled: + body.writeToolsEnabled ?? + body.write_tools_enabled ?? + existingConfig.write_tools_enabled, + }) + const rawBearerToken = body.bearerToken ?? body.bearer_token + const providedBearerToken = typeof rawBearerToken === 'string' ? rawBearerToken : null + const existingCredential = await getOrgRemoteMcpCredential(existing.id) + const bearerToken = + config.auth_type === AuthType.BEARER_TOKEN + ? (providedBearerToken ?? bearerTokenFromCredential(existingCredential)) + : null + + if (config.auth_type === AuthType.BEARER_TOKEN && !bearerToken) { + throw error(400, 'bearerToken is required for bearer auth') + } + + const { shouldBeActive, shouldDeleteCredentials } = remoteMcpPutTransition( + existing.isActive, + existingConfig, + config, + ) + + const probe = await probeRemoteMcpServer({ + endpointUrl: config.endpoint_url, + authType: config.auth_type, + bearerToken, + }) + if (!probe.ok) throw error(400, probe.error ?? 'Remote MCP probe failed') + + const [updated] = await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${`source_slug:${existing.sourceType}`}))`, + ) + + if (shouldBeActive) { + const conflicts = await tx + .select({ id: sources.id }) + .from(sources) + .where( + and( + eq(sources.sourceType, existing.sourceType), + eq(sources.isActive, true), + eq(sources.isDeleted, false), + ), + ) + .limit(2) + if (conflicts.some((row: { id: string }) => row.id !== existing.id)) { + throw error(409, `An active source already uses sourceType ${existing.sourceType}`) + } + } + + const rows = await tx + .update(sources) + .set({ name, config, isActive: shouldBeActive, updatedAt: new Date() }) + .where(eq(sources.id, existing.id)) + .returning() + + if (config.auth_type === AuthType.BEARER_TOKEN && providedBearerToken) { + await upsertRemoteMcpBearerCredential(tx, existing.id, providedBearerToken) + } else if (shouldDeleteCredentials) { + await tx + .delete(serviceCredentials) + .where( + and( + eq(serviceCredentials.sourceId, existing.id), + eq(serviceCredentials.provider, ServiceProvider.REMOTE_MCP), + ), + ) + } + + return rows + }) + + if (existing.isActive && !updated.isActive) { + await pruneRemoteMcpCapabilities(updated.id, fetch) + } + + const credential = await getOrgRemoteMcpCredential(updated.id) + return json({ ...sanitizeRemoteMcpSource(updated, Boolean(credential)), probe }) +} + +export const PATCH = PUT + +export const DELETE: RequestHandler = async ({ params, locals, fetch }) => { + if (!locals.user) throw error(401, 'Unauthorized') + if (locals.user.role !== 'admin') throw error(403, 'Admin access required') + + const source = await loadRemoteMcpSource(params.sourceId) + await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${`source_slug:${source.sourceType}`}))`, + ) + await tx.delete(serviceCredentials).where(eq(serviceCredentials.sourceId, source.id)) + await tx + .update(sources) + .set({ isActive: false, isDeleted: true, updatedAt: new Date() }) + .where(eq(sources.id, source.id)) + }) + + await pruneRemoteMcpCapabilities(source.id, fetch) + + return json({ success: true }) +} diff --git a/web/src/routes/api/remote-mcp/[sourceId]/server.test.ts b/web/src/routes/api/remote-mcp/[sourceId]/server.test.ts new file mode 100644 index 000000000..bab27fe92 --- /dev/null +++ b/web/src/routes/api/remote-mcp/[sourceId]/server.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { AuthType } from '$lib/types' +import { remoteMcpPutTransition } from './+server' + +describe('remote MCP PUT state transitions', () => { + it('preserves OAuth active state and credentials for name-only or write-tools edits', () => { + const transition = remoteMcpPutTransition( + true, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: true, + }, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: false, + }, + ) + + expect(transition).toEqual({ + shouldBeActive: true, + shouldDeleteCredentials: false, + oauthBootstrapRequired: false, + }) + }) + + it('deactivates and drops credentials when OAuth endpoint changes', () => { + const transition = remoteMcpPutTransition( + true, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: true, + }, + { + endpoint_url: 'https://new-mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: true, + }, + ) + + expect(transition).toEqual({ + shouldBeActive: false, + shouldDeleteCredentials: true, + oauthBootstrapRequired: true, + }) + }) + + it('activates public and bearer transitions after successful probe', () => { + expect( + remoteMcpPutTransition( + false, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: true, + }, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: null, + write_tools_enabled: true, + }, + ).shouldBeActive, + ).toBe(true) + + expect( + remoteMcpPutTransition( + false, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.OAUTH, + write_tools_enabled: true, + }, + { + endpoint_url: 'https://mcp.example.com/mcp', + auth_type: AuthType.BEARER_TOKEN, + write_tools_enabled: true, + }, + ).shouldBeActive, + ).toBe(true) + }) +}) diff --git a/web/src/routes/api/remote-mcp/test/+server.ts b/web/src/routes/api/remote-mcp/test/+server.ts new file mode 100644 index 000000000..42bf133bb --- /dev/null +++ b/web/src/routes/api/remote-mcp/test/+server.ts @@ -0,0 +1,30 @@ +import { json, error } from '@sveltejs/kit' +import type { RequestHandler } from './$types' +import { AuthType } from '$lib/types' +import { probeRemoteMcpServer, remoteMcpConfigFromInput } from '$lib/server/mcp/client' + +export const POST: RequestHandler = async ({ request, locals }) => { + if (!locals.user) throw error(401, 'Unauthorized') + if (locals.user.role !== 'admin') throw error(403, 'Admin access required') + + const body = await request.json() + const config = remoteMcpConfigFromInput({ + endpointUrl: String(body.endpointUrl ?? body.endpoint_url ?? ''), + authType: body.authType ?? body.auth_type ?? null, + writeToolsEnabled: body.writeToolsEnabled ?? body.write_tools_enabled, + }) + const rawBearerToken = body.bearerToken ?? body.bearer_token + const bearerToken = typeof rawBearerToken === 'string' ? rawBearerToken : null + + if (config.auth_type === AuthType.BEARER_TOKEN && !bearerToken) { + throw error(400, 'bearerToken is required for bearer auth') + } + + const probe = await probeRemoteMcpServer({ + endpointUrl: config.endpoint_url, + authType: config.auth_type, + bearerToken, + }) + + return json(probe, { status: probe.ok ? 200 : 400 }) +} diff --git a/web/src/routes/api/service-credentials/+server.ts b/web/src/routes/api/service-credentials/+server.ts index 335500f04..85ae6e8c1 100644 --- a/web/src/routes/api/service-credentials/+server.ts +++ b/web/src/routes/api/service-credentials/+server.ts @@ -2,7 +2,7 @@ import { json, error } from '@sveltejs/kit' import type { RequestHandler } from './$types' import { getSourceById } from '$lib/server/db/sources' import { serviceCredentialsRepository } from '$lib/server/repositories/service-credentials' -import { ServiceProvider, AuthType } from '$lib/types' +import { ServiceProvider, AuthType, supportsDataSync } from '$lib/types' export const POST: RequestHandler = async ({ request, locals, fetch }) => { if (!locals.user) { @@ -64,7 +64,7 @@ export const POST: RequestHandler = async ({ request, locals, fetch }) => { config: config || {}, }) - if (triggerSync) { + if (triggerSync && supportsDataSync(source.integrationType)) { try { const syncResponse = await fetch(`/api/sources/${sourceId}/sync`, { method: 'POST', @@ -184,7 +184,7 @@ export const PATCH: RequestHandler = async ({ request, locals, fetch }) => { credentials: hasNewCredentials ? credentials : null, }) - if (hasNewCredentials) { + if (hasNewCredentials && supportsDataSync(source.integrationType)) { try { const syncResponse = await fetch(`/api/sources/${sourceId}/sync`, { method: 'POST', diff --git a/web/src/routes/api/sources/+server.ts b/web/src/routes/api/sources/+server.ts index 4078996d8..34f6142bf 100644 --- a/web/src/routes/api/sources/+server.ts +++ b/web/src/routes/api/sources/+server.ts @@ -2,10 +2,10 @@ import { json, error } from '@sveltejs/kit' import type { RequestHandler } from './$types' import { db } from '$lib/server/db' import { sources, serviceCredentials, syncRuns } from '$lib/server/db/schema' -import { inArray, sql } from 'drizzle-orm' +import { and, eq, inArray, sql } from 'drizzle-orm' import { ulid } from 'ulid' import { logger } from '$lib/server/logger' -import { SourceType, DEFAULT_SYNC_INTERVAL_SECONDS } from '$lib/types' +import { IntegrationType, SourceType, DEFAULT_SYNC_INTERVAL_SECONDS } from '$lib/types' import { getSourcesByType } from '$lib/server/db/sources' export const GET: RequestHandler = async ({ locals }) => { @@ -60,6 +60,7 @@ export const GET: RequestHandler = async ({ locals }) => { id: source.id, name: source.name, sourceType: source.sourceType, + integrationType: source.integrationType, scope: source.scope, config: sourceConfig, syncStatus: latestSync?.status ?? null, @@ -92,6 +93,13 @@ export const POST: RequestHandler = async ({ request, locals }) => { throw error(400, 'Name and sourceType are required') } + if ( + body.integrationType === IntegrationType.REMOTE_MCP || + body.integration_type === IntegrationType.REMOTE_MCP + ) { + throw error(400, 'Remote MCP sources must be created through /api/remote-mcp') + } + if (scope === 'org' && user.role !== 'admin') { throw error(403, 'Only admins can create org-wide sources') } @@ -112,24 +120,53 @@ export const POST: RequestHandler = async ({ request, locals }) => { } } - const [newSource] = await db - .insert(sources) - .values({ - id: ulid(), - name, - sourceType, - scope, - config: config || {}, - createdBy: user.id, - isActive: isActive ?? false, - syncIntervalSeconds: DEFAULT_SYNC_INTERVAL_SECONDS[sourceType as SourceType], - }) - .returning() + const [newSource] = await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${`source_slug:${sourceType}`}))`, + ) + + if (isActive ?? false) { + const remoteConflicts = await tx + .select({ id: sources.id }) + .from(sources) + .where( + and( + eq(sources.sourceType, sourceType), + eq(sources.integrationType, IntegrationType.REMOTE_MCP), + eq(sources.isActive, true), + eq(sources.isDeleted, false), + ), + ) + .limit(1) + if (remoteConflicts.length > 0) { + throw error( + 409, + `An active remote MCP source already uses sourceType ${sourceType}`, + ) + } + } + + return await tx + .insert(sources) + .values({ + id: ulid(), + name, + sourceType, + integrationType: IntegrationType.CONNECTOR, + scope, + config: config || {}, + createdBy: user.id, + isActive: isActive ?? false, + syncIntervalSeconds: DEFAULT_SYNC_INTERVAL_SECONDS[sourceType as SourceType], + }) + .returning() + }) return json({ id: newSource.id, name: newSource.name, sourceType: newSource.sourceType, + integrationType: newSource.integrationType, scope: newSource.scope, config: newSource.config, syncStatus: null, diff --git a/web/src/routes/api/sources/[sourceId]/sync/+server.ts b/web/src/routes/api/sources/[sourceId]/sync/+server.ts index 551b701db..cdd967924 100644 --- a/web/src/routes/api/sources/[sourceId]/sync/+server.ts +++ b/web/src/routes/api/sources/[sourceId]/sync/+server.ts @@ -2,6 +2,8 @@ import { json, error } from '@sveltejs/kit' import type { RequestHandler } from './$types' import { getConfig } from '$lib/server/config' import { logger } from '$lib/server/logger' +import { sourcesRepository } from '$lib/server/repositories/sources' +import { supportsDataSync } from '$lib/types' type SyncMode = 'incremental' | 'full' @@ -34,6 +36,14 @@ export const POST: RequestHandler = async ({ params, request, fetch }) => { } try { + const source = await sourcesRepository.getById(sourceId) + if (!source || source.isDeleted) { + throw error(404, 'Source not found') + } + if (!supportsDataSync(source.integrationType)) { + throw error(400, 'Source does not support data sync') + } + const mode = await getSyncMode(request) const config = getConfig() const connectorManagerUrl = config.services.connectorManagerUrl From 216ca3b41d769ce721b96a3da46e8dfff3cf4aa4 Mon Sep 17 00:00:00 2001 From: Praveen Sampath Date: Fri, 24 Jul 2026 12:31:19 +0000 Subject: [PATCH 02/19] fix(mcp): fix DNS pinning lookup callback when options.all is true - Agent connect.lookup callback must return an array when opts.all is true - Improve error logging to surface underlying cause for fetch failures --- web/src/lib/server/mcp/client.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/web/src/lib/server/mcp/client.ts b/web/src/lib/server/mcp/client.ts index 64cceebfc..80f046f59 100644 --- a/web/src/lib/server/mcp/client.ts +++ b/web/src/lib/server/mcp/client.ts @@ -121,9 +121,17 @@ export async function fetchWithPinnedRemoteMcpDns( let next = 0 const dispatcher = new Agent({ connect: { - lookup(_hostname: string, _options: unknown, callback: (...args: unknown[]) => void) { + lookup( + _hostname: string, + options: { all?: boolean }, + callback: (err: Error | null, result?: unknown) => void, + ) { const record = addresses[next++ % addresses.length] - callback(null, record.address, record.family) + if (options.all) { + callback(null, [{ address: record.address, family: record.family }]) + } else { + callback(null, record.address, record.family) + } }, }, } as any) @@ -230,7 +238,14 @@ async function mcpPost( } function remoteMcpProbeErrorMessage(err: unknown): string { - if (err instanceof Error) return err.message + if (err instanceof Error) { + const cause = err.cause + if (cause instanceof Error) { + return `${err.message}: ${cause.message}${cause.code ? ` (${cause.code})` : ''}` + } + if (typeof cause === 'string') return `${err.message}: ${cause}` + return err.message + } if (err && typeof err === 'object') { const body = 'body' in err ? (err as { body?: unknown }).body : undefined if (body && typeof body === 'object' && 'message' in body) { From 38ff66c95d4ea0bf819d9ba175838cbf2b95fd20 Mon Sep 17 00:00:00 2001 From: Praveen Sampath Date: Fri, 24 Jul 2026 12:39:40 +0000 Subject: [PATCH 03/19] fix(mcp): allow hyphens in slugs and fix 500 on source detail page - Allow hyphens in MCP source slugs (regex and DB constraint) - Show user-friendly error message instead of raw regex - Remove stray 'export' from helper function causing SvelteKit 500 --- services/migrations/106_add_remote_mcp_sources.sql | 2 +- web/src/lib/server/mcp/client.test.ts | 1 + web/src/lib/server/mcp/client.ts | 9 ++++++--- web/src/routes/api/remote-mcp/[sourceId]/+server.ts | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/services/migrations/106_add_remote_mcp_sources.sql b/services/migrations/106_add_remote_mcp_sources.sql index 8c63b5a55..7a8eacb93 100644 --- a/services/migrations/106_add_remote_mcp_sources.sql +++ b/services/migrations/106_add_remote_mcp_sources.sql @@ -33,7 +33,7 @@ ALTER TABLE sources ADD CONSTRAINT sources_source_type_check CHECK ( 'google_ads', 'darwinbox' )) OR - (integration_type = 'remote_mcp' AND source_type ~ '^[a-z][a-z0-9_]{1,49}$') + (integration_type = 'remote_mcp' AND source_type ~ '^[a-z][a-z0-9_-]{1,49}$') ); -- At most one non-deleted remote MCP source may claim an app slug. Native/native diff --git a/web/src/lib/server/mcp/client.test.ts b/web/src/lib/server/mcp/client.test.ts index 0db93d598..913dbc80a 100644 --- a/web/src/lib/server/mcp/client.test.ts +++ b/web/src/lib/server/mcp/client.test.ts @@ -63,6 +63,7 @@ describe('remote MCP client validation', () => { it('validates immutable source slug shape', () => { expect(validateRemoteMcpSlug('github_mcp')).toBe('github_mcp') + expect(validateRemoteMcpSlug('my-mcp-server')).toBe('my-mcp-server') expect(() => validateRemoteMcpSlug('Remote-MCP')).toThrow() expect(() => validateRemoteMcpSlug('1github')).toThrow() }) diff --git a/web/src/lib/server/mcp/client.ts b/web/src/lib/server/mcp/client.ts index 80f046f59..ef32358c3 100644 --- a/web/src/lib/server/mcp/client.ts +++ b/web/src/lib/server/mcp/client.ts @@ -162,12 +162,15 @@ function parseSlugCandidate(name: string | null): string | null { .replace(/[^a-z0-9]+/g, '_') .replace(/^_+|_+$/g, '') .slice(0, 50) - return /^[a-z][a-z0-9_]{1,49}$/.test(slug) ? slug : null + return /^[a-z][a-z0-9_-]{1,49}$/.test(slug) ? slug : null } export function validateRemoteMcpSlug(sourceType: string): string { - if (!/^[a-z][a-z0-9_]{1,49}$/.test(sourceType)) { - throw error(400, 'sourceType must match ^[a-z][a-z0-9_]{1,49}$') + if (!/^[a-z][a-z0-9_-]{1,49}$/.test(sourceType)) { + throw error( + 400, + 'Invalid slug. Use 2-50 lowercase letters, numbers, hyphens, or underscores. Must start with a letter.', + ) } return sourceType } diff --git a/web/src/routes/api/remote-mcp/[sourceId]/+server.ts b/web/src/routes/api/remote-mcp/[sourceId]/+server.ts index f1d16b5e1..7f09a8422 100644 --- a/web/src/routes/api/remote-mcp/[sourceId]/+server.ts +++ b/web/src/routes/api/remote-mcp/[sourceId]/+server.ts @@ -64,7 +64,7 @@ function bearerTokenFromCredential( return typeof token === 'string' ? token : null } -export function remoteMcpPutTransition( +function remoteMcpPutTransition( existingIsActive: boolean, previousConfig: Partial, nextConfig: RemoteMcpConfig, From 337ad386070064c09c2dbf915dd9e6a7b459db6b Mon Sep 17 00:00:00 2001 From: Praveen Sampath Date: Fri, 24 Jul 2026 12:42:05 +0000 Subject: [PATCH 04/19] feat(mcp): show MCP source list with dialog-based creation - List page now loads and displays all configured MCP connections - Add MCP form moved to a dialog instead of a separate page - Shows manifest availability, tool/resource counts, and connection status per source --- .../admin/settings/mcp/+page.server.ts | 48 +++++- .../(admin)/admin/settings/mcp/+page.svelte | 154 +++++++++++++++--- 2 files changed, 175 insertions(+), 27 deletions(-) diff --git a/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts b/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts index 5338b269a..fb362690d 100644 --- a/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts +++ b/web/src/routes/(admin)/admin/settings/mcp/+page.server.ts @@ -1,7 +1,51 @@ import { requireAdmin } from '$lib/server/authHelpers' +import { getConfig } from '$lib/server/config' +import { IntegrationType } from '$lib/types' import type { PageServerLoad } from './$types' -export const load: PageServerLoad = async ({ locals }) => { +type RemoteMcpSourceResponse = { + id: string + name: string + sourceType: string + authType: string | null + config: Record + isActive: boolean + hasCredential: boolean +} + +type ConnectorInfo = { + source_type: string + manifest?: { + integration_type?: string + actions?: unknown[] + resources?: unknown[] + } | null +} + +export const load: PageServerLoad = async ({ locals, fetch }) => { requireAdmin(locals) - return {} + + const sourceResponse = await fetch('/api/remote-mcp') + const sources: RemoteMcpSourceResponse[] = sourceResponse.ok ? await sourceResponse.json() : [] + + let manifestBySourceType = new Map() + try { + const config = getConfig() + const response = await fetch(`${config.services.connectorManagerUrl}/connectors`) + if (response.ok) { + const connectors: ConnectorInfo[] = await response.json() + for (const c of connectors) { + if (c.manifest?.integration_type === IntegrationType.REMOTE_MCP) { + manifestBySourceType.set(c.source_type, { + toolCount: c.manifest.actions?.length ?? 0, + resourceCount: c.manifest.resources?.length ?? 0, + }) + } + } + } + } catch (err) { + locals.logger.warn('Failed to fetch MCP manifest status', err) + } + + return { sources, manifestBySourceType: Object.fromEntries(manifestBySourceType) } } diff --git a/web/src/routes/(admin)/admin/settings/mcp/+page.svelte b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte index ac79c5d08..0683ecefa 100644 --- a/web/src/routes/(admin)/admin/settings/mcp/+page.svelte +++ b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte @@ -8,11 +8,16 @@ CardHeader, CardTitle, } from '$lib/components/ui/card' + import * as Dialog from '$lib/components/ui/dialog' import { Input } from '$lib/components/ui/input' import { Label } from '$lib/components/ui/label' import { Badge } from '$lib/components/ui/badge' import { AuthType } from '$lib/types' + import { Plus, Server } from '@lucide/svelte' import { toast } from 'svelte-sonner' + import type { PageProps } from './$types' + + let { data }: PageProps = $props() type ProbeResult = { ok: boolean @@ -25,6 +30,13 @@ error?: string } + let sources = $state(data.sources) + let manifestBySourceType = $state( + data.manifestBySourceType as Record, + ) + + // Dialog form state + let dialogOpen = $state(false) let name = $state('') let endpointUrl = $state('') let sourceType = $state('') @@ -50,6 +62,23 @@ } } + function resetForm() { + name = '' + endpointUrl = '' + sourceType = '' + authType = '' + bearerToken = '' + writeToolsEnabled = true + probe = null + isTesting = false + isCreating = false + } + + function openDialog() { + resetForm() + dialogOpen = true + } + async function testConnection() { isTesting = true probe = null @@ -86,6 +115,8 @@ const body = await response.json().catch(() => null) if (!response.ok) throw new Error(body?.message || body?.error || 'Create failed') toast.success('Remote MCP server created') + dialogOpen = false + resetForm() await goto(`/admin/settings/mcp/${body.id}`) } catch (error) { toast.error(error instanceof Error ? error.message : 'Create failed') @@ -93,26 +124,98 @@ isCreating = false } } + + function authLabel(authType: string | null): string { + if (authType === AuthType.BEARER_TOKEN) return 'Bearer' + if (authType === AuthType.OAUTH) return 'OAuth' + return 'Public' + } -Add Remote MCP Server +Remote MCP Servers
-
-

Add remote MCP server

-

- Configure a remote Streamable HTTP MCP endpoint. Catalogs are discovered at runtime; - secrets are write-only. -

+
+
+

Remote MCP Servers

+

+ Admin-managed remote Streamable HTTP MCP endpoints. Catalogs are discovered at + runtime; secrets are write-only. +

+
+
- - - Connection - Only remote HTTP(S) MCP endpoints are supported. - - + {#if sources.length > 0} +
+ {#each sources as source} + {@const m = manifestBySourceType[source.sourceType]} + + {/each} +
+ {:else} +
+ +

+ No remote MCP servers configured yet. +

+ +
+ {/if} +
+
+ + { + dialogOpen = o + if (!o) resetForm() + }}> + + + + + Add remote MCP server + + Configure a remote Streamable HTTP MCP endpoint. Only HTTP(S) URLs are + supported. + + + +
@@ -128,7 +231,8 @@

- Immutable after creation. Use lowercase letters, numbers, and underscores. + Immutable after creation. Use 2-50 lowercase letters, numbers, hyphens, or + underscores. Must start with a letter.

@@ -166,13 +270,6 @@ Expose write-capable tools -
- - -
- {#if probe?.ok}
@@ -187,7 +284,14 @@
{/if} - - -
-
+
+ + + + + + + + From f0696a519f6bf8f4183e405f82e8783d929957dc Mon Sep 17 00:00:00 2001 From: Praveen Sampath Date: Fri, 24 Jul 2026 12:46:56 +0000 Subject: [PATCH 05/19] fix(mcp): simplify copy for non-engineer audience MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove internal-infrastructure jargon (Connector Manager, Redis) - Remove redundant security reassurance text - 'Immutable' → 'Can't be changed later' - 'Runtime catalog' → 'Connection status' - 'Bootstrap credential' → 'Admin access' - 'Catalog data discovered at runtime' → 'Available tools detected automatically' --- .../(admin)/admin/settings/mcp/+page.svelte | 15 ++++-------- .../settings/mcp/[sourceId]/+page.svelte | 24 +++++++++---------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/web/src/routes/(admin)/admin/settings/mcp/+page.svelte b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte index 0683ecefa..f79e2dc3e 100644 --- a/web/src/routes/(admin)/admin/settings/mcp/+page.svelte +++ b/web/src/routes/(admin)/admin/settings/mcp/+page.svelte @@ -140,8 +140,8 @@

Remote MCP Servers

- Admin-managed remote Streamable HTTP MCP endpoints. Catalogs are discovered at - runtime; secrets are write-only. + Connect to remote MCP servers. Available tools and resources are detected + automatically.

@@ -254,15 +253,11 @@ type="password" bind:value={bearerToken} placeholder="Paste token" /> -

- Stored encrypted and never returned by the API. -

{/if} {#if authType === AuthType.OAUTH}

- After creation, configure this server's OAuth client and authorize an admin - bootstrap credential on the edit page. + After creation, configure OAuth and authorize admin access on the edit page.

{/if}