diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 74168f05..795a7dc7 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -70,10 +70,82 @@ export function getHandler(scenarioName: string): ScenarioHandler | undefined { } // ============================================================================ -// Basic scenarios (initialize, tools-call) +// Stateless requester (SEP-2575 / 2026-x lifecycle) +// +// Shim for the fact that the SDK Client doesn't support stateless mode yet. +// Carry-forward handlers below pick this when MCP_CONFORMANCE_PROTOCOL_VERSION +// is the draft version, so the same handler exercises both lifecycles. +// ============================================================================ + +const PROTOCOL_VERSION = process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; +const DRAFT_VERSION = 'DRAFT-2026-v1'; + +const STATELESS_META_BASE = { + 'io.modelcontextprotocol/clientInfo': { + name: 'conformance-test-client', + version: '1.0.0' + }, + 'io.modelcontextprotocol/clientCapabilities': { + tools: {}, + roots: {}, + sampling: {}, + elicitation: {} + } +}; + +let _nextStatelessId = 1; +async function statelessRequest( + serverUrl: string, + method: string, + params: Record = {} +): Promise { + const _meta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_VERSION, + ...STATELESS_META_BASE, + ...((params._meta as object | undefined) ?? {}) + }; + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': DRAFT_VERSION + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: _nextStatelessId++, + method, + params: { ...params, _meta } + }) + }); + const body = await response.json(); + if (body.error) { + throw new Error( + `${method} failed: ${body.error.code} ${body.error.message}` + ); + } + return body.result; +} + +// ============================================================================ +// Basic scenarios (initialize, tools_call) // ============================================================================ async function runBasicClient(serverUrl: string): Promise { + if (PROTOCOL_VERSION === DRAFT_VERSION) { + logger.debug('Stateless lifecycle: calling tools/list + tools/call'); + const list = await statelessRequest(serverUrl, 'tools/list'); + logger.debug('Successfully listed tools:', JSON.stringify(list)); + const tool = list?.tools?.[0]; + if (tool) { + const result = await statelessRequest(serverUrl, 'tools/call', { + name: tool.name, + arguments: { a: 2, b: 3 } + }); + logger.debug('Successfully called tool:', JSON.stringify(result)); + } + return; + } + const client = new Client( { name: 'test-client', version: '1.0.0' }, { capabilities: {} } @@ -84,14 +156,20 @@ async function runBasicClient(serverUrl: string): Promise { await client.connect(transport); logger.debug('Successfully connected to MCP server'); - await client.listTools(); + const list = await client.listTools(); logger.debug('Successfully listed tools'); + const tool = list.tools[0]; + if (tool) { + await client.callTool({ name: tool.name, arguments: { a: 2, b: 3 } }); + logger.debug('Successfully called tool'); + } + await transport.close(); logger.debug('Connection closed successfully'); } -registerScenarios(['initialize', 'tools-call'], runBasicClient); +registerScenarios(['initialize', 'tools_call', 'tools-call'], runBasicClient); // SEP-2106: json-schema-ref-no-deref advertises a tool whose inputSchema // contains a network-URI $ref. A conformant client lists tools normally and @@ -106,20 +184,9 @@ registerScenario('json-schema-ref-no-deref', runBasicClient); async function runRequestMetadataClient(serverUrl: string): Promise { logger.debug('Starting request-metadata client flow...'); - const meta = { - 'io.modelcontextprotocol/clientInfo': { - name: 'conformance-test-client', - version: '1.0.0' - }, - 'io.modelcontextprotocol/clientCapabilities': { - tools: {}, - roots: {}, - sampling: {}, - elicitation: {} - } - }; + const meta = STATELESS_META_BASE; - let activeVersion = 'DRAFT-2026-v1'; + let activeVersion = DRAFT_VERSION; const sendRequestWithNegotiation = async ( method: string, @@ -161,7 +228,7 @@ async function runRequestMetadataClient(serverUrl: string): Promise { ); const serverSupported: string[] = errorResult.error.data?.supported || []; - const clientSupported = ['DRAFT-2026-v1']; + const clientSupported = [DRAFT_VERSION]; const mutuallySupported = clientSupported.filter((v) => serverSupported.includes(v) ); diff --git a/src/connection/select.ts b/src/connection/select.ts index a08ec954..27b1868d 100644 --- a/src/connection/select.ts +++ b/src/connection/select.ts @@ -14,10 +14,12 @@ const STATEFUL_VERSIONS: ReadonlySet = new Set([ '2025-11-25' ]); +export function isStatefulVersion(v: SpecVersion): boolean { + return STATEFUL_VERSIONS.has(v); +} + export function connectFor( specVersion: SpecVersion ): (serverUrl: string) => Promise { - return STATEFUL_VERSIONS.has(specVersion) - ? connectStateful - : connectStateless; + return isStatefulVersion(specVersion) ? connectStateful : connectStateless; } diff --git a/src/connection/stateless.ts b/src/connection/stateless.ts index 3b1d7d0b..6949e697 100644 --- a/src/connection/stateless.ts +++ b/src/connection/stateless.ts @@ -98,7 +98,10 @@ async function readFinalSseMessage( response: Response, id: number, sink: JSONRPCNotification[] -): Promise<{ result?: unknown; error?: { code: number; message: string } }> { +): Promise<{ + result?: unknown; + error?: { code: number; message: string; data?: unknown }; +}> { const reader = response.body!.getReader(); const decoder = new TextDecoder(); let buf = ''; diff --git a/src/index.ts b/src/index.ts index 8754a0b1..ad34ffa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -254,7 +254,12 @@ program // If no command provided, run in interactive mode if (!validated.command) { - await runInteractiveMode(validated.scenario, verbose, outputDir); + await runInteractiveMode( + validated.scenario, + verbose, + outputDir, + specVersionFilter + ); process.exit(0); } diff --git a/src/mock-server/index.ts b/src/mock-server/index.ts new file mode 100644 index 00000000..c3e24059 --- /dev/null +++ b/src/mock-server/index.ts @@ -0,0 +1,58 @@ +/** + * Version-aware mock-server abstraction for client-conformance scenarios. + * + * A `MockServer` is the HTTP server a client-under-test connects to. The + * lifecycle scaffold (initialize handshake vs per-request `_meta` validation) + * is supplied by the runner based on `--spec-version`; the scenario only + * provides per-method handlers and asserts on the recorded requests. + * + * This is the client-conformance mirror of `Connection` in `../connection`. + */ + +import type { SpecVersion } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; + +/** + * Per-method response handlers. Called with the request `params` object; + * return value becomes the JSON-RPC `result`. Throw to produce an error + * response. + */ +export type RequestHandlers = Record< + string, + ( + params: Record, + request: JSONRPCRequest + ) => unknown | Promise +>; + +export interface MockServer { + /** Full URL of the `/mcp` endpoint. */ + url: string; + /** Base URL (no `/mcp` suffix), for scenarios that serve sibling routes. */ + baseUrl: string; + /** + * Every JSON-RPC request the client sent, in arrival order, excluding the + * lifecycle preamble (`initialize` / `notifications/initialized` under the + * stateful impl; nothing is excluded under stateless since there is none). + */ + readonly recorded: JSONRPCRequest[]; + close(): Promise; +} + +/** + * Per-run context handed to `Scenario.start()`. The runner constructs this + * from the resolved `--spec-version`. + */ +export interface ScenarioContext { + specVersion: SpecVersion; + /** + * Create a version-appropriate mock server. Scenarios that test the + * lifecycle itself (initialize, SSE-retry) bypass this and build a raw + * `http.createServer`. + */ + createServer(handlers: RequestHandlers): Promise; +} + +export { createServerStateful } from './stateful'; +export { createServerStateless, validateStatelessRequest } from './stateless'; +export { createServerFor } from './select'; diff --git a/src/mock-server/mock-server.test.ts b/src/mock-server/mock-server.test.ts new file mode 100644 index 00000000..cf20cc8b --- /dev/null +++ b/src/mock-server/mock-server.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect } from 'vitest'; +import { createServerFor } from './select'; +import { createServerStateful } from './stateful'; +import { createServerStateless } from './stateless'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; + +describe('createServerFor', () => { + it('returns stateful for dated 2025-x versions', () => { + expect(createServerFor('2025-06-18')).toBe(createServerStateful); + expect(createServerFor('2025-11-25')).toBe(createServerStateful); + }); + it('returns stateless for the draft version', () => { + expect(createServerFor(DRAFT_PROTOCOL_VERSION)).toBe(createServerStateless); + }); +}); + +describe('createServerStateless', () => { + const meta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientInfo': { name: 't', version: '1' }, + 'io.modelcontextprotocol/clientCapabilities': {} + }; + + async function post(url: string, body: object, headers: object = {}) { + const r = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + return { status: r.status, body: await r.json() }; + } + + it('rejects requests missing the version header', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post(srv.url, { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }); + expect(status).toBe(400); + expect(body.error.code).toBe(-32001); + } finally { + await srv.close(); + } + }); + + it('rejects requests missing required _meta keys', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(400); + expect(body.error.code).toBe(-32602); + } finally { + await srv.close(); + } + }); + + it('serves server/discover', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(200); + expect(body.result.supportedVersions).toEqual([DRAFT_PROTOCOL_VERSION]); + expect(body.result.serverInfo.name).toBe('conformance-mock-server'); + } finally { + await srv.close(); + } + }); + + it('routes to handlers and records requests', async () => { + const srv = await createServerStateless({ + 'tools/list': () => ({ tools: [{ name: 'x' }] }) + }); + try { + const { body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(body.result.tools[0].name).toBe('x'); + expect(srv.recorded).toHaveLength(1); + expect(srv.recorded[0].method).toBe('tools/list'); + } finally { + await srv.close(); + } + }); + + it('returns -32601 for unknown methods', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'nope', params: { _meta: meta } }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(404); + expect(body.error.code).toBe(-32601); + } finally { + await srv.close(); + } + }); +}); + +describe('createServerStateful', () => { + async function postInit(url: string) { + const r = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-11-25', + capabilities: {}, + clientInfo: { name: 't', version: '1' } + } + }) + }); + return { + status: r.status, + contentType: r.headers.get('content-type') ?? '' + }; + } + + it('accepts initialize and routes to handlers, recording non-preamble', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + // SDK transport in sessionless mode handles initialize internally; we + // can drive it via the SDK Client. + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client.listTools(); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toEqual(['tools/list']); + } finally { + await srv.close(); + } + }); + + it('derives capabilities from handler keys; non-tools handler does not 500 initialize', async () => { + const srv = await createServerStateful({ + 'prompts/list': () => ({ prompts: [] }) + }); + try { + const { status, contentType } = await postInit(srv.url); + expect(status).toBe(200); + expect(contentType).not.toContain('text/html'); + } finally { + await srv.close(); + } + }); + + it('records requests for unregistered methods (parity with stateless)', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const { ResultSchema } = + await import('@modelcontextprotocol/sdk/types.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client + .request( + { method: 'tools/call', params: { name: 'nope' } }, + ResultSchema + ) + .catch(() => {}); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toContain('tools/call'); + } finally { + await srv.close(); + } + }); +}); diff --git a/src/mock-server/select.ts b/src/mock-server/select.ts new file mode 100644 index 00000000..0145786b --- /dev/null +++ b/src/mock-server/select.ts @@ -0,0 +1,13 @@ +import type { SpecVersion } from '../types'; +import type { MockServer, RequestHandlers } from './index'; +import { isStatefulVersion } from '../connection/select'; +import { createServerStateful } from './stateful'; +import { createServerStateless } from './stateless'; + +export function createServerFor( + specVersion: SpecVersion +): (handlers: RequestHandlers) => Promise { + return isStatefulVersion(specVersion) + ? createServerStateful + : createServerStateless; +} diff --git a/src/mock-server/stateful.ts b/src/mock-server/stateful.ts new file mode 100644 index 00000000..50042812 --- /dev/null +++ b/src/mock-server/stateful.ts @@ -0,0 +1,143 @@ +/** + * Stateful mock server: 2025-x lifecycle (initialize handshake). + * + * Backed by the SDK's `Server` + `StreamableHTTPServerTransport` so we don't + * reimplement the handshake or SSE response framing. The SDK is the scaffold + * here, not the system-under-test; the client-under-test connecting to this + * mock is what's being verified. + */ + +import express from 'express'; +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; +import { z } from 'zod'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; + +const CAPABILITY_BY_PREFIX: Record = { + tools: 'tools', + prompts: 'prompts', + resources: 'resources', + completion: 'completions', + logging: 'logging' +}; + +/** + * Derive the server `capabilities` object from the registered handler method + * names so the SDK's `assertRequestHandlerCapability` gate is always satisfied. + * Shared with the stateless impl for `server/discover`. + */ +export function capabilitiesFromHandlers( + handlers: RequestHandlers +): Record { + const out: Record = {}; + for (const method of Object.keys(handlers)) { + const cap = CAPABILITY_BY_PREFIX[method.split('/')[0]]; + if (cap) out[cap] = {}; + } + return out; +} + +export async function createServerStateful( + handlers: RequestHandlers +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + + // Fresh SDK Server per HTTP request (the SDK transport is single-shot in + // sessionless mode after GHSA-345p-7cg4-v4c7). + function newServer(): Server { + const server = new Server( + { name: 'conformance-mock-server', version: '1.0.0' }, + { capabilities } + ); + for (const [method, handler] of Object.entries(handlers)) { + // The SDK's setRequestHandler matches by parsing against the schema's + // method literal; build a minimal schema so any method string works. + const schema = z.object({ + method: z.literal(method), + params: z.unknown().optional() + }); + server.setRequestHandler(schema, async (request) => { + try { + return (await handler( + (request.params ?? {}) as Record, + request as JSONRPCRequest + )) as Record; + } catch (e) { + if (e instanceof McpError) throw e; + throw new McpError( + ErrorCode.InternalError, + e instanceof Error ? e.message : String(e) + ); + } + }); + } + return server; + } + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + // Record every JSON-RPC request the client sends (excluding the lifecycle + // preamble) at the HTTP layer so unregistered methods are captured too, + // matching the stateless impl and the MockServer.recorded contract. + const body = req.body; + if ( + body?.method && + body.method !== 'initialize' && + body.method !== 'notifications/initialized' + ) { + recorded.push(body as JSONRPCRequest); + } + try { + const server = newServer(); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined + }); + await server.connect(transport); + await transport.handleRequest(req, res, req.body); + res.on('close', () => { + transport.close(); + server.close(); + }); + } catch (e) { + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + id: req.body?.id ?? null, + error: { code: -32603, message: String(e) } + }); + } + } + }); + + return listen(app, recorded); +} + +function listen( + app: express.Application, + recorded: JSONRPCRequest[] +): Promise { + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((res) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => res()); + }) + }); + }); + }); +} diff --git a/src/mock-server/stateless.ts b/src/mock-server/stateless.ts new file mode 100644 index 00000000..75e1370b --- /dev/null +++ b/src/mock-server/stateless.ts @@ -0,0 +1,162 @@ +/** + * Stateless mock server: 2026-x lifecycle (SEP-2575). + * + * No initialize handshake. Validates `_meta` (protocolVersion / clientInfo / + * clientCapabilities) and the `MCP-Protocol-Version` header on every request, + * serves `server/discover`, and routes other methods to the supplied handlers. + * Implemented with raw express so it can front-run SDK support. + */ + +import express from 'express'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; +import { capabilitiesFromHandlers } from './stateful'; + +const META_KEYS = [ + 'io.modelcontextprotocol/protocolVersion', + 'io.modelcontextprotocol/clientInfo', + 'io.modelcontextprotocol/clientCapabilities' +] as const; + +type IncomingHeaders = Record; + +export type StatelessValidation = + | { + ok: true; + id: string | number | null; + method: string; + params: Record; + } + | { ok: false; status: number; body: object }; + +/** + * Shared SEP-2575 request validation: header presence, `_meta` 3-key check, + * header/`_meta` version match, version-supported check, and `server/discover` + * handling. Consumers write `res.status(v.status).json(v.body)` for the + * not-ok branch (which also covers discover) and route on the ok branch. + * + * Exported so any mock server that needs a stateless `/mcp` route (e.g. + * `auth/helpers/createServer.ts`) uses the same validation as this module. + */ +export function validateStatelessRequest( + req: { headers: IncomingHeaders; body: unknown }, + capabilities: Record +): StatelessValidation { + const body = (req.body ?? {}) as Record; + const id = (body.id ?? null) as string | number | null; + const method = body.method as string; + const params = (body.params ?? {}) as Record; + const meta = params._meta as Record | undefined; + + const reject = (status: number, code: number, message: string) => + ({ + ok: false, + status, + body: { jsonrpc: '2.0', id, error: { code, message } } + }) as const; + + const headerVersion = req.headers['mcp-protocol-version']; + if (!headerVersion) { + return reject(400, -32001, 'Missing MCP-Protocol-Version header'); + } + const missing = META_KEYS.filter((k) => meta?.[k] === undefined); + if (missing.length > 0) { + return reject( + 400, + -32602, + `Invalid params: missing _meta keys: ${missing.join(', ')}` + ); + } + if (meta?.[META_KEYS[0]] !== headerVersion) { + return reject( + 400, + -32001, + 'MCP-Protocol-Version header does not match _meta.protocolVersion' + ); + } + if (headerVersion !== DRAFT_PROTOCOL_VERSION) { + return { + ok: false, + status: 400, + body: { + jsonrpc: '2.0', + id, + error: { + code: -32004, + message: 'Unsupported protocol version', + data: { supported: [DRAFT_PROTOCOL_VERSION] } + } + } + }; + } + if (method === 'server/discover') { + return { + ok: false, + status: 200, + body: { + jsonrpc: '2.0', + id, + result: { + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities, + serverInfo: { name: 'conformance-mock-server', version: '1.0.0' } + } + } + }; + } + return { ok: true, id, method, params }; +} + +export async function createServerStateless( + handlers: RequestHandlers +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + const v = validateStatelessRequest(req, capabilities); + if (!v.ok) { + return res.status(v.status).json(v.body); + } + const { id, method, params } = v; + const error = (status: number, code: number, message: string) => + res.status(status).json({ jsonrpc: '2.0', id, error: { code, message } }); + + recorded.push(req.body as JSONRPCRequest); + + const handler = handlers[method]; + if (!handler) { + return error(404, -32601, `Method not found: ${method}`); + } + try { + const result = await handler(params, req.body as JSONRPCRequest); + return res.json({ jsonrpc: '2.0', id, result }); + } catch (e) { + return error(500, -32603, e instanceof Error ? e.message : String(e)); + } + }); + + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((r) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => r()); + }) + }); + }); + }); +} diff --git a/src/mock-server/testing.ts b/src/mock-server/testing.ts new file mode 100644 index 00000000..1e307b8d --- /dev/null +++ b/src/mock-server/testing.ts @@ -0,0 +1,17 @@ +import { LATEST_SPEC_VERSION, type SpecVersion } from '../types'; +import { createServerFor } from './select'; +import type { ScenarioContext } from './index'; + +/** + * Build a ScenarioContext for unit tests that drive a Scenario directly. + * Defaults to the latest dated spec version (stateful lifecycle) so existing + * tests keep their pre-ScenarioContext behaviour. + */ +export function testScenarioContext( + specVersion: SpecVersion = LATEST_SPEC_VERSION +): ScenarioContext { + return { + specVersion, + createServer: (handlers) => createServerFor(specVersion)(handlers) + }; +} diff --git a/src/runner/client.ts b/src/runner/client.ts index 1bf8c9f6..b8683497 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,8 +1,9 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck, SpecVersion } from '../types'; +import { ConformanceCheck, SpecVersion, LATEST_SPEC_VERSION } from '../types'; import { getScenario } from '../scenarios'; +import { createServerFor, type ScenarioContext } from '../mock-server'; import { createResultDir, formatPrettyChecks } from './utils'; export interface ClientExecutionResult { @@ -35,9 +36,7 @@ async function executeClient( // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; env.MCP_CONFORMANCE_SCENARIO = scenarioName; - if (specVersion) { - env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion; - } + env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion ?? LATEST_SPEC_VERSION; if (context) { // Include scenario name in context for discriminated union parsing env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ @@ -114,8 +113,14 @@ export async function runConformanceTest( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + const resolvedVersion = specVersion ?? LATEST_SPEC_VERSION; + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.error(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.error(`Executing client: ${clientCommand} ${urls.serverUrl}`); if (urls.context) { @@ -129,7 +134,7 @@ export async function runConformanceTest( urls.serverUrl, timeout, urls.context, - specVersion + resolvedVersion ); // Print stdout/stderr if client exited with nonzero code @@ -269,7 +274,8 @@ export function printClientResults( export async function runInteractiveMode( scenarioName: string, verbose: boolean = false, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion ): Promise { let resultDir: string | undefined; @@ -281,8 +287,14 @@ export async function runInteractiveMode( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + const resolvedVersion = specVersion ?? LATEST_SPEC_VERSION; + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.log(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.log(`Server URL: ${urls.serverUrl}`); console.log('Press Ctrl+C to stop...'); diff --git a/src/scenarios/client/auth/authorization-server-migration.ts b/src/scenarios/client/auth/authorization-server-migration.ts index 858bddb5..9fe8add2 100644 --- a/src/scenarios/client/auth/authorization-server-migration.ts +++ b/src/scenarios/client/auth/authorization-server-migration.ts @@ -6,6 +6,7 @@ * to AS₂. On the next 401 the client re-discovers PRM, sees a new issuer, and * MUST re-register with AS₂ rather than reuse AS₁'s client credentials. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Request, Response, NextFunction } from 'express'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; @@ -25,7 +26,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, ['mcp:basic']); @@ -125,6 +126,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { }; const app = createServer( + ctx, this.checks, this.server.getUrl, currentAuthServerUrl, diff --git a/src/scenarios/client/auth/basic-cimd.ts b/src/scenarios/client/auth/basic-cimd.ts index a99b4e35..7a9969fc 100644 --- a/src/scenarios/client/auth/basic-cimd.ts +++ b/src/scenarios/client/auth/basic-cimd.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -29,7 +30,7 @@ export class AuthBasicCIMDScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const authApp = createAuthServer(this.checks, this.authServer.getUrl, { @@ -60,6 +61,7 @@ export class AuthBasicCIMDScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/client-credentials.ts b/src/scenarios/client/auth/client-credentials.ts index eaebcb01..1171e5f0 100644 --- a/src/scenarios/client/auth/client-credentials.ts +++ b/src/scenarios/client/auth/client-credentials.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; @@ -42,7 +43,7 @@ export class ClientCredentialsJwtScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate a fresh keypair for this test run @@ -201,6 +202,7 @@ export class ClientCredentialsJwtScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl @@ -263,7 +265,7 @@ export class ClientCredentialsBasicScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const authApp = createAuthServer(this.checks, this.authServer.getUrl, { @@ -364,6 +366,7 @@ export class ClientCredentialsBasicScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/discovery-metadata.ts b/src/scenarios/client/auth/discovery-metadata.ts index 7387ead4..16b3dd9f 100644 --- a/src/scenarios/client/auth/discovery-metadata.ts +++ b/src/scenarios/client/auth/discovery-metadata.ts @@ -6,6 +6,7 @@ * generated from them. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -94,7 +95,7 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { **OAuth metadata:** ${config.oauthMetadataLocation} `, - async start(): Promise { + async start(ctx: ScenarioContext): Promise { checks = []; const authApp = createAuthServer(checks, authServer.getUrl, { @@ -131,7 +132,7 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { ? () => `${authServer.getUrl()}${routePrefix}` : authServer.getUrl; - const app = createServer(checks, server.getUrl, getAuthServerUrl, { + const app = createServer(ctx, checks, server.getUrl, getAuthServerUrl, { prmPath: config.prmLocation, includePrmInWwwAuth: config.inWwwAuth }); diff --git a/src/scenarios/client/auth/enterprise-managed-authorization.ts b/src/scenarios/client/auth/enterprise-managed-authorization.ts index ddadfbf5..b89a2474 100644 --- a/src/scenarios/client/auth/enterprise-managed-authorization.ts +++ b/src/scenarios/client/auth/enterprise-managed-authorization.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import express, { type Request, type Response } from 'express'; @@ -69,7 +70,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { private idpPrivateKey?: CryptoKey; private grantKeypairs: Map = new Map(); - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate IDP keypair @@ -120,6 +121,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { // Start MCP server with shared token verifier const mcpApp = createServer( + ctx, this.checks, this.mcpServer.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/helpers/createServer.ts b/src/scenarios/client/auth/helpers/createServer.ts index c836630d..e18fcddd 100644 --- a/src/scenarios/client/auth/helpers/createServer.ts +++ b/src/scenarios/client/auth/helpers/createServer.ts @@ -10,6 +10,11 @@ import { import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js'; import express, { Request, Response, NextFunction } from 'express'; import type { ConformanceCheck } from '../../../../types'; +import { + validateStatelessRequest, + type ScenarioContext +} from '../../../../mock-server'; +import { isStatefulVersion } from '../../../../connection/select'; import { createRequestLogger } from '../../../request-logger'; import { MockTokenVerifier } from './mockTokenVerifier'; import { SpecReferences } from '../spec-references'; @@ -27,6 +32,7 @@ export interface ServerOptions { } export function createServer( + ctx: ScenarioContext, checks: ConformanceCheck[], getBaseUrl: () => string, getAuthServerUrl: () => string, @@ -157,6 +163,9 @@ export function createServer( authMiddleware(req, res, async (err?: any) => { if (err) return next(err); + if (!isStatefulVersion(ctx.specVersion)) { + return handleStateless(req, res); + } const server = createMcpServer(); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined @@ -186,5 +195,38 @@ export function createServer( }); }); + // Stateless lifecycle for the /mcp route: shared SEP-2575 validation + + // server/discover from mock-server/stateless, then the same tools handlers + // as createMcpServer. Bearer-auth middleware and PRM route above are + // version-independent. + function handleStateless(req: Request, res: Response) { + const v = validateStatelessRequest(req, { tools: {} }); + if (!v.ok) { + return res.status(v.status).json(v.body); + } + const { id, method } = v; + if (method === 'tools/list') { + return res.json({ + jsonrpc: '2.0', + id, + result: { + tools: [{ name: 'test-tool', inputSchema: { type: 'object' } }] + } + }); + } + if (method === 'tools/call') { + return res.json({ + jsonrpc: '2.0', + id, + result: { content: [{ type: 'text', text: 'test' }] } + }); + } + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: 'Method not found' } + }); + } + return app; } diff --git a/src/scenarios/client/auth/issuer-parameter.ts b/src/scenarios/client/auth/issuer-parameter.ts index ecb2bc2b..81dac383 100644 --- a/src/scenarios/client/auth/issuer-parameter.ts +++ b/src/scenarios/client/auth/issuer-parameter.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -30,7 +31,7 @@ export class IssParameterSupportedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; @@ -48,6 +49,7 @@ export class IssParameterSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -99,7 +101,7 @@ export class IssParameterNotAdvertisedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; @@ -117,6 +119,7 @@ export class IssParameterNotAdvertisedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -170,7 +173,7 @@ export class IssParameterSupportedMissingScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -192,6 +195,7 @@ export class IssParameterSupportedMissingScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -256,7 +260,7 @@ export class IssParameterWrongIssuerScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -278,6 +282,7 @@ export class IssParameterWrongIssuerScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -343,7 +348,7 @@ export class IssParameterUnexpectedScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -365,6 +370,7 @@ export class IssParameterUnexpectedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -435,7 +441,7 @@ export class IssParameterNormalizedVariantScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -457,6 +463,7 @@ export class IssParameterNormalizedVariantScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -522,7 +529,7 @@ export class MetadataIssuerMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private metadataEndpointsUsed = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.metadataEndpointsUsed = false; @@ -555,6 +562,7 @@ export class MetadataIssuerMismatchScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/march-spec-backcompat.ts b/src/scenarios/client/auth/march-spec-backcompat.ts index b2502092..68868b03 100644 --- a/src/scenarios/client/auth/march-spec-backcompat.ts +++ b/src/scenarios/client/auth/march-spec-backcompat.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -17,7 +18,7 @@ export class Auth20250326OAuthMetadataBackcompatScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Legacy server, so we create the auth server endpoints on the // same URL as the main server (rather than separating AS / RS). @@ -28,6 +29,7 @@ export class Auth20250326OAuthMetadataBackcompatScenario implements Scenario { routePrefix: '/oauth' }); const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, @@ -81,10 +83,11 @@ export class Auth20250326OEndpointFallbackScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, diff --git a/src/scenarios/client/auth/offline-access.ts b/src/scenarios/client/auth/offline-access.ts index afcd593f..2f927c69 100644 --- a/src/scenarios/client/auth/offline-access.ts +++ b/src/scenarios/client/auth/offline-access.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -33,7 +34,7 @@ export class OfflineAccessScopeScenario implements Scenario { private grantTypesChecked = false; private capturedCimdUrl: string | undefined; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.grantTypesChecked = false; this.capturedCimdUrl = undefined; @@ -101,6 +102,7 @@ export class OfflineAccessScopeScenario implements Scenario { // PRM does NOT include offline_access (per SEP-2207 server guidance: // servers SHOULD NOT include offline_access in PRM scopes_supported) const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -235,7 +237,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, [ @@ -268,6 +270,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/pre-registration.ts b/src/scenarios/client/auth/pre-registration.ts index b482e4f3..eefdd033 100644 --- a/src/scenarios/client/auth/pre-registration.ts +++ b/src/scenarios/client/auth/pre-registration.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; @@ -27,7 +28,7 @@ export class PreRegistrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); @@ -105,6 +106,7 @@ export class PreRegistrationScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts index 9c15d4d6..e59ebcc1 100644 --- a/src/scenarios/client/auth/resource-mismatch.ts +++ b/src/scenarios/client/auth/resource-mismatch.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -37,7 +38,7 @@ export class ResourceMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private authorizationRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationRequestMade = false; @@ -60,6 +61,7 @@ export class ResourceMismatchScenario implements Scenario { // Create server that returns a mismatched resource in PRM const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index c24d51d6..c1154ca2 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -22,7 +23,7 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const expectedScope = 'mcp:basic'; @@ -53,6 +54,7 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -108,7 +110,7 @@ export class ScopeFromScopesSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const scopesSupported = ['mcp:basic', 'mcp:read', 'mcp:write']; @@ -148,6 +150,7 @@ export class ScopeFromScopesSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -204,7 +207,7 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); @@ -232,6 +235,7 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -291,7 +295,7 @@ export class ScopeStepUpAuthScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const initialScope = 'mcp:basic'; @@ -437,6 +441,7 @@ export class ScopeStepUpAuthScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -528,7 +533,7 @@ export class ScopeRetryLimitScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const requiredScope = 'mcp:admin'; @@ -613,6 +618,7 @@ export class ScopeRetryLimitScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/test_helpers/testClient.ts b/src/scenarios/client/auth/test_helpers/testClient.ts index 0dbc8d8d..c0cfd33b 100644 --- a/src/scenarios/client/auth/test_helpers/testClient.ts +++ b/src/scenarios/client/auth/test_helpers/testClient.ts @@ -1,4 +1,5 @@ import { getScenario } from '../../../index'; +import { testScenarioContext } from '../../../../mock-server/testing'; import { spawn } from 'child_process'; const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete @@ -103,7 +104,7 @@ export async function runClientAgainstScenario( } // Start the scenario server - const urls = await scenario.start(); + const urls = await scenario.start(testScenarioContext()); const serverUrl = urls.serverUrl; try { diff --git a/src/scenarios/client/auth/token-endpoint-auth.ts b/src/scenarios/client/auth/token-endpoint-auth.ts index 79d62968..7b7616ff 100644 --- a/src/scenarios/client/auth/token-endpoint-auth.ts +++ b/src/scenarios/client/auth/token-endpoint-auth.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -62,7 +63,7 @@ class TokenEndpointAuthScenario implements Scenario { this.description = `Tests that client uses ${AUTH_METHOD_NAMES[expectedAuthMethod]} when server only supports ${expectedAuthMethod}`; } - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationResource = undefined; this.tokenResource = undefined; @@ -137,6 +138,7 @@ class TokenEndpointAuthScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/elicitation-defaults.ts b/src/scenarios/client/elicitation-defaults.ts index c78f3495..d4170295 100644 --- a/src/scenarios/client/elicitation-defaults.ts +++ b/src/scenarios/client/elicitation-defaults.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-1034: Elicitation defaults test * Validates that clients properly apply default values for omitted fields @@ -13,7 +14,7 @@ import { } from '@modelcontextprotocol/sdk/types.js'; import type { Scenario, ConformanceCheck } from '../../types'; import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; +import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; import { createRequestLogger } from '../request-logger'; import { randomUUID } from 'crypto'; @@ -474,7 +475,10 @@ function createServer(checks: ConformanceCheck[]): { export class ElicitationClientDefaultsScenario implements Scenario { name = 'elicitation-sep1034-client-defaults'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests client applies default values for omitted elicitation fields (SEP-1034)'; private app: express.Application | null = null; @@ -482,7 +486,7 @@ export class ElicitationClientDefaultsScenario implements Scenario { private checks: ConformanceCheck[] = []; private cleanup: (() => void) | null = null; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; const { app, cleanup } = createServer(this.checks); this.app = app; diff --git a/src/scenarios/client/http-base.ts b/src/scenarios/client/http-base.ts index 06c2afaf..9f6187d3 100644 --- a/src/scenarios/client/http-base.ts +++ b/src/scenarios/client/http-base.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * Shared HTTP test-server scaffold for client-under-test SEP-2243 scenarios. * @@ -27,7 +28,7 @@ export abstract class BaseHttpScenario implements Scenario { protected port: number = 0; protected sessionId: string = `session-${Date.now()}`; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/http-custom-headers.test.ts b/src/scenarios/client/http-custom-headers.test.ts index 88714eea..8a68f22a 100644 --- a/src/scenarios/client/http-custom-headers.test.ts +++ b/src/scenarios/client/http-custom-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpCustomHeadersScenario, @@ -43,7 +44,7 @@ function statusesFor( describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('emits exactly the declared requirement IDs as FAILURE when the client never connects', async () => { const scenario = new HttpCustomHeadersScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); expect(idsOf(checks)).toEqual(new Set(CUSTOM_HEADERS_DECLARED_CHECK_IDS)); @@ -57,7 +58,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('maps each parameter kind to its requirement ID on a conforming tool call', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { const nonAscii = 'Hello, 世界'; const nonAsciiB64 = Buffer.from(nonAscii, 'utf-8').toString('base64'); @@ -123,7 +124,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('FAILs client-mirrors-designated-params when an annotated header is missing', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post( serverUrl, @@ -154,7 +155,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('emits every x-mcp-header constraint ID, SUCCESS when only valid_tool is called', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { @@ -176,7 +177,7 @@ describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('FAILs the violated constraint ID when the client calls an invalid tool', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { diff --git a/src/scenarios/client/http-custom-headers.ts b/src/scenarios/client/http-custom-headers.ts index e91cf2c0..25e36610 100644 --- a/src/scenarios/client/http-custom-headers.ts +++ b/src/scenarios/client/http-custom-headers.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * HTTP Custom Headers conformance test scenario for MCP clients (SEP-2243) * @@ -186,8 +187,8 @@ export class HttpCustomHeadersScenario extends BaseHttpScenario { private toolCallReceived: boolean = false; private nullToolCallReceived: boolean = false; - async start(): Promise { - const urls = await super.start(); + async start(_ctx: ScenarioContext): Promise { + const urls = await super.start(_ctx); // Pass test values via context for encoding edge cases. // The conformance client should use these values when calling test_custom_headers. urls.context = { diff --git a/src/scenarios/client/http-standard-headers.test.ts b/src/scenarios/client/http-standard-headers.test.ts index 8c5ff6ad..3478e060 100644 --- a/src/scenarios/client/http-standard-headers.test.ts +++ b/src/scenarios/client/http-standard-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpStandardHeadersScenario } from './http-standard-headers'; @@ -38,7 +39,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('FAILs the initialize Mcp-Method emission when Mcp-Method is missing', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, {}); // no Mcp-Method header const checks = scenario.getChecks(); @@ -53,7 +54,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('SUCCEEDs the initialize Mcp-Method emission when Mcp-Method matches', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const checks = scenario.getChecks(); @@ -68,7 +69,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('getChecks() is idempotent', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const first = scenario.getChecks(); diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index b5e2aef2..be8d7082 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,23 +1,28 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, ScenarioUrls, ConformanceCheck, LATEST_SPEC_VERSION, - NEGOTIABLE_PROTOCOL_VERSIONS + NEGOTIABLE_PROTOCOL_VERSIONS, + DRAFT_PROTOCOL_VERSION } from '../../types'; import { clientChecks } from '../../checks/index'; export class InitializeScenario implements Scenario { name = 'initialize'; - readonly source = { introducedIn: '2025-06-18' } as const; + readonly source = { + introducedIn: '2025-06-18', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests MCP client initialization handshake'; private server: http.Server | null = null; private checks: ConformanceCheck[] = []; private port: number = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/json-schema-ref-deref.test.ts b/src/scenarios/client/json-schema-ref-deref.test.ts index b11d2d25..a3091933 100644 --- a/src/scenarios/client/json-schema-ref-deref.test.ts +++ b/src/scenarios/client/json-schema-ref-deref.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; @@ -92,7 +93,7 @@ describe('json-schema-ref-no-deref (SEP-2106)', () => { test('client that never lists tools fails: requirement cannot be evaluated', async () => { const scenario = new JsonSchemaRefDerefScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const check = checks.find( diff --git a/src/scenarios/client/json-schema-ref-deref.ts b/src/scenarios/client/json-schema-ref-deref.ts index 91ccc9e0..a7b315e0 100644 --- a/src/scenarios/client/json-schema-ref-deref.ts +++ b/src/scenarios/client/json-schema-ref-deref.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; @@ -82,7 +83,7 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at private canaryRequests: Array<{ method: string; userAgent?: string }> = []; private toolsListed = false; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.canaryRequests = []; this.toolsListed = false; diff --git a/src/scenarios/client/mrtr-client.ts b/src/scenarios/client/mrtr-client.ts index 431fafe6..4521ca04 100644 --- a/src/scenarios/client/mrtr-client.ts +++ b/src/scenarios/client/mrtr-client.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-2322: MRTR Client Conformance Tests * @@ -440,7 +441,7 @@ export class MRTRClientScenario implements Scenario { private httpServer: ReturnType | null = null; private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; this.app = createMRTRServer(this.checks); this.httpServer = this.app.listen(0); diff --git a/src/scenarios/client/request-metadata.test.ts b/src/scenarios/client/request-metadata.test.ts index 69384648..303691af 100644 --- a/src/scenarios/client/request-metadata.test.ts +++ b/src/scenarios/client/request-metadata.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { runClientAgainstScenario, @@ -206,7 +207,7 @@ describe('request-metadata client scenario — client never connects', () => { throw new Error('Scenario not found'); } - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const byId = new Map(checks.map((c) => [c.id, c])); diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts index 47d4f00a..0b472851 100644 --- a/src/scenarios/client/request-metadata.ts +++ b/src/scenarios/client/request-metadata.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, @@ -32,7 +33,7 @@ export class RequestMetadataScenario implements Scenario { private hasSimulatedRejection = false; private requestsObserved = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.hasSimulatedRejection = false; this.checks = []; this.requestsObserved = 0; diff --git a/src/scenarios/client/sse-retry.ts b/src/scenarios/client/sse-retry.ts index b90bf40a..e89b23eb 100644 --- a/src/scenarios/client/sse-retry.ts +++ b/src/scenarios/client/sse-retry.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SSE Retry conformance test scenarios for MCP clients (SEP-1699) * @@ -8,11 +9,19 @@ */ import http from 'http'; -import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; +import { + Scenario, + ScenarioUrls, + ConformanceCheck, + DRAFT_PROTOCOL_VERSION +} from '../../types.js'; export class SSERetryScenario implements Scenario { name = 'sse-retry'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests that client respects SSE retry field timing and reconnects properly (SEP-1699)'; @@ -38,7 +47,7 @@ export class SSERetryScenario implements Scenario { private readonly LATE_TOLERANCE = 200; // Allow 200ms late for network/event loop private readonly VERY_LATE_MULTIPLIER = 2; // If >2x retry value, client is likely ignoring it - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/tools_call.ts b/src/scenarios/client/tools_call.ts index 59470f37..31be0832 100644 --- a/src/scenarios/client/tools_call.ts +++ b/src/scenarios/client/tools_call.ts @@ -1,164 +1,82 @@ -import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; -import { - CallToolRequestSchema, - ListToolsRequestSchema -} from '@modelcontextprotocol/sdk/types.js'; -import type { Scenario, ConformanceCheck } from '../../types'; -import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; -import { createRequestLogger } from '../request-logger'; +import type { ScenarioContext, MockServer } from '../../mock-server'; +import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../types'; +import type { CallToolRequest } from '../../spec-types/2025-06-18'; -function createMcpServer(checks: ConformanceCheck[]): Server { - const server = new Server( - { - name: 'add-numbers-server', - version: '1.0.0' - }, - { - capabilities: { - tools: {} - } - } - ); - - server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [ - { - name: 'add_numbers', - description: 'Add two numbers together', - inputSchema: { - type: 'object', - properties: { - a: { - type: 'number', - description: 'First number' - }, - b: { - type: 'number', - description: 'Second number' - } - }, - required: ['a', 'b'] - } - } - ] - }; - }); - - server.setRequestHandler(CallToolRequestSchema, async (request) => { - if (request.params.name === 'add_numbers') { - const { a, b } = request.params.arguments as { a: number; b: number }; - const result = a + b; - - checks.push({ - id: 'tool-add-numbers', - name: 'ToolAddNumbers', - description: 'Validates that the add_numbers tool works correctly', - status: 'SUCCESS', - timestamp: new Date().toISOString(), - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' - } - ], - details: { - a, - b, - result - } - }); - - return { - content: [ - { - type: 'text', - text: `The sum of ${a} and ${b} is ${result}` - } - ] - }; - } - - throw new Error(`Unknown tool: ${request.params.name}`); - }); - - return server; -} - -function createServerApp(checks: ConformanceCheck[]): express.Application { - const app = express(); - app.use(express.json()); - - app.use( - createRequestLogger(checks, { - incomingId: 'incoming-request', - outgoingId: 'outgoing-response', - mcpRoute: '/mcp' - }) - ); - - app.post('/mcp', async (req: Request, res: Response) => { - // Stateless: create a fresh server and transport per request - const server = createMcpServer(checks); - const transport = new StreamableHTTPServerTransport({ - sessionIdGenerator: undefined - }); - await server.connect(transport); - - await transport.handleRequest(req, res, req.body); - }); - - return app; -} +const SPEC_REF = { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' +}; export class ToolsCallScenario implements Scenario { name = 'tools_call'; readonly source = { introducedIn: '2025-06-18' } as const; description = 'Tests calling tools with various parameter types'; - private app: express.Application | null = null; - private httpServer: any = null; + private srv: MockServer | null = null; private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; - this.app = createServerApp(this.checks); - this.httpServer = this.app.listen(0); - const port = this.httpServer.address().port; - return { serverUrl: `http://localhost:${port}/mcp` }; + this.srv = await ctx.createServer({ + 'tools/list': () => ({ + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers together', + inputSchema: { + type: 'object', + properties: { + a: { type: 'number', description: 'First number' }, + b: { type: 'number', description: 'Second number' } + }, + required: ['a', 'b'] + } + } + ] + }), + 'tools/call': (params) => { + const p = params as CallToolRequest['params']; + if (p.name !== 'add_numbers') { + throw new Error(`Unknown tool: ${p.name}`); + } + const { a, b } = p.arguments as { a: number; b: number }; + return { + content: [ + { type: 'text', text: `The sum of ${a} and ${b} is ${a + b}` } + ] + }; + } + }); + return { serverUrl: this.srv.url }; } async stop() { - if (this.httpServer) { - await new Promise((resolve) => this.httpServer.close(resolve)); - this.httpServer = null; - } - this.app = null; + await this.srv?.close(); + this.srv = null; } getChecks(): ConformanceCheck[] { - const expectedSlugs = ['tool-add-numbers']; - // add a failure if not in there already - for (const slug of expectedSlugs) { - if (!this.checks.find((c) => c.id === slug)) { - // TODO: this is duplicated from above, refactor - this.checks.push({ - id: slug, - name: `ToolAddNumbers`, - description: `Validates that the add_numbers tool works correctly`, - status: 'FAILURE', - timestamp: new Date().toISOString(), - details: { message: 'Tool was not called by client' }, - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' - } - ] - }); - } - } + const call = this.srv?.recorded.find((r) => r.method === 'tools/call'); + const args = (call?.params as CallToolRequest['params'] | undefined) + ?.arguments as { a?: unknown; b?: unknown } | undefined; + const ok = + call !== undefined && + typeof args?.a === 'number' && + typeof args?.b === 'number'; + this.checks.push({ + id: 'tool-add-numbers', + name: 'ToolAddNumbers', + description: 'Validates that the add_numbers tool works correctly', + status: ok ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REF], + details: ok + ? { + a: args!.a, + b: args!.b, + result: (args!.a as number) + (args!.b as number) + } + : { message: 'Tool was not called by client' } + }); return this.checks; } } diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index c77e43e7..e39f06c2 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -4,6 +4,7 @@ import { listScenariosForSpec, listDraftScenarios, listExtensionScenarios, + getScenario, getScenarioSpecVersions, resolveSpecVersion, ALL_SPEC_VERSIONS, @@ -54,11 +55,17 @@ describe('specVersions helpers', () => { expect(current.length).toBeGreaterThan(overlap.length); }); - it('the draft spec version is a superset of the latest dated release', () => { + it('every scenario in latest but not in draft is explicitly removedIn: DRAFT', () => { const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION)); for (const name of latest) { - expect(draft.has(name)).toBe(true); + if (!draft.has(name)) { + const s = getScenario(name)!; + expect( + 'removedIn' in s.source && s.source.removedIn, + `"${name}" is in ${LATEST_SPEC_VERSION} but not in draft without removedIn` + ).toBe(DRAFT_PROTOCOL_VERSION); + } } for (const name of listDraftScenarios()) { expect(draft.has(name)).toBe(true); diff --git a/src/types.ts b/src/types.ts index 6c6376e5..2e9dd22a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,5 @@ import type { RunContext } from './connection'; +import type { ScenarioContext } from './mock-server'; export type CheckStatus = | 'SUCCESS' @@ -102,7 +103,7 @@ export interface Scenario { * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). */ allowClientError?: boolean; - start(): Promise; + start(ctx: ScenarioContext): Promise; stop(): Promise; getChecks(): ConformanceCheck[]; }