From d61a33f3738b003137461c459594efd1e711b97d Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Mon, 4 May 2026 13:47:03 +0300 Subject: [PATCH 1/2] test: add MCP and CLI smoke suites Adds live smoke coverage for user-facing MCP tools and CLI commands so text defaults, JSON opt-in, auth handling, and parity affordances stay aligned. --- AGENTS.md | 2 + docs/implementation/mcp-cli-parity.md | 10 + package.json | 2 + scripts/cli-smoke.ts | 498 +++++++++++++++++++++++ scripts/mcp-smoke.ts | 555 ++++++++++++++++++++++++++ 5 files changed, 1067 insertions(+) create mode 100644 scripts/cli-smoke.ts create mode 100644 scripts/mcp-smoke.ts diff --git a/AGENTS.md b/AGENTS.md index 2beab4ea..9a90d004 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,8 @@ Philosophy: "If it is not tested, it is likely broken" **Critical Rules:** - Use `bun test` for running tests +- Use `bun run smoke:mcp` and `bun run smoke:cli` when changing MCP tools, CLI commands, shared formatters, auth/error envelopes, or MCP/CLI parity behavior. These are live smoke suites, not the normal unit suite; they must pass unauthenticated by validating auth handling, and provide deeper coverage when authenticated. +- Maintain smoke coverage when adding or changing user-facing tools/commands. Prefer structural UX assertions over brittle snapshots, and keep MCP `format: "json"` and CLI `--json` behavior aligned. - Keep tests async and isolated - Mock services at the interface level using factory functions - Use mock factories from `test-helpers.ts` (e.g., `createMockGitHitsService()`, `createMockAuthService()`) diff --git a/docs/implementation/mcp-cli-parity.md b/docs/implementation/mcp-cli-parity.md index fc1d1854..482ff100 100644 --- a/docs/implementation/mcp-cli-parity.md +++ b/docs/implementation/mcp-cli-parity.md @@ -10,6 +10,16 @@ default to compact `text-v1` where available, while CLI has human terminal output and `--json`. Structured parity is enforced through CLI `--json` and MCP `format: "json"`. +Live smoke coverage lives in `scripts/mcp-smoke.ts` and +`scripts/cli-smoke.ts`, run with `bun run smoke:mcp` and +`bun run smoke:cli`. These suites intentionally avoid exact-output +snapshots because backend ranking and release metadata can change. They +assert durable UX invariants instead: server/command startup, registered +tools, auth handling, compact default text, parseable JSON opt-in, +MCP-native hints, CLI terminal affordances, and JSON envelope shape. When +adding or changing a dual-surface tool, update both smoke scripts if the +covered live UX contract changes. + The dual-surface tools today are unified `search` / `search_status`, the file-exploration bundle (`code_files`, `code_read`, `code_grep`), the package-intelligence tools (`pkg_info`, diff --git a/package.json b/package.json index 780bc0fb..a98f9453 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,8 @@ "build": "bunup --dts --target node --packages=external --exports && chmod +x dist/cli.js", "dev": "bun run ./src/cli.ts", "inspector": "npx @modelcontextprotocol/inspector bun run dev mcp", + "smoke:cli": "bun run scripts/cli-smoke.ts", + "smoke:mcp": "bun run scripts/mcp-smoke.ts", "test": "bun test", "typecheck": "tsc", "format": "biome format --write .", diff --git a/scripts/cli-smoke.ts b/scripts/cli-smoke.ts new file mode 100644 index 00000000..2706cda7 --- /dev/null +++ b/scripts/cli-smoke.ts @@ -0,0 +1,498 @@ +interface CommandResult { + command: string[]; + exitCode: number; + stdout: string; + stderr: string; +} + +interface ErrorEnvelope { + error: string; + code: string; + retryable: boolean; +} + +const DEFAULT_TEXT_LIMIT = 20_000; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function parseJson(text: string, context: string): unknown { + try { + return JSON.parse(text); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + throw new Error(`${context}: expected parseable JSON (${message})`); + } +} + +function assertRecord( + value: unknown, + context: string, +): asserts value is Record { + assert( + value !== null && typeof value === "object" && !Array.isArray(value), + `${context}: expected object`, + ); +} + +function assertNotJson(text: string, context: string): void { + try { + JSON.parse(text); + } catch { + return; + } + throw new Error(`${context}: terminal output unexpectedly parsed as JSON`); +} + +function assertCleanErrorEnvelope( + text: string, + context: string, +): ErrorEnvelope { + const jsonLine = text + .split("\n") + .map((line) => line.trim()) + .find((line) => line.startsWith("{") && line.endsWith("}")); + assert(jsonLine, `${context}: missing JSON error envelope`); + const payload = parseJson(jsonLine, context); + assertRecord(payload, context); + assert( + typeof payload.error === "string" && payload.error.length > 0, + `${context}: missing error`, + ); + assert( + typeof payload.code === "string" && payload.code.length > 0, + `${context}: missing code`, + ); + assert( + typeof payload.retryable === "boolean", + `${context}: missing retryable`, + ); + return payload as unknown as ErrorEnvelope; +} + +function assertTerminalOutput(result: CommandResult, context: string): string { + assert( + result.exitCode === 0, + `${context}: command failed (${result.exitCode})\n${result.stderr}`, + ); + const text = result.stdout.trim(); + assert(text.length > 0, `${context}: expected stdout`); + assert( + text.length < DEFAULT_TEXT_LIMIT, + `${context}: terminal output too large (${text.length} chars)`, + ); + assertNotJson(text, context); + return text; +} + +function assertJsonOutput(result: CommandResult, context: string): unknown { + assert( + result.exitCode === 0, + `${context}: command failed (${result.exitCode})\n${result.stderr}`, + ); + assert(result.stdout.trim().length > 0, `${context}: expected stdout`); + return parseJson(result.stdout, context); +} + +function assertJsonErrorCode( + result: CommandResult, + context: string, + code: string, +): void { + assert(result.exitCode !== 0, `${context}: expected command failure`); + const envelope = assertCleanErrorEnvelope(result.stderr, context); + assert( + envelope.code === code, + `${context}: expected ${code}, got ${envelope.code}`, + ); +} + +async function runCli(args: string[]): Promise { + const proc = Bun.spawn(["bun", "run", "dev", ...args], { + stdout: "pipe", + stderr: "pipe", + env: { + ...process.env, + NO_COLOR: "1", + }, + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { command: args, exitCode, stdout, stderr }; +} + +async function assertLiveOrAuthRequired(): Promise { + const result = await runCli(["languages", "python", "--json"]); + if (result.exitCode === 0) { + const payload = parseJson(result.stdout, "languages auth probe"); + assert(Array.isArray(payload), "languages auth probe: expected array"); + return true; + } + + assert( + result.exitCode !== 0, + "languages auth probe: expected non-zero auth failure", + ); + const stderr = result.stderr.trim(); + assert( + stderr.includes("Authentication required"), + "auth probe missing authentication guidance", + ); + assert(stderr.includes("githits login"), "auth probe missing login guidance"); + console.log("AUTH_REQUIRED: live CLI smoke skipped"); + return false; +} + +async function runLiveSmoke(): Promise { + const languagesText = assertTerminalOutput( + await runCli(["languages", "python"]), + "languages terminal", + ); + assert(languagesText.includes("python"), "languages terminal missing python"); + assert( + languagesText.includes("Python"), + "languages terminal missing display name", + ); + + const languagesJson = assertJsonOutput( + await runCli(["languages", "python", "--json"]), + "languages json", + ); + assert(Array.isArray(languagesJson), "languages json: expected array"); + + const exampleText = assertTerminalOutput( + await runCli(["example", "express hello world", "--lang", "javascript"]), + "example terminal", + ); + assert(exampleText.length > 20, "example terminal unexpectedly short"); + + const exampleJson = assertJsonOutput( + await runCli([ + "example", + "express hello world", + "--lang", + "javascript", + "--json", + ]), + "example json", + ); + assertRecord(exampleJson, "example json"); + assert(typeof exampleJson.result === "string", "example json missing result"); + + const pkgInfoText = assertTerminalOutput( + await runCli(["pkg", "info", "npm:express"]), + "pkg info terminal", + ); + assert( + pkgInfoText.includes("express"), + "pkg info terminal missing package name", + ); + + const pkgInfoJson = assertJsonOutput( + await runCli(["pkg", "info", "npm:express", "--json"]), + "pkg info json", + ); + assertRecord(pkgInfoJson, "pkg info json"); + assert(pkgInfoJson.registry === "npm", "pkg info json registry mismatch"); + assert(pkgInfoJson.name === "express", "pkg info json name mismatch"); + assert( + typeof pkgInfoJson.version === "string", + "pkg info json missing version", + ); + + const depsText = assertTerminalOutput( + await runCli(["pkg", "deps", "npm:express"]), + "pkg deps terminal", + ); + assert( + depsText.includes("runtime"), + "pkg deps terminal missing runtime deps", + ); + + const depsAllText = assertTerminalOutput( + await runCli(["pkg", "deps", "npm:express", "--lifecycle", "all"]), + "pkg deps lifecycle all terminal", + ); + assert( + depsAllText.includes("development") || + depsAllText.includes("peer") || + depsAllText.includes("optional"), + "pkg deps lifecycle all terminal missing non-runtime groups", + ); + + const depsJson = assertJsonOutput( + await runCli(["pkg", "deps", "npm:express", "--json"]), + "pkg deps json", + ); + assertRecord(depsJson, "pkg deps json"); + assertRecord(depsJson.runtime, "pkg deps json runtime"); + + const vulnsText = assertTerminalOutput( + await runCli(["pkg", "vulns", "npm:express"]), + "pkg vulns terminal", + ); + assert( + vulnsText.includes("express") || vulnsText.includes("vulnerab"), + "pkg vulns terminal missing context", + ); + + const vulnsJson = assertJsonOutput( + await runCli(["pkg", "vulns", "npm:express", "--json"]), + "pkg vulns json", + ); + assertRecord(vulnsJson, "pkg vulns json"); + assert( + "summary" in vulnsJson || "advisories" in vulnsJson, + "pkg vulns json missing vulnerability data", + ); + + const changelogText = assertTerminalOutput( + await runCli(["pkg", "changelog", "npm:express", "--limit", "1"]), + "pkg changelog terminal", + ); + assert( + changelogText.includes("express") || changelogText.includes("changelog"), + "pkg changelog terminal missing context", + ); + + const changelogJson = assertJsonOutput( + await runCli(["pkg", "changelog", "npm:express", "--limit", "1", "--json"]), + "pkg changelog json", + ); + assertRecord(changelogJson, "pkg changelog json"); + assertRecord(changelogJson.entries, "pkg changelog json entries"); + + const docsText = assertTerminalOutput( + await runCli(["docs", "list", "npm:express", "--limit", "2"]), + "docs list terminal", + ); + assert( + docsText.includes("docs") || docsText.includes("page"), + "docs list terminal missing docs context", + ); + + const docsJson = assertJsonOutput( + await runCli(["docs", "list", "npm:express", "--limit", "2", "--json"]), + "docs list json", + ); + assertRecord(docsJson, "docs list json"); + assert(Array.isArray(docsJson.pages), "docs list json missing pages array"); + const firstPage = docsJson.pages[0] as Record | undefined; + assert( + firstPage && typeof firstPage.pageId === "string", + "docs list json missing readable page id", + ); + + const docsReadText = assertTerminalOutput( + await runCli(["docs", "read", firstPage.pageId, "--lines", "1-5"]), + "docs read terminal", + ); + assert(docsReadText.length > 0, "docs read terminal missing content"); + + const docsReadJson = assertJsonOutput( + await runCli([ + "docs", + "read", + firstPage.pageId, + "--lines", + "1-5", + "--json", + ]), + "docs read json", + ); + assertRecord(docsReadJson, "docs read json"); + assert( + typeof docsReadJson.content === "string", + "docs read json missing content", + ); + + const codeFilesText = assertTerminalOutput( + await runCli([ + "code", + "files", + "npm:express", + "package.json", + "--limit", + "1", + ]), + "code files terminal", + ); + assert( + codeFilesText.includes("package.json"), + "code files terminal missing package.json", + ); + + const codeFilesJson = assertJsonOutput( + await runCli([ + "code", + "files", + "npm:express", + "package.json", + "--limit", + "1", + "--json", + ]), + "code files json", + ); + assertRecord(codeFilesJson, "code files json"); + assert( + Array.isArray(codeFilesJson.files), + "code files json missing files array", + ); + + const codeReadText = assertTerminalOutput( + await runCli([ + "code", + "read", + "npm:express", + "package.json", + "--lines", + "1-5", + ]), + "code read terminal", + ); + assert( + codeReadText.includes('"name"'), + "code read terminal missing file content", + ); + + const codeReadJson = assertJsonOutput( + await runCli([ + "code", + "read", + "npm:express", + "package.json", + "--lines", + "1-5", + "--json", + ]), + "code read json", + ); + assertRecord(codeReadJson, "code read json"); + assert(codeReadJson.path === "package.json", "code read json path mismatch"); + assert( + typeof codeReadJson.content === "string", + "code read json missing content", + ); + + const codeGrepText = assertTerminalOutput( + await runCli([ + "code", + "grep", + "npm:express", + "express", + "package.json", + "--limit", + "1", + ]), + "code grep terminal", + ); + assert( + codeGrepText.includes("package.json"), + "code grep terminal missing package.json", + ); + + const codeGrepJson = assertJsonOutput( + await runCli([ + "code", + "grep", + "npm:express", + "express", + "package.json", + "--limit", + "1", + "--json", + ]), + "code grep json", + ); + assertRecord(codeGrepJson, "code grep json"); + assert( + "matches" in codeGrepJson || "totalMatches" in codeGrepJson, + "code grep json missing matches", + ); + + const searchText = assertTerminalOutput( + await runCli(["search", "router", "--in", "npm:express", "--limit", "1"]), + "search terminal", + ); + assert( + searchText.includes("search") || searchText.includes("result"), + "search terminal missing result context", + ); + + const searchJson = assertJsonOutput( + await runCli([ + "search", + "router", + "--in", + "npm:express", + "--limit", + "1", + "--json", + ]), + "search json", + ); + assertRecord(searchJson, "search json"); + assert( + "results" in searchJson || "searchRef" in searchJson, + "search json missing results/searchRef", + ); + if (typeof searchJson.searchRef === "string") { + const statusJson = assertJsonOutput( + await runCli(["search-status", searchJson.searchRef, "--json"]), + "search-status json", + ); + assertRecord(statusJson, "search-status json"); + assert( + "completed" in statusJson || "progress" in statusJson, + "search-status json missing status data", + ); + } else { + assertJsonErrorCode( + await runCli(["search-status", "smoke-invalid-search-ref", "--json"]), + "search-status invalid json error", + "NOT_FOUND", + ); + } + + const feedbackValidation = await runCli([ + "feedback", + "smoke-solution-id", + "--json", + ]); + assert( + feedbackValidation.exitCode !== 0, + "feedback validation should fail before submitting", + ); + assert( + feedbackValidation.stderr.includes("Specify either --accept or --reject"), + "feedback validation missing action guidance", + ); + + const invalidJson = await runCli([ + "pkg", + "info", + "npm:express@4.18.0", + "--json", + ]); + assertJsonErrorCode( + invalidJson, + "pkg info invalid json error", + "INVALID_ARGUMENT", + ); +} + +async function main(): Promise { + if (await assertLiveOrAuthRequired()) { + await runLiveSmoke(); + console.log("CLI smoke passed"); + } +} + +await main(); diff --git a/scripts/mcp-smoke.ts b/scripts/mcp-smoke.ts new file mode 100644 index 00000000..ab3837a1 --- /dev/null +++ b/scripts/mcp-smoke.ts @@ -0,0 +1,555 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; + +interface TextContent { + type: "text"; + text: string; +} + +interface ToolCallResult { + content?: unknown; + isError?: boolean; +} + +interface ErrorEnvelope { + error: string; + code: string; + retryable: boolean; +} + +const EXPECTED_TOOLS = [ + "get_example", + "search_language", + "pkg_info", + "pkg_deps", + "pkg_vulns", + "pkg_changelog", + "docs_list", + "docs_read", + "code_files", + "code_read", + "code_grep", + "search", + "search_status", + "feedback", +] as const; + +const DEFAULT_TEXT_LIMIT = 12_000; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function parseJson(text: string, context: string): unknown { + try { + return JSON.parse(text); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + throw new Error(`${context}: expected parseable JSON (${message})`); + } +} + +function assertNotJson(text: string, context: string): void { + try { + JSON.parse(text); + } catch { + return; + } + throw new Error(`${context}: default response unexpectedly parsed as JSON`); +} + +function assertRecord( + value: unknown, + context: string, +): asserts value is Record { + assert( + value !== null && typeof value === "object" && !Array.isArray(value), + `${context}: expected object`, + ); +} + +function resultText(result: ToolCallResult, context: string): string { + assert(Array.isArray(result.content), `${context}: expected content array`); + const first = result.content[0] as Partial | undefined; + assert(first?.type === "text", `${context}: expected text content`); + assert(typeof first.text === "string", `${context}: expected text string`); + return first.text; +} + +function assertCleanErrorEnvelope( + result: ToolCallResult, + context: string, +): ErrorEnvelope { + assert(result.isError === true, `${context}: expected MCP error result`); + const payload = parseJson(resultText(result, context), context); + assertRecord(payload, context); + assert( + typeof payload.error === "string" && payload.error.length > 0, + `${context}: missing error`, + ); + assert( + typeof payload.code === "string" && payload.code.length > 0, + `${context}: missing code`, + ); + assert( + typeof payload.retryable === "boolean", + `${context}: missing retryable`, + ); + return payload as unknown as ErrorEnvelope; +} + +function assertDefaultText(result: ToolCallResult, context: string): string { + assert(result.isError !== true, `${context}: expected success`); + const text = resultText(result, context); + assert(text.length > 0, `${context}: expected non-empty text`); + assert( + text.length < DEFAULT_TEXT_LIMIT, + `${context}: default text too large (${text.length} chars)`, + ); + assertNotJson(text, context); + assert( + !text.includes("--lifecycle"), + `${context}: leaked CLI lifecycle flag`, + ); + assert(!text.includes("--verbose"), `${context}: leaked CLI verbose flag`); + return text; +} + +function assertJsonResult(result: ToolCallResult, context: string): unknown { + assert(result.isError !== true, `${context}: expected success`); + return parseJson(resultText(result, context), context); +} + +function assertErrorCode( + result: ToolCallResult, + context: string, + code: string, +): void { + const envelope = assertCleanErrorEnvelope(result, context); + assert( + envelope.code === code, + `${context}: expected ${code}, got ${envelope.code}`, + ); +} + +async function callTool( + client: Client, + name: string, + args: Record, +): Promise { + return (await client.callTool({ name, arguments: args })) as ToolCallResult; +} + +async function assertLiveOrAuthRequired(client: Client): Promise { + const result = await callTool(client, "search_language", { query: "python" }); + if (result.isError === true) { + const envelope = assertCleanErrorEnvelope( + result, + "search_language auth probe", + ); + assert( + envelope.code === "AUTH_REQUIRED", + `auth probe returned unexpected code ${envelope.code}`, + ); + console.log("AUTH_REQUIRED: live smoke skipped"); + return false; + } + + const text = assertDefaultText(result, "search_language default"); + assert( + text.includes("python (Python)"), + "search_language default missing Python display name", + ); + assert(text.includes("aliases:"), "search_language default missing aliases"); + return true; +} + +async function runLiveSmoke(client: Client): Promise { + const languageJson = assertJsonResult( + await callTool(client, "search_language", { + query: "python", + format: "json", + }), + "search_language json", + ); + assert(Array.isArray(languageJson), "search_language json: expected array"); + + const exampleText = assertDefaultText( + await callTool(client, "get_example", { + query: "express hello world", + language: "javascript", + }), + "get_example default", + ); + assert( + exampleText.includes("solution_id:"), + "get_example default missing solution_id hint", + ); + + const exampleJson = assertJsonResult( + await callTool(client, "get_example", { + query: "express hello world", + language: "javascript", + format: "json", + }), + "get_example json", + ); + assertRecord(exampleJson, "get_example json"); + assert( + typeof exampleJson.result === "string", + "get_example json missing result", + ); + + const pkgInfoText = assertDefaultText( + await callTool(client, "pkg_info", { + registry: "npm", + package_name: "express", + }), + "pkg_info default", + ); + assert( + pkgInfoText.includes("express"), + "pkg_info default missing package name", + ); + + const pkgInfoJson = assertJsonResult( + await callTool(client, "pkg_info", { + registry: "npm", + package_name: "express", + format: "json", + }), + "pkg_info json", + ); + assertRecord(pkgInfoJson, "pkg_info json"); + assert(pkgInfoJson.registry === "npm", "pkg_info json registry mismatch"); + assert(pkgInfoJson.name === "express", "pkg_info json name mismatch"); + assert( + typeof pkgInfoJson.version === "string", + "pkg_info json missing version", + ); + + const depsText = assertDefaultText( + await callTool(client, "pkg_deps", { + registry: "npm", + package_name: "express", + }), + "pkg_deps default", + ); + assert( + depsText.includes('pass lifecycle="all"'), + "pkg_deps default missing MCP-native lifecycle hint", + ); + + const depsAllText = assertDefaultText( + await callTool(client, "pkg_deps", { + registry: "npm", + package_name: "express", + lifecycle: "all", + }), + "pkg_deps lifecycle all", + ); + assert( + !depsAllText.includes("Hidden groups:"), + "pkg_deps lifecycle all still hides groups", + ); + + const depsJson = assertJsonResult( + await callTool(client, "pkg_deps", { + registry: "npm", + package_name: "express", + format: "json", + }), + "pkg_deps json", + ); + assertRecord(depsJson, "pkg_deps json"); + assertRecord(depsJson.runtime, "pkg_deps json runtime"); + + const vulnsText = assertDefaultText( + await callTool(client, "pkg_vulns", { + registry: "npm", + package_name: "express", + }), + "pkg_vulns default", + ); + assert( + vulnsText.includes("express") || vulnsText.includes("vulnerab"), + "pkg_vulns default missing context", + ); + + const vulnsJson = assertJsonResult( + await callTool(client, "pkg_vulns", { + registry: "npm", + package_name: "express", + format: "json", + }), + "pkg_vulns json", + ); + assertRecord(vulnsJson, "pkg_vulns json"); + assert( + "summary" in vulnsJson || "advisories" in vulnsJson, + "pkg_vulns json missing vulnerability data", + ); + + const changelogText = assertDefaultText( + await callTool(client, "pkg_changelog", { + registry: "npm", + package_name: "express", + limit: 1, + }), + "pkg_changelog default", + ); + assert( + !changelogText.includes("--verbose"), + "pkg_changelog default leaked CLI verbose flag", + ); + if ( + changelogText.includes("truncated") || + changelogText.includes("full bodies") + ) { + assert( + changelogText.includes('pass format="json" for full bodies'), + "pkg_changelog truncation hint is not MCP-native", + ); + } + + const changelogJson = assertJsonResult( + await callTool(client, "pkg_changelog", { + registry: "npm", + package_name: "express", + limit: 1, + format: "json", + }), + "pkg_changelog json", + ); + assertRecord(changelogJson, "pkg_changelog json"); + assertRecord(changelogJson.entries, "pkg_changelog json entries"); + + const docsText = assertDefaultText( + await callTool(client, "docs_list", { + registry: "npm", + package_name: "express", + limit: 2, + }), + "docs_list default", + ); + assert( + docsText.includes("docs_read page_id="), + "docs_list default missing docs_read follow-up", + ); + + const docsJson = assertJsonResult( + await callTool(client, "docs_list", { + registry: "npm", + package_name: "express", + limit: 2, + format: "json", + }), + "docs_list json", + ); + assertRecord(docsJson, "docs_list json"); + assert(Array.isArray(docsJson.pages), "docs_list json missing pages array"); + const firstPage = docsJson.pages[0] as Record | undefined; + assert( + firstPage && typeof firstPage.pageId === "string", + "docs_list json missing readable page id", + ); + + const docReadText = assertDefaultText( + await callTool(client, "docs_read", { + page_id: firstPage.pageId, + start_line: 1, + end_line: 5, + }), + "docs_read default", + ); + assert(docReadText.length > 0, "docs_read default missing content"); + + const docReadJson = assertJsonResult( + await callTool(client, "docs_read", { + page_id: firstPage.pageId, + start_line: 1, + end_line: 5, + format: "json", + }), + "docs_read json", + ); + assertRecord(docReadJson, "docs_read json"); + assert( + typeof docReadJson.content === "string", + "docs_read json missing content", + ); + + const codeFilesText = assertDefaultText( + await callTool(client, "code_files", { + target: { registry: "npm", package_name: "express" }, + path_prefix: "package.json", + limit: 1, + }), + "code_files default", + ); + assert( + codeFilesText.includes("package.json"), + "code_files default missing package.json", + ); + + const codeFilesJson = assertJsonResult( + await callTool(client, "code_files", { + target: { registry: "npm", package_name: "express" }, + path_prefix: "package.json", + limit: 1, + format: "json", + }), + "code_files json", + ); + assertRecord(codeFilesJson, "code_files json"); + assert( + Array.isArray(codeFilesJson.files), + "code_files json missing files array", + ); + + const codeReadText = assertDefaultText( + await callTool(client, "code_read", { + target: { registry: "npm", package_name: "express" }, + path: "package.json", + start_line: 1, + end_line: 5, + }), + "code_read default", + ); + assert(/^1\s+/m.test(codeReadText), "code_read default missing line numbers"); + + const codeReadJson = assertJsonResult( + await callTool(client, "code_read", { + target: { registry: "npm", package_name: "express" }, + path: "package.json", + start_line: 1, + end_line: 5, + format: "json", + }), + "code_read json", + ); + assertRecord(codeReadJson, "code_read json"); + assert(codeReadJson.path === "package.json", "code_read json path mismatch"); + + const codeGrepText = assertDefaultText( + await callTool(client, "code_grep", { + target: { registry: "npm", package_name: "express" }, + pattern: "express", + path: "package.json", + max_matches: 1, + }), + "code_grep default", + ); + assert( + codeGrepText.includes("package.json"), + "code_grep default missing package.json", + ); + + const codeGrepJson = assertJsonResult( + await callTool(client, "code_grep", { + target: { registry: "npm", package_name: "express" }, + pattern: "express", + path: "package.json", + max_matches: 1, + format: "json", + }), + "code_grep json", + ); + assertRecord(codeGrepJson, "code_grep json"); + assert( + "matches" in codeGrepJson || "totalMatches" in codeGrepJson, + "code_grep json missing matches", + ); + + const searchText = assertDefaultText( + await callTool(client, "search", { + target: { registry: "npm", package_name: "express" }, + query: "router", + limit: 1, + }), + "search default", + ); + assert( + searchText.includes("code_read") || + searchText.includes("docs_read") || + searchText.includes("search_status"), + "search default missing ready-to-call follow-up", + ); + + const searchJson = assertJsonResult( + await callTool(client, "search", { + target: { registry: "npm", package_name: "express" }, + query: "router", + limit: 1, + format: "json", + }), + "search json", + ); + assertRecord(searchJson, "search json"); + const searchRef = + typeof searchJson.searchRef === "string" ? searchJson.searchRef : undefined; + if (searchRef) { + const statusJson = assertJsonResult( + await callTool(client, "search_status", { + search_ref: searchRef, + format: "json", + }), + "search_status json", + ); + assertRecord(statusJson, "search_status json"); + assert( + "completed" in statusJson || "progress" in statusJson, + "search_status json missing status data", + ); + } else { + assertErrorCode( + await callTool(client, "search_status", { + search_ref: "smoke-invalid-search-ref", + }), + "search_status invalid ref", + "NOT_FOUND", + ); + } + + const feedbackValidation = await callTool(client, "feedback", { + solution_id: "", + accepted: true, + }); + assert( + feedbackValidation.isError === true, + "feedback validation should fail before submitting", + ); + assert( + resultText(feedbackValidation, "feedback validation").includes("MCP error"), + "feedback validation missing protocol error text", + ); +} + +async function main(): Promise { + const transport = new StdioClientTransport({ + command: "bun", + args: ["run", "dev", "mcp", "start"], + }); + const client = new Client({ name: "githits-mcp-smoke", version: "0.1.0" }); + + try { + await client.connect(transport); + + const toolsResponse = await client.listTools(); + const toolNames = new Set(toolsResponse.tools.map((tool) => tool.name)); + for (const expected of EXPECTED_TOOLS) { + assert(toolNames.has(expected), `listTools missing ${expected}`); + } + + if (await assertLiveOrAuthRequired(client)) { + await runLiveSmoke(client); + console.log("MCP smoke passed"); + } + } finally { + await client.close(); + } +} + +await main(); From 08c98d98795af3af4471602cd0a0abb818c830a5 Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Mon, 4 May 2026 14:03:04 +0300 Subject: [PATCH 2/2] test: strengthen smoke parity checks Forces unauthenticated smoke coverage and compares representative CLI --json payloads against MCP format=json responses so parity drift is caught live. --- docs/implementation/mcp-cli-parity.md | 25 ++- scripts/cli-smoke.ts | 213 +++++++++++++++++++++++++- scripts/mcp-call.ts | 55 +++++++ scripts/mcp-smoke.ts | 83 ++++++++-- 4 files changed, 358 insertions(+), 18 deletions(-) create mode 100644 scripts/mcp-call.ts diff --git a/docs/implementation/mcp-cli-parity.md b/docs/implementation/mcp-cli-parity.md index 482ff100..cbd0524b 100644 --- a/docs/implementation/mcp-cli-parity.md +++ b/docs/implementation/mcp-cli-parity.md @@ -20,11 +20,26 @@ MCP-native hints, CLI terminal affordances, and JSON envelope shape. When adding or changing a dual-surface tool, update both smoke scripts if the covered live UX contract changes. -The dual-surface tools today are unified `search` / `search_status`, -the file-exploration bundle (`code_files`, `code_read`, `code_grep`), -the package-intelligence tools (`pkg_info`, -`pkg_vulns`, `pkg_deps`, `pkg_changelog`), -and the docs surface (`docs_list`, `docs_read`). +The dual-surface tools today are: + +- `get_example` ↔ `githits example` +- `search_language` ↔ `githits languages` +- `feedback` ↔ `githits feedback` +- `search` ↔ `githits search` +- `search_status` ↔ `githits search-status` +- `code_files` ↔ `githits code files` +- `code_read` ↔ `githits code read` +- `code_grep` ↔ `githits code grep` +- `pkg_info` ↔ `githits pkg info` +- `pkg_vulns` ↔ `githits pkg vulns` +- `pkg_deps` ↔ `githits pkg deps` +- `pkg_changelog` ↔ `githits pkg changelog` +- `docs_list` ↔ `githits docs list` +- `docs_read` ↔ `githits docs read` + +`feedback` is mutating, so smoke coverage exercises registration and +validation/auth paths only. It does not submit fake feedback to the live +backend. One deliberate exception: `search_status` does not echo the original structured request because the backend follow-up endpoint does not diff --git a/scripts/cli-smoke.ts b/scripts/cli-smoke.ts index 2706cda7..399eddd0 100644 --- a/scripts/cli-smoke.ts +++ b/scripts/cli-smoke.ts @@ -1,3 +1,7 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + interface CommandResult { command: string[]; exitCode: number; @@ -11,7 +15,117 @@ interface ErrorEnvelope { retryable: boolean; } +interface JsonParityFixture { + name: string; + cliArgs: string[]; + mcpTool: string; + mcpArgs: Record; +} + const DEFAULT_TEXT_LIMIT = 20_000; +const AUTH_ENV_KEYS = ["GITHITS_API_TOKEN", "GITHITS_TOKEN"] as const; +const JSON_PARITY_FIXTURES: JsonParityFixture[] = [ + { + name: "pkg_info", + cliArgs: ["pkg", "info", "npm:express", "--json"], + mcpTool: "pkg_info", + mcpArgs: { registry: "npm", package_name: "express", format: "json" }, + }, + { + name: "pkg_deps", + cliArgs: ["pkg", "deps", "npm:express", "--json"], + mcpTool: "pkg_deps", + mcpArgs: { registry: "npm", package_name: "express", format: "json" }, + }, + { + name: "pkg_vulns", + cliArgs: ["pkg", "vulns", "npm:express", "--json"], + mcpTool: "pkg_vulns", + mcpArgs: { registry: "npm", package_name: "express", format: "json" }, + }, + { + name: "pkg_changelog", + cliArgs: ["pkg", "changelog", "npm:express", "--limit", "1", "--json"], + mcpTool: "pkg_changelog", + mcpArgs: { + registry: "npm", + package_name: "express", + limit: 1, + format: "json", + }, + }, + { + name: "docs_list", + cliArgs: ["docs", "list", "npm:express", "--limit", "2", "--json"], + mcpTool: "docs_list", + mcpArgs: { + registry: "npm", + package_name: "express", + limit: 2, + format: "json", + }, + }, + { + name: "code_files", + cliArgs: [ + "code", + "files", + "npm:express", + "package.json", + "--limit", + "1", + "--json", + ], + mcpTool: "code_files", + mcpArgs: { + target: { registry: "npm", package_name: "express" }, + path_prefix: "package.json", + limit: 1, + format: "json", + }, + }, + { + name: "code_read", + cliArgs: [ + "code", + "read", + "npm:express", + "package.json", + "--lines", + "1-5", + "--json", + ], + mcpTool: "code_read", + mcpArgs: { + target: { registry: "npm", package_name: "express" }, + path: "package.json", + start_line: 1, + end_line: 5, + format: "json", + }, + }, + { + name: "code_grep", + cliArgs: [ + "code", + "grep", + "npm:express", + "express", + "package.json", + "--limit", + "1", + "--json", + ], + mcpTool: "code_grep", + mcpArgs: { + target: { registry: "npm", package_name: "express" }, + pattern: "express", + path_prefix: "package.json", + max_matches: 1, + format: "json", + }, + }, +]; function assert(condition: unknown, message: string): asserts condition { if (!condition) { @@ -111,11 +225,18 @@ function assertJsonErrorCode( } async function runCli(args: string[]): Promise { + return runCliWithEnv(args, process.env); +} + +async function runCliWithEnv( + args: string[], + baseEnv: NodeJS.ProcessEnv | Record, +): Promise { const proc = Bun.spawn(["bun", "run", "dev", ...args], { stdout: "pipe", stderr: "pipe", env: { - ...process.env, + ...baseEnv, NO_COLOR: "1", }, }); @@ -127,6 +248,94 @@ async function runCli(args: string[]): Promise { return { command: args, exitCode, stdout, stderr }; } +async function runMcpJson( + toolName: string, + args: Record, +): Promise { + const proc = Bun.spawn( + ["bun", "run", "scripts/mcp-call.ts", toolName, JSON.stringify(args)], + { + stdout: "pipe", + stderr: "pipe", + env: process.env, + }, + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + assert( + exitCode === 0, + `${toolName} MCP parity call failed (${exitCode})\n${stderr}`, + ); + return parseJson(stdout, `${toolName} MCP parity`); +} + +function assertDeepEqual( + actual: unknown, + expected: unknown, + context: string, +): void { + const actualText = JSON.stringify(actual); + const expectedText = JSON.stringify(expected); + assert( + actualText === expectedText, + `${context}: JSON parity mismatch\nCLI: ${expectedText}\nMCP: ${actualText}`, + ); +} + +async function assertJsonParity(): Promise { + for (const fixture of JSON_PARITY_FIXTURES) { + const cliPayload = assertJsonOutput( + await runCli(fixture.cliArgs), + `${fixture.name} CLI parity`, + ); + const mcpPayload = await runMcpJson(fixture.mcpTool, fixture.mcpArgs); + assertDeepEqual(mcpPayload, cliPayload, `${fixture.name} CLI/MCP`); + } +} + +function isolatedUnauthenticatedEnv(): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if ( + value !== undefined && + !AUTH_ENV_KEYS.includes(key as (typeof AUTH_ENV_KEYS)[number]) + ) { + env[key] = value; + } + } + const dir = mkdtempSync(join(tmpdir(), "githits-cli-smoke-home-")); + env.HOME = dir; + env.USERPROFILE = dir; + env.XDG_CONFIG_HOME = `${dir}/.config`; + env.APPDATA = `${dir}/AppData/Roaming`; + env.GITHITS_AUTH_STORAGE = "file"; + return env; +} + +async function assertUnauthenticatedBehavior(): Promise { + const env = isolatedUnauthenticatedEnv(); + try { + const result = await runCliWithEnv(["languages", "python", "--json"], env); + assert(result.exitCode !== 0, "unauthenticated languages should fail"); + const output = `${result.stdout}\n${result.stderr}`; + assert( + output.includes("Authentication required"), + "unauthenticated probe missing authentication guidance", + ); + assert( + output.includes("githits login"), + "unauthenticated probe missing login guidance", + ); + } finally { + if (env.HOME) { + rmSync(env.HOME, { recursive: true, force: true }); + } + } +} + async function assertLiveOrAuthRequired(): Promise { const result = await runCli(["languages", "python", "--json"]); if (result.exitCode === 0) { @@ -489,8 +698,10 @@ async function runLiveSmoke(): Promise { } async function main(): Promise { + await assertUnauthenticatedBehavior(); if (await assertLiveOrAuthRequired()) { await runLiveSmoke(); + await assertJsonParity(); console.log("CLI smoke passed"); } } diff --git a/scripts/mcp-call.ts b/scripts/mcp-call.ts new file mode 100644 index 00000000..652c7ab8 --- /dev/null +++ b/scripts/mcp-call.ts @@ -0,0 +1,55 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; + +interface TextContent { + type: "text"; + text: string; +} + +interface ToolCallResult { + content?: unknown; + isError?: boolean; +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function resultText(result: ToolCallResult, context: string): string { + assert(Array.isArray(result.content), `${context}: expected content array`); + const first = result.content[0] as Partial | undefined; + assert(first?.type === "text", `${context}: expected text content`); + assert(typeof first.text === "string", `${context}: expected text string`); + return first.text; +} + +const [, , toolName, rawArgs] = process.argv; +assert(toolName, "usage: bun run scripts/mcp-call.ts "); +assert(rawArgs, "usage: bun run scripts/mcp-call.ts "); + +const args = JSON.parse(rawArgs) as Record; +const transport = new StdioClientTransport({ + command: "bun", + args: ["run", "dev", "mcp", "start"], +}); +const client = new Client({ + name: "githits-mcp-parity-smoke", + version: "0.1.0", +}); + +try { + await client.connect(transport); + const result = (await client.callTool({ + name: toolName, + arguments: args, + })) as ToolCallResult; + if (result.isError === true) { + process.stderr.write(`${resultText(result, toolName)}\n`); + process.exit(1); + } + process.stdout.write(resultText(result, toolName)); +} finally { + await client.close(); +} diff --git a/scripts/mcp-smoke.ts b/scripts/mcp-smoke.ts index ab3837a1..e53c68bb 100644 --- a/scripts/mcp-smoke.ts +++ b/scripts/mcp-smoke.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; @@ -35,6 +38,7 @@ const EXPECTED_TOOLS = [ ] as const; const DEFAULT_TEXT_LIMIT = 12_000; +const AUTH_ENV_KEYS = ["GITHITS_API_TOKEN", "GITHITS_TOKEN"] as const; function assert(condition: unknown, message: string): asserts condition { if (!condition) { @@ -142,6 +146,70 @@ async function callTool( return (await client.callTool({ name, arguments: args })) as ToolCallResult; } +function isolatedUnauthenticatedEnv(): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if ( + value !== undefined && + !AUTH_ENV_KEYS.includes(key as (typeof AUTH_ENV_KEYS)[number]) + ) { + env[key] = value; + } + } + const home = mkdtempSync(join(tmpdir(), "githits-mcp-smoke-home-")); + env.HOME = home; + env.USERPROFILE = home; + env.XDG_CONFIG_HOME = join(home, ".config"); + env.APPDATA = join(home, "AppData", "Roaming"); + env.GITHITS_AUTH_STORAGE = "file"; + return env; +} + +async function withMcpClient( + env: Record | undefined, + fn: (client: Client) => Promise, +): Promise { + const transport = new StdioClientTransport({ + command: "bun", + args: ["run", "dev", "mcp", "start"], + env, + }); + const client = new Client({ name: "githits-mcp-smoke", version: "0.1.0" }); + try { + await client.connect(transport); + return await fn(client); + } finally { + await client.close(); + } +} + +async function assertUnauthenticatedBehavior(): Promise { + const env = isolatedUnauthenticatedEnv(); + const home = env.HOME; + try { + await withMcpClient(env, async (client) => { + const toolsResponse = await client.listTools(); + assert( + toolsResponse.tools.length > 0, + "unauthenticated listTools returned no tools", + ); + const result = await callTool(client, "search_language", { + query: "python", + }); + const envelope = assertCleanErrorEnvelope( + result, + "search_language unauthenticated", + ); + assert( + envelope.code === "AUTH_REQUIRED", + `unauthenticated probe returned unexpected code ${envelope.code}`, + ); + }); + } finally { + if (home) rmSync(home, { recursive: true, force: true }); + } +} + async function assertLiveOrAuthRequired(client: Client): Promise { const result = await callTool(client, "search_language", { query: "python" }); if (result.isError === true) { @@ -528,15 +596,8 @@ async function runLiveSmoke(client: Client): Promise { } async function main(): Promise { - const transport = new StdioClientTransport({ - command: "bun", - args: ["run", "dev", "mcp", "start"], - }); - const client = new Client({ name: "githits-mcp-smoke", version: "0.1.0" }); - - try { - await client.connect(transport); - + await assertUnauthenticatedBehavior(); + await withMcpClient(undefined, async (client) => { const toolsResponse = await client.listTools(); const toolNames = new Set(toolsResponse.tools.map((tool) => tool.name)); for (const expected of EXPECTED_TOOLS) { @@ -547,9 +608,7 @@ async function main(): Promise { await runLiveSmoke(client); console.log("MCP smoke passed"); } - } finally { - await client.close(); - } + }); } await main();