Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions .opencode/plugins/crawlith.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import type { Plugin } from '@opencode-ai/plugin';
import { tool } from '@opencode-ai/plugin';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const CLI_PATH = path.resolve(__dirname, '../../packages/cli/dist/index.js');

/**
* Execute a Crawlith CLI command in JSON mode.
*
* @param $ OpenCode shell helper.
* @param commandArgs Command arguments excluding global JSON format flags.
* @returns Command output text.
*/
async function runCli(
$: { (strings: TemplateStringsArray, ...values: any[]): Promise<{ text(): Promise<string> }> },
commandArgs: string
): Promise<string> {
const command = `node ${CLI_PATH} --format json ${commandArgs}`;
const result = await $`sh -c ${command}`;
return result.text();
}

/**
* Crawlith OpenCode plugin exposing Crawlith CLI tools and useful lifecycle hooks.
*/
export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) => {
await client.app.log({
body: {
service: 'crawlith-plugin',
level: 'info',
message: `Initialized for project: ${project.name}`
}
});

return {
tool: {
'crawlith-crawl-site': tool({
description: 'Crawl an entire site and build Crawlith graph snapshot data.',
args: {
url: tool.schema.string().describe('Site URL or domain to crawl'),
limit: tool.schema.number().optional().describe('Maximum number of pages to crawl'),
depth: tool.schema.number().optional().describe('Maximum click depth'),
concurrency: tool.schema.number().optional().describe('Max concurrent requests')
},
async execute({ url, limit, depth, concurrency }) {
const flags = [
typeof limit === 'number' ? `--limit ${limit}` : '',
typeof depth === 'number' ? `--depth ${depth}` : '',
typeof concurrency === 'number' ? `--concurrency ${concurrency}` : ''
]
.filter(Boolean)
.join(' ');

return runCli($, `crawl ${url} ${flags}`.trim());
}
}),

'crawlith-analyze-page': tool({
description: 'Analyze a single page URL for SEO, content, and accessibility signals.',
args: {
url: tool.schema.string().describe('Page URL to analyze'),
live: tool.schema.boolean().optional().describe('Run with --live flag')
},
async execute({ url, live }) {
const flags = live ? '--live' : '';
return runCli($, `page ${url} ${flags}`.trim());
}
}),

'crawlith-probe-domain': tool({
description: 'Inspect TLS/transport and HTTP behavior for a domain or URL.',
args: {
url: tool.schema.string().describe('Domain or URL to inspect'),
timeout: tool.schema.number().optional().describe('Probe timeout in milliseconds')
},
async execute({ url, timeout }) {
const flags = typeof timeout === 'number' ? `--timeout ${timeout}` : '';
return runCli($, `probe ${url} ${flags}`.trim());
}
}),

'crawlith-list-sites': tool({
description: 'List all tracked sites from Crawlith local storage.',
args: {},
async execute() {
return runCli($, 'sites');
}
})
},

event: async ({ event }) => {
if (event.type === 'session.idle') {
await client.app.log({
body: {
service: 'crawlith-plugin',
level: 'info',
message: 'Session reached idle state.'
}
});
}
},

'tool.execute.before': async (input) => {
await client.app.log({
body: {
service: 'crawlith-plugin',
level: 'debug',
message: `Tool called: ${input.tool}`
}
});
},

'tool.execute.after': async (input, output) => {
const crawlithTools = new Set([
'crawlith-crawl-site',
'crawlith-analyze-page',
'crawlith-probe-domain',
'crawlith-list-sites'
]);

if (!crawlithTools.has(input.tool)) {
return;
}

const reportsDir = path.join(directory, 'crawlith-opencode-reports');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const reportPath = path.join(reportsDir, `${input.tool}-${timestamp}.json`);

await $`mkdir -p ${reportsDir}`;
await $`sh -c ${`cat > ${reportPath} <<'JSON'\n${JSON.stringify(output.output, null, 2)}\nJSON`}`;

await client.app.log({
body: {
service: 'crawlith-plugin',
level: 'info',
message: `Saved tool output: ${reportPath}`
}
});
}
};
};
10 changes: 10 additions & 0 deletions opencode.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"$schema": "https://opencode.ai/config.json",
"mcp": {
"crawlith": {
"type": "local",
"command": "npx",
"args": ["tsx", "./packages/mcp/src/mcp-server.ts"]
}
}
}
48 changes: 48 additions & 0 deletions packages/core/src/plugin-system/plugin-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,45 @@ export interface PluginContext {
/** Legacy logger alias — kept for backwards compatibility. */
logger?: CLIWriter;
metadata?: Record<string, any>;
/** MCP discovery registry available during MCP startup discovery. */
mcpDiscovery?: PluginMcpDiscovery;
[key: string]: any;
}

// ─── MCP Discovery ───────────────────────────────────────────────────────────

export interface PluginMcpTool {
name: string;
description: string;
/** Zod-like shape object interpreted by the MCP server package. */
inputSchema?: Record<string, unknown>;
execute: (input: Record<string, unknown>, ctx: PluginContext) => unknown | Promise<unknown>;
}

export interface PluginMcpPrompt {
name: string;
description: string;
/** Zod-like shape object interpreted by the MCP server package. */
argumentsSchema?: Record<string, unknown>;
buildMessages: (
input: Record<string, unknown>,
ctx: PluginContext
) => {
messages: Array<{
role: 'user' | 'assistant' | 'system';
content: {
type: 'text';
text: string;
};
}>;
};
}

export interface PluginMcpDiscovery {
registerTool(tool: PluginMcpTool): void;
registerPrompt(prompt: PluginMcpPrompt): void;
}

// ─── CLI option declaration ───────────────────────────────────────────────────

export interface PluginCliOption {
Expand Down Expand Up @@ -141,6 +177,12 @@ export interface CrawlithPlugin {
*/
register?: (cli: any) => void;

/** Declarative MCP definitions discovered by @crawlith/mcp at startup. */
mcp?: {
tools?: PluginMcpTool[];
prompts?: PluginMcpPrompt[];
};

hooks?: {
/**
* Runs on both `page` and `crawl` scopes.
Expand Down Expand Up @@ -193,5 +235,11 @@ export interface CrawlithPlugin {
* Attach snapshot-level summary data to the result object.
*/
onReport?: (ctx: PluginContext, report: any) => void | Promise<void>;

/**
* MCP discovery phase — executed by @crawlith/mcp server startup.
* Plugins can register MCP tools/prompts through `ctx.mcpDiscovery`.
*/
onMcpDiscovery?: (ctx: PluginContext) => void | Promise<void>;
};
}
56 changes: 56 additions & 0 deletions packages/mcp/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# @crawlith/mcp

MCP server that exposes Crawlith CLI workflows as MCP tools for Claude Desktop.

## Tools

- `ensure_crawlith_cli` — checks whether Crawlith CLI is available and can install `@crawlith/cli` if missing
- `crawl_site` — wraps `crawlith crawl`
- `analyze_page` — wraps `crawlith page`
- `probe_domain` — wraps `crawlith probe`
- `list_sites` — wraps `crawlith sites`

## Prompts

- `full_site_audit`
- `portfolio_status`

## Plugin MCP Discovery

`@crawlith/mcp` now discovers plugin-provided MCP tools and prompts at startup:

- Declarative: plugins can expose `plugin.mcp.tools` and `plugin.mcp.prompts`
- Hook-based: plugins can implement `hooks.onMcpDiscovery(ctx)` and register through `ctx.mcpDiscovery.registerTool(...)` / `ctx.mcpDiscovery.registerPrompt(...)`

## Package dependency

This package depends on `@crawlith/cli`, so when `@crawlith/mcp` is published and installed from npm, it can resolve the Crawlith CLI entrypoint from `node_modules`.

## Run

```bash
pnpm --filter @crawlith/mcp run mcp
```

By default, the server resolves and executes the installed `@crawlith/cli` entrypoint.
When developing inside this monorepo, it falls back to `packages/cli/dist/index.js`.

Override either behavior with `CRAWLITH_CLI_COMMAND` if you need a custom CLI command.

## Claude Desktop configuration

Add the server to `claude_desktop_config.json` with an absolute path:

```json
{
"mcpServers": {
"crawlith": {
"command": "npx",
"args": [
"tsx",
"/absolute/path/to/crawlith/packages/mcp/src/mcp-server.ts"
]
}
}
}
```
20 changes: 20 additions & 0 deletions packages/mcp/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"name": "@crawlith/mcp",
"license": "Apache-2.0",
"version": "0.1.0",
"type": "module",
"description": "MCP server bridge for Crawlith CLI tools.",
"scripts": {
"mcp": "tsx src/mcp-server.ts",
"test:mcp": "tsx src/mcp-server.ts --help"
},
"dependencies": {
"@crawlith/cli": "workspace:*",
"@modelcontextprotocol/sdk": "^1.17.5",
"zod": "^3.25.76"
},
"devDependencies": {
"tsx": "^4.20.5",
"typescript": "^5.9.2"
}
}
Loading
Loading