Plugin migration

.opencode/plugins/crawlith.ts

@@ -26,7 +26,7 @@ async function runCli(
 /**
  * Crawlith OpenCode plugin exposing Crawlith CLI tools and useful lifecycle hooks.
  */
-export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) => {
+export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }: any) => {
   await client.app.log({
     body: {
       service: 'crawlith-plugin',
@@ -45,7 +45,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
           depth: tool.schema.number().optional().describe('Maximum click depth'),
           concurrency: tool.schema.number().optional().describe('Max concurrent requests')
         },
-        async execute({ url, limit, depth, concurrency }) {
+        async execute({ url, limit, depth, concurrency }: any) {
           const flags = [
             typeof limit === 'number' ? `--limit ${limit}` : '',
             typeof depth === 'number' ? `--depth ${depth}` : '',
@@ -64,7 +64,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
           url: tool.schema.string().describe('Page URL to analyze'),
           live: tool.schema.boolean().optional().describe('Run with --live flag')
         },
-        async execute({ url, live }) {
+        async execute({ url, live }: any) {
           const flags = live ? '--live' : '';
           return runCli($, `page ${url} ${flags}`.trim());
         }
@@ -76,7 +76,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
           url: tool.schema.string().describe('Domain or URL to inspect'),
           timeout: tool.schema.number().optional().describe('Probe timeout in milliseconds')
         },
-        async execute({ url, timeout }) {
+        async execute({ url, timeout }: any) {
           const flags = typeof timeout === 'number' ? `--timeout ${timeout}` : '';
           return runCli($, `probe ${url} ${flags}`.trim());
         }
@@ -91,7 +91,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
       })
     },
 
-    event: async ({ event }) => {
+    event: async ({ event }: any) => {
       if (event.type === 'session.idle') {
         await client.app.log({
           body: {
@@ -103,7 +103,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
       }
     },
 
-    'tool.execute.before': async (input) => {
+    'tool.execute.before': async (input: any) => {
       await client.app.log({
         body: {
           service: 'crawlith-plugin',
@@ -113,7 +113,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory })
       });
     },
 
-    'tool.execute.after': async (input, output) => {
+    'tool.execute.after': async (input: any, output: any) => {
       const crawlithTools = new Set([
         'crawlith-crawl-site',
         'crawlith-analyze-page',

packages/cli/package.json

@@ -16,18 +16,6 @@
   "dependencies": {
     "@crawlith/core": "workspace:*",
     "@crawlith/server": "workspace:*",
-    "@crawlith/plugin-pagerank": "workspace:*",
-    "@crawlith/plugin-hits": "workspace:*",
-    "@crawlith/plugin-duplicate-detection": "workspace:*",
-    "@crawlith/plugin-content-clustering": "workspace:*",
-    "@crawlith/plugin-simhash": "workspace:*",
-    "@crawlith/plugin-heading-health": "workspace:*",
-    "@crawlith/plugin-orphan-intelligence": "workspace:*",
-    "@crawlith/plugin-soft404-detector": "workspace:*",
-    "@crawlith/plugin-crawl-trap-analyzer": "workspace:*",
-    "@crawlith/plugin-health-score-engine": "workspace:*",
-    "@crawlith/plugin-snapshot-diff": "workspace:*",
-    "@crawlith/plugin-crawl-policy": "workspace:*",
     "@crawlith/plugin-exporter": "workspace:*",
     "@crawlith/plugin-reporter": "workspace:*",
     "@crawlith/plugin-signals": "workspace:*",

packages/cli/src/commands/crawl.ts

@@ -19,7 +19,33 @@ export const getCrawlCommand = (registry: PluginRegistry) => {
     .option('--no-query', 'strip query params')
     .option('--sitemap [url]', 'sitemap URL (defaults to /sitemap.xml if not specified)')
     .addOption(new Option('--log-level [level]', 'Log level (normal, verbose, debug)').choices(['normal', 'verbose', 'debug']).default('normal'))
-    .option('--force', 'force run (override existing lock)');
+    .option('--force', 'force run (override existing lock)')
+    .option('--allow <domains>', 'comma separated list of domains to allow')
+    .option('--deny <domains>', 'comma separated list of domains to deny')
+    .option('--include-subdomains', 'include subdomains in the default scope')
+    .option('--ignore-robots', 'ignore robots.txt directives')
+    .option('--proxy <url>', 'proxy URL to use for requests')
+    .option('--ua <string>', 'user agent string to use')
+    .option('--rate <number>', 'requests per second limit')
+    .option('--max-bytes <number>', 'maximum bytes to download per page')
+    .option('--max-redirects <number>', 'maximum redirects to follow')
+    // Clustering (Moved from plugin to core)
+    .option('--clustering', 'Enable content clustering analysis')
+    .option('--cluster-threshold <number>', 'Hamming distance for content clusters', '10')
+    .option('--min-cluster-size <number>', 'Minimum pages per cluster', '3')
+    // Heading & Health (Moved from plugin to core)
+    .option('--heading', 'Analyze heading structure and hierarchy health')
+    .option('--health', 'Run health score analysis')
+    .option('--fail-on-critical', 'Exit code 1 if critical issues exist')
+    .option('--score-breakdown', 'Print health score component weights')
+    // Graph Centrality
+    .option('--compute-hits', 'Compute Hub and Authority scores (HITS)')
+    .option('--compute-pagerank', 'Compute PageRank centrality scores')
+    // Orphan Intelligence
+    .option('--orphans', 'Detect orphaned pages')
+    .option('--orphan-severity', 'Enable severity scoring for orphans')
+    .option('--include-soft-orphans', 'Include pages with very few in-links as soft orphans')
+    .option('--min-inbound <number>', 'Minimum inbound links to not be an orphan', '2');
 
   // Let plugins register their flags on this command
   registry.registerPlugins(crawlCommand);
@@ -70,10 +96,34 @@ export const getCrawlCommand = (registry: PluginRegistry) => {
         sitemap: sitemap as string | undefined,
         debug: options.logLevel === 'debug',
         concurrency: options.concurrency ? parseInt(options.concurrency, 10) : 2,
+        ignoreRobots: options.ignoreRobots,
+        allowedDomains: options.allow ? options.allow.split(',').map((d: string) => d.trim()) : undefined,
+        deniedDomains: options.deny ? options.deny.split(',').map((d: string) => d.trim()) : undefined,
+        includeSubdomains: options.includeSubdomains,
+        proxyUrl: options.proxy,
+        userAgent: options.ua,
+        rate: options.rate ? parseFloat(options.rate) : undefined,
+        maxBytes: options.maxBytes ? parseInt(options.maxBytes, 10) : undefined,
+        maxRedirects: options.maxRedirects ? parseInt(options.maxRedirects, 10) : undefined,
+        clustering: options.clustering,
+        clusterThreshold: options.clusterThreshold ? parseInt(options.clusterThreshold, 10) : undefined,
+        minClusterSize: options.minClusterSize ? parseInt(options.minClusterSize, 10) : undefined,
+        heading: options.heading,
+        health: options.health,
+        failOnCritical: options.failOnCritical,
+        scoreBreakdown: options.scoreBreakdown,
+        computeHits: options.computeHits,
+        computePagerank: options.computePagerank,
+        orphans: options.orphans,
+        orphanSeverity: options.orphanSeverity,
+        includeSoftOrphans: options.includeSoftOrphans,
+        minInbound: options.minInbound ? parseInt(options.minInbound, 10) : undefined,
         plugins: registry.getPlugins(),
         context: {
           command: 'crawl',
+          scope: 'crawl' as const,
           flags: options as Record<string, any>,
+          emit: (e: any) => controller.handle(e),
           logger: {
             info: (m: string) => context.emit({ type: 'info', message: m }),
             warn: (m: string) => context.emit({ type: 'warn', message: m, context: undefined }),

packages/cli/src/commands/page.ts

@@ -15,7 +15,31 @@ export const getPageCommand = (registry: PluginRegistry) => {
     .addOption(new Option('--log-level <level>', 'Log level (normal, verbose, debug)').choices(['normal', 'verbose', 'debug']).default('normal'))
     .option('--seo', 'Show only SEO module output')
     .option('--content', 'Show only content module output')
-    .option('--accessibility', 'Show only accessibility module output');
+    .option('--accessibility', 'Show only accessibility module output')
+    // Crawl Policy
+    .option('--proxy <url>', 'proxy URL to use for requests')
+    .option('--ua <string>', 'user agent string to use')
+    .option('--rate <number>', 'requests per second limit')
+    .option('--max-bytes <number>', 'maximum bytes to download per page')
+    .option('--max-redirects <number>', 'maximum redirects to follow')
+    // Clustering
+    .option('--clustering', 'Enable content clustering analysis')
+    .option('--cluster-threshold <number>', 'Hamming distance for content clusters', '10')
+    .option('--min-cluster-size <number>', 'Minimum pages per cluster', '3')
+    .option('--sitemap [url]', 'sitemap URL (defaults to /sitemap.xml if not specified)')
+    // Heading & Health
+    .option('--heading', 'Analyze heading structure and hierarchy health')
+    .option('--health', 'Run health score analysis')
+    .option('--fail-on-critical', 'Exit code 1 if critical issues exist')
+    .option('--score-breakdown', 'Print health score component weights')
+    // Graph Centrality
+    .option('--pagerank', 'Calculate PageRank')
+    .option('--hits', 'Compute Hub and Authority scores (HITS)')
+    // Orphans
+    .option('--orphans', 'Detect orphaned pages')
+    .option('--orphan-severity <value>', 'Severity for orphans (low/medium/high)')
+    .option('--include-soft-orphans', 'Include soft orphans in detection')
+    .option('--min-inbound <value>', 'Minimum inbound links to not be an orphan', '2');
 
   // Let plugins register their flags on this command
   registry.registerPlugins(analyze);
@@ -47,6 +71,25 @@ export const getPageCommand = (registry: PluginRegistry) => {
         seo: options.seo,
         content: options.content,
         accessibility: options.accessibility,
+        proxyUrl: options.proxy,
+        userAgent: options.ua,
+        rate: options.rate ? parseFloat(options.rate) : undefined,
+        maxBytes: options.maxBytes ? parseInt(options.maxBytes, 10) : undefined,
+        maxRedirects: options.maxRedirects ? parseInt(options.maxRedirects, 10) : undefined,
+        clustering: options.clustering,
+        clusterThreshold: options.clusterThreshold ? parseInt(options.clusterThreshold, 10) : undefined,
+        minClusterSize: options.minClusterSize ? parseInt(options.minClusterSize, 10) : undefined,
+        sitemap: options.sitemap,
+        heading: options.heading,
+        health: options.health,
+        failOnCritical: options.failOnCritical,
+        scoreBreakdown: options.scoreBreakdown,
+        computePagerank: options.pagerank,
+        computeHits: options.hits,
+        orphans: options.orphans,
+        orphanSeverity: options.orphanSeverity,
+        includeSoftOrphans: options.includeSoftOrphans,
+        minInbound: options.minInbound ? parseInt(options.minInbound, 10) : undefined,
         debug: options.logLevel === 'debug',
         plugins: registry.getPlugins(),
         context: {