diff --git a/.opencode/plugins/crawlith.ts b/.opencode/plugins/crawlith.ts index c6dd668..aad5879 100644 --- a/.opencode/plugins/crawlith.ts +++ b/.opencode/plugins/crawlith.ts @@ -26,7 +26,7 @@ async function runCli( /** * Crawlith OpenCode plugin exposing Crawlith CLI tools and useful lifecycle hooks. */ -export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) => { +export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }: any) => { await client.app.log({ body: { service: 'crawlith-plugin', @@ -45,7 +45,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) depth: tool.schema.number().optional().describe('Maximum click depth'), concurrency: tool.schema.number().optional().describe('Max concurrent requests') }, - async execute({ url, limit, depth, concurrency }) { + async execute({ url, limit, depth, concurrency }: any) { const flags = [ typeof limit === 'number' ? `--limit ${limit}` : '', typeof depth === 'number' ? `--depth ${depth}` : '', @@ -64,7 +64,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) url: tool.schema.string().describe('Page URL to analyze'), live: tool.schema.boolean().optional().describe('Run with --live flag') }, - async execute({ url, live }) { + async execute({ url, live }: any) { const flags = live ? '--live' : ''; return runCli($, `page ${url} ${flags}`.trim()); } @@ -76,7 +76,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) url: tool.schema.string().describe('Domain or URL to inspect'), timeout: tool.schema.number().optional().describe('Probe timeout in milliseconds') }, - async execute({ url, timeout }) { + async execute({ url, timeout }: any) { const flags = typeof timeout === 'number' ? `--timeout ${timeout}` : ''; return runCli($, `probe ${url} ${flags}`.trim()); } @@ -91,7 +91,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) }) }, - event: async ({ event }) => { + event: async ({ event }: any) => { if (event.type === 'session.idle') { await client.app.log({ body: { @@ -103,7 +103,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) } }, - 'tool.execute.before': async (input) => { + 'tool.execute.before': async (input: any) => { await client.app.log({ body: { service: 'crawlith-plugin', @@ -113,7 +113,7 @@ export const CrawlithPlugin: Plugin = async ({ project, client, $, directory }) }); }, - 'tool.execute.after': async (input, output) => { + 'tool.execute.after': async (input: any, output: any) => { const crawlithTools = new Set([ 'crawlith-crawl-site', 'crawlith-analyze-page', diff --git a/packages/cli/package.json b/packages/cli/package.json index 1b7d968..ca688dd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -16,18 +16,6 @@ "dependencies": { "@crawlith/core": "workspace:*", "@crawlith/server": "workspace:*", - "@crawlith/plugin-pagerank": "workspace:*", - "@crawlith/plugin-hits": "workspace:*", - "@crawlith/plugin-duplicate-detection": "workspace:*", - "@crawlith/plugin-content-clustering": "workspace:*", - "@crawlith/plugin-simhash": "workspace:*", - "@crawlith/plugin-heading-health": "workspace:*", - "@crawlith/plugin-orphan-intelligence": "workspace:*", - "@crawlith/plugin-soft404-detector": "workspace:*", - "@crawlith/plugin-crawl-trap-analyzer": "workspace:*", - "@crawlith/plugin-health-score-engine": "workspace:*", - "@crawlith/plugin-snapshot-diff": "workspace:*", - "@crawlith/plugin-crawl-policy": "workspace:*", "@crawlith/plugin-exporter": "workspace:*", "@crawlith/plugin-reporter": "workspace:*", "@crawlith/plugin-signals": "workspace:*", diff --git a/packages/cli/src/commands/crawl.ts b/packages/cli/src/commands/crawl.ts index 5c02e12..5391869 100644 --- a/packages/cli/src/commands/crawl.ts +++ b/packages/cli/src/commands/crawl.ts @@ -19,7 +19,33 @@ export const getCrawlCommand = (registry: PluginRegistry) => { .option('--no-query', 'strip query params') .option('--sitemap [url]', 'sitemap URL (defaults to /sitemap.xml if not specified)') .addOption(new Option('--log-level [level]', 'Log level (normal, verbose, debug)').choices(['normal', 'verbose', 'debug']).default('normal')) - .option('--force', 'force run (override existing lock)'); + .option('--force', 'force run (override existing lock)') + .option('--allow ', 'comma separated list of domains to allow') + .option('--deny ', 'comma separated list of domains to deny') + .option('--include-subdomains', 'include subdomains in the default scope') + .option('--ignore-robots', 'ignore robots.txt directives') + .option('--proxy ', 'proxy URL to use for requests') + .option('--ua ', 'user agent string to use') + .option('--rate ', 'requests per second limit') + .option('--max-bytes ', 'maximum bytes to download per page') + .option('--max-redirects ', 'maximum redirects to follow') + // Clustering (Moved from plugin to core) + .option('--clustering', 'Enable content clustering analysis') + .option('--cluster-threshold ', 'Hamming distance for content clusters', '10') + .option('--min-cluster-size ', 'Minimum pages per cluster', '3') + // Heading & Health (Moved from plugin to core) + .option('--heading', 'Analyze heading structure and hierarchy health') + .option('--health', 'Run health score analysis') + .option('--fail-on-critical', 'Exit code 1 if critical issues exist') + .option('--score-breakdown', 'Print health score component weights') + // Graph Centrality + .option('--compute-hits', 'Compute Hub and Authority scores (HITS)') + .option('--compute-pagerank', 'Compute PageRank centrality scores') + // Orphan Intelligence + .option('--orphans', 'Detect orphaned pages') + .option('--orphan-severity', 'Enable severity scoring for orphans') + .option('--include-soft-orphans', 'Include pages with very few in-links as soft orphans') + .option('--min-inbound ', 'Minimum inbound links to not be an orphan', '2'); // Let plugins register their flags on this command registry.registerPlugins(crawlCommand); @@ -70,10 +96,34 @@ export const getCrawlCommand = (registry: PluginRegistry) => { sitemap: sitemap as string | undefined, debug: options.logLevel === 'debug', concurrency: options.concurrency ? parseInt(options.concurrency, 10) : 2, + ignoreRobots: options.ignoreRobots, + allowedDomains: options.allow ? options.allow.split(',').map((d: string) => d.trim()) : undefined, + deniedDomains: options.deny ? options.deny.split(',').map((d: string) => d.trim()) : undefined, + includeSubdomains: options.includeSubdomains, + proxyUrl: options.proxy, + userAgent: options.ua, + rate: options.rate ? parseFloat(options.rate) : undefined, + maxBytes: options.maxBytes ? parseInt(options.maxBytes, 10) : undefined, + maxRedirects: options.maxRedirects ? parseInt(options.maxRedirects, 10) : undefined, + clustering: options.clustering, + clusterThreshold: options.clusterThreshold ? parseInt(options.clusterThreshold, 10) : undefined, + minClusterSize: options.minClusterSize ? parseInt(options.minClusterSize, 10) : undefined, + heading: options.heading, + health: options.health, + failOnCritical: options.failOnCritical, + scoreBreakdown: options.scoreBreakdown, + computeHits: options.computeHits, + computePagerank: options.computePagerank, + orphans: options.orphans, + orphanSeverity: options.orphanSeverity, + includeSoftOrphans: options.includeSoftOrphans, + minInbound: options.minInbound ? parseInt(options.minInbound, 10) : undefined, plugins: registry.getPlugins(), context: { command: 'crawl', + scope: 'crawl' as const, flags: options as Record, + emit: (e: any) => controller.handle(e), logger: { info: (m: string) => context.emit({ type: 'info', message: m }), warn: (m: string) => context.emit({ type: 'warn', message: m, context: undefined }), diff --git a/packages/cli/src/commands/page.ts b/packages/cli/src/commands/page.ts index 8fd84d3..fc62ef0 100644 --- a/packages/cli/src/commands/page.ts +++ b/packages/cli/src/commands/page.ts @@ -15,7 +15,31 @@ export const getPageCommand = (registry: PluginRegistry) => { .addOption(new Option('--log-level ', 'Log level (normal, verbose, debug)').choices(['normal', 'verbose', 'debug']).default('normal')) .option('--seo', 'Show only SEO module output') .option('--content', 'Show only content module output') - .option('--accessibility', 'Show only accessibility module output'); + .option('--accessibility', 'Show only accessibility module output') + // Crawl Policy + .option('--proxy ', 'proxy URL to use for requests') + .option('--ua ', 'user agent string to use') + .option('--rate ', 'requests per second limit') + .option('--max-bytes ', 'maximum bytes to download per page') + .option('--max-redirects ', 'maximum redirects to follow') + // Clustering + .option('--clustering', 'Enable content clustering analysis') + .option('--cluster-threshold ', 'Hamming distance for content clusters', '10') + .option('--min-cluster-size ', 'Minimum pages per cluster', '3') + .option('--sitemap [url]', 'sitemap URL (defaults to /sitemap.xml if not specified)') + // Heading & Health + .option('--heading', 'Analyze heading structure and hierarchy health') + .option('--health', 'Run health score analysis') + .option('--fail-on-critical', 'Exit code 1 if critical issues exist') + .option('--score-breakdown', 'Print health score component weights') + // Graph Centrality + .option('--pagerank', 'Calculate PageRank') + .option('--hits', 'Compute Hub and Authority scores (HITS)') + // Orphans + .option('--orphans', 'Detect orphaned pages') + .option('--orphan-severity ', 'Severity for orphans (low/medium/high)') + .option('--include-soft-orphans', 'Include soft orphans in detection') + .option('--min-inbound ', 'Minimum inbound links to not be an orphan', '2'); // Let plugins register their flags on this command registry.registerPlugins(analyze); @@ -47,6 +71,25 @@ export const getPageCommand = (registry: PluginRegistry) => { seo: options.seo, content: options.content, accessibility: options.accessibility, + proxyUrl: options.proxy, + userAgent: options.ua, + rate: options.rate ? parseFloat(options.rate) : undefined, + maxBytes: options.maxBytes ? parseInt(options.maxBytes, 10) : undefined, + maxRedirects: options.maxRedirects ? parseInt(options.maxRedirects, 10) : undefined, + clustering: options.clustering, + clusterThreshold: options.clusterThreshold ? parseInt(options.clusterThreshold, 10) : undefined, + minClusterSize: options.minClusterSize ? parseInt(options.minClusterSize, 10) : undefined, + sitemap: options.sitemap, + heading: options.heading, + health: options.health, + failOnCritical: options.failOnCritical, + scoreBreakdown: options.scoreBreakdown, + computePagerank: options.pagerank, + computeHits: options.hits, + orphans: options.orphans, + orphanSeverity: options.orphanSeverity, + includeSoftOrphans: options.includeSoftOrphans, + minInbound: options.minInbound ? parseInt(options.minInbound, 10) : undefined, debug: options.logLevel === 'debug', plugins: registry.getPlugins(), context: { diff --git a/packages/core/src/analysis/analyze.ts b/packages/core/src/analysis/analyze.ts index 157cbda..57a3d99 100644 --- a/packages/core/src/analysis/analyze.ts +++ b/packages/core/src/analysis/analyze.ts @@ -1,7 +1,9 @@ import { load } from 'cheerio'; import { crawl } from '../crawler/crawl.js'; +import { UrlResolver } from '../crawler/resolver.js'; +import { Fetcher } from '../crawler/fetcher.js'; import { loadGraphFromSnapshot } from '../db/graphLoader.js'; -import { normalizeUrl } from '../crawler/normalize.js'; +import { normalizeUrl, UrlUtil } from '../crawler/normalize.js'; import { calculateMetrics, Metrics } from '../graph/metrics.js'; import { Graph } from '../graph/graph.js'; import { analyzeContent, calculateThinContentScore } from './content.js'; @@ -10,13 +12,24 @@ import { analyzeImageAlts, ImageAltAnalysis } from './images.js'; import { analyzeLinks, LinkRatioAnalysis } from './links.js'; import { analyzeStructuredData, StructuredDataResult } from './structuredData.js'; import { aggregateSiteScore, scorePageSeo } from './scoring.js'; +import { ClusteringService } from './clustering.js'; +import { DuplicateService } from './duplicate.js'; +import { Soft404Service } from './soft404.js'; import { getDb } from '../db/index.js'; import { SiteRepository } from '../db/repositories/SiteRepository.js'; import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js'; import { PageRepository } from '../db/repositories/PageRepository.js'; +import { MetricsRepository } from '../db/repositories/MetricsRepository.js'; import { ANALYSIS_LIST_TEMPLATE, ANALYSIS_PAGE_TEMPLATE } from './templates.js'; import { EngineContext } from '../events.js'; +import { DEFAULTS } from '../constants.js'; + +import { PageRankService } from '../graph/pagerank.js'; +import { HITSService } from '../graph/hits.js'; +import { HeadingHealthService } from './heading.js'; +import { annotateOrphans } from './orphan.js'; +import { HealthService } from '../scoring/health.js'; export interface CrawlPage { url: string; @@ -39,10 +52,23 @@ export interface AnalyzeOptions { proxyUrl?: string; userAgent?: string; maxRedirects?: number; + maxBytes?: number; debug?: boolean; + heading?: boolean; + clustering?: boolean; clusterThreshold?: number; minClusterSize?: number; allPages?: boolean; + sitemap?: string | boolean; + health?: boolean; + failOnCritical?: boolean; + scoreBreakdown?: boolean; + computeHits?: boolean; + computePagerank?: boolean; + orphans?: boolean; + orphanSeverity?: 'low' | 'medium' | 'high'; + includeSoftOrphans?: boolean; + minInbound?: number; plugins?: any[]; // CrawlithPlugin[] pluginContext?: any; // PluginContext } @@ -65,6 +91,8 @@ export interface PageAnalysis { nofollow?: boolean; crawlStatus?: string; }; + soft404?: { score: number; reason: string }; + headingScore?: number; plugins?: Record; } @@ -87,6 +115,8 @@ export interface AnalysisResult { snapshotId?: number; crawledAt?: string; + clusters?: any[]; + duplicates?: any[]; plugins?: Record; } @@ -103,11 +133,36 @@ interface CrawlData { * Supports live crawling or loading from a database snapshot. */ export async function analyzeSite(url: string, options: AnalyzeOptions, context?: EngineContext): Promise { - const normalizedRoot = normalizeUrl(url, '', { stripQuery: false }); - if (!normalizedRoot) { + // 1. Parse siteOrigin (e.g. https://example.com) and targetPath (e.g. /stats) from the URL. + // We resolve the *origin* — not the full page URL — so rootOrigin is always just the + // scheme+host and normalizedPath is always the pathname. + let parsedUrl: URL | null = null; + try { parsedUrl = new URL(url); } catch { /* bare domain fallback below */ } + + const inputOrigin = parsedUrl ? `${parsedUrl.protocol}//${parsedUrl.host}` : url; + const inputPath = parsedUrl?.pathname || '/'; + + let rootOrigin = inputOrigin; + if (options.live !== false) { + const resolver = new UrlResolver(); + const fetcher = new Fetcher({ rate: options.rate, proxyUrl: options.proxyUrl, userAgent: options.userAgent }); + try { + const resolved = await resolver.resolve(inputOrigin, fetcher); + rootOrigin = resolved.url; + } catch { + // Fallback to basic normalization if resolution fails + } + } + + // Normalize the resolved origin + const normalizedAbs = normalizeUrl(rootOrigin, '', { stripQuery: false }); + if (!normalizedAbs) { throw new Error('Invalid URL for analysis'); } + // normalizedPath: use the input pathname (e.g. '/stats'), falling back to '/' for root + const normalizedPath = inputPath && inputPath !== '/' ? inputPath : UrlUtil.toPath(normalizedAbs, rootOrigin); + const start = Date.now(); let crawlData: CrawlData; let robots: any = null; @@ -115,10 +170,10 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? // 1. Robots fetch (live-mode only to keep snapshot analysis deterministic and fast) if (options.live) { try { - const robotsUrl = new URL('/robots.txt', normalizedRoot).toString(); + const robotsUrl = new URL('/robots.txt', rootOrigin).toString(); const { Fetcher } = await import('../crawler/fetcher.js'); const fetcher = new Fetcher({ - rate: 10, + rate: DEFAULTS.RATE_LIMIT, proxyUrl: options.proxyUrl, userAgent: options.userAgent }); @@ -137,25 +192,25 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? // 2. Data Acquisition if (options.live) { const crawlStart = Date.now(); - crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); + crawlData = await runLiveCrawl(normalizedAbs, rootOrigin, options, context, robots); if (context) context.emit({ type: 'debug', message: `[analyze] runLiveCrawl took ${Date.now() - crawlStart}ms` }); } else { try { const loadStart = Date.now(); - crawlData = await loadCrawlData(normalizedRoot, options.snapshotId); + crawlData = await loadCrawlData(normalizedAbs, options.snapshotId); if (context) context.emit({ type: 'debug', message: `[analyze] loadCrawlData took ${Date.now() - loadStart}ms` }); const allPages = Array.from(crawlData.pages); crawlData.pages = allPages; - const exists = allPages.some(p => p.url === normalizedRoot); + const exists = allPages.some(p => p.url === normalizedPath); if (!exists) { - if (context) context.emit({ type: 'info', message: `URL ${normalizedRoot} not found. Fetching live...` }); - crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); + if (context) context.emit({ type: 'info', message: `URL ${normalizedAbs} not found. Fetching live...` }); + crawlData = await runLiveCrawl(normalizedAbs, rootOrigin, options, context, robots); } } catch (_error: any) { if (context) context.emit({ type: 'info', message: 'No local crawl data found. Switching to live...' }); - crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); + crawlData = await runLiveCrawl(normalizedAbs, rootOrigin, options, context, robots); } } @@ -165,9 +220,21 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? const pagesStart = Date.now(); - const pages = analyzePages(normalizedRoot, crawlData.pages, robots, options); + const pages = analyzePages(normalizedPath, rootOrigin, crawlData.pages, robots, options); if (context) context.emit({ type: 'debug', message: `[analyze] analyzePages took ${Date.now() - pagesStart}ms` }); + // Sync basic page analysis results back to graph nodes for persistence + for (const pageAnalysis of pages) { + const node = crawlData.graph.nodes.get(pageAnalysis.url); + if (node) { + node.soft404Score = pageAnalysis.soft404?.score; + node.wordCount = pageAnalysis.content.wordCount; + node.externalLinkRatio = pageAnalysis.links.externalRatio; + node.thinContentScore = pageAnalysis.thinScore; + node.title = pageAnalysis.title.value || undefined; + } + } + const activeModules = { seo: !!options.seo, content: !!options.content, @@ -177,7 +244,7 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? const hasFilters = activeModules.seo || activeModules.content || activeModules.accessibility; const filteredPages = hasFilters ? pages.map((page) => filterPageModules(page, activeModules)) : pages; - const targetPage = filteredPages.find(p => p.url === normalizedRoot); + const targetPage = filteredPages.find(p => p.url === normalizedPath || p.url === normalizedAbs); let resultPages: PageAnalysis[]; if (options.allPages) { @@ -186,11 +253,141 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? resultPages = targetPage ? [targetPage] : (options.live ? filteredPages.slice(0, 1) : []); } + + let clusters: any[] = []; + let duplicates: any[] = []; + let prResults = new Map(); + let hitsResults = new Map(); + let headingPayloads: Record = {}; + + if (options.clustering) { + const clustering = new ClusteringService(); + clusters = clustering.detectContentClusters(crawlData.graph, options.clusterThreshold, options.minClusterSize); + } + + if (options.allPages) { + const duplication = new DuplicateService(); + duplicates = duplication.detectDuplicates(crawlData.graph, { collapse: false }); + } + + if (options.computePagerank) { + const prService = new PageRankService(); + prResults = prService.evaluate(crawlData.graph); + } + + if (options.computeHits) { + const hitsService = new HITSService(); + hitsResults = hitsService.evaluate(crawlData.graph); + } + + if (options.heading) { + const headingService = new HeadingHealthService(); + const { payloadsByUrl } = headingService.evaluateNodes(crawlData.graph.getNodes()); + headingPayloads = payloadsByUrl; + } + + if (options.orphans) { + const edges = crawlData.graph.getEdges(); + annotateOrphans(crawlData.graph.getNodes(), edges, { + enabled: true, + severityEnabled: !!options.orphanSeverity, + includeSoftOrphans: !!options.includeSoftOrphans, + minInbound: options.minInbound || 2, + rootUrl: normalizedAbs + }); + } + + // Run HealthService when --health is enabled + let healthBreakdown: ReturnType | undefined; + if (options.health) { + const healthService = new HealthService(); + const issues = healthService.collectCrawlIssues(crawlData.graph, crawlData.metrics, rootOrigin); + healthBreakdown = healthService.calculateHealthScore(crawlData.graph.nodes.size, issues); + } + + // Update nodes in graph with results + for (const node of crawlData.graph.getNodes()) { + const pr = prResults.get(node.url); + if (pr) node.pagerankScore = pr.score; + + const hits = hitsResults.get(node.url); + if (hits) { + node.hubScore = hits.hub_score; + node.authScore = hits.authority_score; + node.linkRole = hits.link_role; + } + + const heading = headingPayloads[node.url]; + if (heading) { + node.headingScore = heading.score; + node.headingData = JSON.stringify(heading); + } + } + + // Synchronize graph-level final scores back to PageAnalysis models + for (const page of pages) { + const node = crawlData.graph.nodes.get(page.url); + if (node) { + if (node.headingScore !== undefined) page.headingScore = node.headingScore; + page.seoScore = scorePageSeo(page); + } + } + const duplicateTitles = pages.filter((page) => page.title.status === 'duplicate').length; const thinPages = pages.filter((page) => page.thinScore >= 70).length; const siteScores = aggregateSiteScore(crawlData.metrics, resultPages.length === 1 ? resultPages : pages); if (context) context.emit({ type: 'debug', message: `[analyze] Total analysis completed in ${Date.now() - start}ms` }); + + + // Persist to Database + const db = getDb(); + const metricsRepo = new MetricsRepository(db); + const pageRepo = new PageRepository(db); + + // Efficiently map URLs to IDs for this snapshot + const pagesIdentity = pageRepo.getPagesIdentityBySnapshot(snapshotId); + const urlToIdMap = new Map(pagesIdentity.map(p => [p.normalized_url, p.id])); + + const metricsToSave = crawlData.graph.getNodes().map(node => { + const pageId = urlToIdMap.get(node.url); + if (!pageId) return null; + + return { + snapshot_id: snapshotId, + page_id: pageId, + crawl_status: node.crawlStatus || null, + word_count: node.wordCount || null, + thin_content_score: node.thinContentScore || null, + external_link_ratio: node.externalLinkRatio || null, + pagerank_score: node.pagerankScore || null, + hub_score: node.hubScore || null, + auth_score: node.authScore || null, + link_role: node.linkRole || null, + duplicate_cluster_id: (node as any).duplicateClusterId || null, + duplicate_type: (node as any).duplicateType || null, + cluster_id: (node as any).clusterId || null, + soft404_score: node.soft404Score || null, + heading_score: node.headingScore || null, + orphan_score: node.orphanScore || null, + orphan_type: node.orphanType || null, + impact_level: node.impactLevel || null, + heading_data: node.headingData || null, + is_cluster_primary: (node as any).isClusterPrimary ? 1 : 0 + }; + }).filter(m => m !== null); + + // Persist health score to snapshot if computed + if (healthBreakdown && snapshotId) { + const db2 = getDb(); + const snapshotRepo = new SnapshotRepository(db2); + snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', { + health_score: healthBreakdown.score + }); + } + + metricsRepo.insertMany(metricsToSave as any); + const result: AnalysisResult = { site_summary: { pages_analyzed: resultPages.length, @@ -205,47 +402,49 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? active_modules: activeModules, snapshotId, - crawledAt + crawledAt, + clusters, + duplicates }; return result; } -export function analyzePages(rootUrl: string, pages: Iterable | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] { +export function analyzePages(targetPath: string, rootOrigin: string, pages: Iterable | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] { const titleCounts = new Map(); const metaCounts = new Map(); const sentenceCountFrequency = new Map(); const results: PageAnalysis[] = []; - const normalizedRoot = rootUrl; for (const page of pages) { - const isTarget = page.url === normalizedRoot; + // page.url is a root-relative path (e.g. '/about') — compare to targetPath + const isTarget = page.url === targetPath; // In single-page mode, if it's not the target, we skip it entirely for speed. - // Duplicate title detection is sacrificed for single-page live analysis speed. - // Full site audits will correctly handle site-wide duplication. if (!options.allPages && !isTarget) continue; const html = page.html || ''; const $ = load(html || ''); + // Reconstruct absolute URL from stored path for robots & link resolution + const pageAbsUrl = UrlUtil.toAbsolute(page.url, rootOrigin); + let crawlStatus = page.crawlStatus; if (robots) { - const isBlocked = !robots.isAllowed(page.url, 'crawlith') || - (!page.url.endsWith('/') && !robots.isAllowed(page.url + '/', 'crawlith')); + const isBlocked = !robots.isAllowed(pageAbsUrl, 'crawlith') || + (!pageAbsUrl.endsWith('/') && !robots.isAllowed(pageAbsUrl + '/', 'crawlith')); if (isBlocked) crawlStatus = 'blocked_by_robots'; } // Shared DOM Analysis const title = analyzeTitle($); const metaDescription = analyzeMetaDescription($); - const h1 = analyzeH1($, title.value); const content = analyzeContent($); const images = analyzeImageAlts($); - const links = analyzeLinks($, page.url, rootUrl); + const links = analyzeLinks($, pageAbsUrl, rootOrigin); const structuredData = analyzeStructuredData($); if (title.value) { @@ -258,7 +457,10 @@ export function analyzePages(rootUrl: string, pages: Iterable | Crawl } sentenceCountFrequency.set(content.uniqueSentenceCount, (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) + 1); - results.push({ + const soft404Service = new Soft404Service(); + const soft404 = soft404Service.analyze(html, links.externalLinks + links.internalLinks); + + const resultPage: PageAnalysis = { url: page.url, status: page.status || 0, title, @@ -275,8 +477,12 @@ export function analyzePages(rootUrl: string, pages: Iterable | Crawl noindex: page.noindex, nofollow: page.nofollow, crawlStatus - } - }); + }, + soft404 + }; + + Object.defineProperty(resultPage, 'html', { value: html, enumerable: false }); + results.push(resultPage); } for (const analysis of results) { @@ -297,17 +503,21 @@ export function analyzePages(rootUrl: string, pages: Iterable | Crawl } function filterPageModules(page: PageAnalysis, modules: { seo: boolean; content: boolean; accessibility: boolean }): PageAnalysis { - return { + const filtered: PageAnalysis = { ...page, title: modules.seo ? page.title : { value: null, length: 0, status: 'missing' }, metaDescription: modules.seo ? page.metaDescription : { value: null, length: 0, status: 'missing' }, - h1: (modules.seo || modules.content) ? page.h1 : { count: 0, status: 'critical', matchesTitle: false }, + h1: (modules.seo || modules.content) ? page.h1 : { count: 0, status: 'critical', matchesTitle: false, value: null }, links: modules.seo ? page.links : { internalLinks: 0, externalLinks: 0, nofollowCount: 0, externalRatio: 0 }, structuredData: modules.seo ? page.structuredData : { present: false, valid: false, types: [] }, content: modules.content ? page.content : { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 }, thinScore: modules.content ? page.thinScore : 0, images: modules.accessibility ? page.images : { totalImages: 0, missingAlt: 0, emptyAlt: 0 } }; + if ((page as any).html) { + Object.defineProperty(filtered, 'html', { value: (page as any).html, enumerable: false }); + } + return filtered; } async function loadCrawlData(rootUrl: string, snapshotId?: number): Promise { @@ -355,7 +565,7 @@ async function loadCrawlData(rootUrl: string, snapshotId?: number): Promise { +async function runLiveCrawl(url: string, origin: string, options: AnalyzeOptions, context?: EngineContext, robots?: any): Promise { const snapshotId = await crawl(url, { limit: 1, depth: 0, @@ -366,6 +576,7 @@ async function runLiveCrawl(url: string, options: AnalyzeOptions, context?: Engi debug: options.debug, snapshotType: 'partial', robots, + sitemap: options.sitemap, plugins: options.plugins }, context) as number; const graph = loadGraphFromSnapshot(snapshotId); diff --git a/packages/plugins/content-clustering/src/Service.ts b/packages/core/src/analysis/clustering.ts similarity index 84% rename from packages/plugins/content-clustering/src/Service.ts rename to packages/core/src/analysis/clustering.ts index 6062f10..57a6a12 100644 --- a/packages/plugins/content-clustering/src/Service.ts +++ b/packages/core/src/analysis/clustering.ts @@ -1,6 +1,18 @@ -import { Graph, SimHash } from '@crawlith/core'; -import { load } from 'cheerio'; -import { ClusterInfo } from './types.js'; +import { Graph, SimHash, analyzeH1, analyzeTitle } from '../index.js'; + +export interface ClusterInfo { + id: number; + count: number; + primaryUrl: string; + risk: 'low' | 'medium' | 'high'; + sharedPathPrefix?: string; + nodes?: string[]; +} + +export interface ClusteringOptions { + threshold?: number; + minSize?: number; +} export class ClusteringService { /** @@ -107,9 +119,6 @@ export class ClusteringService { const clusterId = index + 1; const clusterNodes = memberUrls.map(url => graph.nodes.get(url)!); - // In the plugin world, we might not want to mutate the graph nodes directly - // but we'll return the cluster info and let the plugin handle storage. - // However, some core components might still expect node.clusterId for now. for (const node of clusterNodes) { (node as any).clusterId = clusterId; } @@ -156,21 +165,18 @@ export class ClusteringService { for (const node of nodes) { if (!node.html) continue; - try { - const $ = load(node.html); - const title = $('title').text().trim().toLowerCase(); - const h1 = $('h1').first().text().trim().toLowerCase(); + const titleRes = analyzeTitle(node.html); + const h1Res = analyzeH1(node.html, titleRes.value); + const title = titleRes.value || ''; + const h1 = h1Res.value || ''; - if (title) { - titleCounts.set(title, (titleCounts.get(title) || 0) + 1); - } - if (h1) { - h1Counts.set(h1, (h1Counts.get(h1) || 0) + 1); - } - processedCount++; - } catch { - // Ignore parsing errors + if (title) { + titleCounts.set(title.toLowerCase(), (titleCounts.get(title.toLowerCase()) || 0) + 1); + } + if (h1) { + h1Counts.set(h1.toLowerCase(), (h1Counts.get(h1.toLowerCase()) || 0) + 1); } + processedCount++; } if (processedCount < nodes.length * 0.5) { @@ -207,7 +213,17 @@ export class ClusteringService { if (urls.length < 2) return undefined; try { - const paths = urls.map(u => new URL(u).pathname.split('/').filter(Boolean)); + // Works for both absolute URLs (https://example.com/foo) and root-relative paths (/foo) + const getPathname = (u: string): string[] => { + try { + return new URL(u).pathname.split('/').filter(Boolean); + } catch { + // root-relative path like '/foo/bar' + return u.split('/').filter(Boolean); + } + }; + + const paths = urls.map(getPathname); const first = paths[0]; const common: string[] = []; diff --git a/packages/plugins/duplicate-detection/src/Service.ts b/packages/core/src/analysis/duplicate.ts similarity index 99% rename from packages/plugins/duplicate-detection/src/Service.ts rename to packages/core/src/analysis/duplicate.ts index a59767d..d0ebdd7 100644 --- a/packages/plugins/duplicate-detection/src/Service.ts +++ b/packages/core/src/analysis/duplicate.ts @@ -1,4 +1,4 @@ -import { Graph, GraphNode, SimHash } from '@crawlith/core'; +import { Graph, GraphNode, SimHash } from '../index.js'; export interface DuplicateOptions { collapse?: boolean; @@ -247,7 +247,8 @@ export class DuplicateService { } private checkTemplateHeavy(cluster: DuplicateCluster) { - const avgRatio = cluster.nodes.reduce((sum, n) => sum + ((n as any).uniqueTokenRatio || 0), 0) / cluster.nodes.length; + + const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length; if (avgRatio < 0.3) { cluster.type = 'template_heavy'; cluster.nodes.forEach(n => (n as any).duplicateType = 'template_heavy'); diff --git a/packages/core/src/analysis/heading.ts b/packages/core/src/analysis/heading.ts new file mode 100644 index 0000000..934a886 --- /dev/null +++ b/packages/core/src/analysis/heading.ts @@ -0,0 +1,503 @@ +import { createHash } from 'node:crypto'; + +/** + * Supported heading levels within HTML content. + */ +export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6; + +/** + * Represents a normalized heading node extracted from the DOM. + */ +export interface HeadingNode { + level: HeadingLevel; + text: string; + index: number; + parentIndex?: number; +} + +/** + * Represents content statistics for a section under a heading. + */ +export interface SectionMetrics { + headingIndex: number; + headingText: string; + words: number; + keywordConcentration: number; + thin: boolean; + duplicateRisk: number; +} + +/** + * Raw heading analysis generated for a single URL. + */ +export interface LocalPageAnalysis { + url: string; + headingNodes: HeadingNode[]; + sections: SectionMetrics[]; + h1Norm: string; + h2SetHash: string; + patternHash: string; + issues: string[]; + metrics: { + entropy: number; + maxDepth: number; + avgDepth: number; + headingDensity: number; + fragmentation: number; + levelVolatility: number; + hierarchySkips: number; + reverseJumps: number; + missingH1: number; + multipleH1: number; + }; +} + +/** + * Final heading-health payload attached to a page node. + */ +export interface HeadingHealthPayload { + score: number; + status: 'Healthy' | 'Moderate' | 'Poor'; + issues: string[]; + map: HeadingNode[]; + missing_h1: number; + multiple_h1: number; + entropy: number; + max_depth: number; + avg_depth: number; + heading_density: number; + fragmentation: number; + volatility: number; + hierarchy_skips: number; + reverse_jumps: number; + thin_sections: number; + duplicate_h1_group: number; + similar_h1_group: number; + identical_h2_set_group: number; + duplicate_pattern_group: number; + template_risk: number; +} + +/** + * Snapshot-level summary emitted by the plugin. + */ +export interface HeadingHealthSummary { + avgScore: number; + evaluatedPages: number; + totalMissing: number; + totalMultiple: number; + totalSkips: number; + totalReverseJumps: number; + totalThinSections: number; + avgEntropy: number; + poorPages: number; +} + +const STOPWORDS = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'your', 'about', 'into', 'over', 'under', 'are', 'was', 'were', 'can', 'has', 'have', 'had', 'you', 'our', 'out', 'all']); +const THIN_SECTION_WORDS = 80; +const HEADING_PATTERN = /]*>([\s\S]*?)<\/h\1>/gi; +const TITLE_PATTERN = /]*>([\s\S]*?)<\/title>/i; + +const normalizeText = (input: string) => input.replace(/<[^<>]*>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/\s+/g, ' ').trim(); +const normalizeComparable = (input: string) => normalizeText(input).toLowerCase(); +const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((token) => token.length > 2 && !STOPWORDS.has(token)); +const stableHash = (input: string) => createHash('sha1').update(input).digest('hex').slice(0, 16); + +/** + * Calculates token-level Jaccard similarity between two text values. + */ +export function jaccardSimilarity(a: string, b: string): number { + const aSet = new Set(tokenize(a)); + const bSet = new Set(tokenize(b)); + + if (!aSet.size || !bSet.size) { + return 0; + } + + let intersection = 0; + for (const token of aSet) { + if (bSet.has(token)) { + intersection += 1; + } + } + + return intersection / (aSet.size + bSet.size - intersection); +} + +/** + * Performs per-page heading extraction and structural scoring signal generation. + */ +export function analyzeHeadingHealth(html: string, fallbackTitle?: string): LocalPageAnalysis { + const segments: Array<{ level: HeadingLevel; text: string; start: number; end: number }> = []; + for (const match of html.matchAll(HEADING_PATTERN)) { + const level = Number(match[1]) as HeadingLevel; + segments.push({ + level, + text: normalizeText(match[2] || ''), + start: match.index || 0, + end: (match.index || 0) + match[0].length + }); + } + + const headingNodes: HeadingNode[] = []; + const stack: HeadingNode[] = []; + segments.forEach((segment, index) => { + const node: HeadingNode = { level: segment.level, text: segment.text, index }; + while (stack.length > 0 && stack[stack.length - 1].level >= node.level) { + stack.pop(); + } + if (stack.length > 0) { + node.parentIndex = stack[stack.length - 1].index; + } + stack.push(node); + headingNodes.push(node); + }); + + const sections: SectionMetrics[] = []; + const pageWords = tokenize(html); + const frequency = new Map(); + for (const word of pageWords) { + frequency.set(word, (frequency.get(word) || 0) + 1); + } + + for (let i = 0; i < segments.length; i += 1) { + const current = segments[i]; + const next = segments[i + 1]; + const textChunk = html.slice(current.end, next ? next.start : html.length); + const words = tokenize(textChunk); + const headingTokens = tokenize(current.text); + const concentration = headingTokens.reduce((sum, token) => sum + (frequency.get(token) || 0), 0); + + sections.push({ + headingIndex: headingNodes[i]?.index ?? i, + headingText: current.text, + words: words.length, + keywordConcentration: words.length > 0 ? Number((concentration / words.length).toFixed(3)) : 0, + thin: words.length > 0 && words.length < THIN_SECTION_WORDS, + duplicateRisk: 0 + }); + } + + const levelCounts = [0, 0, 0, 0, 0, 0]; + headingNodes.forEach((node) => { + levelCounts[node.level - 1] += 1; + }); + + const entropyScore = Number(calculateEntropy(levelCounts).toFixed(3)); + const missingH1 = levelCounts[0] === 0 ? 1 : 0; + const multipleH1 = levelCounts[0] > 1 ? 1 : 0; + + let hierarchySkips = 0; + let reverseJumps = 0; + let volatilitySum = 0; + for (let i = 1; i < headingNodes.length; i += 1) { + const delta = headingNodes[i].level - headingNodes[i - 1].level; + volatilitySum += Math.abs(delta); + if (delta > 1) { + hierarchySkips += 1; + } + if (delta < -1) { + reverseJumps += 1; + } + } + + const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((node) => node.level)) : 0; + const avgDepth = headingNodes.length ? Number((headingNodes.reduce((sum, node) => sum + node.level, 0) / headingNodes.length).toFixed(2)) : 0; + const headingDensity = pageWords.length ? Number((headingNodes.length / pageWords.length).toFixed(4)) : 0; + const fragmentation = headingNodes.length ? Number((headingNodes.filter((node) => node.level <= 2).length / headingNodes.length).toFixed(3)) : 0; + const levelVolatility = headingNodes.length > 1 ? Number((volatilitySum / (headingNodes.length - 1)).toFixed(3)) : 0; + + const h1Nodes = headingNodes.filter((node) => node.level === 1); + const issues: string[] = []; + if (missingH1) issues.push('Missing H1'); + if (multipleH1) issues.push('Multiple H1 found'); + if (h1Nodes.some((node) => node.text.length < 6)) issues.push('Empty or near-empty H1'); + + const title = fallbackTitle || getTitleFromHtml(html); + if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) { + issues.push('H1 diverges from '); + } + if (hierarchySkips > 0) issues.push(`${hierarchySkips} hierarchy skips detected`); + if (reverseJumps > 0) issues.push(`${reverseJumps} reverse hierarchy jumps detected`); + for (const thin of sections.filter((section) => section.thin).slice(0, 2)) { + issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`); + } + if (entropyScore > 2.1) issues.push('High structural entropy'); + if (fragmentation > 0.65) issues.push('Section fragmentation is high'); + + const h1Norm = normalizeComparable(h1Nodes[0]?.text || ''); + const h2SetHash = stableHash( + headingNodes + .filter((node) => node.level === 2) + .map((node) => normalizeComparable(node.text)) + .filter(Boolean) + .sort() + .join('|') + ); + const patternHash = stableHash(headingNodes.map((node) => node.level).join('>')); + + return { + url: '', + headingNodes, + sections, + h1Norm, + h2SetHash, + patternHash, + issues, + metrics: { + entropy: entropyScore, + maxDepth, + avgDepth, + headingDensity, + fragmentation, + levelVolatility, + hierarchySkips, + reverseJumps, + missingH1, + multipleH1 + } + }; +} + +/** + * Enriches section-level duplicate risk by comparing normalized section signatures across pages. + */ +export function enrichDuplicateRisk(pages: LocalPageAnalysis[]): void { + const buckets = new Map<string, string[]>(); + + for (const page of pages) { + for (const section of page.sections) { + const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); + const bucket = buckets.get(key) || []; + bucket.push(page.url); + buckets.set(key, bucket); + } + } + + for (const page of pages) { + for (const section of page.sections) { + const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); + const size = (buckets.get(key) || []).length; + section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3)); + } + } +} + +function calculateEntropy(values: number[]): number { + const total = values.reduce((a, b) => a + b, 0); + if (!total) { + return 0; + } + + return values.reduce((sum, value) => { + if (value === 0) { + return sum; + } + + const probability = value / total; + return sum - probability * Math.log2(probability); + }, 0); +} + +function getTitleFromHtml(html: string): string { + const match = html.match(TITLE_PATTERN); + return match ? normalizeText(match[1]) : ''; +} + +/** + * Coordinates heading analysis across graph nodes and builds report-safe payloads. + */ +export class HeadingHealthService { + /** + * Builds page-level payloads plus a snapshot summary for every eligible node. + */ + public evaluateNodes(nodes: Array<Record<string, any>>): { + payloadsByUrl: Map<string, HeadingHealthPayload>; + summary: HeadingHealthSummary; + } { + const analyzedPages = this.collectAnalyses(nodes); + enrichDuplicateRisk(analyzedPages); + + const exactH1Buckets = this.buildBuckets(analyzedPages, (page) => page.h1Norm); + const h2SetBuckets = this.buildBuckets(analyzedPages, (page) => page.h2SetHash); + const patternBuckets = this.buildBuckets(analyzedPages, (page) => page.patternHash); + const similarH1GroupSizes = this.computeSimilarH1GroupSizes(analyzedPages); + + const payloadsByUrl = new Map<string, HeadingHealthPayload>(); + + let totalScore = 0; + let totalMissing = 0; + let totalMultiple = 0; + let totalSkips = 0; + let totalReverseJumps = 0; + let totalThinSections = 0; + let totalEntropy = 0; + let poorPages = 0; + + for (const page of analyzedPages) { + const duplicateH1GroupSize = page.h1Norm ? exactH1Buckets.get(page.h1Norm)?.length || 1 : 0; + const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0; + const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1; + const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1; + const templateRisk = this.computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize); + const thinSectionCount = page.sections.filter((section) => section.thin).length; + + const health = this.scoreHealth({ + metrics: page.metrics, + thinSectionCount, + duplicateH1GroupSize, + similarH1GroupSize, + identicalH2SetGroupSize, + duplicatePatternGroupSize, + templateRisk, + issues: page.issues + }); + + if (health.status === 'Poor') { + poorPages += 1; + } + + totalScore += health.score; + totalMissing += page.metrics.missingH1; + totalMultiple += page.metrics.multipleH1; + totalSkips += page.metrics.hierarchySkips; + totalReverseJumps += page.metrics.reverseJumps; + totalThinSections += thinSectionCount; + totalEntropy += page.metrics.entropy; + + payloadsByUrl.set(page.url, { + score: health.score, + status: health.status, + issues: health.issues, + map: page.headingNodes, + missing_h1: page.metrics.missingH1, + multiple_h1: page.metrics.multipleH1, + entropy: page.metrics.entropy, + max_depth: page.metrics.maxDepth, + avg_depth: page.metrics.avgDepth, + heading_density: page.metrics.headingDensity, + fragmentation: page.metrics.fragmentation, + volatility: page.metrics.levelVolatility, + hierarchy_skips: page.metrics.hierarchySkips, + reverse_jumps: page.metrics.reverseJumps, + thin_sections: thinSectionCount, + duplicate_h1_group: duplicateH1GroupSize, + similar_h1_group: similarH1GroupSize, + identical_h2_set_group: identicalH2SetGroupSize, + duplicate_pattern_group: duplicatePatternGroupSize, + template_risk: templateRisk + }); + } + + const evaluatedPages = analyzedPages.length; + const summary: HeadingHealthSummary = { + avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0, + evaluatedPages, + totalMissing, + totalMultiple, + totalSkips, + totalReverseJumps, + totalThinSections, + avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0, + poorPages + }; + + return { payloadsByUrl, summary }; + } + + private collectAnalyses(nodes: Array<Record<string, any>>): LocalPageAnalysis[] { + const analyses: LocalPageAnalysis[] = []; + + for (const node of nodes) { + if (node.status < 200 || node.status >= 300 || !node.html || !node.url) { + continue; + } + + const analysis = analyzeHeadingHealth(node.html, node.title || node.rawTitle); + analysis.url = node.url; + analyses.push(analysis); + } + + return analyses; + } + + private buildBuckets(pages: LocalPageAnalysis[], selector: (page: LocalPageAnalysis) => string): Map<string, string[]> { + const buckets = new Map<string, string[]>(); + for (const page of pages) { + const key = selector(page); + if (!key) { + continue; + } + buckets.set(key, [...(buckets.get(key) || []), page.url]); + } + return buckets; + } + + private computeSimilarH1GroupSizes(pages: LocalPageAnalysis[]): Map<string, number> { + const uniqueH1 = Array.from(new Set(pages.map((page) => page.h1Norm).filter(Boolean))); + const similarBuckets = new Map<string, Set<string>>(); + + for (const h1 of uniqueH1) { + similarBuckets.set(h1, new Set([h1])); + } + + for (let i = 0; i < uniqueH1.length; i += 1) { + for (let j = i + 1; j < uniqueH1.length; j += 1) { + const a = uniqueH1[i]; + const b = uniqueH1[j]; + if (jaccardSimilarity(a, b) >= 0.7) { + similarBuckets.get(a)?.add(b); + similarBuckets.get(b)?.add(a); + } + } + } + + const groupSizes = new Map<string, number>(); + for (const page of pages) { + groupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0)); + } + + return groupSizes; + } + + private scoreHealth(input: { + metrics: LocalPageAnalysis['metrics']; + thinSectionCount: number; + duplicateH1GroupSize: number; + similarH1GroupSize: number; + identicalH2SetGroupSize: number; + duplicatePatternGroupSize: number; + templateRisk: number; + issues: string[]; + }): { score: number; status: 'Healthy' | 'Moderate' | 'Poor'; issues: string[] } { + let score = 100; + const metrics = input.metrics; + + if (metrics.missingH1) score -= 20; + if (metrics.multipleH1) score -= 6; + score -= metrics.hierarchySkips * 8; + score -= metrics.reverseJumps * 6; + score -= Math.round(metrics.entropy * 7); + score -= Math.round(metrics.fragmentation * 20); + score -= Math.round(metrics.levelVolatility * 6); + score -= input.thinSectionCount * 4; + + if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3); + if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2); + if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2); + if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2); + score -= Math.round(input.templateRisk * 12); + + score = Math.max(0, Math.min(100, score)); + + return { + score, + status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor', + issues: input.issues + }; + } + + private computeTemplateRisk(similar: number, h2set: number, pattern: number): number { + return Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3)); + } +} diff --git a/packages/core/src/analysis/links.ts b/packages/core/src/analysis/links.ts index 205b9f9..512c54b 100644 --- a/packages/core/src/analysis/links.ts +++ b/packages/core/src/analysis/links.ts @@ -1,5 +1,5 @@ import { load } from 'cheerio'; -import { normalizeUrl } from '../crawler/normalize.js'; +import { normalizeUrl, UrlUtil } from '../crawler/normalize.js'; export interface LinkRatioAnalysis { internalLinks: number; @@ -11,7 +11,6 @@ export interface LinkRatioAnalysis { export function analyzeLinks($: any, pageUrl: string, rootUrl: string): LinkRatioAnalysis { const isString = typeof $ === 'string'; const cheerioObj = isString ? load($ || '<html></html>') : $; - const rootOrigin = new URL(rootUrl).origin; let internalLinks = 0; let externalLinks = 0; @@ -28,13 +27,9 @@ export function analyzeLinks($: any, pageUrl: string, rootUrl: string): LinkRati nofollowCount += 1; } - try { - if (new URL(normalized).origin === rootOrigin) { - internalLinks += 1; - } else { - externalLinks += 1; - } - } catch { + if (UrlUtil.isInternal(normalized, rootUrl)) { + internalLinks += 1; + } else { externalLinks += 1; } }); diff --git a/packages/core/src/analysis/orphan.ts b/packages/core/src/analysis/orphan.ts new file mode 100644 index 0000000..292d6c2 --- /dev/null +++ b/packages/core/src/analysis/orphan.ts @@ -0,0 +1,172 @@ +import type { GraphNode, GraphEdge } from '../index.js'; + +export interface ExtendedGraphNode extends GraphNode { + pageType?: string; + hasStructuredData?: boolean; + isProductOrCommercial?: boolean; + duplicateContent?: boolean; + isHomepage?: boolean; + robotsExcluded?: boolean; + discoveredViaSitemap?: boolean; +} + +export type OrphanType = 'hard' | 'near' | 'soft' | 'crawl-only'; +export type ImpactLevel = 'low' | 'medium' | 'high' | 'critical'; + +export interface OrphanScoringOptions { + enabled: boolean; + severityEnabled: boolean; + includeSoftOrphans: boolean; + minInbound: number; + rootUrl?: string; +} + +export type AnnotatedNode = GraphNode & { + orphan: boolean; + orphanType?: OrphanType; + orphanSeverity?: number; + impactLevel?: ImpactLevel; +}; + +const LOW_VALUE_PATTERNS = [ + /[?&](page|p)=\d+/i, + /\/(page|tag|tags|category|categories)\//i, + /[?&](q|query|search|filter|sort)=/i, + /\/search(\/|\?|$)/i +]; + +function isLowValuePage(node: ExtendedGraphNode): boolean { + const type = (node.pageType || '').toLowerCase(); + if (['pagination', 'tag', 'category', 'filter', 'search', 'archive'].includes(type)) { + return true; + } + if (node.noindex) { + return true; + } + return LOW_VALUE_PATTERNS.some((pattern) => pattern.test(node.url)); +} + +function clampScore(score: number): number { + return Math.max(0, Math.min(100, Math.round(score))); +} + +export function mapImpactLevel(score: number): ImpactLevel { + if (score <= 39) return 'low'; + if (score <= 69) return 'medium'; + if (score <= 89) return 'high'; + return 'critical'; +} + +export function calculateOrphanSeverity(orphanType: OrphanType, node: ExtendedGraphNode): number { + let score = 0; + + switch (orphanType) { + case 'hard': + score = 90; + break; + case 'crawl-only': + // Sitemap-only URLs are less severe if we haven't even tried to crawl them yet. + score = node.status === 0 ? 50 : 70; + break; + case 'near': + score = node.inLinks <= 1 ? 70 : 60; + break; + case 'soft': + score = 50; + break; + } + + let positiveModifier = 0; + if ((node.wordCount || 0) > 800) positiveModifier += 10; + if (node.hasStructuredData) positiveModifier += 10; + if (node.depth <= 2) positiveModifier += 10; + if (node.isProductOrCommercial) positiveModifier += 10; + positiveModifier = Math.min(20, positiveModifier); + + let negativeModifier = 0; + if ((node.wordCount || 0) > 0 && (node.wordCount || 0) < 300) negativeModifier += 20; + if (node.noindex) negativeModifier += 20; + if (node.duplicateContent) negativeModifier += 20; + if ((node.pageType || '').toLowerCase() === 'archive' || (node.pageType || '').toLowerCase() === 'pagination') negativeModifier += 20; + negativeModifier = Math.min(20, negativeModifier); + + score += positiveModifier; + score -= negativeModifier; + + // Safety: unvisited nodes should not be flagged as high-severity orphans + // because we haven't confirmed they are real pages or fully explored their context. + if (node.status === 0) { + score = Math.min(score, 60); + } + + return clampScore(score); +} + +function consolidateInboundByCanonical(nodes: ExtendedGraphNode[]): Map<string, number> { + const canonicalInbound = new Map<string, number>(); + for (const node of nodes) { + const canonical = node.canonical || node.url; + canonicalInbound.set(canonical, (canonicalInbound.get(canonical) || 0) + node.inLinks); + } + return canonicalInbound; +} + +export function annotateOrphans(nodes: ExtendedGraphNode[], edges: GraphEdge[], options: OrphanScoringOptions): AnnotatedNode[] { + if (!options.enabled) { + return nodes.map((node) => ({ ...node, orphan: false } as AnnotatedNode)); + } + + const canonicalInbound = consolidateInboundByCanonical(nodes); + const nodeByUrl = new Map(nodes.map((node) => [node.url, node])); + + return nodes.map((node) => { + const isHomepage = node.isHomepage || (options.rootUrl ? node.url === options.rootUrl : node.depth === 0); + if (isHomepage || node.robotsExcluded) { + return { ...node, orphan: false } as AnnotatedNode; + } + + const canonical = node.canonical || node.url; + const inbound = canonicalInbound.get(canonical) || 0; + + let orphanType: OrphanType | undefined; + + if (inbound === 0) { + orphanType = node.discoveredViaSitemap ? 'crawl-only' : 'hard'; + } else if (inbound <= options.minInbound) { + orphanType = 'near'; + } + + if (!orphanType && options.includeSoftOrphans && inbound > 0) { + const inboundSources = edges + .filter((edge) => edge.target === node.url) + .map((edge) => nodeByUrl.get(edge.source)) + .filter((source): source is GraphNode => Boolean(source)); + + if (inboundSources.length > 0 && inboundSources.every((source) => isLowValuePage(source))) { + orphanType = 'soft'; + } + } + + if (!orphanType) { + return { ...node, orphan: false } as AnnotatedNode; + } + + if (!options.severityEnabled) { + return { + ...node, + orphan: true, + orphanType + } as AnnotatedNode; + } + + const orphanSeverity = calculateOrphanSeverity(orphanType, { ...node, inLinks: inbound }); + + return { + ...node, + orphan: true, + orphanType, + orphanSeverity, + impactLevel: mapImpactLevel(orphanSeverity) + } as AnnotatedNode; + }); +} diff --git a/packages/core/src/analysis/scoring.ts b/packages/core/src/analysis/scoring.ts index dcc8344..3a701d6 100644 --- a/packages/core/src/analysis/scoring.ts +++ b/packages/core/src/analysis/scoring.ts @@ -12,7 +12,10 @@ export function scorePageSeo(page: PageAnalysis): number { return 0; } const titleMeta = (scoreTextStatus(page.title.status) + scoreTextStatus(page.metaDescription.status)) / 2; - const h1 = page.h1.status === 'ok' ? 100 : page.h1.status === 'warning' ? 60 : 10; + let h1 = page.h1.status === 'ok' ? 100 : page.h1.status === 'warning' ? 60 : 10; + if (page.headingScore !== undefined && page.headingScore !== null) { + h1 = page.headingScore; + } const wordQuality = Math.min(100, (page.content.wordCount / 600) * 100) * 0.7 + Math.min(100, page.content.textHtmlRatio * 500) * 0.3; const thin = 100 - page.thinScore; const imageDen = Math.max(1, page.images.totalImages); diff --git a/packages/core/src/analysis/seo.ts b/packages/core/src/analysis/seo.ts index 465c988..2707957 100644 --- a/packages/core/src/analysis/seo.ts +++ b/packages/core/src/analysis/seo.ts @@ -12,6 +12,7 @@ export interface H1Analysis { count: number; status: 'ok' | 'critical' | 'warning'; matchesTitle: boolean; + value: string | null; } function normalizedText(value: string | null): string { @@ -61,12 +62,12 @@ export function analyzeH1($: any, titleValue: string | null): H1Analysis { const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue)); if (count === 0) { - return { count, status: 'critical', matchesTitle }; + return { count, status: 'critical', matchesTitle, value: null }; } if (count > 1) { - return { count, status: 'warning', matchesTitle }; + return { count, status: 'warning', matchesTitle, value: first }; } - return { count, status: 'ok', matchesTitle }; + return { count, status: 'ok', matchesTitle, value: first }; } export function applyDuplicateStatuses<T extends { value: string | null; status: string }>(items: T[]): T[] { diff --git a/packages/plugins/soft404-detector/src/Service.ts b/packages/core/src/analysis/soft404.ts similarity index 96% rename from packages/plugins/soft404-detector/src/Service.ts rename to packages/core/src/analysis/soft404.ts index 4c2e8ee..99bb117 100644 --- a/packages/plugins/soft404-detector/src/Service.ts +++ b/packages/core/src/analysis/soft404.ts @@ -1,5 +1,9 @@ import * as cheerio from 'cheerio'; -import { Soft404Result } from './types.js'; + +export interface Soft404Result { + score: number; + reason: string; +} /** * Service to analyze HTML content for soft 404 signals. diff --git a/packages/core/src/application/usecases.ts b/packages/core/src/application/usecases.ts index 1c3d7af..415a737 100644 --- a/packages/core/src/application/usecases.ts +++ b/packages/core/src/application/usecases.ts @@ -1,4 +1,5 @@ import { crawl } from '../crawler/crawl.js'; +import { DEFAULTS } from '../constants.js'; import { runPostCrawlMetrics } from '../crawler/metricsRunner.js'; import { analyzeSite, type AnalyzeOptions, type AnalysisResult } from '../analysis/analyze.js'; import { loadGraphFromSnapshot } from '../db/graphLoader.js'; @@ -22,7 +23,7 @@ export interface SiteCrawlInput { concurrency?: number; stripQuery?: boolean; ignoreRobots?: boolean; - sitemap?: string; + sitemap?: string | boolean; debug?: boolean; detectSoft404?: boolean; detectTraps?: boolean; @@ -34,6 +35,19 @@ export interface SiteCrawlInput { proxyUrl?: string; maxRedirects?: number; userAgent?: string; + clustering?: boolean; + clusterThreshold?: number; + minClusterSize?: number; + heading?: boolean; + health?: boolean; + failOnCritical?: boolean; + scoreBreakdown?: boolean; + computeHits?: boolean; + computePagerank?: boolean; + orphans?: boolean; + orphanSeverity?: 'low' | 'medium' | 'high'; + includeSoftOrphans?: boolean; + minInbound?: number; plugins?: CrawlithPlugin[]; context?: PluginContext; } @@ -57,8 +71,8 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes // Map the unified DTO into the underlying CrawlOptions const crawlOpts: any = { // Temporary any to avoid mismatch until crawl() is updated - limit: input.limit ?? 500, - depth: input.depth ?? 5, + limit: input.limit ?? DEFAULTS.CRAWL_LIMIT, + depth: input.depth ?? DEFAULTS.MAX_DEPTH, concurrency: input.concurrency, stripQuery: input.stripQuery, ignoreRobots: policy.ignoreRobots !== undefined ? policy.ignoreRobots : input.ignoreRobots, @@ -77,13 +91,34 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes registry: registry }; - const snapshotId = await crawl(input.url, crawlOpts as any); + // Build an EngineContext from the plugin context's emit so per-page logs reach OutputController + const engineContext: EngineContext | undefined = ctx.emit + ? { emit: (e: any) => ctx.emit!(e) } + : undefined; + + const snapshotId = await crawl(input.url, crawlOpts as any, engineContext); const graph = loadGraphFromSnapshot(snapshotId); await registry.runHook('onGraphBuilt', ctx, graph); await registry.runHook('onMetrics', ctx, graph); - runPostCrawlMetrics(snapshotId, crawlOpts.depth, undefined, false, graph); + runPostCrawlMetrics(snapshotId, crawlOpts.depth, { + context: undefined, + limitReached: false, + graphInstance: graph, + clustering: input.clustering ?? true, + clusterThreshold: input.clusterThreshold, + minClusterSize: input.minClusterSize, + heading: input.heading ?? true, + health: input.health ?? true, + computeHits: input.computeHits ?? true, + computePagerank: input.computePagerank ?? true, + orphans: input.orphans ?? true, + orphanSeverity: input.orphanSeverity ?? true, + includeSoftOrphans: input.includeSoftOrphans ?? true, + minInbound: input.minInbound, + rootOrigin: input.url.startsWith('http') ? new URL(input.url).origin : `https://${input.url}` + }); if (ctx.db) { ctx.db.aggregateScoreProviders(snapshotId, registry.pluginsList); @@ -125,10 +160,23 @@ export interface PageAnalysisInput { proxyUrl?: string; userAgent?: string; maxRedirects?: number; + maxBytes?: number; + clustering?: boolean; clusterThreshold?: number; minClusterSize?: number; debug?: boolean; allPages?: boolean; + sitemap?: string | boolean; + heading?: boolean; + health?: boolean; + failOnCritical?: boolean; + scoreBreakdown?: boolean; + computeHits?: boolean; + computePagerank?: boolean; + orphans?: boolean; + orphanSeverity?: 'low' | 'medium' | 'high'; + includeSoftOrphans?: boolean; + minInbound?: number; plugins?: CrawlithPlugin[]; context?: PluginContext; } @@ -137,18 +185,41 @@ export class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisR constructor(private readonly context?: EngineContext) { } async execute(input: PageAnalysisInput): Promise<AnalysisResult> { + // When running a live single-page analysis, enable all content modules + // by default (they all work well on a single page). Callers can still + // explicitly pass false to disable any of them. + const isLive = !!input.live; + const seo = input.seo ?? (isLive ? true : undefined); + const content = input.content ?? (isLive ? true : undefined); + const accessibility = input.accessibility ?? (isLive ? true : undefined); + const health = input.health ?? (isLive ? true : undefined); + const heading = input.heading ?? (isLive ? true : undefined); + const result = await analyzeSite(input.url, { live: input.live, snapshotId: input.snapshotId, - seo: input.seo, - content: input.content, - accessibility: input.accessibility, + seo, + content, + accessibility, rate: input.rate, proxyUrl: input.proxyUrl, userAgent: input.userAgent, maxRedirects: input.maxRedirects, + maxBytes: input.maxBytes, + clustering: input.clustering, clusterThreshold: input.clusterThreshold, minClusterSize: input.minClusterSize, + sitemap: input.sitemap, + heading, + health, + failOnCritical: input.failOnCritical, + scoreBreakdown: input.scoreBreakdown, + computeHits: input.computeHits, + computePagerank: input.computePagerank, + orphans: input.orphans, + orphanSeverity: input.orphanSeverity, + includeSoftOrphans: input.includeSoftOrphans, + minInbound: input.minInbound, debug: input.debug, allPages: input.allPages, }, this.context); @@ -172,9 +243,11 @@ export class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisR await registry.runHook('onInit', pluginCtx); // Fire onPage once per analyzed page (normally just 1 for the page command) + const inputOrigin = new URL(input.url).origin; for (const page of result.pages) { + const absoluteUrl = page.url.startsWith('/') ? `${inputOrigin}${page.url}` : page.url; await registry.runHook('onPage', pluginCtx, { - url: page.url, + url: absoluteUrl, html: (page as any).html ?? '', status: page.status, }); diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts new file mode 100644 index 0000000..bc65923 --- /dev/null +++ b/packages/core/src/constants.ts @@ -0,0 +1,21 @@ +import { version } from './utils/version.js'; + +export const DEFAULTS = { + // Crawler defaults + MAX_DEPTH: 5, + MAX_DEPTH_LIMIT: 10, + CONCURRENCY: 2, + CONCURRENCY_LIMIT: 10, + CRAWL_LIMIT: 500, + + // Network/Fetcher defaults + USER_AGENT: `crawlith/${version}`, + RATE_LIMIT: 10, + MAX_BYTES: 2000000, // 2MB + MAX_REDIRECTS: 5, + MAX_REDIRECTS_LIMIT: 11, + + // Network timeouts + HEADERS_TIMEOUT: 10000, + BODY_TIMEOUT: 10000, +} as const; diff --git a/packages/core/src/core/scope/scopeManager.ts b/packages/core/src/core/scope/scopeManager.ts index 60acc30..1228a33 100644 --- a/packages/core/src/core/scope/scopeManager.ts +++ b/packages/core/src/core/scope/scopeManager.ts @@ -26,6 +26,9 @@ export class ScopeManager { } isUrlEligible(url: string): EligibilityResult { + // Root-relative paths (e.g. '/about', '/?q=foo') are always internal + if (url.startsWith('/')) return 'allowed'; + let hostname: string; try { hostname = new URL(url).hostname.toLowerCase(); diff --git a/packages/core/src/crawler/crawler.ts b/packages/core/src/crawler/crawler.ts index 60f2d9e..cc8f5e1 100644 --- a/packages/core/src/crawler/crawler.ts +++ b/packages/core/src/crawler/crawler.ts @@ -5,7 +5,8 @@ import { Graph, GraphNode } from '../graph/graph.js'; import { Fetcher, FetchResult } from './fetcher.js'; import { Parser } from './parser.js'; import { Sitemap } from './sitemap.js'; -import { normalizeUrl } from './normalize.js'; +import { normalizeUrl, UrlUtil } from './normalize.js'; +import { UrlResolver } from './resolver.js'; import { ScopeManager } from '../core/scope/scopeManager.js'; import { getDb } from '../db/index.js'; import { SiteRepository } from '../db/repositories/SiteRepository.js'; @@ -17,6 +18,7 @@ import { analyzeContent, calculateThinContentScore } from '../analysis/content.j import { analyzeLinks } from '../analysis/links.js'; import { EngineContext } from '../events.js'; import { PluginRegistry } from '../plugin-system/plugin-registry.js'; +import { DEFAULTS } from '../constants.js'; export interface CrawlOptions { limit: number; @@ -25,7 +27,7 @@ export interface CrawlOptions { ignoreRobots?: boolean; stripQuery?: boolean; previousGraph?: Graph; - sitemap?: string; + sitemap?: string | boolean; debug?: boolean; detectSoft404?: boolean; detectTraps?: boolean; @@ -96,6 +98,7 @@ export class Crawler { private pageBuffer: Map<string, any> = new Map(); private edgeBuffer: { sourceUrl: string; targetUrl: string; weight: number; rel: string }[] = []; private metricsBuffer: any[] = []; + private pendingSitemaps: number = 0; // Modules private scopeManager: ScopeManager | null = null; @@ -115,8 +118,8 @@ export class Crawler { this.active = 0; this.pagesCrawled = 0; this.reachedLimit = false; - this.maxDepthInCrawl = Math.min(options.depth, 10); - this.concurrency = Math.min(options.concurrency || 2, 10); + this.maxDepthInCrawl = Math.min(options.depth || DEFAULTS.MAX_DEPTH, DEFAULTS.MAX_DEPTH_LIMIT); + this.concurrency = Math.min(options.concurrency || DEFAULTS.CONCURRENCY, DEFAULTS.CONCURRENCY_LIMIT); this.limitConcurrency = pLimit(this.concurrency); } @@ -128,14 +131,44 @@ export class Crawler { this.edgeRepo = new EdgeRepository(db); this.metricsRepo = new MetricsRepository(db); - const rootUrl = normalizeUrl(this.startUrl, '', { stripQuery: this.options.stripQuery }); + // Use resolver to find canonical origin and SSL + const resolver = new UrlResolver(); + const tempFetcher = new Fetcher({ userAgent: this.options.userAgent, rate: this.options.rate }); + const resolved = await resolver.resolve(this.startUrl, tempFetcher); + this.rootOrigin = resolved.url; + + // Use the resolved absolute URL as the base — NOT this.startUrl which may be + // a bare domain (e.g. 'callforpaper.org') that would be treated as a relative + // path when passed to normalizeUrl, producing '/callforpaper.org'. + const rootUrl = normalizeUrl(this.rootOrigin, '', { stripQuery: this.options.stripQuery }); if (!rootUrl) throw new Error('Invalid start URL'); - const urlObj = new URL(rootUrl); + const urlObj = new URL(this.rootOrigin); const domain = urlObj.hostname.replace('www.', ''); const site = this.siteRepo.firstOrCreateSite(domain); this.siteId = site.id; + // Persist the resolved preferred URL and SSL status + this.siteRepo.updateSitePreference(this.siteId, { + preferred_url: this.rootOrigin, + ssl: this.rootOrigin.startsWith('https') ? 1 : 0 + }); + + // Update normalized start URL as a path + this.startUrl = UrlUtil.toPath(rootUrl, this.rootOrigin); + + // Now that rootOrigin is resolved, initialize ScopeManager with the correct absolute origin + this.scopeManager = new ScopeManager({ + allowedDomains: this.options.allowedDomains || [], + deniedDomains: this.options.deniedDomains || [], + includeSubdomains: this.options.includeSubdomains || false, + rootUrl: this.rootOrigin + }); + // Update fetcher with the now-initialized scopeManager + if (this.fetcher) { + (this.fetcher as any).scopeManager = this.scopeManager; + } + // For partial snapshots (page --live), reuse the latest partial snapshot // instead of creating a new one each time if (this.options.snapshotType === 'partial') { @@ -163,28 +196,20 @@ export class Crawler { } setupModules(): void { - this.scopeManager = new ScopeManager({ - allowedDomains: this.options.allowedDomains || [], - deniedDomains: this.options.deniedDomains || [], - includeSubdomains: this.options.includeSubdomains || false, - rootUrl: this.startUrl - }); - this.fetcher = new Fetcher({ rate: this.options.rate, proxyUrl: this.options.proxyUrl, - scopeManager: this.scopeManager, + scopeManager: this.scopeManager ?? undefined, maxRedirects: this.options.maxRedirects, userAgent: this.options.userAgent }); this.parser = new Parser(); - this.sitemapFetcher = new Sitemap(this.context); + this.sitemapFetcher = new Sitemap(this.context, this.fetcher!); } - - async fetchRobots(): Promise<void> { + private async fetchRobots(): Promise<void> { + const robotsUrl = new URL('/robots.txt', this.rootOrigin).toString(); try { - const robotsUrl = new URL('/robots.txt', this.rootOrigin).toString(); const res = await this.fetcher!.fetch(robotsUrl, { maxBytes: 500000 }); if (res && typeof res.status === 'number' && res.status >= 200 && res.status < 300) { this.robots = (robotsParser as any)(robotsUrl, res.body); @@ -226,25 +251,76 @@ export class Crawler { } async seedQueue(): Promise<void> { - // Seed from Sitemap + // Seed from startUrl first to ensure it's prioritized in the queue + this.addToQueue(this.startUrl, 0); + + const sitemapsToFetch = new Set<string>(); + + // 1. Explicitly configured sitemap if (this.options.sitemap) { - try { - const sitemapUrl = this.options.sitemap === 'true' ? new URL('/sitemap.xml', this.rootOrigin).toString() : this.options.sitemap; - if (sitemapUrl.startsWith('http')) { - this.context.emit({ type: 'info', message: 'Fetching sitemap', context: { url: sitemapUrl } }); - const sitemapUrls = await this.sitemapFetcher!.fetch(sitemapUrl); - for (const u of sitemapUrls) { - const normalized = normalizeUrl(u, '', this.options); - if (normalized) this.addToQueue(normalized, 0, { discovered_via_sitemap: 1 }); - } - } - } catch (e) { - this.context.emit({ type: 'warn', message: 'Sitemap fetch failed', context: e }); + const explicitUrl = this.options.sitemap === 'true' || (this.options.sitemap as any) === true + ? new URL('/sitemap.xml', this.rootOrigin).toString() + : this.options.sitemap; + + if (typeof explicitUrl === 'string' && explicitUrl.startsWith('http')) { + sitemapsToFetch.add(explicitUrl); } } - // Seed from startUrl - this.addToQueue(this.startUrl, 0); + // 2. Discover sitemaps from robots.txt (unless explicitly disabled) + // Only auto-fetch on the FIRST real crawl (full/incremental). + // page --live reuses partial snapshots and should NOT trigger sitemap fetch. + const isFirstCrawl = !this.snapshotRepo?.hasFullCrawl(this.siteId!); + if (this.options.sitemap !== false && (this.options.sitemap || isFirstCrawl) && this.robots) { + const robotsSitemaps = this.robots.getSitemaps(); + for (const s of robotsSitemaps) { + if (s) sitemapsToFetch.add(s); + } + } + + // Process all discovered sitemaps in background + if (sitemapsToFetch.size > 0) { + for (const sitemapUrl of sitemapsToFetch) { + this.pendingSitemaps++; + // KICK OFF BACKGROUND TASK (Un-awaited) + (async () => { + try { + this.context.emit({ type: 'debug', message: 'Fetching sitemap in background', context: { url: sitemapUrl } }); + const sitemapUrls = await this.sitemapFetcher!.fetch(sitemapUrl); + + if (sitemapUrls.length > 0) { + this.context.emit({ type: 'debug', message: `Mapping ${sitemapUrls.length} URLs from sitemap... (Background)` }); + const sitemapEntries = sitemapUrls.map(u => { + const normalized = normalizeUrl(u, this.rootOrigin, this.options); + if (!normalized) return null; + const path = UrlUtil.toPath(normalized, this.rootOrigin); + return { + site_id: this.siteId!, + normalized_url: path, + first_seen_snapshot_id: this.snapshotId!, + last_seen_snapshot_id: this.snapshotId!, + discovered_via_sitemap: 1, + depth: 0, + http_status: 0 + }; + }).filter((p): p is any => p !== null); + + // Bulk register to DB + this.pageRepo!.upsertMany(sitemapEntries); + + // Add to queue for Actual Crawling + for (const entry of sitemapEntries) { + this.addToQueue(entry.normalized_url, 0, { discovered_via_sitemap: 1 }); + } + } + } catch (e) { + this.context.emit({ type: 'warn', message: 'Sitemap fetch failed', context: { url: sitemapUrl, error: String(e) } }); + } finally { + this.pendingSitemaps--; + } + })(); + } + } } private bufferPage(url: string, depth: number, status: number, data: any = {}): void { @@ -357,18 +433,23 @@ export class Crawler { return { snapshot_id: this.snapshotId!, page_id: pageId, - authority_score: null, - hub_score: null, - pagerank: null, - pagerank_score: null, - link_role: null, crawl_status: null, word_count: null, thin_content_score: null, external_link_ratio: null, - orphan_score: null, + pagerank_score: null, + hub_score: null, + auth_score: null, + link_role: null, duplicate_cluster_id: null, duplicate_type: null, + cluster_id: null, + soft404_score: null, + heading_score: null, + orphan_score: null, + orphan_type: null, + impact_level: null, + heading_data: null, is_cluster_primary: 0, ...item.data }; @@ -415,33 +496,41 @@ export class Crawler { } private handleCachedResponse(url: string, finalUrl: string, depth: number, prevNode: GraphNode): void { - this.bufferPage(finalUrl, depth, 200, { + const path = UrlUtil.toPath(url, this.rootOrigin); + const finalPath = UrlUtil.toPath(finalUrl, this.rootOrigin); + this.bufferPage(finalPath, depth, prevNode.status, { html: prevNode.html, canonical_url: prevNode.canonical, + noindex: prevNode.noindex ? 1 : 0, + nofollow: prevNode.nofollow ? 1 : 0, content_hash: prevNode.contentHash, simhash: prevNode.simhash, etag: prevNode.etag, - last_modified: prevNode.lastModified, - noindex: prevNode.noindex ? 1 : 0, - nofollow: prevNode.nofollow ? 1 : 0 - }); - this.bufferMetrics(finalUrl, { - crawl_status: 'cached' + last_modified: prevNode.lastModified }); + this.bufferMetrics(finalPath, { + crawl_status: 'cached', + word_count: prevNode.wordCount, + thin_content_score: prevNode.thinContentScore, + external_link_ratio: prevNode.externalLinkRatio + }); // Re-discovery links from previous graph to continue crawling if needed const prevLinks = this.options.previousGraph?.getEdges() - .filter(e => e.source === url) + .filter(e => e.source === path) .map(e => e.target); if (prevLinks) { for (const link of prevLinks) { - const normalizedLink = normalizeUrl(link, '', this.options); - if (normalizedLink && normalizedLink !== finalUrl) { - this.bufferPage(normalizedLink, depth + 1, 0); - this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal'); - if (this.shouldEnqueue(normalizedLink, depth + 1)) { - this.addToQueue(normalizedLink, depth + 1); + const normalizedLink = normalizeUrl(link, this.rootOrigin, this.options); + if (normalizedLink) { + const path = UrlUtil.toPath(normalizedLink, this.rootOrigin); + if (path !== url) { + this.bufferPage(path, depth + 1, 0); + this.bufferEdge(url, path, 1.0, 'internal'); + if (this.shouldEnqueue(path, depth + 1)) { + this.addToQueue(path, depth + 1); + } } } } @@ -450,29 +539,31 @@ export class Crawler { private handleRedirects(chain: FetchResult['redirectChain'], depth: number): void { for (const step of chain) { - const source = normalizeUrl(step.url, '', this.options); - const target = normalizeUrl(step.target, '', this.options); - if (source && target) { - this.bufferPage(source, depth, step.status); - this.bufferPage(target, depth, 0); - this.bufferEdge(source, target); + const sourceAbs = normalizeUrl(step.url, this.rootOrigin, this.options); + const targetAbs = normalizeUrl(step.target, this.rootOrigin, this.options); + if (sourceAbs && targetAbs) { + const sourcePath = UrlUtil.toPath(sourceAbs, this.rootOrigin); + const targetPath = UrlUtil.toPath(targetAbs, this.rootOrigin); + this.bufferPage(sourcePath, depth, step.status); + this.bufferPage(targetPath, depth, 0); + this.bufferEdge(sourcePath, targetPath); } } } - private handleSuccessResponse(res: FetchResult, finalUrl: string, depth: number, isBlocked: boolean = false): void { + private handleSuccessResponse(res: FetchResult, path: string, absoluteUrl: string, depth: number, isBlocked: boolean = false): void { const contentTypeHeader = res.headers['content-type']; const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || ''); if (!contentType || !contentType.toLowerCase().includes('text/html')) { - this.bufferPage(finalUrl, depth, typeof res.status === 'number' ? res.status : 0); + this.bufferPage(path, depth, typeof res.status === 'number' ? res.status : 0); return; } - const parseResult = this.parser!.parse(res.body, finalUrl, res.status as number); + const parseResult = this.parser!.parse(res.body, absoluteUrl, res.status as number); if (this.registry) { this.registry.runHook('onPageParsed', this.context as any, { - url: finalUrl, + url: absoluteUrl, status: res.status, depth: depth, headers: res.headers, @@ -480,7 +571,7 @@ export class Crawler { }); } - this.bufferPage(finalUrl, depth, res.status as number, { + this.bufferPage(path, depth, res.status as number, { html: parseResult.html, canonical_url: parseResult.canonical || undefined, noindex: parseResult.noindex ? 1 : 0, @@ -494,26 +585,32 @@ export class Crawler { try { const contentAnalysis = analyzeContent(parseResult.html); - const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, this.rootOrigin); + const linkAnalysis = analyzeLinks(parseResult.html, absoluteUrl, this.rootOrigin); const thinScore = calculateThinContentScore(contentAnalysis, 0); - this.bufferMetrics(finalUrl, { + this.bufferMetrics(path, { crawl_status: isBlocked ? 'blocked_by_robots' : 'fetched', word_count: contentAnalysis.wordCount, thin_content_score: thinScore, external_link_ratio: linkAnalysis.externalRatio }); } catch (e) { - this.context.emit({ type: 'error', message: 'Error calculating per-page metrics', error: e, context: { url: finalUrl } }); + this.context.emit({ type: 'error', message: 'Error calculating per-page metrics', error: e, context: { url: absoluteUrl } }); } for (const linkItem of parseResult.links) { - const normalizedLink = normalizeUrl(linkItem.url, '', this.options); - if (normalizedLink && normalizedLink !== finalUrl) { - this.bufferPage(normalizedLink, depth + 1, 0); - this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal'); - if (this.shouldEnqueue(normalizedLink, depth + 1)) { - this.addToQueue(normalizedLink, depth + 1); + const normalizedLink = normalizeUrl(linkItem.url, absoluteUrl, this.options); + if (normalizedLink) { + const targetPath = UrlUtil.toPath(normalizedLink, this.rootOrigin); + + if (targetPath !== path) { + const isInternal = UrlUtil.isInternal(normalizedLink, this.rootOrigin); + this.bufferPage(targetPath, depth + 1, 0); + this.bufferEdge(path, targetPath, 1.0, isInternal ? 'internal' : 'external'); + + if (isInternal && this.shouldEnqueue(targetPath, depth + 1)) { + this.addToQueue(targetPath, depth + 1); + } } } } @@ -526,15 +623,22 @@ export class Crawler { return; } + // Convert stored path to absolute URL for fetching. + // External/subdomain URLs are already absolute (UrlUtil.toPath returns them as-is). + const fetchUrl = UrlUtil.toAbsolute(url, this.rootOrigin); + try { const prevNode = this.options.previousGraph?.nodes.get(url); - const res = await this.fetchPage(url, depth, prevNode); + const res = await this.fetchPage(fetchUrl, depth, prevNode); if (!res) return; - const finalUrl = normalizeUrl(res.finalUrl, '', this.options); + const finalUrl = normalizeUrl(res.finalUrl, this.rootOrigin, this.options); if (!finalUrl) return; + const fullUrl = finalUrl; // Already absolute + const finalPath = UrlUtil.toPath(finalUrl, this.rootOrigin); + if (res.status === 304 && prevNode) { this.handleCachedResponse(url, finalUrl, depth, prevNode); return; @@ -545,18 +649,18 @@ export class Crawler { const isStringStatus = typeof res.status === 'string'; if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) { const statusNum = typeof res.status === 'number' ? res.status : 0; - this.bufferPage(finalUrl, depth, statusNum, { + this.bufferPage(finalPath, depth, statusNum, { security_error: isStringStatus ? res.status : undefined, retries: res.retries }); - this.bufferMetrics(finalUrl, { + this.bufferMetrics(finalPath, { crawl_status: isStringStatus ? res.status : 'fetched_error' }); return; } if (res.status === 200) { - this.handleSuccessResponse(res, finalUrl, depth, isBlocked); + this.handleSuccessResponse(res, finalPath, fullUrl, depth, isBlocked); } } catch (e) { this.context.emit({ type: 'crawl:error', url, error: String(e), depth }); @@ -564,8 +668,12 @@ export class Crawler { } async run(): Promise<number> { - await this.initialize(); + // 1. Setup fetcher and basic modules this.setupModules(); + + // 2. Initialize repositories, resolve URL (SSL/WWW), and set up site context + await this.initialize(); + if (this.options.robots) { this.robots = this.options.robots; } else { @@ -575,7 +683,7 @@ export class Crawler { return new Promise((resolve) => { const checkDone = async () => { - if (this.queue.length === 0 && this.active === 0) { + if (this.queue.length === 0 && this.active === 0 && this.pendingSitemaps === 0) { await this.flushAll(); this.snapshotRepo!.updateSnapshotStatus(this.snapshotId!, 'completed', { limit_reached: this.reachedLimit ? 1 : 0 @@ -612,11 +720,12 @@ export class Crawler { const item = this.queue.shift()!; if (this.visited.has(item.url)) continue; - // Robust robots check: if path doesn't end in /, check both /path and /path/ - // to handle cases where normalization stripped a slash that robots.txt relies on. + // Robust robots check: reconstruct absolute URL since robots-parser needs full URLs, + // not root-relative paths. Also check /path/ variant in case robots.txt uses trailing slash. + const absUrlForRobots = UrlUtil.toAbsolute(item.url, this.rootOrigin); const isBlocked = this.robots && ( - !this.robots.isAllowed(item.url, 'crawlith') || - (!item.url.endsWith('/') && !this.robots.isAllowed(item.url + '/', 'crawlith')) + !this.robots.isAllowed(absUrlForRobots, 'crawlith') || + (!absUrlForRobots.endsWith('/') && !this.robots.isAllowed(absUrlForRobots + '/', 'crawlith')) ); if (isBlocked) { diff --git a/packages/core/src/crawler/fetcher.ts b/packages/core/src/crawler/fetcher.ts index 90bc0dd..3bcca17 100644 --- a/packages/core/src/crawler/fetcher.ts +++ b/packages/core/src/crawler/fetcher.ts @@ -7,7 +7,7 @@ import { ResponseLimiter } from '../core/network/responseLimiter.js'; import { RedirectController } from '../core/network/redirectController.js'; import { ProxyAdapter } from '../core/network/proxyAdapter.js'; import { ScopeManager } from '../core/scope/scopeManager.js'; -import { version } from '../utils/version.js'; +import { DEFAULTS } from '../constants.js'; export interface RedirectStep { url: string; @@ -45,7 +45,7 @@ export interface FetchOptions { } export class Fetcher { - private userAgent = 'crawlith/1.0'; + private userAgent: string = DEFAULTS.USER_AGENT; private rateLimiter: RateLimiter; private proxyAdapter: ProxyAdapter; private secureDispatcher: Dispatcher; @@ -59,7 +59,7 @@ export class Fetcher { maxRedirects?: number; userAgent?: string; } = {}) { - this.rateLimiter = new RateLimiter(options.rate || 2); + this.rateLimiter = new RateLimiter(options.rate || DEFAULTS.RATE_LIMIT); this.proxyAdapter = new ProxyAdapter(options.proxyUrl); if (this.proxyAdapter.dispatcher) { @@ -69,12 +69,12 @@ export class Fetcher { } this.scopeManager = options.scopeManager; - this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11); - this.userAgent = options.userAgent || `crawlith/${version}`; + this.maxRedirects = Math.min(options.maxRedirects ?? DEFAULTS.MAX_REDIRECTS, DEFAULTS.MAX_REDIRECTS_LIMIT); + this.userAgent = options.userAgent || DEFAULTS.USER_AGENT; } async fetch(url: string, options: FetchOptions = {}): Promise<FetchResult> { - const maxBytes = options.maxBytes || 2000000; + const maxBytes = options.maxBytes || DEFAULTS.MAX_BYTES; const redirectChain: RedirectStep[] = []; const redirectController = new RedirectController(this.maxRedirects, url); diff --git a/packages/core/src/crawler/metricsRunner.ts b/packages/core/src/crawler/metricsRunner.ts index c95b315..b38fac2 100644 --- a/packages/core/src/crawler/metricsRunner.ts +++ b/packages/core/src/crawler/metricsRunner.ts @@ -8,8 +8,40 @@ import { EngineContext } from '../events.js'; import { Graph } from '../graph/graph.js'; +import { PageRankService } from '../graph/pagerank.js'; +import { HITSService } from '../graph/hits.js'; +import { TrapDetector } from './trap.js'; +import { ClusteringService } from '../analysis/clustering.js'; +import { DuplicateService } from '../analysis/duplicate.js'; +import { annotateOrphans } from '../analysis/orphan.js'; +import { Soft404Service } from '../analysis/soft404.js'; +import { HeadingHealthService } from '../analysis/heading.js'; +import { HealthService } from '../scoring/health.js'; +import { analyzeContent } from '../analysis/content.js'; +import { load } from 'cheerio'; -export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, context?: EngineContext, limitReached: boolean = false, graphInstance?: Graph) { +export interface PostCrawlOptions { + context?: EngineContext; + limitReached?: boolean; + graphInstance?: Graph; + clustering?: boolean; + clusterThreshold?: number; + minClusterSize?: number; + health?: boolean; + computePagerank?: boolean; + computeHits?: boolean; + heading?: boolean; + orphans?: boolean; + orphanSeverity?: 'low' | 'medium' | 'high' | boolean; + includeSoftOrphans?: boolean; + minInbound?: number; + rootOrigin?: string; +} + +export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, options: PostCrawlOptions = {}) { + const context = options.context; + const limitReached = options.limitReached || false; + const graphInstance = options.graphInstance; const db = getDb(); const metricsRepo = new MetricsRepository(db); const snapshotRepo = new SnapshotRepository(db); @@ -43,37 +75,160 @@ export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, contex + emit({ type: 'metrics:start', phase: 'Running core algorithms' }); + + // 1. Graph Algorithms + const prResults = options.computePagerank ? new PageRankService().evaluate(graph) : new Map(); + const hitsResults = options.computeHits ? new HITSService().evaluate(graph, { iterations: 20 }) : new Map(); + + // 2. Crawler Safety + new TrapDetector().analyze(graph); + + // 3. Analysis / Intelligence + if (options.clustering) { + const contentClusters = new ClusteringService().detectContentClusters(graph, options.clusterThreshold, options.minClusterSize); + if (contentClusters.length > 0) { + const insertCluster = db.prepare(` + INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix) + VALUES (@id, @snapshot_id, @count, @primary_url, @risk, @shared_path_prefix) + `); + const insertContentTx = db.transaction((clusters: any[]) => { + for (const c of clusters) { + insertCluster.run({ + id: c.id, + snapshot_id: snapshotId, + count: c.count, + primary_url: c.primaryUrl, + risk: c.risk, + shared_path_prefix: c.sharedPathPrefix ?? null + }); + } + }); + insertContentTx(contentClusters); + } + } + const duplicateClusters = new DuplicateService().detectDuplicates(graph, { collapse: false }); + if (duplicateClusters.length > 0) { + const insertCluster = db.prepare(` + INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity) + VALUES (@id, @snapshot_id, @type, @size, @representative, @severity) + `); + const insertDuplicateTx = db.transaction((clusters: any[]) => { + for (const c of clusters) { + insertCluster.run({ + id: c.id, + snapshot_id: snapshotId, + type: c.type, // valid: 'exact' | 'near' | 'template_heavy' + size: c.size, + representative: c.representative, + severity: c.severity || 'low' + }); + } + }); + insertDuplicateTx(duplicateClusters); + } + + let annotatedNodes: any[] = []; + if (options.orphans) { + const orphanOptions = { + enabled: true, + severityEnabled: !!options.orphanSeverity || options.orphanSeverity === undefined, + includeSoftOrphans: options.includeSoftOrphans ?? true, + minInbound: options.minInbound ?? 2 + }; + annotatedNodes = annotateOrphans(graph.getNodes(), graph.getEdges(), orphanOptions) as any[]; + } + + const soft404Service = new Soft404Service(); + const headingService = new HeadingHealthService(); + // Pre-calculate heading health for all nodes with HTML + let headingPayloads = new Map(); + if (options.heading) { + const result = headingService.evaluateNodes(graph.getNodes()); + headingPayloads = result.payloadsByUrl; + } + + // Apply signals to nodes + for (const node of graph.getNodes()) { + const pr = prResults.get(node.url); + if (pr) node.pagerankScore = pr.score; + + const hits = hitsResults.get(node.url); + if (hits) { + node.authScore = hits.authority_score; + node.hubScore = hits.hub_score; + node.linkRole = hits.link_role; + } + + if (options.orphans) { + const annotated = annotatedNodes.find((n: any) => n.url === node.url); + if (annotated) { + node.orphanScore = annotated.orphanSeverity; + node.orphanType = annotated.orphanType; + node.impactLevel = annotated.impactLevel; + } + } + + if (options.heading) { + const heading = headingPayloads.get(node.url); + if (heading) { + node.headingScore = heading.score; + node.headingData = JSON.stringify(heading); + } + } + + if (node.html) { + const soft404 = soft404Service.analyze(node.html, node.outLinks); + node.soft404Score = soft404.score; + + const $ = load(node.html); + const content = analyzeContent($); + node.wordCount = content.wordCount; + } + } + emit({ type: 'metrics:start', phase: 'Updating metrics in DB' }); - const nodes = graph.getNodes(); // Pre-fetch all page IDs to avoid N+1 queries - // Use getPagesIdentityBySnapshot to avoid loading full page content (HTML) into memory again - const pages = pageRepo.getPagesIdentityBySnapshot(snapshotId); + const pagesIdentity = pageRepo.getPagesIdentityBySnapshot(snapshotId); const urlToId = new Map<string, number>(); - for (const p of pages) { + for (const p of pagesIdentity) { urlToId.set(p.normalized_url, p.id); } + const metricsToSave = graph.getNodes().map(node => { + const pageId = urlToId.get(node.url); + if (!pageId) return null; + return { + snapshot_id: snapshotId, + page_id: pageId, + crawl_status: node.crawlStatus ?? null, + word_count: node.wordCount ?? null, + thin_content_score: node.thinContentScore ?? null, + external_link_ratio: node.externalLinkRatio ?? null, + pagerank_score: node.pagerankScore ?? null, + hub_score: node.hubScore ?? null, + auth_score: node.authScore ?? null, + link_role: node.linkRole ?? null, + duplicate_cluster_id: (node as any).duplicateClusterId ?? null, + duplicate_type: (node as any).duplicateType ?? null, + cluster_id: (node as any).clusterId ?? null, + soft404_score: node.soft404Score ?? null, + heading_score: node.headingScore ?? null, + orphan_score: node.orphanScore ?? null, + orphan_type: node.orphanType ?? null, + impact_level: node.impactLevel ?? null, + heading_data: node.headingData ?? null, + is_cluster_primary: (node as any).isClusterPrimary ? 1 : 0 + }; + }).filter(m => m !== null); - const tx = db.transaction(() => { - for (const node of nodes) { - const pageId = urlToId.get(node.url); - if (!pageId) continue; - - - metricsRepo.insertMetrics({ - snapshot_id: snapshotId, - page_id: pageId, + metricsRepo.insertMany(metricsToSave as any); - crawl_status: node.crawlStatus ?? null, - word_count: node.wordCount ?? null, - thin_content_score: node.thinContentScore ?? null, - external_link_ratio: node.externalLinkRatio ?? null, - orphan_score: node.orphanScore ?? null - }); - - // Update page-level crawl trap data + // Update page-level metadata in transaction + const tx = db.transaction(() => { + for (const node of graph.getNodes()) { if (node.crawlTrapFlag || node.redirectChain?.length || node.bytesReceived) { pageRepo.upsertPage({ site_id: snapshot.site_id, @@ -87,18 +242,31 @@ export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, contex }); } } - - }); tx(); emit({ type: 'metrics:start', phase: 'Computing aggregate stats' }); const metrics = calculateMetrics(graph, maxDepth); + // Compute health score if enabled + let healthScore: number | null = null; + if (options.health) { + try { + const rootOrigin = options.rootOrigin ?? ''; + const healthService = new HealthService(); + const issues = healthService.collectCrawlIssues(graph, metrics, rootOrigin); + const breakdown = healthService.calculateHealthScore(metrics.totalPages, issues); + healthScore = breakdown.score; + } catch (e) { + emit({ type: 'error', message: 'Error computing health score', error: e }); + } + } + snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', { node_count: metrics.totalPages, edge_count: metrics.totalEdges, - limit_reached: limitReached ? 1 : 0 + limit_reached: limitReached ? 1 : 0, + ...(healthScore !== null ? { health_score: healthScore } : {}) }); emit({ type: 'metrics:complete', durationMs: 0 }); diff --git a/packages/core/src/crawler/normalize.ts b/packages/core/src/crawler/normalize.ts index f8a28de..7db154a 100644 --- a/packages/core/src/crawler/normalize.ts +++ b/packages/core/src/crawler/normalize.ts @@ -3,6 +3,7 @@ */ export interface NormalizeOptions { stripQuery?: boolean; + toPath?: boolean; } const TRACKING_PARAMS = new Set([ @@ -18,7 +19,7 @@ const TRACKING_PARAMS = new Set([ const SKIP_EXTENSIONS = new Set([ '.pdf', '.jpg', '.png', '.svg', '.webp', '.gif', - '.zip', '.xml', '.json', '.mp4' + '.zip', '.xml', '.json', '.mp4', '.avif', '.ics' ]); export function normalizeUrl(input: string, base: string, options: NormalizeOptions = {}): string | null { @@ -89,6 +90,7 @@ export function normalizeUrl(input: string, base: string, options: NormalizeOpti } u.pathname = pathname; + const finalUrl = u.toString(); // 9. Skip non-HTML assets by extension const lastDotIndex = u.pathname.lastIndexOf('.'); @@ -99,10 +101,68 @@ export function normalizeUrl(input: string, base: string, options: NormalizeOpti } } - // 10. Return final string - return u.toString(); + // 10. Return path if requested + if (options.toPath) { + return u.pathname + u.search; + } + + // 11. Return final string + return finalUrl; } catch (_e) { return null; } } + +/** + * Utility for converting between absolute URLs and relative paths + * primarily used for database storage. + */ +export class UrlUtil { + /** + * Converts a full URL to a root-relative path if it matches the origin. + * If it doesn't match the origin, it's considered external and kept absolute. + */ + static toPath(urlStr: string, origin: string): string { + try { + const url = new URL(urlStr); + const originUrl = new URL(origin); + + if (url.origin === originUrl.origin) { + return url.pathname + url.search; + } + return urlStr; + } catch { + return urlStr; + } + } + + /** + * Converts a root-relative path back to an absolute URL relative to the origin. + * If the input is already an absolute URL, it is returned as-is. + */ + static toAbsolute(pathOrUrl: string, origin: string): string { + if (pathOrUrl.startsWith('http://') || pathOrUrl.startsWith('https://')) { + return pathOrUrl; + } + try { + return new URL(pathOrUrl, origin).toString(); + } catch { + return pathOrUrl; + } + } + + /** + * Determines if a URL (or path) is internal relative to the origin. + */ + static isInternal(pathOrUrl: string, origin: string): boolean { + if (!pathOrUrl.startsWith('http')) return true; + try { + const url = new URL(pathOrUrl); + const originUrl = new URL(origin); + return url.origin === originUrl.origin; + } catch { + return false; + } + } +} diff --git a/packages/core/src/crawler/resolver.ts b/packages/core/src/crawler/resolver.ts new file mode 100644 index 0000000..e2f2f1d --- /dev/null +++ b/packages/core/src/crawler/resolver.ts @@ -0,0 +1,82 @@ +import { Fetcher } from './fetcher.js'; +import { SiteRepository, Site } from '../db/repositories/SiteRepository.js'; +import { getDb } from '../db/index.js'; + +export interface ResolvedUrl { + url: string; + site: Site; +} + +export class UrlResolver { + private siteRepo: SiteRepository; + + constructor() { + this.siteRepo = new SiteRepository(getDb()); + } + + async resolve(inputUrl: string, fetcher: Fetcher): Promise<ResolvedUrl> { + const hasProtocol = inputUrl.startsWith('http://') || inputUrl.startsWith('https://'); + const workingUrl = hasProtocol ? inputUrl : `https://${inputUrl}`; + + let hostname: string; + try { + hostname = new URL(workingUrl).hostname; + } catch { + throw new Error(`Invalid URL or domain: ${inputUrl}`); + } + + const domain = hostname.replace(/^www\./, ''); + let site = this.siteRepo.firstOrCreateSite(domain); + + // If protocol was omitted, we use our discovery logic or stored preference + if (!hasProtocol) { + if (site.ssl !== null && site.preferred_url) { + return { + url: site.preferred_url, + site + }; + } + + // No protocol provided and no stored preference: Probe HTTPS first + try { + const res = await fetcher.fetch(`https://${hostname}/`); + if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) { + const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0; + this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl }); + + // Refresh site object + site = this.siteRepo.getSiteById(site.id)!; + return { url: res.finalUrl, site }; + } + } catch { + // Fallback to HTTP + } + + // Try HTTP + try { + const res = await fetcher.fetch(`http://${hostname}/`); + if (typeof res.status === 'number' && res.status >= 200 && res.status < 400) { + const isSsl = res.finalUrl.startsWith('https:') ? 1 : 0; + this.siteRepo.updateSitePreference(site.id, { preferred_url: res.finalUrl, ssl: isSsl }); + + site = this.siteRepo.getSiteById(site.id)!; + return { url: res.finalUrl, site }; + } + } catch { + // If both fail, we still default to the provided input as https + return { url: workingUrl, site }; + } + } + + // Protocol was provided, we just return it but ensure site is in sync if it's the first time + if (site.ssl === null) { + this.siteRepo.updateSitePreference(site.id, { + preferred_url: inputUrl, + ssl: inputUrl.startsWith('https:') ? 1 : 0 + }); + site = this.siteRepo.getSiteById(site.id)!; + } + + return { url: inputUrl, site }; + } +} diff --git a/packages/core/src/crawler/sitemap.ts b/packages/core/src/crawler/sitemap.ts index 0f986b8..b1083d8 100644 --- a/packages/core/src/crawler/sitemap.ts +++ b/packages/core/src/crawler/sitemap.ts @@ -1,11 +1,11 @@ -import { request } from 'undici'; import * as cheerio from 'cheerio'; import pLimit from 'p-limit'; import { normalizeUrl } from './normalize.js'; import { EngineContext } from '../events.js'; +import { Fetcher } from './fetcher.js'; export class Sitemap { - constructor(private context?: EngineContext) { } + constructor(private context?: EngineContext, private fetcher?: Fetcher) { } /** * Fetches and parses a sitemap (or sitemap index) to extract URLs. @@ -28,15 +28,17 @@ export class Sitemap { if (visited.size > 50) return; try { - const res = await request(url, { - maxRedirections: 3, - headers: { 'User-Agent': 'crawlith/1.0' }, - headersTimeout: 10000, - bodyTimeout: 10000 - }); + const res = this.fetcher + ? await this.fetcher.fetch(url, { maxBytes: 5000000 }) + : await (async () => { + const { request } = await import('undici'); + const r = await request(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }); + const b = await r.body.text(); + return { status: r.statusCode, body: b }; + })(); - if (res.statusCode >= 200 && res.statusCode < 300) { - const xml = await res.body.text(); + if (typeof res.status === 'number' && res.status >= 200 && res.status < 300) { + const xml = res.body; // Basic validation: must verify it looks like XML if (!xml.trim().startsWith('<')) return; @@ -68,11 +70,9 @@ export class Sitemap { } }); } - } else { - await res.body.dump(); } } catch (e) { - this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url}`, context: e }); + this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url} (${String(e)})`, context: e }); } } } diff --git a/packages/plugins/crawl-trap-analyzer/src/trap.ts b/packages/core/src/crawler/trap.ts similarity index 86% rename from packages/plugins/crawl-trap-analyzer/src/trap.ts rename to packages/core/src/crawler/trap.ts index 5546e61..4cb8fff 100644 --- a/packages/plugins/crawl-trap-analyzer/src/trap.ts +++ b/packages/core/src/crawler/trap.ts @@ -86,6 +86,23 @@ export class TrapDetector { return { risk, type }; } + /** + * Iterates over all nodes in the graph and flags potential traps. + */ + analyze(graph: any) { + const nodes = graph.getNodes(); + for (const node of nodes) { + if (node.status === 200 || node.status === 0) { + const res = this.checkTrap(node.url, node.depth || 0); + if (res.risk > 0.4) { + node.crawlTrapFlag = true; + node.crawlTrapRisk = res.risk; + node.trapType = res.type; + } + } + } + } + /** * Resets internal state (useful for multi-crawl sessions if needed) */ diff --git a/packages/core/src/db/CrawlithDB.ts b/packages/core/src/db/CrawlithDB.ts index 44df382..147a556 100644 --- a/packages/core/src/db/CrawlithDB.ts +++ b/packages/core/src/db/CrawlithDB.ts @@ -192,7 +192,8 @@ export class CrawlithDB { } public getPageIdByUrl(snapshotId: number | string, url: string): number | null { - const normalized = normalizeUrl(url, '', { stripQuery: false }); + // Find by path. In standard crawling, we use root-relative paths like /engineering-computer-science + const normalized = normalizeUrl(url, '', { stripQuery: false, toPath: true }); if (!normalized) return null; const row = this.statements.getPageIdByUrl.get(snapshotId, normalized) as { id: number } | undefined; diff --git a/packages/core/src/db/graphLoader.ts b/packages/core/src/db/graphLoader.ts index 904f51c..eaba16f 100644 --- a/packages/core/src/db/graphLoader.ts +++ b/packages/core/src/db/graphLoader.ts @@ -81,7 +81,16 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph { wordCount: m?.word_count != null ? m.word_count : undefined, thinContentScore: m?.thin_content_score != null ? m.thin_content_score : undefined, externalLinkRatio: m?.external_link_ratio != null ? m.external_link_ratio : undefined, - orphanScore: m?.orphan_score != null ? m.orphan_score : undefined, + pagerankScore: m?.pagerank_score ?? undefined, + hubScore: m?.hub_score ?? undefined, + authScore: m?.auth_score ?? undefined, + linkRole: m?.link_role ?? undefined, + soft404Score: m?.soft404_score ?? undefined, + headingScore: m?.heading_score ?? undefined, + orphanScore: m?.orphan_score ?? undefined, + orphanType: m?.orphan_type ?? undefined, + impactLevel: m?.impact_level ?? undefined, + headingData: m?.heading_data ?? undefined, }); } diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts index dead079..e2c3738 100644 --- a/packages/core/src/db/index.ts +++ b/packages/core/src/db/index.ts @@ -10,7 +10,6 @@ let crawlithDbInstance: CrawlithDB | null = null; export * from './repositories/SiteRepository.js'; export * from './repositories/SnapshotRepository.js'; export * from './CrawlithDB.js'; -export { initSchema } from './schema.js'; export function getDbPath(): string { if (process.env.NODE_ENV === 'test') { diff --git a/packages/core/src/db/migrations.ts b/packages/core/src/db/migrations.ts index ecd2d34..d1fb353 100644 --- a/packages/core/src/db/migrations.ts +++ b/packages/core/src/db/migrations.ts @@ -6,6 +6,8 @@ export function runBaseMigrations(db: Database) { CREATE TABLE IF NOT EXISTS sites ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, + preferred_url TEXT, + ssl INTEGER, created_at TEXT DEFAULT (datetime('now')), settings_json TEXT, is_active INTEGER DEFAULT 1 @@ -101,8 +103,18 @@ export function runBaseMigrations(db: Database) { thin_content_score REAL, external_link_ratio REAL, orphan_score INTEGER, + pagerank_score REAL, + hub_score REAL, + auth_score REAL, + link_role TEXT, duplicate_cluster_id TEXT, duplicate_type TEXT, + cluster_id INTEGER, + soft404_score REAL, + heading_score REAL, + orphan_type TEXT, + impact_level TEXT, + heading_data TEXT, is_cluster_primary INTEGER DEFAULT 0, PRIMARY KEY(snapshot_id, page_id), FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE, @@ -165,4 +177,25 @@ export function runBaseMigrations(db: Database) { db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_snapshot ON plugin_reports(snapshot_id);`); db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_composite ON plugin_reports(snapshot_id, plugin_name);`); + + // Migrations for metrics + const metricsCols = [ + ['pagerank_score', 'REAL'], + ['hub_score', 'REAL'], + ['auth_score', 'REAL'], + ['link_role', 'TEXT'], + ['cluster_id', 'INTEGER'], + ['soft404_score', 'REAL'], + ['heading_score', 'REAL'], + ['orphan_type', 'TEXT'], + ['impact_level', 'TEXT'], + ['heading_data', 'TEXT'], + ]; + for (const [col, type] of metricsCols) { + try { db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`); } catch { /* ignore */ } + } + + // Final site column migrations + try { db.exec('ALTER TABLE sites ADD COLUMN preferred_url TEXT'); } catch { /* ignore */ } + try { db.exec('ALTER TABLE sites ADD COLUMN ssl INTEGER'); } catch { /* ignore */ } } diff --git a/packages/core/src/db/repositories/MetricsRepository.ts b/packages/core/src/db/repositories/MetricsRepository.ts index e775a27..a9e80cc 100644 --- a/packages/core/src/db/repositories/MetricsRepository.ts +++ b/packages/core/src/db/repositories/MetricsRepository.ts @@ -9,8 +9,20 @@ export interface DbMetrics { word_count: number | null; thin_content_score: number | null; external_link_ratio: number | null; + pagerank_score: number | null; + hub_score: number | null; + auth_score: number | null; + link_role: string | null; + duplicate_cluster_id: string | null; + duplicate_type: string | null; + cluster_id: number | null; + soft404_score: number | null; + heading_score: number | null; orphan_score: number | null; - + orphan_type: string | null; + impact_level: string | null; + heading_data: string | null; + is_cluster_primary: number | null; } export class MetricsRepository { @@ -23,11 +35,19 @@ export class MetricsRepository { INSERT OR REPLACE INTO metrics ( snapshot_id, page_id, crawl_status, word_count, thin_content_score, external_link_ratio, - orphan_score + pagerank_score, hub_score, auth_score, link_role, + duplicate_cluster_id, duplicate_type, cluster_id, + soft404_score, heading_score, + orphan_score, orphan_type, impact_level, + heading_data, is_cluster_primary ) VALUES ( @snapshot_id, @page_id, @crawl_status, @word_count, @thin_content_score, @external_link_ratio, - @orphan_score + @pagerank_score, @hub_score, @auth_score, @link_role, + @duplicate_cluster_id, @duplicate_type, @cluster_id, + @soft404_score, @heading_score, + @orphan_score, @orphan_type, @impact_level, + @heading_data, @is_cluster_primary ) `); } diff --git a/packages/core/src/db/repositories/SiteRepository.ts b/packages/core/src/db/repositories/SiteRepository.ts index 8186bfd..f36d01c 100644 --- a/packages/core/src/db/repositories/SiteRepository.ts +++ b/packages/core/src/db/repositories/SiteRepository.ts @@ -3,6 +3,8 @@ import { Database } from 'better-sqlite3'; export interface Site { id: number; domain: string; + preferred_url: string | null; + ssl: number | null; created_at: string; settings_json: string | null; is_active: number; @@ -29,6 +31,11 @@ export class SiteRepository { return info.lastInsertRowid as number; } + updateSitePreference(id: number, prefs: { preferred_url: string; ssl: number }): void { + const stmt = this.db.prepare('UPDATE sites SET preferred_url = ?, ssl = ? WHERE id = ?'); + stmt.run(prefs.preferred_url, prefs.ssl, id); + } + firstOrCreateSite(domain: string): Site { let site = this.getSite(domain); if (!site) { @@ -37,6 +44,7 @@ export class SiteRepository { } return site!; } + deleteSite(id: number): void { this.db.prepare('DELETE FROM sites WHERE id = ?').run(id); } diff --git a/packages/core/src/db/repositories/SnapshotRepository.ts b/packages/core/src/db/repositories/SnapshotRepository.ts index 82e9c71..4bbcb0b 100644 --- a/packages/core/src/db/repositories/SnapshotRepository.ts +++ b/packages/core/src/db/repositories/SnapshotRepository.ts @@ -62,6 +62,17 @@ export class SnapshotRepository { return result.count; } + /** + * Returns true if the site has ever had a completed full or incremental crawl. + * Partial snapshots (from page --live) do NOT count as a "first crawl". + */ + hasFullCrawl(siteId: number): boolean { + const result = this.db.prepare( + `SELECT COUNT(*) as count FROM snapshots WHERE site_id = ? AND type IN ('full', 'incremental') AND status = 'completed'` + ).get(siteId) as { count: number }; + return result.count > 0; + } + updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats: Partial<Snapshot> = {}) { const sets: string[] = ['status = ?']; const params: any[] = [status]; diff --git a/packages/core/src/db/schema.ts b/packages/core/src/db/schema.ts deleted file mode 100644 index aa0d42f..0000000 --- a/packages/core/src/db/schema.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { Database } from 'better-sqlite3'; - -export function initSchema(db: Database) { - // Sites Table - db.exec(` - CREATE TABLE IF NOT EXISTS sites ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - created_at TEXT DEFAULT (datetime('now')), - settings_json TEXT, - is_active INTEGER DEFAULT 1 - ); - `); - - // Snapshots Table - db.exec(` - CREATE TABLE IF NOT EXISTS snapshots ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - site_id INTEGER NOT NULL, - type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL, - created_at TEXT DEFAULT (datetime('now')), - node_count INTEGER DEFAULT 0, - edge_count INTEGER DEFAULT 0, - status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running', - limit_reached INTEGER DEFAULT 0, - health_score REAL, - orphan_count INTEGER, - thin_content_count INTEGER, - FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE - ); - `); - - // Pages Table - db.exec(` - CREATE TABLE IF NOT EXISTS pages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - site_id INTEGER NOT NULL, - normalized_url TEXT NOT NULL, - first_seen_snapshot_id INTEGER, - last_seen_snapshot_id INTEGER, - http_status INTEGER, - canonical_url TEXT, - content_hash TEXT, - simhash TEXT, - etag TEXT, - last_modified TEXT, - html TEXT, - noindex INTEGER DEFAULT 0, - nofollow INTEGER DEFAULT 0, - security_error TEXT, - retries INTEGER DEFAULT 0, - depth INTEGER DEFAULT 0, - discovered_via_sitemap INTEGER DEFAULT 0, - redirect_chain TEXT, - bytes_received INTEGER, - crawl_trap_flag INTEGER DEFAULT 0, - crawl_trap_risk REAL, - trap_type TEXT, - created_at TEXT DEFAULT (datetime('now')), - updated_at TEXT DEFAULT (datetime('now')), - FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE, - FOREIGN KEY(first_seen_snapshot_id) REFERENCES snapshots(id), - FOREIGN KEY(last_seen_snapshot_id) REFERENCES snapshots(id), - UNIQUE(site_id, normalized_url) - ); - `); - - // Index for Pages - db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`); - - // Edges Table - db.exec(` - CREATE TABLE IF NOT EXISTS edges ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - snapshot_id INTEGER NOT NULL, - source_page_id INTEGER NOT NULL, - target_page_id INTEGER NOT NULL, - weight REAL DEFAULT 1.0, - rel TEXT CHECK(rel IN ('nofollow', 'sponsored', 'ugc', 'internal', 'external', 'unknown')) DEFAULT 'internal', - FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE, - FOREIGN KEY(source_page_id) REFERENCES pages(id) ON DELETE CASCADE, - FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE - ); - `); - - // Index for Edges - db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`); - - // Metrics Table - db.exec(` - CREATE TABLE IF NOT EXISTS metrics ( - snapshot_id INTEGER NOT NULL, - page_id INTEGER NOT NULL, - crawl_status TEXT, - word_count INTEGER, - thin_content_score REAL, - external_link_ratio REAL, - orphan_score INTEGER, - PRIMARY KEY(snapshot_id, page_id), - FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE, - FOREIGN KEY(page_id) REFERENCES pages(id) ON DELETE CASCADE - ); - `); - - db.exec(`CREATE INDEX IF NOT EXISTS idx_metrics_snapshot ON metrics(snapshot_id);`); - - - - // Plugin Reports Table - db.exec(` - CREATE TABLE IF NOT EXISTS plugin_reports ( - snapshot_id INTEGER NOT NULL, - plugin_name TEXT NOT NULL, - data TEXT NOT NULL, - PRIMARY KEY (snapshot_id, plugin_name), - FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE - ); - `); - - db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_snapshot ON plugin_reports(snapshot_id);`); - - // Migration: add columns to existing DBs that were created before this update - migrateSchema(db); -} - -function migrateSchema(db: Database) { - // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch) - const pageColumns = [ - ['depth', 'INTEGER DEFAULT 0'], - ['discovered_via_sitemap', 'INTEGER DEFAULT 0'], - ['redirect_chain', 'TEXT'], - ['bytes_received', 'INTEGER'], - ['crawl_trap_flag', 'INTEGER DEFAULT 0'], - ['crawl_trap_risk', 'REAL'], - ['trap_type', 'TEXT'], - ]; - - for (const [col, type] of pageColumns) { - try { db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`); } catch { /* already exists */ } - } - - // Add missing columns to edges - try { db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0'); } catch { /* already exists */ } - - // Add missing columns to metrics - const metricsColumns: [string, string][] = []; - - for (const [col, type] of metricsColumns) { - try { db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`); } catch { /* already exists */ } - } -} diff --git a/packages/core/src/diff/service.ts b/packages/core/src/diff/service.ts new file mode 100644 index 0000000..9fa0458 --- /dev/null +++ b/packages/core/src/diff/service.ts @@ -0,0 +1,64 @@ +import { Graph } from '../index.js'; + +export interface DiffOptions { + onlyCritical?: boolean; +} + +export interface SnapshotDiff { + newPages: string[]; + removedPages: string[]; + changedPages: { + url: string; + changes: string[]; + severity: 'low' | 'medium' | 'high'; + }[]; +} + +export class DiffService { + public compare(oldGraph: Graph | undefined, newGraph: Graph, _options: DiffOptions = {}): SnapshotDiff { + if (!oldGraph) { + return { + newPages: Array.from(newGraph.nodes.keys()), + removedPages: [], + changedPages: [] + }; + } + + const oldUrls = new Set(oldGraph.nodes.keys()); + const newUrls = new Set(newGraph.nodes.keys()); + + const newPages = Array.from(newUrls).filter(u => !oldUrls.has(u)); + const removedPages = Array.from(oldUrls).filter(u => !newUrls.has(u)); + const changedPages: SnapshotDiff['changedPages'] = []; + + for (const url of newUrls) { + if (oldUrls.has(url)) { + const oldNode = oldGraph.nodes.get(url)!; + const newNode = newGraph.nodes.get(url)!; + const changes: string[] = []; + let severity: 'low' | 'medium' | 'high' = 'low'; + + if (oldNode.status !== newNode.status) { + changes.push(`status: ${oldNode.status} -> ${newNode.status}`); + severity = 'high'; + } + + if (oldNode.contentHash !== newNode.contentHash) { + changes.push('content changed'); + if (severity !== 'high') severity = 'medium'; + } + + if (oldNode.noindex !== newNode.noindex) { + changes.push(`noindex: ${oldNode.noindex} -> ${newNode.noindex}`); + severity = 'high'; + } + + if (changes.length > 0) { + changedPages.push({ url, changes, severity }); + } + } + } + + return { newPages, removedPages, changedPages }; + } +} diff --git a/packages/core/src/graph/graph.ts b/packages/core/src/graph/graph.ts index 915ed2f..a4e7955 100644 --- a/packages/core/src/graph/graph.ts +++ b/packages/core/src/graph/graph.ts @@ -29,11 +29,25 @@ export interface GraphNode { wordCount?: number; thinContentScore?: number; externalLinkRatio?: number; - orphanScore?: number; h1Count?: number; h2Count?: number; title?: string; + clusterId?: number; + duplicateClusterId?: string; + duplicateType?: 'exact' | 'near' | 'template_heavy'; + pagerankScore?: number; + hubScore?: number; + authScore?: number; + linkRole?: string; + soft404Score?: number; + headingScore?: number; + orphanScore?: number; + orphanType?: string; + impactLevel?: string; + headingData?: string; + isClusterPrimary?: boolean; + isCollapsed?: boolean; } export interface GraphEdge { @@ -66,15 +80,18 @@ export class Graph { * Generates a unique key for an edge. */ static getEdgeKey(source: string, target: string): string { - return JSON.stringify([source, target]); + return source + '\x00' + target; } /** * Parses an edge key back into source and target. */ static parseEdgeKey(key: string): { source: string; target: string } { - const [source, target] = JSON.parse(key); - return { source, target }; + const splitIndex = key.indexOf('\x00'); + return { + source: key.slice(0, splitIndex), + target: key.slice(splitIndex + 1) + }; } /** diff --git a/packages/plugins/hits/src/Service.ts b/packages/core/src/graph/hits.ts similarity index 94% rename from packages/plugins/hits/src/Service.ts rename to packages/core/src/graph/hits.ts index 6f0c3a9..0319ff1 100644 --- a/packages/plugins/hits/src/Service.ts +++ b/packages/core/src/graph/hits.ts @@ -1,5 +1,16 @@ -import { Graph } from '@crawlith/core'; -import { HITSOptions, HITSRow, LinkRole } from './types.js'; +import { Graph } from './graph.js'; + +export type LinkRole = 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral'; + +export interface HITSRow { + authority_score: number; + hub_score: number; + link_role: LinkRole; +} + +export interface HITSOptions { + iterations?: number; +} /** * Service to compute Hub and Authority scores using the HITS algorithm. diff --git a/packages/plugins/pagerank/src/Service.ts b/packages/core/src/graph/pagerank.ts similarity index 83% rename from packages/plugins/pagerank/src/Service.ts rename to packages/core/src/graph/pagerank.ts index ac5259e..e3f647d 100644 --- a/packages/plugins/pagerank/src/Service.ts +++ b/packages/core/src/graph/pagerank.ts @@ -1,5 +1,16 @@ -import { Graph } from '@crawlith/core'; -import { PageRankOptions, PageRankRow } from './types.js'; +import { Graph } from './graph.js'; + +export interface PageRankRow { + raw_rank: number; + score: number; +} + +export interface PageRankOptions { + dampingFactor?: number; + maxIterations?: number; + convergenceThreshold?: number; + soft404WeightThreshold?: number; +} /** * Service to analyze a site's link graph and compute PageRank metrics. @@ -24,10 +35,20 @@ export class PageRankService { // 1. Filter Eligible Nodes const eligibleNodes = allNodes.filter(node => { if (node.noindex) return false; - if ((node as any).isCollapsed) return false; + if (node.isCollapsed) return false; // Keep compat with other plugins mutating soft404Score onto nodes - if ((node as any).soft404Score && (node as any).soft404Score > soft404Threshold) return false; - if (node.canonical && node.canonical !== node.url) return false; + + if (node.soft404Score && node.soft404Score > soft404Threshold) return false; + // canonical is stored as absolute URL; extract pathname for path-based comparison + if (node.canonical) { + try { + const canonicalPath = new URL(node.canonical).pathname; + if (canonicalPath !== node.url) return false; + } catch { + // if canonical isn't a valid URL, compare as-is + if (node.canonical !== node.url) return false; + } + } if (node.status >= 400) return false; // Don't pass rank to broken pages if (node.status === 0) return false; // Don't pass rank to uncrawled/external pages return true; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8cadc5b..8064b5c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,23 +1,33 @@ export * from './crawler/crawl.js'; export * from './crawler/normalize.js'; export * from './crawler/metricsRunner.js'; +export * from './crawler/trap.js'; export * from './graph/metrics.js'; -export * from './report/html.js'; -export * from './report/crawl_template.js'; -export * from './report/crawlExport.js'; -export * from './report/export.js'; -export * from './report/insight.js'; export * from './graph/graph.js'; -export * from './diff/compare.js'; export * from './graph/simhash.js'; +export * from './graph/pagerank.js'; +export * from './graph/hits.js'; +export * from './diff/compare.js'; +export * from './diff/service.js'; export * from './analysis/analyze.js'; export * from './analysis/content.js'; export * from './analysis/seo.js'; export * from './analysis/images.js'; export * from './analysis/links.js'; export * from './analysis/scoring.js'; +export * from './analysis/clustering.js'; +export * from './analysis/duplicate.js'; +export * from './analysis/soft404.js'; +export * from './analysis/heading.js'; +export * from './analysis/orphan.js'; export * from './audit/index.js'; export * from './audit/types.js'; +export * from './scoring/health.js'; +export * from './report/html.js'; +export * from './report/crawl_template.js'; +export * from './report/crawlExport.js'; +export * from './report/export.js'; +export * from './report/insight.js'; export * from './db/index.js'; export * from './db/reset.js'; export * from './db/graphLoader.js'; diff --git a/packages/plugins/health-score-engine/src/Service.ts b/packages/core/src/scoring/health.ts similarity index 83% rename from packages/plugins/health-score-engine/src/Service.ts rename to packages/core/src/scoring/health.ts index 05487d9..3d2c475 100644 --- a/packages/plugins/health-score-engine/src/Service.ts +++ b/packages/core/src/scoring/health.ts @@ -1,9 +1,46 @@ -import { Graph, analyzeContent, analyzeH1, analyzeImageAlts, analyzeLinks } from '@crawlith/core'; -import { - HealthScoreWeights, - CrawlIssueCounts, - HealthScoreBreakdown -} from './types.js'; +import { Graph, analyzeContent, analyzeH1, analyzeImageAlts, analyzeLinks, UrlUtil } from '../index.js'; + +export interface HealthScoreWeights { + orphans: number; + brokenLinks: number; + redirectChains: number; + duplicateClusters: number; + thinContent: number; + missingH1: number; + noindexMisuse: number; + canonicalConflicts: number; + lowInternalLinks: number; + excessiveLinks: number; + blockedByRobots: number; +} + +export interface CrawlIssueCounts { + orphanPages: number; + brokenInternalLinks: number; + redirectChains: number; + duplicateClusters: number; + canonicalConflicts: number; + accidentalNoindex: number; + missingH1: number; + thinContent: number; + lowInternalLinkCount: number; + excessiveInternalLinkCount: number; + highExternalLinkRatio: number; + imageAltMissing: number; + strongPagesUnderLinking: number; + cannibalizationClusters: number; + nearAuthorityThreshold: number; + underlinkedHighAuthorityPages: number; + externalLinks: number; + blockedByRobots: number; +} + +export interface HealthScoreBreakdown { + score: number; + status: string; + weightedPenalties: Record<string, number>; + weights: HealthScoreWeights; +} export const THIN_CONTENT_THRESHOLD = 300; export const LOW_INTERNAL_LINK_THRESHOLD = 2; @@ -68,7 +105,7 @@ export class HealthService { }; } - public collectCrawlIssues(graph: Graph, metrics: any): CrawlIssueCounts { + public collectCrawlIssues(graph: Graph, metrics: any, rootOrigin: string = ''): CrawlIssueCounts { const nodes = graph.getNodes(); let brokenInternalLinks = 0; @@ -109,7 +146,7 @@ export class HealthService { if ((node.redirectChain?.length || 0) > 1) { redirectChains += 1; } - if (node.canonical && node.canonical !== node.url) { + if (node.canonical && node.canonical !== node.url && node.canonical !== (rootOrigin ? new URL(node.url, rootOrigin).toString() : node.url)) { canonicalConflicts += 1; } if (node.noindex && node.status >= 200 && node.status < 300) { @@ -143,7 +180,8 @@ export class HealthService { } } - const links = analyzeLinks(node.html || '', node.url, node.url); + const pageAbsUrl = rootOrigin ? UrlUtil.toAbsolute(node.url, rootOrigin) : node.url; + const links = analyzeLinks(node.html || '', pageAbsUrl, rootOrigin || node.url); externalLinks += links.externalLinks; if (links.externalRatio > HIGH_EXTERNAL_LINK_RATIO_THRESHOLD) { highExternalLinkRatio += 1; @@ -162,9 +200,6 @@ export class HealthService { const cannibalizationClusters = clusters.filter((cluster: any) => cluster.risk === 'high' || cluster.type === 'near').length; for (const node of nodes) { - // Since PageRank is now a plugin, we use in-links as a basic authority signal here - // or we could check ctx.metadata.pagerank if we want to be more integrated. - // For now, let's use in-links normalized against hypothetical max. const authority = node.inLinks > 5 ? 0.8 : 0.2; if (authority >= OPPORTUNITY_AUTHORITY_THRESHOLD && node.outLinks < 3) { strongPagesUnderLinking += 1; diff --git a/packages/core/tests/db.test.ts b/packages/core/tests/db.test.ts index fb77a4d..693e814 100644 --- a/packages/core/tests/db.test.ts +++ b/packages/core/tests/db.test.ts @@ -141,7 +141,20 @@ describe('Database Layer', () => { word_count: 100, thin_content_score: 0.1, external_link_ratio: 0.0, - orphan_score: 0 + orphan_score: 0, + pagerank_score: 0, + hub_score: 0, + auth_score: 0, + link_role: null, + duplicate_cluster_id: null, + duplicate_type: null, + cluster_id: null, + soft404_score: 0, + heading_score: 0, + orphan_type: null, + impact_level: null, + heading_data: null, + is_cluster_primary: 0 }); const metrics = metricsRepo.getMetricsForPage(snapshotId, p1.id); diff --git a/packages/core/tests/graphLoader.test.ts b/packages/core/tests/graphLoader.test.ts index 956e11a..260996e 100644 --- a/packages/core/tests/graphLoader.test.ts +++ b/packages/core/tests/graphLoader.test.ts @@ -48,7 +48,20 @@ describe('GraphLoader', () => { word_count: 500, thin_content_score: 10, external_link_ratio: 0.1, - orphan_score: 5 + orphan_score: 5, + pagerank_score: 0.5, + hub_score: 0.3, + auth_score: 0.2, + link_role: 'authority', + duplicate_cluster_id: null, + duplicate_type: null, + cluster_id: 1, + soft404_score: 0.05, + heading_score: 0.9, + orphan_type: null, + impact_level: 'high', + heading_data: null, + is_cluster_primary: 0 }); // Load Graph @@ -91,7 +104,20 @@ describe('GraphLoader', () => { word_count: null, thin_content_score: null, external_link_ratio: null, - orphan_score: null + orphan_score: null, + pagerank_score: null, + hub_score: null, + auth_score: null, + link_role: null, + duplicate_cluster_id: null, + duplicate_type: null, + cluster_id: null, + soft404_score: null, + heading_score: null, + orphan_type: null, + impact_level: null, + heading_data: null, + is_cluster_primary: null }); const graph = loadGraphFromSnapshot(snapshotId); diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 5bfc7f6..0b4d3b7 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@crawlith/cli": "workspace:*", + "@crawlith/core": "workspace:*", "@modelcontextprotocol/sdk": "^1.17.5", "zod": "^3.25.76" }, @@ -17,4 +18,4 @@ "tsx": "^4.20.5", "typescript": "^5.9.2" } -} +} \ No newline at end of file diff --git a/packages/mcp/src/mcp-server.ts b/packages/mcp/src/mcp-server.ts index 5b23f5a..77a1a75 100644 --- a/packages/mcp/src/mcp-server.ts +++ b/packages/mcp/src/mcp-server.ts @@ -424,6 +424,29 @@ server.tool( } ); +server.tool( + 'full_site_audit', + 'Perform a complete site audit: crawl the site with standard limits and build the graph.', + { + url: z.string().describe('Site URL or domain to audit.') + }, + async ({ url }: { url: string }) => { + const args = ['crawl', url, '--limit', '2000', '--depth', '10']; + const result = await runCliCommand(args); + return asTextContent(result); + } +); + +server.tool( + 'portfolio_status', + 'View all tracked domains and their crawl health summary across the local database.', + {}, + async () => { + const result = await runCliCommand(['sites']); + return asTextContent(result); + } +); + registerPrompts(server); /** diff --git a/packages/plugins/STRATEGY.md b/packages/plugins/STRATEGY.md new file mode 100644 index 0000000..dd06deb --- /dev/null +++ b/packages/plugins/STRATEGY.md @@ -0,0 +1,50 @@ +# Crawlith Plugin Strategy: Core vs Plugin Architecture + +This document outlines the architectural recommendations for each Crawlith plugin. The goal is to move foundational "intelligence" features into the core engine while maintaining a lightweight plugin system for external integrations and delivery formats. + +## Architectural Principles + +1. **Core**: Foundational intelligence, graph algorithms, crawler safety, and primary SEO signals. +2. **Plugin**: External APIs, niche formats, large/heavy dependencies, and delivery/notification layers. + +--- + +## Detailed Review & Recommendations + +| Plugin | Recommendation | Rationale | +| :--- | :--- | :--- | +| **Content Clustering** | **Core** | Content similarity (SimHash) is a primary audit signal. Essential for finding thin content/internal competition. | +| **Duplicate Detection** | **Core** | Fundamental for graph hygiene. Handling exact/near duplicates and collapsing nodes should be native. | +| **Health Score Engine** | **Core** | This is the "brain" that aggregates all signals. A unified health score should be a core result. | +| **HITS** | **Core** | Standard graph algorithm for Hubs and Authorities; foundational to link intelligence. | +| **PageRank** | **Core** | Standard graph algorithm for centrality; foundational to link intelligence. | +| **Orphan Intelligence** | **Core** | Finding pages with zero in-links is a primary use case for crawling. | +| **Crawl Policy** | **Core** | Defining path rules and crawler constraints is better handled natively by the engine. | +| **Crawl Trap Analyzer** | **Core** | A critical safety mechanism to prevent infinite loops (e.g., dynamic calendar pages). | +| **Heading Health** | **Core** | A simple but essential structural audit that every SEO tool should have by default. | +| **Soft 404 Detector** | **Core** | Enhances crawling accuracy by identifying fake 200 OK responses. | +| **Snapshot Diff** | **Core** | Foundational for monitoring site changes over time (Monitoring/Alerting). | +| **SimHash** | **Core** | Redundant as a plugin; core utility already used by clustering and duplicate detection. | +| **Exporter** | **Plugin** | Diverse targets (CSV, S3, SQL, BigQuery) are perfect for an extensible plugin ecosystem. | +| **Reporter** | **Plugin** | Presentation logic (HTML Reports, PDF) varies widely per user; keeps core headless. | +| **PageSpeed** | **Plugin** | Heavy external dependency (PageSpeed Insights API); better as an optional module. | +| **Signals** | **Plugin** | Integrates with external platforms (GSC, Ahrefs, Moz); high maintenance/variance. | + +--- + +## Implementation Plan + +### Phase 1: Core Consolidation +- [x] Move `hits` and `pagerank` logic into `@crawlith/core/graph`. +- [x] Integrate `crawl-trap-analyzer` and `crawl-policy` into the `@crawlith/core/crawler`. +- [x] Incorporate `health-score-engine` into `@crawlith/core/scoring`. + +### Phase 2: Feature Integration +- [x] Refactor `content-clustering` and `duplicate-detection` into a dedicated `@crawlith/core/analysis` module. +- [x] Add `snapshot-diff` as a native capability in `@crawlith/core/diff`. +- [x] Add `heading-health`, `soft404-detector`, `orphan-intelligence` into core. + +### Phase 3: Plugin Ecosystem Cleanup +- [x] Retain `exporter`, `reporter`, `pagespeed`, and `signals` as standalone packages in `packages/plugins/`. +- [x] Update plugin loader in `core` to reflect the refined scope. +- [x] Remove migrated packages from `packages/plugins/`. diff --git a/packages/plugins/content-clustering/CHANGELOG.md b/packages/plugins/content-clustering/CHANGELOG.md deleted file mode 100644 index a6416b1..0000000 --- a/packages/plugins/content-clustering/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-content-clustering - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/content-clustering/README.md b/packages/plugins/content-clustering/README.md deleted file mode 100644 index e9b5da3..0000000 --- a/packages/plugins/content-clustering/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Content Clustering Plugin - -Crawlith plugin for content clustering - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/content-clustering/index.ts b/packages/plugins/content-clustering/index.ts deleted file mode 100644 index 2dd5ab6..0000000 --- a/packages/plugins/content-clustering/index.ts +++ /dev/null @@ -1,34 +0,0 @@ - -import { CrawlithPlugin } from '@crawlith/core'; -import { ClusteringHooks } from './src/plugin.js'; - -/** - * Content Clustering Plugin - * Crawlith plugin for content clustering - */ -export const ContentClusteringPlugin: CrawlithPlugin = { - name: 'content-clustering', - description: 'Detects near-duplicate content clusters using SimHash LSH', - - cli: { - flag: '--clustering', - description: 'Enable content clustering analysis', - options: [ - { flag: '--cluster-threshold <number>', description: 'Hamming distance for content clusters', defaultValue: '10' }, - { flag: '--min-cluster-size <number>', description: 'Minimum pages per cluster', defaultValue: '3' } - ] - }, - - storage: { - fetchMode: 'local', - perPage: { - columns: { - cluster_id: 'INTEGER' - } - } - }, - - hooks: ClusteringHooks -}; - -export default ContentClusteringPlugin; diff --git a/packages/plugins/content-clustering/package.json b/packages/plugins/content-clustering/package.json deleted file mode 100644 index 6a59dd5..0000000 --- a/packages/plugins/content-clustering/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@crawlith/plugin-content-clustering", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*", - "cheerio": "^1.2.0" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for content clustering" -} \ No newline at end of file diff --git a/packages/plugins/content-clustering/src/plugin.ts b/packages/plugins/content-clustering/src/plugin.ts deleted file mode 100644 index 5dfabaf..0000000 --- a/packages/plugins/content-clustering/src/plugin.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { PluginContext, Graph } from '@crawlith/core'; -import { ClusteringService } from './Service.js'; - -const service = new ClusteringService(); - -export const ClusteringHooks = { - /** - * On metrics, we compute the clusters and store them. - */ - onMetrics: async (ctx: PluginContext, graph: Graph) => { - const threshold = Number(ctx.flags?.clusterThreshold ?? 10); - const minSize = Number(ctx.flags?.minClusterSize ?? 3); - - ctx.logger?.info(`Detecting content clusters (threshold=${threshold}, minSize=${minSize})...`); - const clusters = service.detectContentClusters(graph, threshold, minSize); - ctx.logger?.info(`Found ${clusters.length} content clusters.`); - - // Store per-page cluster IDs - if (ctx.db) { - const nodes = graph.getNodes(); - for (const node of nodes) { - if ((node as any).clusterId) { - ctx.db.data.save({ - url: node.url, - data: { - cluster_id: (node as any).clusterId - } - }); - } - } - - // Store cluster summary in the report context for the onReport hook - ctx.metadata = ctx.metadata || {}; - ctx.metadata.clusters = clusters; - - // Also save to database report for persistence - ctx.db.report.save({ - clusters - }); - } - }, - - /** - * Inject cluster info into the final report. - */ - onReport: async (ctx: PluginContext, result: any) => { - const clusters = ctx.metadata?.clusters; - if (clusters) { - result.plugins = result.plugins || {}; - result.plugins.clusters = clusters; - } - } -}; diff --git a/packages/plugins/content-clustering/src/types.ts b/packages/plugins/content-clustering/src/types.ts deleted file mode 100644 index f87b04a..0000000 --- a/packages/plugins/content-clustering/src/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface ClusterInfo { - id: number; - count: number; - primaryUrl: string; - risk: 'low' | 'medium' | 'high'; - sharedPathPrefix?: string; - nodes?: string[]; -} - -export interface ClusteringOptions { - threshold?: number; - minSize?: number; -} diff --git a/packages/plugins/content-clustering/tests/clustering.test.ts b/packages/plugins/content-clustering/tests/clustering.test.ts deleted file mode 100644 index e6bae9e..0000000 --- a/packages/plugins/content-clustering/tests/clustering.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { Graph } from '@crawlith/core'; -import { ClusteringService } from '../src/Service.js'; - -describe('ClusteringService', () => { - let service: ClusteringService; - let graph: Graph; - - beforeEach(() => { - service = new ClusteringService(); - graph = new Graph(); - }); - - it('should detect clusters from nodes with similar simhashes', () => { - const h1 = 0b101010n.toString(); - const h2 = 0b101011n.toString(); // distance 1 - const h3 = (0b1111111111111111n << 32n).toString(); // distance far - - graph.addNode('https://a.com', 0, 200); - graph.addNode('https://b.com', 0, 200); - graph.addNode('https://c.com', 0, 200); - - graph.updateNodeData('https://a.com', { simhash: h1 }); - graph.updateNodeData('https://b.com', { simhash: h2 }); - graph.updateNodeData('https://c.com', { simhash: h3 }); - - const clusters = service.detectContentClusters(graph, 3, 2); - - expect(clusters.length).toBe(1); - expect(clusters[0].nodes).toContain('https://a.com'); - expect(clusters[0].nodes).toContain('https://b.com'); - expect(clusters[0].nodes).not.toContain('https://c.com'); - expect(clusters[0].count).toBe(2); - }); - - it('should calculate shared path prefix correctly', () => { - const h = 0b111111n.toString(); - graph.addNode('https://example.com/blog/p1', 0, 200); - graph.addNode('https://example.com/blog/p2', 0, 200); - graph.addNode('https://example.com/blog/p3', 0, 200); - - graph.updateNodeData('https://example.com/blog/p1', { simhash: h }); - graph.updateNodeData('https://example.com/blog/p2', { simhash: h }); - graph.updateNodeData('https://example.com/blog/p3', { simhash: h }); - - const clusters = service.detectContentClusters(graph, 3, 3); - expect(clusters[0].sharedPathPrefix).toBe('/blog'); - }); - - it('should identify the best primary URL based on in-links', () => { - const h = 0b111111n.toString(); - graph.addNode('https://example.com/p1', 0, 200); - graph.addNode('https://example.com/p2', 0, 200); - - graph.updateNodeData('https://example.com/p1', { simhash: h }); - graph.updateNodeData('https://example.com/p2', { simhash: h }); - - // Give p2 more in-links - graph.nodes.get('https://example.com/p2')!.inLinks = 5; - - const clusters = service.detectContentClusters(graph, 3, 2); - expect(clusters[0].primaryUrl).toBe('https://example.com/p2'); - }); - - describe('Cluster Risk Heuristic', () => { - it('should assign HIGH risk to clusters with identical titles', () => { - const html = '<html><head><title>Duplicate TitleContent'; - const h = 0b101010n.toString(); - - graph.addNode('https://example.com/p1', 0, 200); - graph.addNode('https://example.com/p2', 0, 200); - graph.addNode('https://example.com/p3', 0, 200); - - graph.updateNodeData('https://example.com/p1', { simhash: h, html }); - graph.updateNodeData('https://example.com/p2', { simhash: h, html }); - graph.updateNodeData('https://example.com/p3', { simhash: h, html }); - - const clusters = service.detectContentClusters(graph, 2, 2); - - expect(clusters.length).toBe(1); - expect(clusters[0].risk).toBe('high'); - }); - - it('should assign HIGH risk to clusters with identical H1s', () => { - const h = 0b101010n.toString(); - - graph.addNode('https://example.com/p1', 0, 200); - graph.addNode('https://example.com/p2', 0, 200); - graph.addNode('https://example.com/p3', 0, 200); - - graph.updateNodeData('https://example.com/p1', { - simhash: h, - html: 'Page 1

Duplicate Header

' - }); - graph.updateNodeData('https://example.com/p2', { - simhash: h, - html: 'Page 2

Duplicate Header

' - }); - graph.updateNodeData('https://example.com/p3', { - simhash: h, - html: 'Page 3

Duplicate Header

' - }); - - const clusters = service.detectContentClusters(graph, 2, 2); - - expect(clusters.length).toBe(1); - expect(clusters[0].risk).toBe('high'); - }); - - it('should assign MEDIUM risk to large clusters even with unique titles', () => { - const h = 0b101010n.toString(); - - for (let i = 0; i < 12; i++) { - const url = `https://example.com/p${i}`; - graph.addNode(url, 0, 200); - graph.updateNodeData(url, { - simhash: h, - html: `Page ${i}

Header ${i}

` - }); - } - - const clusters = service.detectContentClusters(graph, 2, 2); - - expect(clusters.length).toBe(1); - expect(clusters[0].risk).toBe('medium'); - }); - }); -}); diff --git a/packages/plugins/content-clustering/tsconfig.json b/packages/plugins/content-clustering/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/content-clustering/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/crawl-policy/CHANGELOG.md b/packages/plugins/crawl-policy/CHANGELOG.md deleted file mode 100644 index d3b9560..0000000 --- a/packages/plugins/crawl-policy/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-crawl-policy - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/crawl-policy/README.md b/packages/plugins/crawl-policy/README.md deleted file mode 100644 index c4c6f06..0000000 --- a/packages/plugins/crawl-policy/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Crawl Policy Plugin - -Crawlith plugin for crawl policy - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/crawl-policy/index.ts b/packages/plugins/crawl-policy/index.ts deleted file mode 100644 index 4387d63..0000000 --- a/packages/plugins/crawl-policy/index.ts +++ /dev/null @@ -1,58 +0,0 @@ - -import { CrawlithPlugin, PluginContext } from '@crawlith/core'; -import { Command } from '@crawlith/core'; - -/** - * Crawl Policy Plugin - * Crawlith plugin for crawl policy - */ -export const CrawlPolicyPlugin: CrawlithPlugin = { - name: 'crawl-policy', - register: (cli: Command) => { - if (cli.name() === 'crawl' || cli.name() === 'page') { - cli - .option("--allow ", "comma separated list of domains to allow") - .option("--deny ", "comma separated list of domains to deny") - .option("--include-subdomains", "include subdomains in the default scope") - .option("--ignore-robots", "ignore robots.txt directives") - .option("--proxy ", "proxy URL to use for requests") - .option("--ua ", "user agent string to use") - .option("--rate ", "requests per second limit") - .option("--max-bytes ", "maximum bytes to download per page") - .option("--max-redirects ", "maximum redirects to follow"); - } - }, - - hooks: { - onInit: async (ctx: PluginContext) => { - const flags = ctx.flags || {}; - - const allowedDomains = flags.allow ? String(flags.allow).split(',').map(d => d.trim()) : []; - const deniedDomains = flags.deny ? String(flags.deny).split(',').map(d => d.trim()) : []; - - const proxyUrl = flags.proxy ? String(flags.proxy) : undefined; - if (proxyUrl) { - try { - new URL(proxyUrl); - } catch { - throw new Error(`Invalid proxy URL: ${proxyUrl}`); - } - } - - if (!ctx.metadata) ctx.metadata = {}; - ctx.metadata.crawlPolicy = { - allowedDomains, - deniedDomains, - includeSubdomains: !!flags.includeSubdomains, - ignoreRobots: !!flags.ignoreRobots, - proxyUrl, - userAgent: flags.ua ? String(flags.ua) : undefined, - rate: flags.rate ? parseFloat(String(flags.rate)) : undefined, - maxBytes: flags.maxBytes ? parseInt(String(flags.maxBytes), 10) : undefined, - maxRedirects: flags.maxRedirects ? parseInt(String(flags.maxRedirects), 10) : undefined - }; - } - } -}; - -export default CrawlPolicyPlugin; diff --git a/packages/plugins/crawl-policy/package.json b/packages/plugins/crawl-policy/package.json deleted file mode 100644 index 18b6449..0000000 --- a/packages/plugins/crawl-policy/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@crawlith/plugin-crawl-policy", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*", - "chalk": "^5.6.2" - }, - "scripts": { - "build": "tsc", - "test": "echo No tests specified" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for crawl policy" -} diff --git a/packages/plugins/crawl-policy/tsconfig.json b/packages/plugins/crawl-policy/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/crawl-policy/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/plugins/crawl-trap-analyzer/CHANGELOG.md b/packages/plugins/crawl-trap-analyzer/CHANGELOG.md deleted file mode 100644 index b4eb6d0..0000000 --- a/packages/plugins/crawl-trap-analyzer/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-crawl-trap-analyzer - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/crawl-trap-analyzer/README.md b/packages/plugins/crawl-trap-analyzer/README.md deleted file mode 100644 index a0c23a9..0000000 --- a/packages/plugins/crawl-trap-analyzer/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Crawl Trap Analyzer Plugin - -Crawlith plugin for crawl trap analyzer - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/crawl-trap-analyzer/index.ts b/packages/plugins/crawl-trap-analyzer/index.ts deleted file mode 100644 index cab8266..0000000 --- a/packages/plugins/crawl-trap-analyzer/index.ts +++ /dev/null @@ -1,74 +0,0 @@ - -import { CrawlithPlugin, PluginContext } from '@crawlith/core'; -import { Command } from '@crawlith/core'; -import { TrapDetector, TrapResult } from './src/trap.js'; - -let detector: TrapDetector | null = null; -const trapResults = new Map(); - -/** - * Crawl Trap Analyzer Plugin - * Crawlith plugin for crawl trap analyzer - */ -export const CrawlTrapAnalyzerPlugin: CrawlithPlugin = { - name: 'crawl-trap-analyzer', - register: (cli: Command) => { - if (cli.name() === 'crawl') { - cli.option("--detect-traps", "Detect and cluster crawl traps"); - } - }, - - hooks: { - onInit: async (ctx: PluginContext) => { - const flags = ctx.flags || {}; - if (flags.detectTraps) { - detector = new TrapDetector(); - trapResults.clear(); - } - }, - shouldEnqueueUrl: (ctx: PluginContext, url: string, depth: number) => { - if (!detector) return true; - const trap = detector.checkTrap(url, depth); - if (trap.risk > 0) { - trapResults.set(url, trap); - } - if (trap.risk > 0.8) { - ctx.logger?.info(`🪤 Caught potential crawl trap: ${url} (risk: ${trap.risk.toFixed(2)})`); - return false; - } - return true; - }, - onMetrics: async (ctx: PluginContext, graph: any) => { - if (!detector) return; - - ctx.logger?.info('🔍 Processing crawl traps...'); - - const nodes = graph.getNodes(); - let trapCount = 0; - - for (const node of nodes) { - // Re-evaluate in case any weren't checked during shouldEnqueueUrl, or use cached - let trap = trapResults.get(node.url); - if (!trap) { - trap = detector.checkTrap(node.url, node.depth); - } - if (trap && trap.risk > 0.8) { - node.crawlTrapFlag = true; - node.crawlTrapRisk = trap.risk; - node.trapType = trap.type; - trapCount++; - } else if (trap && trap.risk > 0) { - node.crawlTrapFlag = false; - node.crawlTrapRisk = trap.risk; - node.trapType = trap.type; - } - } - - if (trapCount > 0) { - ctx.logger?.info(`🪤 Identified ${trapCount} crawl traps`); - } - } - } -}; - -export default CrawlTrapAnalyzerPlugin; diff --git a/packages/plugins/crawl-trap-analyzer/package.json b/packages/plugins/crawl-trap-analyzer/package.json deleted file mode 100644 index 69af62a..0000000 --- a/packages/plugins/crawl-trap-analyzer/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-crawl-trap-analyzer", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for crawl trap analyzer" -} diff --git a/packages/plugins/crawl-trap-analyzer/tests/trap.test.ts b/packages/plugins/crawl-trap-analyzer/tests/trap.test.ts deleted file mode 100644 index 006c962..0000000 --- a/packages/plugins/crawl-trap-analyzer/tests/trap.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { TrapDetector } from '../src/trap.js'; - -describe('TrapDetector', () => { - const detector = new TrapDetector(); - - it('should detect session ID traps', () => { - const result = detector.checkTrap('https://example.com/page?sid=12345', 1); - expect(result.risk).toBeGreaterThan(0.8); - expect(result.type).toBe('session_trap'); - }); - - it('should detect calendar patterns', () => { - const result = detector.checkTrap('https://example.com/archive/2023/12/01/', 1); - expect(result.risk).toBeGreaterThan(0.6); - expect(result.type).toBe('calendar_trap'); - }); - - it('should detect pagination loops', () => { - // Simulate many pages - for (let i = 1; i <= 60; i++) { - detector.checkTrap(`https://example.com/blog?page=${i}`, 1); - } - const result = detector.checkTrap('https://example.com/blog?page=61', 1); - expect(result.risk).toBeGreaterThan(0.8); - expect(result.type).toBe('pagination_loop'); - }); - - it('should detect faceted navigation / parameter explosion', () => { - detector.reset(); - const basePath = 'https://example.com/products'; - for (let i = 1; i <= 35; i++) { - detector.checkTrap(`${basePath}?color=red&size=${i}`, 1); - } - const result = detector.checkTrap(`${basePath}?color=blue&size=large`, 1); - expect(result.risk).toBeGreaterThan(0.9); - expect(result.type).toBe('faceted_navigation'); - }); -}); diff --git a/packages/plugins/crawl-trap-analyzer/tsconfig.json b/packages/plugins/crawl-trap-analyzer/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/crawl-trap-analyzer/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/plugins/duplicate-detection/CHANGELOG.md b/packages/plugins/duplicate-detection/CHANGELOG.md deleted file mode 100644 index 50630be..0000000 --- a/packages/plugins/duplicate-detection/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-duplicate-detection - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/duplicate-detection/README.md b/packages/plugins/duplicate-detection/README.md deleted file mode 100644 index af15eba..0000000 --- a/packages/plugins/duplicate-detection/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Duplicate Detection Plugin - -Crawlith plugin for duplicate detection - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/duplicate-detection/index.ts b/packages/plugins/duplicate-detection/index.ts deleted file mode 100644 index bedde07..0000000 --- a/packages/plugins/duplicate-detection/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { CrawlithPlugin, PluginContext, Graph } from '@crawlith/core'; -import { Command } from '@crawlith/core'; -import { DuplicateService } from './src/Service.js'; - -/** - * Duplicate Detection Plugin - * Crawlith plugin for duplicate detection - */ -export const DuplicateDetectionPlugin: CrawlithPlugin = { - name: 'duplicate-detection', - - cli: { - flag: '--duplicate-detection', - description: 'Detect exact and near duplicates' - }, - - register: (cli: Command) => { - if (cli.name() === 'crawl') { - cli.option('--no-collapse', 'Do not collapse duplicate clusters before PageRank'); - } - }, - - hooks: { - onMetrics: async (ctx: PluginContext, graph: Graph) => { - const flags = ctx.flags || {}; - const collapse = flags.collapse !== false; - - const service = new DuplicateService(); - service.detectDuplicates(graph, { collapse }); - } - } -}; - -export default DuplicateDetectionPlugin; diff --git a/packages/plugins/duplicate-detection/package.json b/packages/plugins/duplicate-detection/package.json deleted file mode 100644 index 89bc543..0000000 --- a/packages/plugins/duplicate-detection/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-duplicate-detection", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for duplicate detection" -} \ No newline at end of file diff --git a/packages/plugins/duplicate-detection/tests/duplicate.test.ts b/packages/plugins/duplicate-detection/tests/duplicate.test.ts deleted file mode 100644 index fa8003d..0000000 --- a/packages/plugins/duplicate-detection/tests/duplicate.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest'; -import { Graph, SimHash } from '@crawlith/core'; -import { DuplicateService } from '../src/Service.js'; - -describe('Duplicate Detection', () => { - let service: DuplicateService; - let graph: Graph; - - beforeEach(() => { - service = new DuplicateService(); - graph = new Graph(); - }); - - it('should detect exact duplicates based on contentHash', () => { - graph.addNode('https://example.com/a', 0, 200); - graph.addNode('https://example.com/b', 0, 200); - graph.addNode('https://example.com/c', 0, 200); - - graph.updateNodeData('https://example.com/a', { contentHash: 'hash1', uniqueTokenRatio: 1.0 }); - graph.updateNodeData('https://example.com/b', { contentHash: 'hash1', uniqueTokenRatio: 1.0 }); - graph.updateNodeData('https://example.com/c', { contentHash: 'hash2', uniqueTokenRatio: 1.0 }); - - service.detectDuplicates(graph); - - const clusters = (graph as any).duplicateClusters; - expect(clusters).toHaveLength(1); - const cluster = clusters[0]; - expect(cluster.type).toBe('exact'); - expect(cluster.size).toBe(2); - - const nodeA = graph.nodes.get('https://example.com/a')!; - const nodeB = graph.nodes.get('https://example.com/b')!; - expect((nodeA as any).duplicateClusterId).toBeDefined(); - expect((nodeA as any).duplicateClusterId).toBe((nodeB as any).duplicateClusterId); - - // One should be primary, one should be collapsed - expect(!(nodeA as any).isCollapsed !== !(nodeB as any).isCollapsed).toBe(true); - }); - - it('should detect near duplicates using SimHash', () => { - graph.addNode('https://example.com/x', 0, 200); - graph.addNode('https://example.com/y', 0, 200); - - const tokens1 = ['hello', 'world', 'this', 'is', 'a', 'test', 'document']; - const tokens2 = ['hello', 'world', 'this', 'is', 'a', 'test', 'document2']; - - const h1 = SimHash.generate(tokens1); - const h2 = SimHash.generate(tokens2); - - graph.updateNodeData('https://example.com/x', { contentHash: 'x', simhash: h1.toString(), uniqueTokenRatio: 1.0 }); - graph.updateNodeData('https://example.com/y', { contentHash: 'y', simhash: h2.toString(), uniqueTokenRatio: 1.0 }); - - service.detectDuplicates(graph, { simhashThreshold: 10 }); - - const clusters = (graph as any).duplicateClusters; - expect(clusters).toHaveLength(1); - expect(clusters[0].type).toBe('near'); - }); - - it('should identify template-heavy clusters', () => { - graph.addNode('https://example.com/1', 0, 200); - graph.addNode('https://example.com/2', 0, 200); - - graph.updateNodeData('https://example.com/1', { contentHash: 'h1', uniqueTokenRatio: 0.2 }); - graph.updateNodeData('https://example.com/2', { contentHash: 'h1', uniqueTokenRatio: 0.2 }); - - service.detectDuplicates(graph); - - const clusters = (graph as any).duplicateClusters; - expect(clusters[0].type).toBe('template_heavy'); - }); - - it('should mark high severity on missing canonicals', () => { - graph.addNode('https://example.com/a', 0, 200); - graph.addNode('https://example.com/b', 0, 200); - - graph.updateNodeData('https://example.com/a', { contentHash: 'h1', canonical: 'https://example.com/a' }); - graph.updateNodeData('https://example.com/b', { contentHash: 'h1', canonical: undefined }); - - service.detectDuplicates(graph); - - const clusters = (graph as any).duplicateClusters; - expect(clusters[0].severity).toBe('high'); - }); - - it('should transfer edges during collapse', () => { - graph.addNode('https://example.com/a', 0, 200); - graph.addNode('https://example.com/b', 0, 200); - graph.addNode('https://example.com/source', 0, 200); - - graph.updateNodeData('https://example.com/a', { contentHash: 'h1' }); - graph.updateNodeData('https://example.com/b', { contentHash: 'h1' }); - - graph.addEdge('https://example.com/source', 'https://example.com/b', 1); - - graph.nodes.get('https://example.com/a')!.inLinks = 10; - - service.detectDuplicates(graph); - - const a = graph.nodes.get('https://example.com/a')!; - const b = graph.nodes.get('https://example.com/b')!; - - expect((a as any).isClusterPrimary).toBe(true); - expect((a as any).isCollapsed).toBe(false); - expect((b as any).isCollapsed).toBe(true); - expect((b as any).collapseInto).toBe('https://example.com/a'); - - expect(graph.edges.has(Graph.getEdgeKey('https://example.com/source', 'https://example.com/a'))).toBe(true); - }); -}); diff --git a/packages/plugins/duplicate-detection/tsconfig.json b/packages/plugins/duplicate-detection/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/duplicate-detection/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/heading-health/CHANGELOG.md b/packages/plugins/heading-health/CHANGELOG.md deleted file mode 100644 index 1071eaf..0000000 --- a/packages/plugins/heading-health/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-heading-health - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/heading-health/README.md b/packages/plugins/heading-health/README.md deleted file mode 100644 index 58de142..0000000 --- a/packages/plugins/heading-health/README.md +++ /dev/null @@ -1,124 +0,0 @@ -# `@crawlith/plugin-heading-health` - -Analyzes the heading structure, hierarchy health, and content distribution of every page in a crawl or single-page analysis. - ---- - -## What It Does - -The heading-health plugin parses the raw HTML of each crawled page, extracts all `

` through `

` tags, and runs a suite of structural checks. It scores each page from **0–100**, assigns a status (`Healthy`, `Moderate`, `Poor`), and surfaces actionable issues. A snapshot-level summary is also produced, giving you a site-wide view of heading quality. - ---- - -## How It Works - -The plugin runs across three lifecycle hooks: - -### `onInit` -Registers the plugin's database schema columns (`score`, `status`, `analysis_json`) so per-URL results can be persisted across runs. - -### `onMetrics` -Triggered after crawling, once the full page graph is available. For each page node in the graph: - -1. **Extracts headings** from raw HTML using regex-based parsing, preserving heading level and text. -2. **Builds a tree** tracking parent–child relationships between headings. -3. **Segments content** between headings to measure word count and keyword concentration per section. -4. **Computes structural metrics** (see below). -5. **Detects site-wide duplicates** by comparing H1 norms, H2 set hashes, and structural pattern hashes across all pages (cross-page duplicate risk for sections). -6. **Scores and statuses** each page, attaches the payload to the graph node, and persists data to the database. -7. Emits a **snapshot-level summary** to the context. - -### `onReport` -Attaches the snapshot-level `HeadingHealthSummary` to the final result object under `result.plugins.headingHealth`. - ---- - -## CLI Usage - -Add `--heading` to the `crawl` or `page` commands to activate the plugin: - -```bash -# Single page -crawlith page https://example.com --heading - -# Full crawl -crawlith crawl https://example.com --heading - -# Force a fresh recompute (bypass 24h cache) -crawlith crawl https://example.com --heading --heading-force-refresh -``` - ---- - -## Metrics Explained - -Each page receives a `HeadingHealthPayload` with the following fields: - -| Field | Description | -| :--- | :--- | -| `score` | Overall heading health score, 0–100 | -| `status` | `Healthy` (≥70), `Moderate` (≥40), or `Poor` (<40) | -| `issues` | List of human-readable issue strings detected | -| `map` | Ordered array of all heading nodes with level, text, index, and parent index | -| `missing_h1` | `1` if no H1 exists on the page, else `0` | -| `multiple_h1` | `1` if more than one H1 exists, else `0` | -| `entropy` | Shannon entropy of heading level distribution — higher = more chaotic structure | -| `max_depth` | Deepest heading level used (e.g., `4` means H4 is the deepest) | -| `avg_depth` | Average heading level across all headings | -| `heading_density` | Ratio of heading count to total word count | -| `fragmentation` | Proportion of headings that are H1 or H2 — high fragmentation = too many top-level headings | -| `volatility` | Average level-jump between consecutive headings — high = erratic structure | -| `hierarchy_skips` | Number of times heading level jumps by more than 1 (e.g., H2 → H4) | -| `reverse_jumps` | Number of times heading level drops by more than 1 (e.g., H4 → H2) | -| `thin_sections` | Number of content sections containing fewer than 80 words | -| `duplicate_h1_group` | Number of other pages sharing the same normalized H1 text | -| `similar_h1_group` | Number of pages with a Jaccard similarity > threshold to this page's H1 | -| `identical_h2_set_group` | Number of pages sharing the exact same ordered set of H2 text | -| `duplicate_pattern_group` | Number of pages with the same heading level pattern (e.g., `1>2>2>3`) | -| `template_risk` | Combined risk score indicating the page may use a templated heading structure | - ---- - -## Site-Wide Summary (`HeadingHealthSummary`) - -Attached to `result.plugins.headingHealth` after a crawl: - -| Field | Description | -| :--- | :--- | -| `avgScore` | Mean heading health score across all evaluated pages | -| `evaluatedPages` | Number of pages that had headings and were scored | -| `totalMissing` | Total pages missing an H1 | -| `totalMultiple` | Total pages with multiple H1 tags | -| `totalSkips` | Sum of hierarchy skips across all pages | -| `totalReverseJumps` | Sum of reverse hierarchy jumps across all pages | -| `totalThinSections` | Total count of thin content sections across the site | -| `avgEntropy` | Mean structural entropy — a site-wide indicator of heading consistency | -| `poorPages` | Number of pages rated `Poor` | - ---- - -## Detected Issues - -The plugin surfaces plain-language issues on each page: - -- `Missing H1` — No H1 tag found -- `Multiple H1 found` — More than one H1 on the page -- `Empty or near-empty H1` — H1 text is fewer than 6 characters -- `H1 diverges from ` — H1 and page title have low Jaccard similarity (< 0.3) -- `N hierarchy skips detected` — Heading level jumps more than 1 step forward -- `N reverse hierarchy jumps detected` — Heading level drops more than 1 step -- `Thin section under "..."` — A content section has fewer than 80 words -- `High structural entropy` — Entropy score exceeds 2.1 -- `Section fragmentation is high` — More than 65% of headings are H1/H2 - ---- - -## Data Storage - -Per-URL results are persisted in the plugin's scoped database table. The stored columns are: - -- `score` — Integer health score -- `status` — `Healthy`, `Moderate`, or `Poor` -- `analysis_json` — Full `HeadingHealthPayload` serialized as JSON - -This data is available for future queries via `ctx.db.data.find()` in other plugins or custom tooling. diff --git a/packages/plugins/heading-health/index.test.ts b/packages/plugins/heading-health/index.test.ts deleted file mode 100644 index 80f571b..0000000 --- a/packages/plugins/heading-health/index.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import type { PluginContext } from '@crawlith/core'; -import HeadingHealthPlugin from './index.js'; - -describe('heading-health plugin', () => { - it('computes per-page heading payload and summary metrics', async () => { - const rawNodes = [ - { - url: 'https://example.com/a', - status: 200, - title: 'Product A Overview', - html: ` - <html><body> - <h1>Product A</h1> - <p>${'intro '.repeat(50)}</p> - <h3>Features</h3> - <p>${'feature '.repeat(20)}</p> - <h2>FAQ</h2> - <p>${'faq '.repeat(90)}</p> - </body></html> - ` - }, - { - url: 'https://example.com/b', - status: 200, - title: 'Product A Details', - html: ` - <html><body> - <h1>Product A</h1> - <h2>Specs</h2> - <p>${'spec '.repeat(30)}</p> - </body></html> - ` - } - ]; - - const graph = { getNodes: () => rawNodes }; - - const getOrFetch = vi.fn(async (url, fetchFn) => fetchFn()); - - const ctx: PluginContext = { - flags: { heading: true }, - db: { - data: { getOrFetch } - } as any - }; - - await HeadingHealthPlugin.hooks?.onMetrics?.(ctx, graph); - - const pageA = (rawNodes.find((node) => node.url === 'https://example.com/a') as any)?.headingHealth; - expect(pageA).toBeDefined(); - expect(Array.isArray(pageA.map)).toBe(true); - expect(Array.isArray(pageA.issues)).toBe(true); - expect(pageA.hierarchy_skips).toBe(1); - expect(pageA.reverse_jumps).toBe(0); - expect(pageA.thin_sections).toBeGreaterThan(0); - expect(pageA.score).toBeLessThan(100); - - const summary = ctx.metadata?.headingHealthSummary; - expect(summary).toBeDefined(); - expect(summary.evaluatedPages).toBe(2); - expect(summary.totalMissing).toBe(0); - expect(summary.totalSkips).toBeGreaterThan(0); - - expect(getOrFetch).toHaveBeenCalledTimes(2); - }); - - it('attaches summary payload on report output', async () => { - const ctx: PluginContext = { - flags: { heading: true }, - metadata: { - headingHealthSummary: { avgScore: 50, evaluatedPages: 1, poorPages: 1 } - } - }; - - const result: any = { pages: [] }; - - await HeadingHealthPlugin.hooks?.onReport?.(ctx, result); - - expect(result.plugins.headingHealth).toBeDefined(); - expect(result.plugins.headingHealth.avgScore).toBe(50); - }); -}); diff --git a/packages/plugins/heading-health/index.ts b/packages/plugins/heading-health/index.ts deleted file mode 100644 index 74183be..0000000 --- a/packages/plugins/heading-health/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { CrawlithPlugin } from '@crawlith/core'; -import { HeadingHealthHooks } from './src/plugin.js'; - -/** - * Heading Health Plugin. - * - * Analyzes heading hierarchy quality, content thinness, and template duplication risk. - * Persists URL-scoped analysis records via ctx.db for 24h cache reuse. - */ -export const HeadingHealthPlugin: CrawlithPlugin = { - name: 'heading-health', - description: 'Analyzes heading structure, hierarchy health, and content distribution', - - cli: { - flag: '--heading', - description: 'Analyze heading structure and hierarchy health', - for: ['page', 'crawl'], - options: [ - { flag: '--heading-force-refresh', description: 'Bypass the 24h heading-health cache and recompute' } - ] - }, - - scoreProvider: true, - - storage: { - fetchMode: 'local', - perPage: { - - columns: { - status: 'TEXT', - analysis_json: 'TEXT' - } - } - }, - - hooks: { - onPage: HeadingHealthHooks.onPage, - onMetrics: HeadingHealthHooks.onMetrics, - onReport: HeadingHealthHooks.onReport - } -}; - -export default HeadingHealthPlugin; diff --git a/packages/plugins/heading-health/package.json b/packages/plugins/heading-health/package.json deleted file mode 100644 index a973eaf..0000000 --- a/packages/plugins/heading-health/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "@crawlith/plugin-heading-health", - "license": "Apache-2.0", - "private": true, - "version": "1.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": { - ".": "./index.ts", - "./package.json": "./package.json" - }, - "description": "Analyzes heading structure, hierarchy health, and content distribution" -} \ No newline at end of file diff --git a/packages/plugins/heading-health/src/HeadingHealthService.ts b/packages/plugins/heading-health/src/HeadingHealthService.ts deleted file mode 100644 index 4c11812..0000000 --- a/packages/plugins/heading-health/src/HeadingHealthService.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { analyzeHeadingHealth, enrichDuplicateRisk, jaccardSimilarity } from './analyzer.js'; -import type { HeadingHealthPayload, HeadingHealthSummary, LocalPageAnalysis } from './types.js'; - -/** - * Coordinates heading analysis across graph nodes and builds report-safe payloads. - */ -export class HeadingHealthService { - /** - * Builds page-level payloads plus a snapshot summary for every eligible node. - */ - public evaluateNodes(nodes: Array<Record<string, any>>): { - payloadsByUrl: Map<string, HeadingHealthPayload>; - summary: HeadingHealthSummary; - } { - const analyzedPages = this.collectAnalyses(nodes); - enrichDuplicateRisk(analyzedPages); - - const exactH1Buckets = this.buildBuckets(analyzedPages, (page) => page.h1Norm); - const h2SetBuckets = this.buildBuckets(analyzedPages, (page) => page.h2SetHash); - const patternBuckets = this.buildBuckets(analyzedPages, (page) => page.patternHash); - const similarH1GroupSizes = this.computeSimilarH1GroupSizes(analyzedPages); - - const payloadsByUrl = new Map<string, HeadingHealthPayload>(); - - let totalScore = 0; - let totalMissing = 0; - let totalMultiple = 0; - let totalSkips = 0; - let totalReverseJumps = 0; - let totalThinSections = 0; - let totalEntropy = 0; - let poorPages = 0; - - for (const page of analyzedPages) { - const duplicateH1GroupSize = page.h1Norm ? exactH1Buckets.get(page.h1Norm)?.length || 1 : 0; - const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0; - const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1; - const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1; - const templateRisk = computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize); - const thinSectionCount = page.sections.filter((section) => section.thin).length; - - const health = scoreHealth({ - metrics: page.metrics, - thinSectionCount, - duplicateH1GroupSize, - similarH1GroupSize, - identicalH2SetGroupSize, - duplicatePatternGroupSize, - templateRisk, - issues: page.issues - }); - - if (health.status === 'Poor') { - poorPages += 1; - } - - totalScore += health.score; - totalMissing += page.metrics.missingH1; - totalMultiple += page.metrics.multipleH1; - totalSkips += page.metrics.hierarchySkips; - totalReverseJumps += page.metrics.reverseJumps; - totalThinSections += thinSectionCount; - totalEntropy += page.metrics.entropy; - - payloadsByUrl.set(page.url, { - score: health.score, - status: health.status, - issues: health.issues, - map: page.headingNodes, - missing_h1: page.metrics.missingH1, - multiple_h1: page.metrics.multipleH1, - entropy: page.metrics.entropy, - max_depth: page.metrics.maxDepth, - avg_depth: page.metrics.avgDepth, - heading_density: page.metrics.headingDensity, - fragmentation: page.metrics.fragmentation, - volatility: page.metrics.levelVolatility, - hierarchy_skips: page.metrics.hierarchySkips, - reverse_jumps: page.metrics.reverseJumps, - thin_sections: thinSectionCount, - duplicate_h1_group: duplicateH1GroupSize, - similar_h1_group: similarH1GroupSize, - identical_h2_set_group: identicalH2SetGroupSize, - duplicate_pattern_group: duplicatePatternGroupSize, - template_risk: templateRisk - }); - } - - const evaluatedPages = analyzedPages.length; - const summary: HeadingHealthSummary = { - avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0, - evaluatedPages, - totalMissing, - totalMultiple, - totalSkips, - totalReverseJumps, - totalThinSections, - avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0, - poorPages - }; - - return { payloadsByUrl, summary }; - } - - /** - * Evaluates a single page's heading health directly from raw HTML. - * Used by the `onPage` hook (page command). Cross-page duplicate signals are omitted - * since there is only one page to compare against. - */ - public evaluateSinglePage(url: string, html: string): HeadingHealthPayload { - const analysis = analyzeHeadingHealth(html); - analysis.url = url; - - const thinSectionCount = analysis.sections.filter((section) => section.thin).length; - const health = scoreHealth({ - metrics: analysis.metrics, - thinSectionCount, - duplicateH1GroupSize: 1, - similarH1GroupSize: 0, - identicalH2SetGroupSize: 1, - duplicatePatternGroupSize: 1, - templateRisk: 0, - issues: analysis.issues - }); - - return { - score: health.score, - status: health.status, - issues: health.issues, - map: analysis.headingNodes, - missing_h1: analysis.metrics.missingH1, - multiple_h1: analysis.metrics.multipleH1, - entropy: analysis.metrics.entropy, - max_depth: analysis.metrics.maxDepth, - avg_depth: analysis.metrics.avgDepth, - heading_density: analysis.metrics.headingDensity, - fragmentation: analysis.metrics.fragmentation, - volatility: analysis.metrics.levelVolatility, - hierarchy_skips: analysis.metrics.hierarchySkips, - reverse_jumps: analysis.metrics.reverseJumps, - thin_sections: thinSectionCount, - duplicate_h1_group: 1, - similar_h1_group: 0, - identical_h2_set_group: 1, - duplicate_pattern_group: 1, - template_risk: 0 - }; - } - - private collectAnalyses(nodes: Array<Record<string, any>>): LocalPageAnalysis[] { - const analyses: LocalPageAnalysis[] = []; - - for (const node of nodes) { - if (node.status < 200 || node.status >= 300 || !node.html || !node.url) { - continue; - } - - const analysis = analyzeHeadingHealth(node.html, node.title || node.rawTitle); - analysis.url = node.url; - analyses.push(analysis); - } - - return analyses; - } - - private buildBuckets(pages: LocalPageAnalysis[], selector: (page: LocalPageAnalysis) => string): Map<string, string[]> { - const buckets = new Map<string, string[]>(); - for (const page of pages) { - const key = selector(page); - if (!key) { - continue; - } - buckets.set(key, [...(buckets.get(key) || []), page.url]); - } - return buckets; - } - - private computeSimilarH1GroupSizes(pages: LocalPageAnalysis[]): Map<string, number> { - const uniqueH1 = Array.from(new Set(pages.map((page) => page.h1Norm).filter(Boolean))); - const similarBuckets = new Map<string, Set<string>>(); - - for (const h1 of uniqueH1) { - similarBuckets.set(h1, new Set([h1])); - } - - for (let i = 0; i < uniqueH1.length; i += 1) { - for (let j = i + 1; j < uniqueH1.length; j += 1) { - const a = uniqueH1[i]; - const b = uniqueH1[j]; - if (jaccardSimilarity(a, b) >= 0.7) { - similarBuckets.get(a)?.add(b); - similarBuckets.get(b)?.add(a); - } - } - } - - const groupSizes = new Map<string, number>(); - for (const page of pages) { - groupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0)); - } - - return groupSizes; - } -} - -function scoreHealth(input: { - metrics: LocalPageAnalysis['metrics']; - thinSectionCount: number; - duplicateH1GroupSize: number; - similarH1GroupSize: number; - identicalH2SetGroupSize: number; - duplicatePatternGroupSize: number; - templateRisk: number; - issues: string[]; -}): { score: number; status: 'Healthy' | 'Moderate' | 'Poor'; issues: string[] } { - let score = 100; - const metrics = input.metrics; - - if (metrics.missingH1) score -= 20; - if (metrics.multipleH1) score -= 6; - score -= metrics.hierarchySkips * 8; - score -= metrics.reverseJumps * 6; - score -= Math.round(metrics.entropy * 7); - score -= Math.round(metrics.fragmentation * 20); - score -= Math.round(metrics.levelVolatility * 6); - score -= input.thinSectionCount * 4; - - if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3); - if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2); - if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2); - if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2); - score -= Math.round(input.templateRisk * 12); - - score = Math.max(0, Math.min(100, score)); - - return { - score, - status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor', - issues: input.issues - }; -} - -function computeTemplateRisk(similar: number, h2set: number, pattern: number): number { - return Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3)); -} diff --git a/packages/plugins/heading-health/src/Output.ts b/packages/plugins/heading-health/src/Output.ts deleted file mode 100644 index b60707e..0000000 --- a/packages/plugins/heading-health/src/Output.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { PluginContext } from '@crawlith/core'; -import type { HeadingHealthPayload, HeadingHealthSummary } from './types.js'; - -/** - * Responsible for plugin logging output. - */ -export class HeadingHealthOutput { - /** - * Emits a single-line result for a single-page heading-health analysis (page command). - */ - public static emitSingle(ctx: PluginContext, payload: HeadingHealthPayload): void { - ctx.logger?.info( - `[plugin:heading-health] score=${payload.score} (${payload.status})` + - (payload.issues.length ? ` — ${payload.issues.join(', ')}` : '') - ); - } - - /** - * Emits a single-line summary for heading-health metrics (crawl command). - */ - public static emitSummary(ctx: PluginContext, summary: HeadingHealthSummary): void { - ctx.logger?.info( - `[plugin:heading-health] analyzed=${summary.evaluatedPages}, avgScore=${summary.avgScore}, poorPages=${summary.poorPages}` - ); - } -} diff --git a/packages/plugins/heading-health/src/analyzer.ts b/packages/plugins/heading-health/src/analyzer.ts deleted file mode 100644 index 5535476..0000000 --- a/packages/plugins/heading-health/src/analyzer.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { createHash } from 'node:crypto'; -import type { HeadingLevel, HeadingNode, LocalPageAnalysis, SectionMetrics } from './types.js'; - -const STOPWORDS = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'your', 'about', 'into', 'over', 'under', 'are', 'was', 'were', 'can', 'has', 'have', 'had', 'you', 'our', 'out', 'all']); -const THIN_SECTION_WORDS = 80; -const HEADING_PATTERN = /<h([1-6])\b[^<>]*>([\s\S]*?)<\/h\1>/gi; -const TITLE_PATTERN = /<title\b[^<>]*>([\s\S]*?)<\/title>/i; - -const normalizeText = (input: string) => input.replace(/<[^<>]*>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/\s+/g, ' ').trim(); -const normalizeComparable = (input: string) => normalizeText(input).toLowerCase(); -const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((token) => token.length > 2 && !STOPWORDS.has(token)); -const stableHash = (input: string) => createHash('sha1').update(input).digest('hex').slice(0, 16); - -/** - * Calculates token-level Jaccard similarity between two text values. - */ -export function jaccardSimilarity(a: string, b: string): number { - const aSet = new Set(tokenize(a)); - const bSet = new Set(tokenize(b)); - - if (!aSet.size || !bSet.size) { - return 0; - } - - let intersection = 0; - for (const token of aSet) { - if (bSet.has(token)) { - intersection += 1; - } - } - - return intersection / (aSet.size + bSet.size - intersection); -} - -/** - * Performs per-page heading extraction and structural scoring signal generation. - */ -export function analyzeHeadingHealth(html: string, fallbackTitle?: string): LocalPageAnalysis { - const segments: Array<{ level: HeadingLevel; text: string; start: number; end: number }> = []; - for (const match of html.matchAll(HEADING_PATTERN)) { - const level = Number(match[1]) as HeadingLevel; - segments.push({ - level, - text: normalizeText(match[2] || ''), - start: match.index || 0, - end: (match.index || 0) + match[0].length - }); - } - - const headingNodes: HeadingNode[] = []; - const stack: HeadingNode[] = []; - segments.forEach((segment, index) => { - const node: HeadingNode = { level: segment.level, text: segment.text, index }; - while (stack.length > 0 && stack[stack.length - 1].level >= node.level) { - stack.pop(); - } - if (stack.length > 0) { - node.parentIndex = stack[stack.length - 1].index; - } - stack.push(node); - headingNodes.push(node); - }); - - const sections: SectionMetrics[] = []; - const pageWords = tokenize(html); - const frequency = new Map<string, number>(); - for (const word of pageWords) { - frequency.set(word, (frequency.get(word) || 0) + 1); - } - - for (let i = 0; i < segments.length; i += 1) { - const current = segments[i]; - const next = segments[i + 1]; - const textChunk = html.slice(current.end, next ? next.start : html.length); - const words = tokenize(textChunk); - const headingTokens = tokenize(current.text); - const concentration = headingTokens.reduce((sum, token) => sum + (frequency.get(token) || 0), 0); - - sections.push({ - headingIndex: headingNodes[i]?.index ?? i, - headingText: current.text, - words: words.length, - keywordConcentration: words.length > 0 ? Number((concentration / words.length).toFixed(3)) : 0, - thin: words.length > 0 && words.length < THIN_SECTION_WORDS, - duplicateRisk: 0 - }); - } - - const levelCounts = [0, 0, 0, 0, 0, 0]; - headingNodes.forEach((node) => { - levelCounts[node.level - 1] += 1; - }); - - const entropyScore = Number(calculateEntropy(levelCounts).toFixed(3)); - const missingH1 = levelCounts[0] === 0 ? 1 : 0; - const multipleH1 = levelCounts[0] > 1 ? 1 : 0; - - let hierarchySkips = 0; - let reverseJumps = 0; - let volatilitySum = 0; - for (let i = 1; i < headingNodes.length; i += 1) { - const delta = headingNodes[i].level - headingNodes[i - 1].level; - volatilitySum += Math.abs(delta); - if (delta > 1) { - hierarchySkips += 1; - } - if (delta < -1) { - reverseJumps += 1; - } - } - - const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((node) => node.level)) : 0; - const avgDepth = headingNodes.length ? Number((headingNodes.reduce((sum, node) => sum + node.level, 0) / headingNodes.length).toFixed(2)) : 0; - const headingDensity = pageWords.length ? Number((headingNodes.length / pageWords.length).toFixed(4)) : 0; - const fragmentation = headingNodes.length ? Number((headingNodes.filter((node) => node.level <= 2).length / headingNodes.length).toFixed(3)) : 0; - const levelVolatility = headingNodes.length > 1 ? Number((volatilitySum / (headingNodes.length - 1)).toFixed(3)) : 0; - - const h1Nodes = headingNodes.filter((node) => node.level === 1); - const issues: string[] = []; - if (missingH1) issues.push('Missing H1'); - if (multipleH1) issues.push('Multiple H1 found'); - if (h1Nodes.some((node) => node.text.length < 6)) issues.push('Empty or near-empty H1'); - - const title = fallbackTitle || getTitleFromHtml(html); - if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) { - issues.push('H1 diverges from <title>'); - } - if (hierarchySkips > 0) issues.push(`${hierarchySkips} hierarchy skips detected`); - if (reverseJumps > 0) issues.push(`${reverseJumps} reverse hierarchy jumps detected`); - for (const thin of sections.filter((section) => section.thin).slice(0, 2)) { - issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`); - } - if (entropyScore > 2.1) issues.push('High structural entropy'); - if (fragmentation > 0.65) issues.push('Section fragmentation is high'); - - const h1Norm = normalizeComparable(h1Nodes[0]?.text || ''); - const h2SetHash = stableHash( - headingNodes - .filter((node) => node.level === 2) - .map((node) => normalizeComparable(node.text)) - .filter(Boolean) - .sort() - .join('|') - ); - const patternHash = stableHash(headingNodes.map((node) => node.level).join('>')); - - return { - url: '', - headingNodes, - sections, - h1Norm, - h2SetHash, - patternHash, - issues, - metrics: { - entropy: entropyScore, - maxDepth, - avgDepth, - headingDensity, - fragmentation, - levelVolatility, - hierarchySkips, - reverseJumps, - missingH1, - multipleH1 - } - }; -} - -/** - * Enriches section-level duplicate risk by comparing normalized section signatures across pages. - */ -export function enrichDuplicateRisk(pages: LocalPageAnalysis[]): void { - const buckets = new Map<string, string[]>(); - - for (const page of pages) { - for (const section of page.sections) { - const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); - const bucket = buckets.get(key) || []; - bucket.push(page.url); - buckets.set(key, bucket); - } - } - - for (const page of pages) { - for (const section of page.sections) { - const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); - const size = (buckets.get(key) || []).length; - section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3)); - } - } -} - -function calculateEntropy(values: number[]): number { - const total = values.reduce((a, b) => a + b, 0); - if (!total) { - return 0; - } - - return values.reduce((sum, value) => { - if (value === 0) { - return sum; - } - - const probability = value / total; - return sum - probability * Math.log2(probability); - }, 0); -} - -function getTitleFromHtml(html: string): string { - const match = html.match(TITLE_PATTERN); - return match ? normalizeText(match[1]) : ''; -} diff --git a/packages/plugins/heading-health/src/cli.ts b/packages/plugins/heading-health/src/cli.ts deleted file mode 100644 index 9e78d3b..0000000 --- a/packages/plugins/heading-health/src/cli.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Command } from '@crawlith/core'; - -/** - * Registers CLI options exposed by the heading-health plugin. - */ -export function registerHeadingHealthCli(cli: Command): void { - if (cli.name() === 'crawl' || cli.name() === 'page') { - cli.option('--heading', 'Analyze heading structure and hierarchy health'); - cli.option('--heading-force-refresh', 'Bypass the 24h heading-health cache and recompute'); - } -} diff --git a/packages/plugins/heading-health/src/plugin.ts b/packages/plugins/heading-health/src/plugin.ts deleted file mode 100644 index 26ec749..0000000 --- a/packages/plugins/heading-health/src/plugin.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { PluginContext, PageInput } from '@crawlith/core'; -import { HeadingHealthOutput } from './Output.js'; -import { HeadingHealthService } from './HeadingHealthService.js'; -import type { HeadingHealthPayload, HeadingHealthRow, HeadingHealthSummary } from './types.js'; - -/** - * Lifecycle hooks for the heading-health plugin. - * Schema registration is handled declaratively via plugin.storage — no onInit needed. - */ -export const HeadingHealthHooks = { - /** - * Single-page hook (page command). Analyzes one URL's HTML directly. - * No cross-page duplicate signals since only one page is available. - */ - onPage: async (ctx: PluginContext, page: PageInput): Promise<void> => { - if (!ctx.flags?.heading || !ctx.db) return; - - const service = new HeadingHealthService(); - - const row = await ctx.db.data.getOrFetch<HeadingHealthRow>( - page.url, - async () => { - const payload = service.evaluateSinglePage(page.url, page.html); - return { - score: payload.score, - status: payload.status, - // Store as string, but getOrFetch/find will auto-parse to object later - analysis_json: JSON.stringify(payload) - } as unknown as HeadingHealthRow; - } - ); - - if (!row) return; - - // analysis_json gets auto-parsed by CrawlithDB._parseRow - const payload = (typeof row.analysis_json === 'string' - ? JSON.parse(row.analysis_json) - : row.analysis_json) as HeadingHealthPayload; - - HeadingHealthOutput.emitSingle(ctx, payload); - }, - - /** - * Graph-level hook (crawl command). Evaluates all nodes with cross-page duplicate detection. - */ - onMetrics: async (ctx: PluginContext, graph: any): Promise<void> => { - if (!ctx.flags?.heading || !ctx.db) return; - - const nodes = graph?.getNodes?.(); - if (!Array.isArray(nodes)) return; - - const service = new HeadingHealthService(); - const { payloadsByUrl, summary } = service.evaluateNodes(nodes); - - for (const node of nodes) { - const url = node?.url; - if (!url) continue; - - const livePayload = payloadsByUrl.get(url); - if (!livePayload) continue; - - const row = await ctx.db.data.getOrFetch<HeadingHealthRow>( - url, - async () => ({ - score: livePayload.score, - status: livePayload.status, - analysis_json: JSON.stringify(livePayload) - } as unknown as HeadingHealthRow) - ); - - if (row) { - const payload = (typeof row.analysis_json === 'string' - ? JSON.parse(row.analysis_json) - : row.analysis_json) as HeadingHealthPayload; - - (node as any).headingHealth = payload; - } - } - - ctx.metadata = ctx.metadata || {}; - ctx.metadata.headingHealthSummary = summary; - HeadingHealthOutput.emitSummary(ctx, summary); - }, - - /** - * Attaches snapshot-level heading-health summary to the final report object. - */ - onReport: async (ctx: PluginContext, result: any): Promise<void> => { - if (!ctx.flags?.heading) return; - - const summary = ctx.metadata?.headingHealthSummary as HeadingHealthSummary | undefined; - if (!summary) return; - - result.plugins = result.plugins || {}; - result.plugins.headingHealth = summary; - } -}; diff --git a/packages/plugins/heading-health/src/types.ts b/packages/plugins/heading-health/src/types.ts deleted file mode 100644 index 4ab8998..0000000 --- a/packages/plugins/heading-health/src/types.ts +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Supported heading levels within HTML content. - */ -export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6; - -/** - * Represents a normalized heading node extracted from the DOM. - */ -export interface HeadingNode { - level: HeadingLevel; - text: string; - index: number; - parentIndex?: number; -} - -/** - * Represents content statistics for a section under a heading. - */ -export interface SectionMetrics { - headingIndex: number; - headingText: string; - words: number; - keywordConcentration: number; - thin: boolean; - duplicateRisk: number; -} - -/** - * Raw heading analysis generated for a single URL. - */ -export interface LocalPageAnalysis { - url: string; - headingNodes: HeadingNode[]; - sections: SectionMetrics[]; - h1Norm: string; - h2SetHash: string; - patternHash: string; - issues: string[]; - metrics: { - entropy: number; - maxDepth: number; - avgDepth: number; - headingDensity: number; - fragmentation: number; - levelVolatility: number; - hierarchySkips: number; - reverseJumps: number; - missingH1: number; - multipleH1: number; - }; -} - -/** - * Final heading-health payload attached to a page node. - */ -export interface HeadingHealthPayload { - score: number; - status: 'Healthy' | 'Moderate' | 'Poor'; - issues: string[]; - map: HeadingNode[]; - missing_h1: number; - multiple_h1: number; - entropy: number; - max_depth: number; - avg_depth: number; - heading_density: number; - fragmentation: number; - volatility: number; - hierarchy_skips: number; - reverse_jumps: number; - thin_sections: number; - duplicate_h1_group: number; - similar_h1_group: number; - identical_h2_set_group: number; - duplicate_pattern_group: number; - template_risk: number; -} - -/** - * Snapshot-level summary emitted by the plugin. - */ -export interface HeadingHealthSummary { - avgScore: number; - evaluatedPages: number; - totalMissing: number; - totalMultiple: number; - totalSkips: number; - totalReverseJumps: number; - totalThinSections: number; - avgEntropy: number; - poorPages: number; -} - -/** - * Persisted row shape used by ctx.db.data.find/save. - */ -export interface HeadingHealthRow { - analysis_json: string; - score: number; - status: 'Healthy' | 'Moderate' | 'Poor'; -} diff --git a/packages/plugins/heading-health/tests/headings.test.ts b/packages/plugins/heading-health/tests/headings.test.ts deleted file mode 100644 index f72cf8d..0000000 --- a/packages/plugins/heading-health/tests/headings.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { analyzeHeadingHealth } from '../src/analyzer.js'; - -describe('heading health analyzer', () => { - it('detects missing H1 and hierarchy skips', () => { - const html = ` - <title>Test Page -

Subheading

-

Jumped level

- `; - const analysis = analyzeHeadingHealth(html); - expect(analysis.issues).toContain('Missing H1'); - expect(analysis.issues).toContain('1 hierarchy skips detected'); - expect(analysis.metrics.missingH1).toBe(1); - expect(analysis.metrics.hierarchySkips).toBe(1); - }); - - it('detects multiple H1s and title divergence', () => { - const html = ` - Correct Title -

First H1

-

Second H1 Divergent

- `; - const analysis = analyzeHeadingHealth(html); - expect(analysis.issues).toContain('Multiple H1 found'); - expect(analysis.issues).toContain('H1 diverges from '); - expect(analysis.metrics.multipleH1).toBe(1); - }); - - it('extracts heading structure correctly', () => { - const html = ` - <h1>Main</h1> - <h2>Section 1</h2> - <h3>Detail 1.1</h3> - <h2>Section 2</h2> - `; - const analysis = analyzeHeadingHealth(html); - expect(analysis.headingNodes).toHaveLength(4); - expect(analysis.headingNodes[0].level).toBe(1); - expect(analysis.headingNodes[1].parentIndex).toBe(0); - expect(analysis.headingNodes[2].parentIndex).toBe(1); - expect(analysis.headingNodes[3].parentIndex).toBe(0); - }); -}); diff --git a/packages/plugins/heading-health/tsconfig.json b/packages/plugins/heading-health/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/heading-health/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/health-score-engine/CHANGELOG.md b/packages/plugins/health-score-engine/CHANGELOG.md deleted file mode 100644 index 6a3039e..0000000 --- a/packages/plugins/health-score-engine/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-health-score-engine - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/health-score-engine/README.md b/packages/plugins/health-score-engine/README.md deleted file mode 100644 index ec0f777..0000000 --- a/packages/plugins/health-score-engine/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Health Score Engine Plugin - -The Health Score Engine plugin provides a penalty-based scoring system for evaluating the overall SEO health of a website. It identifies critical issues such as orphan pages, broken links, redirect chains, and duplicate content. - -## Features -- **ScoreProvider Integration**: Contributes to the aggregate site score by evaluating per-page SEO health. -- **Penalty-based Global Score**: calculates a 0-100 health score based on the frequency of various crawl issues. -- **Issue Collection**: Aggregates SEO warnings and critical errors across the entire crawl. - -## CLI Flags -- `--health`: Enable health score analysis (default). -- `--fail-on-critical`: Exit with code 1 if the health score falls below 50. -- `--score-breakdown`: Print a detailed breakdown of the penalties affecting the health score. - -## Output Details -When enabled, this plugin adds a `health` section to the crawl results, including: -- `score`: The overall health score. -- `status`: A label (e.g., Excellent, Good, Critical) based on the score and presence of critical issues. -- `weightedPenalties`: A breakdown of points deducted for each category of issue. -- `issues`: Raw counts for each type of SEO issue discovered. diff --git a/packages/plugins/health-score-engine/index.ts b/packages/plugins/health-score-engine/index.ts deleted file mode 100644 index 6b34046..0000000 --- a/packages/plugins/health-score-engine/index.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { CrawlithPlugin } from '@crawlith/core'; -import { HealthScoreHooks } from './src/plugin.js'; - -/** - * Health Score Engine Plugin - * Calculates site-wide SEO health, identifies critical issues, and provides a penalty-based scoring system. - */ -export const HealthScoreEnginePlugin: CrawlithPlugin = { - name: 'health-score-engine', - description: 'Iterative penalty-based health scoring and issue collection', - - cli: { - flag: '--health', // We can add a flag to explicitly enable it if it wasn't default - description: 'Run health score analysis', - options: [ - { flag: '--fail-on-critical', description: 'Exit code 1 if critical issues exist' }, - { flag: '--score-breakdown', description: 'Print health score component weights' } - ] - }, - - scoreProvider: true, - - storage: { - fetchMode: 'local', - perPage: { - columns: { - score: 'REAL', - weight: 'REAL' - } - } - }, - - hooks: HealthScoreHooks -}; - -export default HealthScoreEnginePlugin; diff --git a/packages/plugins/health-score-engine/package.json b/packages/plugins/health-score-engine/package.json deleted file mode 100644 index eaaabee..0000000 --- a/packages/plugins/health-score-engine/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@crawlith/plugin-health-score-engine", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*", - "chalk": "^5.6.2" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for health score engine" -} diff --git a/packages/plugins/health-score-engine/src/plugin.ts b/packages/plugins/health-score-engine/src/plugin.ts deleted file mode 100644 index 240a4dc..0000000 --- a/packages/plugins/health-score-engine/src/plugin.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { PluginContext, Graph, analyzePages } from '@crawlith/core'; -import { HealthService, DEFAULT_HEALTH_WEIGHTS } from './Service.js'; - -const service = new HealthService(); - -export const HealthScoreHooks = { - /** - * For single-page analysis, provide a basic health breakdown. - */ - onPage: async (ctx: PluginContext, page: any) => { - // Single page analysis doesn't have a link graph for full health score, - // so we use the SEO scorer from core. - const analysisResults = analyzePages(page.url, [page], undefined, { allPages: false }); - if (analysisResults.length > 0) { - const seoScore = analysisResults[0].seoScore; - ctx.logger?.info(`Page Health Score: ${seoScore}`); - } - }, - - /** - * On metrics, we compute the site-wide health score and issues. - */ - onMetrics: async (ctx: PluginContext, graph: Graph) => { - const metrics = (graph as any).metrics || {}; - const clusters = ctx.metadata?.clusters || []; - const issues = service.collectCrawlIssues(graph, { ...metrics, clusters }); - const health = service.calculateHealthScore(graph.nodes.size, issues, DEFAULT_HEALTH_WEIGHTS); - - // Save summary to plugin-scoped storage - // Since this is a site-wide metric, we store it under a special key or just in the report - ctx.metadata = ctx.metadata || {}; - ctx.metadata.healthReport = { health, issues }; - - // To satisfy ScoreProvider, we should also store per-page scores if we want to contribute - // to the aggregate snapshots.total_score. - if (ctx.db) { - const nodes = graph.getNodes(); - - // Map graph nodes to PageAnalysis for scorePageSeo - // In a real scenario, we might want to perform full analysis, but here we can - // synthesize a PageAnalysis object from the graph node metrics. - const results = analyzePages('', nodes.map(node => ({ - url: node.url, - status: node.status, - html: node.html || '', - depth: node.depth, - crawlStatus: node.crawlStatus - })), undefined, { allPages: true }); - - for (const analysis of results) { - ctx.db.data.save({ - url: analysis.url, - data: { - score: analysis.seoScore, - weight: 1.0 - } - }); - } - - // The logic for snapshots.health_score in core is global. - // Let's update that column directly for compatibility. - const rawDb = (ctx.db as any).unsafeGetRawDb(); - if (rawDb) { - rawDb.prepare(`UPDATE snapshots SET health_score = ?, orphan_count = ?, thin_content_count = ? WHERE id = ?`) - .run(health.score, issues.orphanPages, issues.thinContent, ctx.snapshotId); - } - } - }, - - /** - * Emit the health report. - */ - onReport: async (ctx: PluginContext, result: any) => { - const report = ctx.metadata?.healthReport; - if (!report) return; - - result.plugins = result.plugins || {}; - result.plugins.health = report; - - const flags = ctx.flags || {}; - if (flags.scoreBreakdown) { - ctx.logger?.info('\nHealth Score Breakdown:'); - ctx.logger?.info(` Overall Score: ${report.health.score} (${report.health.status})`); - ctx.logger?.info(` Penalties: ${JSON.stringify(report.health.weightedPenalties, null, 2)}`); - } - - if (flags.failOnCritical && report.health.score < 50) { - ctx.logger?.error(`\n❌ Fail-on-critical: Health score ${report.health.score} is below threshold.`); - process.exit(1); - } - } -}; diff --git a/packages/plugins/health-score-engine/src/types.ts b/packages/plugins/health-score-engine/src/types.ts deleted file mode 100644 index 39dfdbd..0000000 --- a/packages/plugins/health-score-engine/src/types.ts +++ /dev/null @@ -1,47 +0,0 @@ -export interface HealthScoreWeights { - orphans: number; - brokenLinks: number; - redirectChains: number; - duplicateClusters: number; - thinContent: number; - missingH1: number; - noindexMisuse: number; - canonicalConflicts: number; - lowInternalLinks: number; - excessiveLinks: number; - blockedByRobots: number; -} - -export interface CrawlIssueCounts { - orphanPages: number; - brokenInternalLinks: number; - redirectChains: number; - duplicateClusters: number; - canonicalConflicts: number; - accidentalNoindex: number; - missingH1: number; - thinContent: number; - lowInternalLinkCount: number; - excessiveInternalLinkCount: number; - highExternalLinkRatio: number; - imageAltMissing: number; - strongPagesUnderLinking: number; - cannibalizationClusters: number; - nearAuthorityThreshold: number; - underlinkedHighAuthorityPages: number; - externalLinks: number; - blockedByRobots: number; -} - -export interface HealthScoreBreakdown { - score: number; - status: string; - weightedPenalties: Record<keyof HealthScoreWeights, number>; - weights: HealthScoreWeights; -} - -export interface HealthRow { - score: number; - weight: number; - issues_json: string; -} diff --git a/packages/plugins/health-score-engine/tests/health.test.ts b/packages/plugins/health-score-engine/tests/health.test.ts deleted file mode 100644 index 8cbc2a9..0000000 --- a/packages/plugins/health-score-engine/tests/health.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { HealthScoreEnginePlugin } from '../index.js'; -import { HealthService } from '../src/Service.js'; -import { Graph } from '@crawlith/core'; - -describe('HealthScoreEnginePlugin', () => { - let service: HealthService; - - beforeEach(() => { - service = new HealthService(); - vi.restoreAllMocks(); - }); - - it('should calculate health score correctly', () => { - const issues = { - orphanPages: 2, - brokenInternalLinks: 1, - redirectChains: 0, - duplicateClusters: 0, - canonicalConflicts: 0, - accidentalNoindex: 0, - missingH1: 0, - thinContent: 0, - lowInternalLinkCount: 0, - excessiveInternalLinkCount: 0, - blockedByRobots: 0 - }; - - const result = service.calculateHealthScore(10, issues as any); - expect(result.score).toBeLessThan(100); - expect(result.status).toBeDefined(); - }); - - it('should collect issues from graph', () => { - const graph = new Graph(); - graph.addNode('https://example.com/', 0, 200); - graph.addNode('https://example.com/broken', 1, 404); - graph.addEdge('https://example.com/', 'https://example.com/broken'); - - const metrics = { - orphanPages: [], - totalPages: 2 - }; - - const issues = service.collectCrawlIssues(graph, metrics); - expect(issues.brokenInternalLinks).toBeGreaterThanOrEqual(1); - }); - - it('should exit with code 1 if failOnCritical is set and score < 50', async () => { - // Mock process.exit - const exitSpy = vi.spyOn(process, 'exit').mockImplementation((code?: string | number | null) => { - throw new Error(`exit:${code}`); - }); - - const mockLogger = { - info: vi.fn(), - error: vi.fn(), - warn: vi.fn(), - debug: vi.fn() - }; - - const ctx = { - flags: { failOnCritical: true }, - logger: mockLogger, - metadata: { - healthReport: { - health: { score: 45, status: 'Critical' }, - issues: {} - } - } - }; - - const mockResult = {}; - - await expect( - HealthScoreEnginePlugin.hooks!.onReport!(ctx as any, mockResult) - ).rejects.toThrow('exit:1'); - - expect(mockLogger.error).toHaveBeenCalled(); - exitSpy.mockRestore(); - }); - - it('should not exit if score is >= 50', async () => { - const exitSpy = vi.spyOn(process, 'exit').mockImplementation((code?: string | number | null) => { - throw new Error(`exit:${code}`); - }); - - const ctx = { - flags: { failOnCritical: true }, - metadata: { - healthReport: { - health: { score: 80, status: 'Good' }, - issues: {} - } - } - }; - - const mockResult = {}; - - await HealthScoreEnginePlugin.hooks!.onReport!(ctx as any, mockResult); - - expect(exitSpy).not.toHaveBeenCalled(); - exitSpy.mockRestore(); - }); -}); diff --git a/packages/plugins/health-score-engine/tsconfig.json b/packages/plugins/health-score-engine/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/health-score-engine/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/plugins/hits/CHANGELOG.md b/packages/plugins/hits/CHANGELOG.md deleted file mode 100644 index 99189d9..0000000 --- a/packages/plugins/hits/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-hits - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/hits/README.md b/packages/plugins/hits/README.md deleted file mode 100644 index db92d25..0000000 --- a/packages/plugins/hits/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# HITS Plugin (Hubs & Authorities) - -Crawlith plugin that computes Hub and Authority scores using the HITS algorithm to identify relative importance and structural roles of pages in a crawl graph. - -## Architecture - -This plugin follows the **ScoreProvider** architecture: -- **Service**: `HITSService` contains the iterative algorithm logic. -- **Storage**: Maps results to a plugin-scoped SQLite table. -- **Aggregation**: Participates in snapshot-level score aggregation (mapping authority scores). - -## Score Logic - -HITS identifies two types of important pages: -1. **Authorities**: Pages that are linked to by many high-quality hubs. They contain valuable information. -2. **Hubs**: Pages that link to many high-quality authorities. They serve as catalogs or directories. - -The plugin also classifies nodes into roles: -- **Power**: High Authority AND High Hub score. -- **Authority**: High Authority score only. -- **Hub**: High Hub score only. -- **Balanced**: Moderate scores in both. -- **Peripheral**: Low scores or isolated. - -## Usage - -Enable during a crawl by passing the flag: -```bash -crawlith crawl https://example.com --compute-hits -``` - -## Output - -Results are included in the JSON report under `plugins.hits` and stored locally in the plugin database. - -```json -{ - "plugins": { - "hits": { - "authorityCount": 42, - "hubCount": 15, - "topAuthorities": [...], - "topHubs": [...] - } - } -} -``` diff --git a/packages/plugins/hits/index.ts b/packages/plugins/hits/index.ts deleted file mode 100644 index 2260292..0000000 --- a/packages/plugins/hits/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { CrawlithPlugin } from '@crawlith/core'; -import { HitsHooks } from './src/plugin.js'; - -/** - * Description: Hubs and Authorities (HITS) is a link analysis algorithm that rates web pages. - * Authority measures the valuable information on a page, while Hub measures the quality of the links to other pages. - * @requirements Use --compute-hits to enable during crawl. - */ -export const HitsPlugin: CrawlithPlugin = { - name: 'hits', - - cli: { - flag: '--compute-hits', - description: 'Compute Hub and Authority scores (HITS)' - }, - - scoreProvider: true, - - storage: { - fetchMode: 'local', - perPage: { - columns: { - authority_score: 'REAL', - hub_score: 'REAL', - link_role: 'TEXT', - score: 'REAL' // Composite score for aggregation provider - } - } - }, - - hooks: HitsHooks -}; - -export default HitsPlugin; diff --git a/packages/plugins/hits/package.json b/packages/plugins/hits/package.json deleted file mode 100644 index a811062..0000000 --- a/packages/plugins/hits/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-hits", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for hits" -} \ No newline at end of file diff --git a/packages/plugins/hits/src/plugin.ts b/packages/plugins/hits/src/plugin.ts deleted file mode 100644 index 4197897..0000000 --- a/packages/plugins/hits/src/plugin.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { PluginContext, Graph } from '@crawlith/core'; -import { HITSService } from './Service.js'; -import { HITSRow } from './types.js'; - -export const HitsHooks = { - /** - * Called on the fully built graph during post-crawl metrics. - */ - onMetrics: async (ctx: PluginContext, graph: Graph) => { - const flags = ctx.flags || {}; - if (!flags.computeHits || !ctx.db) return; - - ctx.logger?.info?.('🧮 Computing HITS Hub/Authority scores...'); - - const service = new HITSService(); - const hitsResults = service.evaluate(graph); - - let count = 0; - for (const node of graph.getNodes()) { - const res = hitsResults.get(node.url); - if (res) { - // Mutate node for downstream in-memory use (core metrics might still look at these) - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - - // Save to plugin scoped table - await ctx.db.data.save({ - url: node.url, - data: { - ...res, - score: res.authority_score * 100, // Normalized for aggregator (0-100 range) - weight: 1.0 - } - }); - count++; - } - } - - ctx.logger?.info?.(`🧮 HITS computation complete for ${count} nodes.`); - }, - - /** - * Post-crawl reporting hook. - */ - onReport: async (ctx: PluginContext, result: any) => { - const flags = ctx.flags || {}; - if (!flags.computeHits || !ctx.db) return; - - const allRows = ctx.db.data.all<HITSRow & { url: string }>(); - if (allRows.length === 0) return; - - if (!result.plugins) result.plugins = {}; - - // Summary metrics - const authorities = allRows.filter(r => r.link_role === 'authority' || r.link_role === 'power'); - const hubs = allRows.filter(r => r.link_role === 'hub' || r.link_role === 'power'); - - result.plugins.hits = { - authorityCount: authorities.length, - hubCount: hubs.length, - topAuthorities: authorities - .sort((a, b) => b.authority_score - a.authority_score) - .slice(0, 10) - .map(a => ({ url: (a as any).url, score: a.authority_score })), - topHubs: hubs - .sort((a, b) => b.hub_score - a.hub_score) - .slice(0, 10) - .map(h => ({ url: (h as any).url, score: h.hub_score })) - }; - } -}; diff --git a/packages/plugins/hits/src/types.ts b/packages/plugins/hits/src/types.ts deleted file mode 100644 index 4b4a34f..0000000 --- a/packages/plugins/hits/src/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type LinkRole = 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral'; - -export interface HITSRow { - authority_score: number; - hub_score: number; - link_role: LinkRole; -} - -export interface HITSOptions { - iterations?: number; -} diff --git a/packages/plugins/hits/tests/plugin.test.ts b/packages/plugins/hits/tests/plugin.test.ts deleted file mode 100644 index e4d2cde..0000000 --- a/packages/plugins/hits/tests/plugin.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { Graph } from '@crawlith/core'; -import { HITSService } from '../src/Service.js'; - -describe('HITS Scoring', () => { - it('should compute scores for a simple star topology', () => { - const graph = new Graph(); - // Hub - graph.addNode('http://hub.com', 0, 200); - // Authorities - graph.addNode('http://auth1.com', 1, 200); - graph.addNode('http://auth2.com', 1, 200); - graph.addNode('http://auth3.com', 1, 200); - - graph.addEdge('http://hub.com', 'http://auth1.com'); - graph.addEdge('http://hub.com', 'http://auth2.com'); - graph.addEdge('http://hub.com', 'http://auth3.com'); - - const service = new HITSService(); - const results = service.evaluate(graph, { iterations: 10 }); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - } - - const hub = graph.nodes.get('http://hub.com')!; - const auth1 = graph.nodes.get('http://auth1.com')!; - - // In a star topology: - // Hub should have max hub score - // Authorities should have max authority scores - expect((hub as any).hubScore).toBeGreaterThan(0.9); - expect((hub as any).authorityScore).toBe(0); // No one links to hub - - expect((auth1 as any).authorityScore).toBeGreaterThan(0.5); - expect((auth1 as any).hubScore).toBe(0); // Auth1 links to no one - }); - - it('should handle exclusion rules', () => { - const graph = new Graph(); - graph.addNode('http://valid.com', 0, 200); - graph.addNode('http://noindex.com', 0, 200); - graph.updateNodeData('http://noindex.com', { noindex: true }); - graph.addNode('http://redirect.com', 0, 200); - graph.updateNodeData('http://redirect.com', { redirectChain: ['http://target.com'] }); - graph.addNode('http://external.com', 0, 200); // Eligibility check marks it as eligible if status is 200 - // but typically external wouldn't have status 200 in the graph if we don't crawl them or they are marked as external. - // The current hits logic relies on: status === 200 && no redirectChain && !noindex - - graph.addEdge('http://valid.com', 'http://noindex.com'); - graph.addEdge('http://valid.com', 'http://redirect.com'); - - const service = new HITSService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - } - - expect((graph.nodes.get('http://noindex.com') as any)?.hubScore).toBeUndefined(); - expect((graph.nodes.get('http://redirect.com') as any)?.hubScore).toBeUndefined(); - expect((graph.nodes.get('http://valid.com') as any)?.hubScore).toBe(0); // Valid hub but its targets are ineligible - }); - - it('should respect edge weights', () => { - const graph = new Graph(); - graph.addNode('http://hub.com', 0, 200); - graph.addNode('http://auth-high.com', 1, 200); - graph.addNode('http://auth-low.com', 1, 200); - - graph.addEdge('http://hub.com', 'http://auth-high.com', 1.0); - graph.addEdge('http://hub.com', 'http://auth-low.com', 0.1); - - const service = new HITSService(); - const results = service.evaluate(graph, { iterations: 10 }); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - } - - const authHigh = graph.nodes.get('http://auth-high.com')!; - const authLow = graph.nodes.get('http://auth-low.com')!; - - expect((authHigh as any).authorityScore).toBeGreaterThan((authLow as any).authorityScore!); - }); - - it('should classify link roles correctly', () => { - const graph = new Graph(); - for (let i = 0; i < 11; i++) { - graph.addNode(`http://node${i}.com`, 0, 200); - } - - // AUTHORITY: node1 (linked by 0,2,3... no outlinks) - graph.addEdge('http://node0.com', 'http://node1.com'); - graph.addEdge('http://node2.com', 'http://node1.com'); - graph.addEdge('http://node3.com', 'http://node1.com'); - graph.addEdge('http://node4.com', 'http://node1.com'); - - // HUB: node4 (links to 1,5,6,7... few inlinks) - graph.addEdge('http://node4.com', 'http://node5.com'); - graph.addEdge('http://node4.com', 'http://node6.com'); - graph.addEdge('http://node4.com', 'http://node7.com'); - - // POWER: node2 (linked by 0, power is often recursive... link to authority and be linked by hub) - graph.addEdge('http://node0.com', 'http://node2.com'); - graph.addEdge('http://node2.com', 'http://node1.com'); - graph.addEdge('http://node2.com', 'http://node5.com'); - - // PERIPHERAL: node10 (no links) - // Some filler nodes to push medians down - graph.addEdge('http://node8.com', 'http://node9.com'); - - const service = new HITSService(); - const results = service.evaluate(graph, { iterations: 20 }); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - } - - const roles = graph.getNodes().map(n => (n as any).linkRole).filter(Boolean); - expect(roles).toContain('authority'); - expect(roles).toContain('hub'); - expect(roles).toContain('power'); - expect(roles).toContain('peripheral'); - }); - - it('should handle large synthetic graphs (Performance Test)', () => { - const graph = new Graph(); - const nodeCount = 5000; - - // Create 5000 nodes - for (let i = 0; i < nodeCount; i++) { - graph.addNode(`http://page${i}.com`, 1, 200); - } - - // Create random edges (avg 10 per node) - for (let i = 0; i < nodeCount; i++) { - for (let j = 0; j < 10; j++) { - const target = Math.floor(Math.random() * nodeCount); - if (i !== target) { - graph.addEdge(`http://page${i}.com`, `http://page${target}.com`); - } - } - } - - const start = Date.now(); - const service = new HITSService(); - const results = service.evaluate(graph, { iterations: 20 }); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).authorityScore = res.authority_score; - (node as any).hubScore = res.hub_score; - (node as any).linkRole = res.link_role; - } - const duration = Date.now() - start; - - console.log(`HITS on 5000 nodes took ${duration}ms`); - expect(duration).toBeLessThan(2000); // Should be very fast, but allow buffer for CI environments - expect((graph.nodes.get('http://page0.com') as any)?.hubScore).toBeDefined(); - }); -}); diff --git a/packages/plugins/hits/tsconfig.json b/packages/plugins/hits/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/hits/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/orphan-intelligence/CHANGELOG.md b/packages/plugins/orphan-intelligence/CHANGELOG.md deleted file mode 100644 index 3dd0586..0000000 --- a/packages/plugins/orphan-intelligence/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-orphan-intelligence - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/orphan-intelligence/README.md b/packages/plugins/orphan-intelligence/README.md deleted file mode 100644 index 0a8bbc1..0000000 --- a/packages/plugins/orphan-intelligence/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Orphan Intelligence Plugin - -Crawlith plugin for orphan intelligence - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/orphan-intelligence/index.ts b/packages/plugins/orphan-intelligence/index.ts deleted file mode 100644 index 3bbe891..0000000 --- a/packages/plugins/orphan-intelligence/index.ts +++ /dev/null @@ -1,77 +0,0 @@ - -import { CrawlithPlugin, PluginContext } from '@crawlith/core'; -import { Command } from '@crawlith/core'; -import { annotateOrphans, OrphanScoringOptions } from './src/orphanSeverity.js'; - -export * from './src/orphanSeverity.js'; - -/** - * Orphan Intelligence Plugin - * Crawlith plugin for orphan intelligence - */ -export const OrphanIntelligencePlugin: CrawlithPlugin = { - name: 'orphan-intelligence', - register: (cli: Command) => { - if (cli.name() === 'crawl') { - cli - .option("--orphans", "Detect orphaned pages") - .option("--orphan-severity", "Severity for orphans (low/medium/high)") - .option("--include-soft-orphans", "Include soft orphans") - .option("--min-inbound <value>", "Minimum inbound links to not be an orphan", "2"); - } - }, - - hooks: { - onMetrics: async (ctx: PluginContext, graph: any) => { - const flags = ctx.flags || {}; - - if (!flags.orphans) { - return; - } - - ctx.logger?.info('🔍 Detecting orphaned pages...'); - - const options: OrphanScoringOptions = { - enabled: true, - severityEnabled: !!flags.orphanSeverity, - includeSoftOrphans: !!flags.includeSoftOrphans, - minInbound: parseInt(flags.minInbound as string ?? '2', 10), - rootUrl: undefined - }; - - const nodes = graph.getNodes(); - const edges = graph.getEdges(); - - const annotatedNodes = annotateOrphans(nodes, edges, options); - - // Mutate the graph nodes in place - for (const annotated of annotatedNodes) { - if (annotated.orphan) { - const graphNode = graph.getNode(annotated.url); - if (graphNode) { - graphNode.orphanScore = annotated.orphanSeverity; - graphNode.orphanType = annotated.orphanType; - } - } - } - - ctx.logger?.info(`🔍 Orphan detection complete.`); - }, - onReport: async (ctx: PluginContext, result: any) => { - const isCrawl = !!result.snapshotId && !!result.graph; - if (isCrawl && result.graph) { - const nodes = result.graph.getNodes(); - const orphans = nodes.filter((n: any) => n.orphanScore && n.orphanScore !== 'low'); - if (orphans.length > 0) { - if (!result.plugins) result.plugins = {}; - result.plugins.orphanIntelligence = { - criticalOrphans: orphans.length, - sampleOrphans: orphans.slice(0, 10).map((n: any) => ({ url: n.url, severity: n.orphanScore })) - }; - } - } - } - } -}; - -export default OrphanIntelligencePlugin; diff --git a/packages/plugins/orphan-intelligence/package.json b/packages/plugins/orphan-intelligence/package.json deleted file mode 100644 index 68fb324..0000000 --- a/packages/plugins/orphan-intelligence/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-orphan-intelligence", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for orphan intelligence" -} diff --git a/packages/plugins/orphan-intelligence/src/orphanSeverity.ts b/packages/plugins/orphan-intelligence/src/orphanSeverity.ts deleted file mode 100644 index 33677f9..0000000 --- a/packages/plugins/orphan-intelligence/src/orphanSeverity.ts +++ /dev/null @@ -1,172 +0,0 @@ -import type { GraphNode, GraphEdge } from '@crawlith/core'; - -export interface ExtendedGraphNode extends GraphNode { - pageType?: string; - hasStructuredData?: boolean; - isProductOrCommercial?: boolean; - duplicateContent?: boolean; - isHomepage?: boolean; - robotsExcluded?: boolean; - discoveredViaSitemap?: boolean; -} - -export type OrphanType = 'hard' | 'near' | 'soft' | 'crawl-only'; -export type ImpactLevel = 'low' | 'medium' | 'high' | 'critical'; - -export interface OrphanScoringOptions { - enabled: boolean; - severityEnabled: boolean; - includeSoftOrphans: boolean; - minInbound: number; - rootUrl?: string; -} - -export type AnnotatedNode = GraphNode & { - orphan: boolean; - orphanType?: OrphanType; - orphanSeverity?: number; - impactLevel?: ImpactLevel; -}; - -const LOW_VALUE_PATTERNS = [ - /[?&](page|p)=\d+/i, - /\/(page|tag|tags|category|categories)\//i, - /[?&](q|query|search|filter|sort)=/i, - /\/search(\/|\?|$)/i -]; - -function isLowValuePage(node: ExtendedGraphNode): boolean { - const type = (node.pageType || '').toLowerCase(); - if (['pagination', 'tag', 'category', 'filter', 'search', 'archive'].includes(type)) { - return true; - } - if (node.noindex) { - return true; - } - return LOW_VALUE_PATTERNS.some((pattern) => pattern.test(node.url)); -} - -function clampScore(score: number): number { - return Math.max(0, Math.min(100, Math.round(score))); -} - -export function mapImpactLevel(score: number): ImpactLevel { - if (score <= 39) return 'low'; - if (score <= 69) return 'medium'; - if (score <= 89) return 'high'; - return 'critical'; -} - -export function calculateOrphanSeverity(orphanType: OrphanType, node: ExtendedGraphNode): number { - let score = 0; - - switch (orphanType) { - case 'hard': - score = 90; - break; - case 'crawl-only': - // Sitemap-only URLs are less severe if we haven't even tried to crawl them yet. - score = node.status === 0 ? 50 : 70; - break; - case 'near': - score = node.inLinks <= 1 ? 70 : 60; - break; - case 'soft': - score = 50; - break; - } - - let positiveModifier = 0; - if ((node.wordCount || 0) > 800) positiveModifier += 10; - if (node.hasStructuredData) positiveModifier += 10; - if (node.depth <= 2) positiveModifier += 10; - if (node.isProductOrCommercial) positiveModifier += 10; - positiveModifier = Math.min(20, positiveModifier); - - let negativeModifier = 0; - if ((node.wordCount || 0) > 0 && (node.wordCount || 0) < 300) negativeModifier += 20; - if (node.noindex) negativeModifier += 20; - if (node.duplicateContent) negativeModifier += 20; - if ((node.pageType || '').toLowerCase() === 'archive' || (node.pageType || '').toLowerCase() === 'pagination') negativeModifier += 20; - negativeModifier = Math.min(20, negativeModifier); - - score += positiveModifier; - score -= negativeModifier; - - // Safety: unvisited nodes should not be flagged as high-severity orphans - // because we haven't confirmed they are real pages or fully explored their context. - if (node.status === 0) { - score = Math.min(score, 60); - } - - return clampScore(score); -} - -function consolidateInboundByCanonical(nodes: ExtendedGraphNode[]): Map<string, number> { - const canonicalInbound = new Map<string, number>(); - for (const node of nodes) { - const canonical = node.canonical || node.url; - canonicalInbound.set(canonical, (canonicalInbound.get(canonical) || 0) + node.inLinks); - } - return canonicalInbound; -} - -export function annotateOrphans(nodes: ExtendedGraphNode[], edges: GraphEdge[], options: OrphanScoringOptions): AnnotatedNode[] { - if (!options.enabled) { - return nodes.map((node) => ({ ...node, orphan: false })); - } - - const canonicalInbound = consolidateInboundByCanonical(nodes); - const nodeByUrl = new Map(nodes.map((node) => [node.url, node])); - - return nodes.map((node) => { - const isHomepage = node.isHomepage || (options.rootUrl ? node.url === options.rootUrl : node.depth === 0); - if (isHomepage || node.robotsExcluded) { - return { ...node, orphan: false }; - } - - const canonical = node.canonical || node.url; - const inbound = canonicalInbound.get(canonical) || 0; - - let orphanType: OrphanType | undefined; - - if (inbound === 0) { - orphanType = node.discoveredViaSitemap ? 'crawl-only' : 'hard'; - } else if (inbound <= options.minInbound) { - orphanType = 'near'; - } - - if (!orphanType && options.includeSoftOrphans && inbound > 0) { - const inboundSources = edges - .filter((edge) => edge.target === node.url) - .map((edge) => nodeByUrl.get(edge.source)) - .filter((source): source is GraphNode => Boolean(source)); - - if (inboundSources.length > 0 && inboundSources.every((source) => isLowValuePage(source))) { - orphanType = 'soft'; - } - } - - if (!orphanType) { - return { ...node, orphan: false }; - } - - if (!options.severityEnabled) { - return { - ...node, - orphan: true, - orphanType - }; - } - - const orphanSeverity = calculateOrphanSeverity(orphanType, { ...node, inLinks: inbound }); - - return { - ...node, - orphan: true, - orphanType, - orphanSeverity, - impactLevel: mapImpactLevel(orphanSeverity) - }; - }); -} diff --git a/packages/plugins/orphan-intelligence/tests/__snapshots__/orphanSeverity.test.ts.snap b/packages/plugins/orphan-intelligence/tests/__snapshots__/orphanSeverity.test.ts.snap deleted file mode 100644 index e838b0a..0000000 --- a/packages/plugins/orphan-intelligence/tests/__snapshots__/orphanSeverity.test.ts.snap +++ /dev/null @@ -1,49 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`orphan detection and severity scoring > canonical consolidation, robots exclusion, and deterministic JSON output snapshot 1`] = ` -"[ - { - "url": "https://example.com/canonical", - "depth": 1, - "inLinks": 0, - "outLinks": 0, - "status": 200, - "orphan": true, - "orphanType": "near", - "orphanSeverity": 80, - "impactLevel": "high" - }, - { - "url": "https://example.com/variant?a=1", - "depth": 1, - "inLinks": 1, - "outLinks": 0, - "status": 200, - "canonical": "https://example.com/canonical", - "orphan": true, - "orphanType": "near", - "orphanSeverity": 80, - "impactLevel": "high" - }, - { - "url": "https://example.com/blocked", - "depth": 1, - "inLinks": 0, - "outLinks": 0, - "status": 200, - "robotsExcluded": true, - "orphan": false - }, - { - "url": "https://example.com/redirect-target", - "depth": 1, - "inLinks": 1, - "outLinks": 0, - "status": 200, - "orphan": true, - "orphanType": "near", - "orphanSeverity": 80, - "impactLevel": "high" - } -]" -`; diff --git a/packages/plugins/orphan-intelligence/tests/orphanSeverity.test.ts b/packages/plugins/orphan-intelligence/tests/orphanSeverity.test.ts deleted file mode 100644 index bb5075f..0000000 --- a/packages/plugins/orphan-intelligence/tests/orphanSeverity.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { annotateOrphans, calculateOrphanSeverity, mapImpactLevel, type ExtendedGraphNode } from '../src/orphanSeverity.js'; -import type { GraphEdge } from '@crawlith/core'; - -function baseNode(url: string, overrides: Partial<ExtendedGraphNode> = {}): ExtendedGraphNode { - return { - url, - depth: 1, - inLinks: 0, - outLinks: 0, - status: 200, - ...overrides - }; -} - -describe('orphan detection and severity scoring', () => { - test('hard orphan detection and homepage exclusion', () => { - const nodes: ExtendedGraphNode[] = [ - baseNode('https://example.com/', { depth: 0, inLinks: 0 }), - baseNode('https://example.com/orphan', { inLinks: 0 }) - ]; - const edges: GraphEdge[] = []; - - const result = annotateOrphans(nodes, edges, { - enabled: true, - severityEnabled: false, - includeSoftOrphans: false, - minInbound: 2, - rootUrl: 'https://example.com/' - }); - - expect(result[0]).toMatchObject({ orphan: false }); - expect(result[1]).toMatchObject({ orphan: true, orphanType: 'hard' }); - }); - - test('near orphan threshold override', () => { - const nodes = [baseNode('https://example.com/near', { inLinks: 2 })]; - const edges: GraphEdge[] = []; - - const resultDefault = annotateOrphans(nodes, edges, { - enabled: true, - severityEnabled: false, - includeSoftOrphans: false, - minInbound: 2 - }); - const resultStrict = annotateOrphans(nodes, edges, { - enabled: true, - severityEnabled: false, - includeSoftOrphans: false, - minInbound: 1 - }); - - expect(resultDefault[0]).toMatchObject({ orphan: true, orphanType: 'near' }); - expect(resultStrict[0]).toMatchObject({ orphan: false }); - }); - - test('soft orphan detection only when enabled and inbound only from low-value sources', () => { - const nodes: ExtendedGraphNode[] = [ - baseNode('https://example.com/tag/seo', { pageType: 'tag', outLinks: 1 }), - baseNode('https://example.com/list?page=2', { pageType: 'pagination', outLinks: 1 }), - baseNode('https://example.com/target', { inLinks: 2 }), - baseNode('https://example.com/normal', { outLinks: 1 }) - ]; - - const edges: GraphEdge[] = [ - { source: 'https://example.com/tag/seo', target: 'https://example.com/target', weight: 1 }, - { source: 'https://example.com/list?page=2', target: 'https://example.com/target', weight: 1 } - ]; - - const withSoft = annotateOrphans(nodes, edges, { - enabled: true, - severityEnabled: false, - includeSoftOrphans: true, - minInbound: 1 - }); - - const withoutSoft = annotateOrphans(nodes, edges, { - enabled: true, - severityEnabled: false, - includeSoftOrphans: false, - minInbound: 1 - }); - - expect(withSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: true, orphanType: 'soft' }); - expect(withoutSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: false }); - }); - - test('crawl-only orphan detection', () => { - const nodes = [baseNode('https://example.com/sitemap-only', { inLinks: 0, discoveredViaSitemap: true })]; - const result = annotateOrphans(nodes, [], { - enabled: true, - severityEnabled: false, - includeSoftOrphans: false, - minInbound: 2 - }); - - expect(result[0]).toMatchObject({ orphan: true, orphanType: 'crawl-only' }); - }); - - test('severity calculation modifiers and score clamping', () => { - const high = calculateOrphanSeverity('hard', baseNode('https://example.com/high', { - inLinks: 0, - wordCount: 1500, - hasStructuredData: true, - depth: 1, - isProductOrCommercial: true - })); - - const low = calculateOrphanSeverity('hard', baseNode('https://example.com/low', { - inLinks: 0, - wordCount: 120, - noindex: true, - duplicateContent: true, - pageType: 'archive' - })); - - expect(high).toBe(100); - expect(low).toBe(80); - }); - - test('impact level mapping', () => { - expect(mapImpactLevel(0)).toBe('low'); - expect(mapImpactLevel(39)).toBe('low'); - expect(mapImpactLevel(40)).toBe('medium'); - expect(mapImpactLevel(69)).toBe('medium'); - expect(mapImpactLevel(70)).toBe('high'); - expect(mapImpactLevel(89)).toBe('high'); - expect(mapImpactLevel(90)).toBe('critical'); - expect(mapImpactLevel(100)).toBe('critical'); - }); - - test('canonical consolidation, robots exclusion, and deterministic JSON output snapshot', () => { - const nodes: ExtendedGraphNode[] = [ - baseNode('https://example.com/canonical', { inLinks: 0 }), - baseNode('https://example.com/variant?a=1', { canonical: 'https://example.com/canonical', inLinks: 1 }), - baseNode('https://example.com/blocked', { inLinks: 0, robotsExcluded: true }), - baseNode('https://example.com/redirect-target', { inLinks: 1 }) - ]; - - const edges: GraphEdge[] = [ - { source: 'https://example.com/redirect-source', target: 'https://example.com/redirect-target', weight: 1 } - ]; - - const options = { - enabled: true, - severityEnabled: true, - includeSoftOrphans: true, - minInbound: 2 - }; - - const first = annotateOrphans(nodes, edges, options); - const second = annotateOrphans(nodes, edges, options); - - expect(first).toEqual(second); - expect(first.find((n) => n.url.endsWith('/canonical'))).toMatchObject({ orphan: true, orphanType: 'near' }); - expect(first.find((n) => n.url.endsWith('/blocked'))).toMatchObject({ orphan: false }); - - const normalized = JSON.stringify(first, null, 2).replace(/\r\n/g, '\n'); - expect(normalized).toMatchSnapshot(); - }); -}); diff --git a/packages/plugins/orphan-intelligence/tsconfig.json b/packages/plugins/orphan-intelligence/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/orphan-intelligence/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/plugins/pagerank/CHANGELOG.md b/packages/plugins/pagerank/CHANGELOG.md deleted file mode 100644 index 28ac46d..0000000 --- a/packages/plugins/pagerank/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-pagerank - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/pagerank/README.md b/packages/plugins/pagerank/README.md deleted file mode 100644 index 88611bf..0000000 --- a/packages/plugins/pagerank/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Pagerank Plugin - -Crawlith plugin for pagerank - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/pagerank/index.ts b/packages/plugins/pagerank/index.ts deleted file mode 100644 index 5de0efc..0000000 --- a/packages/plugins/pagerank/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { CrawlithPlugin } from '@crawlith/core'; -import { PageRankHooks } from './src/plugin.js'; - -/** - * Description: PageRank calculates the relative authority of pages across a snapshot based on internal link weight and distribution. - * @requirements Must have a fully built graph from a 'crawl' execution spanning 100+ edges to be precise. - */ -export const PageRankPlugin: CrawlithPlugin = { - name: 'pagerank', - - // Implicitly activated inside core due to its legacy importance, - // but mapping 'flag' cleanly for new PluginRegistry loaders. - cli: { - flag: '--pagerank', - description: 'Calculate PageRank' - }, - - scoreProvider: true, // Automatically sums total pagerank score distributed natively across snapshots - - storage: { - fetchMode: 'local', - perPage: { - columns: { - raw_rank: 'REAL', - score: 'REAL' - } - } - }, - - hooks: PageRankHooks -}; - -export default PageRankPlugin; diff --git a/packages/plugins/pagerank/package.json b/packages/plugins/pagerank/package.json deleted file mode 100644 index 1a579ce..0000000 --- a/packages/plugins/pagerank/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-pagerank", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for pagerank" -} \ No newline at end of file diff --git a/packages/plugins/pagerank/src/plugin.ts b/packages/plugins/pagerank/src/plugin.ts deleted file mode 100644 index 02b48bc..0000000 --- a/packages/plugins/pagerank/src/plugin.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { PluginContext, Graph } from '@crawlith/core'; -import { PageRankService } from './Service.js'; -import { PageRankRow } from './types.js'; - -export const PageRankHooks = { - /** - * Called during `crawl` execution. Analyzes the fully formed graph. - * Caches results into SQLite natively via `ctx.db.data.getOrFetch`. - */ - onMetrics: async (ctx: PluginContext, graph: Graph) => { - // We only execute PageRank if there's actually a DB session connected (crawl context) - // and if the crawler explicitly loaded the graph structure. - if (!ctx.db) return; - - ctx.logger?.info?.('🧮 Computing Production-Grade PageRank distribution...'); - - // We compute the entire graph globally once - const service = new PageRankService(); - const rankPayloads = service.evaluate(graph); - - let evaluatedCount = 0; - - for (const node of graph.getNodes()) { - if (!node.url) continue; - - const payload = rankPayloads.get(node.url); - - // We still update graph node natively because downstream clustering/scoring processes depend on it - // in memory before getting into the CLI reporter. - if (payload) { - (node as any).pageRank = payload.raw_rank; - (node as any).pageRankScore = payload.score; - evaluatedCount++; - - // Cache internally against `<snapshotId>, <url_id>` into `pagerank_plugin` - await ctx.db.data.getOrFetch<PageRankRow>( - node.url, - async () => payload - ); - } - } - - ctx.logger?.info?.(`🧮 PageRank distribution calculated across ${evaluatedCount} interconnected nodes.`); - }, - - /** - * Evaluates the active snapshot results and attaches them statically mapped onto the final JSON reporter. - */ - onReport: async (ctx: PluginContext, result: any) => { - if (!ctx.db) return; - - // Pull directly from our scoped SQLite plugin table - const allRows = ctx.db.data.all<{ url_id: number; raw_rank: number; score: number }>(); - - if (allRows.length > 0) { - if (!result.plugins) result.plugins = {}; - - // Inject directly back to final JSON pages footprint - if (Array.isArray(result.pages)) { - for (const page of result.pages) { - if ((page as any).pageRankScore !== undefined) { - // Usually already injected if doing stringification, but ensuring strict format - page.plugins = page.plugins || {}; - page.plugins.pagerank = { - score: (page as any).pageRankScore, - raw_rank: (page as any).pageRank - }; - } - } - } - - const highValueEntities = [...allRows].sort((a, b) => b.score - a.score).slice(0, 5); - - result.plugins.pagerank = { - totalEvaluated: allRows.length, - topEntities: highValueEntities.map(r => ({ score: r.score })) // IDs would need mapped, but raw scores work - }; - } - } -}; diff --git a/packages/plugins/pagerank/src/types.ts b/packages/plugins/pagerank/src/types.ts deleted file mode 100644 index f8b4867..0000000 --- a/packages/plugins/pagerank/src/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface PageRankRow { - raw_rank: number; - score: number; -} - -export interface PageRankOptions { - dampingFactor?: number; - maxIterations?: number; - convergenceThreshold?: number; - soft404WeightThreshold?: number; -} diff --git a/packages/plugins/pagerank/tests/plugin.test.ts b/packages/plugins/pagerank/tests/plugin.test.ts deleted file mode 100644 index d8f7455..0000000 --- a/packages/plugins/pagerank/tests/plugin.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { Graph } from '@crawlith/core'; -import { PageRankService } from '../src/Service.js'; - -describe('PageRank Engine', () => { - it('should calculate identical PageRank for a simple loop', () => { - const graph = new Graph(); - graph.addNode('https://a.com', 0, 200); - graph.addNode('https://b.com', 1, 200); - graph.addEdge('https://a.com', 'https://b.com'); - graph.addEdge('https://b.com', 'https://a.com'); - - const service = new PageRankService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).pageRank = res.raw_rank; - (node as any).pageRankScore = res.score; - } - const nodes = graph.getNodes(); - - expect((nodes[0] as any).pageRank).toBeCloseTo(0.5, 4); - expect((nodes[1] as any).pageRank).toBeCloseTo(0.5, 4); - expect((nodes[0] as any).pageRankScore).toBe(100); - expect((nodes[1] as any).pageRankScore).toBe(100); - }); - - it('should identify the center of a star graph as most important', () => { - const graph = new Graph(); - graph.addNode('https://center.com', 0, 200); - graph.addNode('https://p1.com', 1, 200); - graph.addNode('https://p2.com', 1, 200); - graph.addNode('https://p3.com', 1, 200); - - // Star in: all link to center - graph.addEdge('https://p1.com', 'https://center.com'); - graph.addEdge('https://p2.com', 'https://center.com'); - graph.addEdge('https://p3.com', 'https://center.com'); - - const service = new PageRankService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).pageRank = res.raw_rank; - (node as any).pageRankScore = res.score; - } - const nodes = graph.getNodes(); - - const center = nodes.find(n => n.url.includes('center'))!; - const leaves = nodes.filter(n => !n.url.includes('center')); - - expect((center as any).pageRankScore).toBe(100); - leaves.forEach(leaf => { - expect((leaf as any).pageRankScore).toBeLessThan(100); - expect((leaf as any).pageRank!).toBeLessThan((center as any).pageRank!); - }); - }); - - it('should respect link weights (Body > Nav > Footer)', () => { - const graph = new Graph(); - graph.addNode('https://source.com', 0, 200); - graph.addNode('https://body-target.com', 1, 200); - graph.addNode('https://footer-target.com', 1, 200); - - // Body weight 1.0, Footer weight 0.4 - graph.addEdge('https://source.com', 'https://body-target.com', 1.0); - graph.addEdge('https://source.com', 'https://footer-target.com', 0.4); - - const service = new PageRankService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).pageRank = res.raw_rank; - (node as any).pageRankScore = res.score; - } - - const bodyTarget = graph.nodes.get('https://body-target.com')!; - const footerTarget = graph.nodes.get('https://footer-target.com')!; - - expect((bodyTarget as any).pageRank!).toBeGreaterThan((footerTarget as any).pageRank!); - }); - - it('should handle sink nodes by redistributing rank', () => { - const graph = new Graph(); - graph.addNode('https://a.com', 0, 200); - graph.addNode('https://b.com', 1, 200); // b is a sink - graph.addEdge('https://a.com', 'https://b.com'); - - const service = new PageRankService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).pageRank = res.raw_rank; - (node as any).pageRankScore = res.score; - } - - const nodeA = graph.nodes.get('https://a.com')!; - const nodeB = graph.nodes.get('https://b.com')!; - - // Without redistribution, A would lose all rank. - // With redistribution, A should still have some rank. - expect((nodeA as any).pageRank).toBeGreaterThan(0); - expect((nodeB as any).pageRank).toBeGreaterThan((nodeA as any).pageRank!); - }); - - it('should exclude noindex pages from receiving or passing rank', () => { - const graph = new Graph(); - graph.addNode('https://a.com', 0, 200); - graph.addNode('https://no-index.com', 1, 200); - graph.nodes.get('https://no-index.com')!.noindex = true; - - graph.addEdge('https://a.com', 'https://no-index.com'); - - const service = new PageRankService(); - const results = service.evaluate(graph); - for (const [url, res] of results) { - const node = graph.nodes.get(url)!; - (node as any).pageRank = res.raw_rank; - (node as any).pageRankScore = res.score; - } - - const nodeA = graph.nodes.get('https://a.com')!; - const nodeNoIndex = graph.nodes.get('https://no-index.com')!; - - expect((nodeNoIndex as any).pageRank).toBeUndefined(); - expect((nodeA as any).pageRank).toBe(1.0); // Only one eligible node - }); -}); diff --git a/packages/plugins/pagerank/tsconfig.json b/packages/plugins/pagerank/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/pagerank/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/pagespeed/src/PageSpeedService.ts b/packages/plugins/pagespeed/src/PageSpeedService.ts index 25bab29..b711acd 100644 --- a/packages/plugins/pagespeed/src/PageSpeedService.ts +++ b/packages/plugins/pagespeed/src/PageSpeedService.ts @@ -29,7 +29,7 @@ export class PageSpeedService { for (let attempt = 0; attempt <= maxRetries; attempt++) { try { const response = await fetch(`${PAGESPEED_ENDPOINT}?${params.toString()}`, { - signal: AbortSignal.timeout(15000) + signal: AbortSignal.timeout(60000) }); const raw = await response.json(); @@ -38,6 +38,10 @@ export class PageSpeedService { throw new Error(message); } + if (raw?.lighthouseResult?.runtimeError) { + throw new Error(`Lighthouse returned error: ${raw.lighthouseResult.runtimeError.message || 'Unknown error'}`); + } + return raw; } catch (error) { if (attempt >= maxRetries) throw error; diff --git a/packages/plugins/simhash/CHANGELOG.md b/packages/plugins/simhash/CHANGELOG.md deleted file mode 100644 index 1627acf..0000000 --- a/packages/plugins/simhash/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-simhash - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/simhash/README.md b/packages/plugins/simhash/README.md deleted file mode 100644 index 19b476e..0000000 --- a/packages/plugins/simhash/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Simhash Plugin - -Crawlith plugin for simhash - -## Installation -This plugin is built-in. - -## Usage -Include it in your Crawlith configuration or CLI usage. diff --git a/packages/plugins/simhash/index.ts b/packages/plugins/simhash/index.ts deleted file mode 100644 index 1fc375f..0000000 --- a/packages/plugins/simhash/index.ts +++ /dev/null @@ -1,24 +0,0 @@ - -import { SimHash, CrawlithPlugin, PluginContext } from '@crawlith/core'; -import { Command } from '@crawlith/core'; - -/** - * Simhash Plugin - * Crawlith plugin for simhash - */ -export const SimhashPlugin: CrawlithPlugin = { - name: 'simhash', register: (_cli: Command) => { - // Default for crawl in original, no new flags added. - }, - hooks: { - onMetrics: async (ctx: PluginContext, graph: any) => { - for (const node of graph.getNodes()) { - const title = (node as any).title || ''; - const tokens = (title || node.url).toLowerCase().split(/\W+/).filter(Boolean); - node.simhash = SimHash.generate(tokens).toString(16); - } - } - } -}; - -export default SimhashPlugin; diff --git a/packages/plugins/simhash/package.json b/packages/plugins/simhash/package.json deleted file mode 100644 index f279f77..0000000 --- a/packages/plugins/simhash/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@crawlith/plugin-simhash", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*" - }, - "scripts": { - "build": "tsc", - "test": "echo No tests specified" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for simhash" -} diff --git a/packages/plugins/simhash/tsconfig.json b/packages/plugins/simhash/tsconfig.json deleted file mode 100644 index 4eb6260..0000000 --- a/packages/plugins/simhash/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} \ No newline at end of file diff --git a/packages/plugins/snapshot-diff/CHANGELOG.md b/packages/plugins/snapshot-diff/CHANGELOG.md deleted file mode 100644 index d304a62..0000000 --- a/packages/plugins/snapshot-diff/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-snapshot-diff - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/snapshot-diff/README.md b/packages/plugins/snapshot-diff/README.md deleted file mode 100644 index 359541f..0000000 --- a/packages/plugins/snapshot-diff/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Snapshot Diff Plugin - -The `snapshot-diff` plugin adds snapshot-aware graph comparison and incremental baseline loading to the `crawl` command. - -## What this plugin does - -- Adds `--compare <oldSnapshotId> <newSnapshotId>` to compare **two saved Crawlith snapshots**. -- Adds `--incremental` to preload the latest completed snapshot graph for the same domain before a new crawl. -- Produces either: - - a human-readable terminal summary (`pretty` default), or - - a raw JSON diff payload when `--format json` is used. - -## CLI options - -- `--compare <oldSnapshotId> <newSnapshotId>` - Internal flag that compares two persisted snapshot IDs and exits before crawling. -- `--incremental` - Resolves the latest completed snapshot for the target domain and stores it in plugin metadata for incremental graph diffing. - -## Usage examples - -### Compare two snapshots - -```bash -crawlith crawl https://example.com --compare 101 102 -``` - -### Compare two snapshots in JSON format - -```bash -crawlith crawl https://example.com --compare 101 102 --format json -``` - -### Run a normal incremental crawl - -```bash -crawlith crawl https://example.com --incremental -``` - -## Output details - -### Pretty mode output - -The plugin prints: - -- Compared snapshot IDs. -- Added URL count. -- Removed URL count. -- Status change count. -- Metric delta table (positive/negative/zero colorized values). - -### JSON mode output - -The plugin emits the full diff object from Crawlith core (the same payload returned by `compareGraphs`). - -## Notes - -- `--compare` requires exactly two **numeric** snapshot IDs. -- The plugin does not read graph JSON files from disk. diff --git a/packages/plugins/snapshot-diff/index.ts b/packages/plugins/snapshot-diff/index.ts deleted file mode 100644 index 522750c..0000000 --- a/packages/plugins/snapshot-diff/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { CrawlithPlugin, Command } from '@crawlith/core'; -import { registerSnapshotDiffCli } from './src/cli.js'; -import { snapshotDiffHooks } from './src/plugin.js'; - -/** - * Snapshot Diff Plugin. - * - * Provides two crawl-time features: - * - snapshot-to-snapshot graph comparison via `--compare <oldSnapshotId> <newSnapshotId>` - * - incremental baseline loading via `--incremental` - */ -export const SnapshotDiffPlugin: CrawlithPlugin = { - name: 'snapshot-diff', - register: (cli: Command) => { - registerSnapshotDiffCli(cli); - }, - hooks: snapshotDiffHooks -}; - -export default SnapshotDiffPlugin; diff --git a/packages/plugins/snapshot-diff/package.json b/packages/plugins/snapshot-diff/package.json deleted file mode 100644 index c125dd6..0000000 --- a/packages/plugins/snapshot-diff/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@crawlith/plugin-snapshot-diff", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*", - "chalk": "^5.6.2" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for snapshot-to-snapshot graph diffing and incremental baselines" -} diff --git a/packages/plugins/snapshot-diff/src/Output.ts b/packages/plugins/snapshot-diff/src/Output.ts deleted file mode 100644 index a8de20f..0000000 --- a/packages/plugins/snapshot-diff/src/Output.ts +++ /dev/null @@ -1,40 +0,0 @@ -import chalk from 'chalk'; - -/** - * Renders human-readable output for snapshot comparison. - * @param oldSnapshotId Baseline snapshot ID. - * @param newSnapshotId Target snapshot ID. - * @param diffResult Graph diff payload from core. - */ -export function renderPrettyDiffOutput(oldSnapshotId: number, newSnapshotId: number, diffResult: any): void { - console.log(chalk.cyan('\n🔍 Comparing Snapshots')); - console.log(`${chalk.gray('Old snapshot:')} #${oldSnapshotId}`); - console.log(`${chalk.gray('New snapshot:')} #${newSnapshotId}\n`); - - console.log(chalk.bold('📈 Comparison Results:')); - console.log(`- Added URLs: ${chalk.green(diffResult.addedUrls.length)}`); - console.log(`- Removed URLs: ${chalk.red(diffResult.removedUrls.length)}`); - console.log(`- Status Changes: ${chalk.yellow(diffResult.changedStatus.length)}`); - - console.log(chalk.bold('\n📉 Metric Deltas:')); - Object.entries(diffResult.metricDeltas).forEach(([metric, delta]) => { - const numericDelta = Number(delta); - const deltaLabel = numericDelta > 0 - ? chalk.green(`+${numericDelta.toFixed(3)}`) - : numericDelta < 0 - ? chalk.red(numericDelta.toFixed(3)) - : chalk.gray('0'); - - console.log(` ${metric.padEnd(20)}: ${deltaLabel}`); - }); - - console.log(`\n${chalk.gray('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')}\n`); -} - -/** - * Emits machine-readable JSON output. - * @param diffResult Graph diff payload from core. - */ -export function renderJsonDiffOutput(diffResult: unknown): void { - console.log(JSON.stringify(diffResult, null, 2)); -} diff --git a/packages/plugins/snapshot-diff/src/Service.ts b/packages/plugins/snapshot-diff/src/Service.ts deleted file mode 100644 index 0c13efb..0000000 --- a/packages/plugins/snapshot-diff/src/Service.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { compareGraphs, getDb, loadGraphFromSnapshot, SiteRepository, SnapshotRepository } from '@crawlith/core'; -import type { Graph, PluginContext } from '@crawlith/core'; -import type { SnapshotCompareRequest, SnapshotDiffFlags, SnapshotDiffOutputFormat } from './types.js'; - -/** - * Encapsulates business logic for snapshot comparison and incremental baseline loading. - */ -export class SnapshotDiffService { - /** - * Parses and validates `--compare` payload as a pair of numeric snapshot IDs. - * @param flags CLI flags available on plugin context. - * @returns Parsed compare request or `null` when compare mode is disabled. - */ - parseCompareRequest(flags: SnapshotDiffFlags): SnapshotCompareRequest | null { - if (!flags.compare) return null; - - const rawValues = Array.isArray(flags.compare) - ? flags.compare - : [flags.compare]; - - if (rawValues.length !== 2) { - throw new Error('Error: --compare requires exactly two snapshot IDs (<oldSnapshotId> <newSnapshotId>).'); - } - - const [oldRaw, newRaw] = rawValues.map(String); - const oldSnapshotId = Number.parseInt(oldRaw, 10); - const newSnapshotId = Number.parseInt(newRaw, 10); - - if (!Number.isFinite(oldSnapshotId) || !Number.isFinite(newSnapshotId)) { - throw new Error('Error: --compare only accepts numeric snapshot IDs.'); - } - - return { oldSnapshotId, newSnapshotId }; - } - - /** - * Resolves output format from CLI flags. - * @param flags CLI flags from plugin context. - * @returns Selected output format. - */ - parseOutputFormat(flags: SnapshotDiffFlags): SnapshotDiffOutputFormat { - return flags.format === 'json' ? 'json' : 'pretty'; - } - - /** - * Compares two persisted snapshots. - * @param request Pair of snapshot IDs. - * @returns Diff result generated by Crawlith core. - */ - compareSnapshots(request: SnapshotCompareRequest) { - const oldGraph = loadGraphFromSnapshot(request.oldSnapshotId); - const newGraph = loadGraphFromSnapshot(request.newSnapshotId); - return compareGraphs(oldGraph, newGraph); - } - - /** - * Resolves previous completed snapshot graph for incremental crawl mode. - * @param targetUrl URL passed to crawl command. - * @returns Previous graph when found, otherwise `null`. - */ - resolvePreviousGraph(targetUrl: string): Graph | null { - const db = getDb(); - const siteRepo = new SiteRepository(db); - const snapshotRepo = new SnapshotRepository(db); - - const domain = new URL(targetUrl).hostname.replace('www.', ''); - const site = siteRepo.getSite(domain); - if (!site) return null; - - const latestSnapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed'); - if (!latestSnapshot) return null; - - return loadGraphFromSnapshot(latestSnapshot.id); - } - - /** - * Reads a valid target URL from context flags for incremental mode. - * @param ctx Plugin context. - * @returns Candidate URL or `null` when missing. - */ - getTargetUrl(ctx: PluginContext): string | null { - const flags = (ctx.flags || {}) as SnapshotDiffFlags; - const fromFlags = typeof flags.url === 'string' ? flags.url : null; - if (fromFlags && !fromFlags.startsWith('-')) return fromFlags; - - return typeof (ctx as PluginContext & { url?: unknown }).url === 'string' - ? String((ctx as PluginContext & { url?: unknown }).url) - : null; - } -} diff --git a/packages/plugins/snapshot-diff/src/cli.ts b/packages/plugins/snapshot-diff/src/cli.ts deleted file mode 100644 index 0402ed7..0000000 --- a/packages/plugins/snapshot-diff/src/cli.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Command } from '@crawlith/core'; - -/** - * Registers snapshot-diff CLI options on the crawl command. - * @param cli Crawlith command instance. - */ -export function registerSnapshotDiffCli(cli: Command): void { - if (cli.name() !== 'crawl') return; - - cli - .option('--incremental', 'incremental crawl using previous completed snapshot') - .option('--compare <snapshots...>', 'internal: compare two snapshot IDs'); -} diff --git a/packages/plugins/snapshot-diff/src/plugin.ts b/packages/plugins/snapshot-diff/src/plugin.ts deleted file mode 100644 index 2a95f3e..0000000 --- a/packages/plugins/snapshot-diff/src/plugin.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { PluginContext } from '@crawlith/core'; -import { renderJsonDiffOutput, renderPrettyDiffOutput } from './Output.js'; -import { SnapshotDiffService } from './Service.js'; -import type { SnapshotDiffFlags } from './types.js'; - -const service = new SnapshotDiffService(); - -/** - * Hook implementation for snapshot diff and incremental baseline resolution. - */ -export const snapshotDiffHooks = { - /** - * Handles `--compare` execution before crawl starts. - * @param ctx Plugin context for current command invocation. - */ - onInit: async (ctx: PluginContext) => { - const flags = (ctx.flags || {}) as SnapshotDiffFlags; - const compareRequest = service.parseCompareRequest(flags); - if (!compareRequest) return; - - const diffResult = service.compareSnapshots(compareRequest); - const outputFormat = service.parseOutputFormat(flags); - - if (outputFormat === 'json') { - renderJsonDiffOutput(diffResult); - } else { - renderPrettyDiffOutput(compareRequest.oldSnapshotId, compareRequest.newSnapshotId, diffResult); - } - - ctx.terminate = true; - }, - - /** - * Loads previous completed snapshot graph when incremental mode is enabled. - * @param ctx Plugin context for current crawl execution. - */ - onCrawlStart: async (ctx: PluginContext) => { - const flags = (ctx.flags || {}) as SnapshotDiffFlags; - if (!flags.incremental) return; - - const targetUrl = service.getTargetUrl(ctx); - if (!targetUrl) return; - - ctx.logger?.info('🔍 Resolving previous snapshot for incremental crawl...'); - - try { - const previousGraph = service.resolvePreviousGraph(targetUrl); - if (!previousGraph) return; - - if (!ctx.metadata) ctx.metadata = {}; - ctx.metadata.previousGraph = previousGraph; - ctx.logger?.info('Loaded previous snapshot graph for incremental diffing.'); - } catch { - ctx.logger?.warn('Unable to resolve previous snapshot graph for incremental crawl.'); - } - } -}; diff --git a/packages/plugins/snapshot-diff/src/types.ts b/packages/plugins/snapshot-diff/src/types.ts deleted file mode 100644 index 9eca389..0000000 --- a/packages/plugins/snapshot-diff/src/types.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Parsed flag subset used by the snapshot-diff plugin. - */ -export interface SnapshotDiffFlags { - incremental?: boolean; - compare?: unknown; - format?: string; - url?: string; -} - -/** - * Output modes supported by the plugin formatter. - */ -export type SnapshotDiffOutputFormat = 'json' | 'pretty'; - -/** - * Normalized compare request containing exactly two snapshot IDs. - */ -export interface SnapshotCompareRequest { - oldSnapshotId: number; - newSnapshotId: number; -} diff --git a/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts b/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts deleted file mode 100644 index a2acd4f..0000000 --- a/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; - -const compareGraphs = vi.fn().mockReturnValue({ - addedUrls: ['https://example.com/new'], - removedUrls: [], - changedStatus: [], - metricDeltas: { totalPages: 1 } -}); - -const loadGraphFromSnapshot = vi.fn((snapshotId: number) => ({ snapshotId })); - -vi.mock('@crawlith/core', () => { - return { - compareGraphs, - loadGraphFromSnapshot, - getDb: vi.fn(), - SiteRepository: class { - getSite = vi.fn().mockReturnValue({ id: 1 }); - }, - SnapshotRepository: class { - getLatestSnapshot = vi.fn().mockReturnValue({ id: 5 }); - } - }; -}); - -const { SnapshotDiffPlugin } = await import('../index.js'); - -describe('SnapshotDiffPlugin', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('resolves previous graph during onCrawlStart when --incremental is passed', async () => { - const ctx = { - flags: { incremental: true, url: 'https://example.com' }, - metadata: {}, - logger: { info: vi.fn(), warn: vi.fn() } - }; - - await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); - - expect(ctx.metadata).toHaveProperty('previousGraph', { snapshotId: 5 }); - expect(loadGraphFromSnapshot).toHaveBeenCalledWith(5); - }); - - it('does not resolve previous graph when --incremental is not passed', async () => { - const ctx = { - flags: { incremental: false, url: 'https://example.com' }, - metadata: {}, - logger: { info: vi.fn(), warn: vi.fn() } - }; - - await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); - - expect(ctx.metadata).not.toHaveProperty('previousGraph'); - }); - - it('compares two snapshot IDs during onInit and terminates execution', async () => { - const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); - const ctx = { - flags: { compare: ['10', '11'] }, - logger: { info: vi.fn(), warn: vi.fn() } - }; - - await SnapshotDiffPlugin.hooks!.onInit!(ctx as any); - - expect(loadGraphFromSnapshot).toHaveBeenCalledWith(10); - expect(loadGraphFromSnapshot).toHaveBeenCalledWith(11); - expect(compareGraphs).toHaveBeenCalled(); - expect(ctx).toHaveProperty('terminate', true); - logSpy.mockRestore(); - }); - - it('throws when --compare is not a pair of snapshot IDs', async () => { - const ctx = { - flags: { compare: ['10'] }, - logger: { info: vi.fn(), warn: vi.fn() } - }; - - await expect(SnapshotDiffPlugin.hooks!.onInit!(ctx as any)).rejects.toThrow( - '--compare requires exactly two snapshot IDs' - ); - }); -}); diff --git a/packages/plugins/snapshot-diff/tsconfig.json b/packages/plugins/snapshot-diff/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/snapshot-diff/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/plugins/soft404-detector/CHANGELOG.md b/packages/plugins/soft404-detector/CHANGELOG.md deleted file mode 100644 index 3d20423..0000000 --- a/packages/plugins/soft404-detector/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# @crawlith/plugin-soft404-detector - -## 0.0.1 - -### Patch Changes - -- Add doc blocks, READMEs, and dynamic version/description mapping to all plugins -- Updated dependencies - - @crawlith/core@0.1.2 diff --git a/packages/plugins/soft404-detector/README.md b/packages/plugins/soft404-detector/README.md deleted file mode 100644 index fa8c169..0000000 --- a/packages/plugins/soft404-detector/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# 🕵️ Soft 404 Detector Plugin - -The **Soft 404 Detector** plugin automatically analyzes site pages to detect "Soft 404" responses—pages that return a HTTP `200 OK` status code but conceptually behave like an error page (e.g., stating "Page Not Found", lacking content, or providing no outbound navigation). - -It scores pages between `0.0` and `1.0` and outputs detailed reasoning regarding *why* the page triggered an alert. As a registered **Score Provider** natively mapped into Crawlith's core, its analyses are aggregated continuously across the crawl footprint. - -## Features - -- **Text Pattern Matching:** Flags `<title>` tags and `<h1>` headers containing error phrases (e.g., `not found`, `error`, `unavailable`). -- **Body Context Extraction:** Detects explicit in-text error declarations (e.g., "page not found"). -- **Content Density Checking:** Severely low word counts (< 50 words) heavily increase the soft 404 probability. -- **Link Isolation:** Flags pages acting as dead-ends, lacking any outbound links entirely. -- **Smart Caching (`fetchMode: 'local'`):** Computes entirely on the local device, safely executing against previously downloaded `html` payloads, supporting instant execution during `page` and `crawl` snapshot pipelines without re-triggering remote requests unless bypassing cache thresholds (`--live`). - -## Usage - -Enable the plugin via the CLI using the `--detect-soft404` flag. - -### Command Line Interface - -```bash -# Analyze a single page locally for Soft 404 violations -crawlith page --detect-soft404 https://example.com/missing-product - -# Deep crawl a fully site, scanning all discovered URLs for Soft 404 structures -crawlith crawl --detect-soft404 https://example.com -``` - -## Output & Interpretation - -### Single Page (`page` command) -For specific URL lookups, the plugin outputs a simple terminal read-out on execution: -```plaintext -[Soft404] https://example.com/broken-page | Score: 1.0 | Reason: title_contains_not found, h1_contains_404, very_low_word_count -``` - -### Full Site Crawl JSON Output (`crawl` command) -During extensive crawls, nodes heavily penalized map into the JSON reporter object (`--format json` or `--export json`): - -```json -{ - "plugins": { - "soft404": { - "totalDetected": 3, - "topSample": [ - { - "url": "https://example.com/missing", - "score": 0.8 - } - ] - } - } -} -``` - -The plugin also explicitly injects `soft404Score` directly onto graph `Node` objects, enabling graph algorithms to dynamically down-weight links originating from dead-end error patterns. - -## Architecture Guidelines - -This plugin strictly implements the [AI Agent Guide's](../../../docs/plugins/ai-agent-guide.md) architectural standards: -- All business logic lives inside a standalone `Soft404Service` object (`src/Service.ts`). -- Storage leverages the `ctx.db.data.getOrFetch` fluent API. -- Implements `scoreProvider: true` natively inside its manifest for top-level aggregation logic. diff --git a/packages/plugins/soft404-detector/index.ts b/packages/plugins/soft404-detector/index.ts deleted file mode 100644 index 9776790..0000000 --- a/packages/plugins/soft404-detector/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { CrawlithPlugin } from '@crawlith/core'; -import { Soft404Hooks } from './src/plugin.js'; - -/** - * Description: Soft404 Detector Plugin calculates the likelihood of a Soft 404 response. - * @requirements None. - */ -export const Soft404DetectorPlugin: CrawlithPlugin = { - name: 'soft404-detector', - - cli: { - flag: '--detect-soft404', - description: 'Detect soft 404 pages' - }, - - scoreProvider: true, - - storage: { - fetchMode: 'local', - perPage: { - columns: { - reason: 'TEXT' - } - } - }, - - hooks: Soft404Hooks -}; - -export default Soft404DetectorPlugin; diff --git a/packages/plugins/soft404-detector/package.json b/packages/plugins/soft404-detector/package.json deleted file mode 100644 index 9ff2c66..0000000 --- a/packages/plugins/soft404-detector/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@crawlith/plugin-soft404-detector", - "license": "Apache-2.0", - "private": true, - "version": "0.0.1", - "type": "module", - "dependencies": { - "@crawlith/core": "workspace:*", - "cheerio": "^1.2.0" - }, - "scripts": { - "build": "tsc", - "test": "vitest run" - }, - "exports": "./index.ts", - "description": "Crawlith plugin for soft404 detector" -} diff --git a/packages/plugins/soft404-detector/src/Output.ts b/packages/plugins/soft404-detector/src/Output.ts deleted file mode 100644 index 9fab8b3..0000000 --- a/packages/plugins/soft404-detector/src/Output.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { PluginContext } from '@crawlith/core'; -import { Soft404Result } from './types.js'; - -/** - * Handles presentation and standardized logging for the Soft404 plugin. - */ -export class Soft404Output { - /** - * Emits logging output for a single evaluated page. - * @param {PluginContext} ctx - Plugin context for logging access. - * @param {string} url - Target URL. - * @param {Soft404Result} result - The evaluation payload. - */ - public static emitSingle(ctx: PluginContext, url: string, result: Soft404Result): void { - if (!ctx.logger) return; - - if (result.score === 0) { - ctx.logger.info(`[Soft404] ${url} | Score: 0 (No issues detected)`); - } else { - ctx.logger.warn(`[Soft404] ${url} | Score: ${result.score} | Reason: ${result.reason}`); - } - } -} diff --git a/packages/plugins/soft404-detector/src/plugin.ts b/packages/plugins/soft404-detector/src/plugin.ts deleted file mode 100644 index d0e3014..0000000 --- a/packages/plugins/soft404-detector/src/plugin.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { PluginContext, PageInput } from '@crawlith/core'; -import { Soft404Service } from './Service.js'; -import { Soft404Row } from './types.js'; - -export const Soft404Hooks = { - /** - * Called during `crawlith page` command to analyze the single page immediately. - */ - onPage: async (ctx: PluginContext, page: PageInput) => { - if (!ctx.flags?.detectSoft404 || !ctx.db) return; - - const service = new Soft404Service(); - // Cache the soft404 result natively inside SQLite via getOrFetch. - const row = await ctx.db.data.getOrFetch<Soft404Row>( - page.url, - async () => service.analyze(page.html, 0 /* onPage command doesn't provide fully built links graph yet */) - ); - - if (row && ctx.logger) { - if (row.score > 0) { - ctx.logger.warn(`[soft404-detector] Score for ${page.url}: ${row.score} (${row.reason})`); - } else { - ctx.logger.info(`[soft404-detector] Score for ${page.url}: 0 (No issues detected)`); - } - } - }, - - /** - * Called on the fully built graph when a `crawl` evaluates metrics across the snapshot. - */ - onMetrics: async (ctx: PluginContext, graph: any) => { - const flags = ctx.flags || {}; - if (!flags.detectSoft404 || !ctx.db) return; - - ctx.logger?.info?.('🕵️ Detecting soft 404 pages...'); - - const service = new Soft404Service(); - const nodes = graph.getNodes(); - - let issueCount = 0; - - for (const node of nodes) { - if (node.status === 200) { - // Caches internally inside 'soft404_detector_plugin' SQLite table. - const row = await ctx.db.data.getOrFetch<Soft404Row>( - node.url, - async () => service.analyze(node.html, node.outLinks || 0) - ); - - if (row && row.score > 0) { - (node as any).soft404 = { - score: row.score, - reason: row.reason - }; - // Map back to node so pagerank/crawlers can read it downstream. - node.soft404Score = row.score; - issueCount++; - } - } - } - - if (issueCount > 0) { - ctx.logger?.warn(`🕵️ Soft 404 detection complete. Found ${issueCount} issues!`); - } else { - ctx.logger?.info(`🕵️ Soft 404 detection complete.`); - } - }, - - /** - * Evaluates the active snapshot results and attaches them statically mapped onto the final JSON reporter. - */ - onReport: async (ctx: PluginContext, result: any) => { - const flags = ctx.flags || {}; - if (!flags.detectSoft404 || !ctx.db) return; - - // Pull directly from our scoped SQLite plugin table - const allRows = ctx.db.data.all<{ url_id: number; score: number; reason: string }>(); - const soft404Pages = allRows.filter(r => r.score > 0.5); - - if (soft404Pages.length > 0) { - if (!result.plugins) result.plugins = {}; - - const mappedPages = (result.pages || []).filter((p: any) => p.soft404Score && p.soft404Score > 0.5) - .map((p: any) => ({ url: p.url, score: p.soft404Score })); - - result.plugins.soft404 = { - totalDetected: mappedPages.length > 0 ? mappedPages.length : soft404Pages.length, - topSample: mappedPages.slice(0, 5) - }; - } - } -}; diff --git a/packages/plugins/soft404-detector/src/types.ts b/packages/plugins/soft404-detector/src/types.ts deleted file mode 100644 index 0ece4fe..0000000 --- a/packages/plugins/soft404-detector/src/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface Soft404Result { - score: number; - reason: string; -} - -export interface Soft404Row { - score: number; - reason: string; -} diff --git a/packages/plugins/soft404-detector/tests/soft404.test.ts b/packages/plugins/soft404-detector/tests/soft404.test.ts deleted file mode 100644 index bc96b07..0000000 --- a/packages/plugins/soft404-detector/tests/soft404.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { Soft404DetectorPlugin } from '../index.js'; -import { Graph } from '@crawlith/core'; - -describe('Soft 404 Detection Plugin', () => { - const baseUrl = 'https://example.com'; - - const runPlugin = async (html: string, outLinks: number = 0) => { - const graph = new Graph(); - graph.addNode(baseUrl, 0, 200); - const node = graph.nodes.get(baseUrl)!; - node.html = html; - node.outLinks = outLinks; - - const ctx: any = { - flags: { detectSoft404: true }, - db: { - data: { - getOrFetch: async (url: string, fetchFn: () => Promise<string>): Promise<string> => { - return await fetchFn(); - } - } - } - }; - - await Soft404DetectorPlugin.hooks!.onMetrics!(ctx, graph); - return node as any; - }; - - it('should detect soft 404 by title pattern', async () => { - const html = '<html><head><title>Page Not FoundWelcome to the site'; - const result = await runPlugin(html, 1); - expect(result.soft404.score).toBeGreaterThan(0.3); - expect(result.soft404.reason).toContain('title_contains_not found'); - }); - - it('should detect soft 404 by H1 pattern', async () => { - const html = '

404 Error

'; - const result = await runPlugin(html, 1); - expect(result.soft404.score).toBeGreaterThan(0.2); - expect(result.soft404.reason).toContain('h1_contains_404'); - }); - - it('should detect soft 404 by very low word count', async () => { - const html = 'Short text'; - const result = await runPlugin(html, 1); - expect(result.soft404.score).toBeGreaterThan(0.2); - expect(result.soft404.reason).toContain('very_low_word_count'); - }); - - it('should detect soft 404 by lack of outbound links', async () => { - const html = 'A page with some text but no links.'; - const result = await runPlugin(html, 0); - expect(result.soft404.reason).toContain('no_outbound_links'); - }); - - it('should combine multiple signals for high score', async () => { - const html = 'Error

Not Found

The requested page was not found.

'; - const result = await runPlugin(html, 1); - // title (0.4) + h1 (0.3) + low word count (0.3) = 1.0 (capped) - expect(result.soft404.score).toBe(1.0); - }); - - it('should not do anything if flag is false', async () => { - const graph = new Graph(); - graph.addNode(baseUrl, 0, 200); - const node = graph.nodes.get(baseUrl)!; - node.html = 'Page Not FoundWelcome to the site'; - - await Soft404DetectorPlugin.hooks!.onMetrics!({ flags: { detectSoft404: false } } as any, graph); - - expect((node as any).soft404).toBeUndefined(); - }); -}); diff --git a/packages/plugins/soft404-detector/tsconfig.json b/packages/plugins/soft404-detector/tsconfig.json deleted file mode 100644 index 8228c28..0000000 --- a/packages/plugins/soft404-detector/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./" - }, - "include": [ - "./**/*.ts" - ] -} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 44dc087..ddbf12d 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -221,7 +221,6 @@ export function startServer(options: ServerOptions): Promise { p.http_status, p.noindex, p.redirect_chain, - m.pagerank as rawPageRank, m.pagerank_score as pageRankScore, m.duplicate_type, m.thin_content_score, @@ -300,7 +299,7 @@ export function startServer(options: ServerOptions): Promise { issueType, severity: sev, impactScore: Math.round(impactFactor * importanceMultiplier), - pageRank: r.rawPageRank, + pageRank: r.pageRankScore, pageRankScore: r.pageRankScore, lastSeen: r.lastSeen, isProblematic @@ -333,7 +332,7 @@ export function startServer(options: ServerOptions): Promise { api.get('/metrics/top-pagerank', validateSnapshot, (req, res) => { const currentSnapshotId = (req as any).snapshotId as number; const rows = db.prepare(` - SELECT p.normalized_url as url, m.pagerank_score as pageRank, m.authority_score as authorityScore, m.hub_score as hubScore + SELECT p.normalized_url as url, m.pagerank_score as pageRank, m.auth_score as authorityScore, m.hub_score as hubScore FROM metrics m JOIN pages p ON m.page_id = p.id WHERE m.snapshot_id = ? @@ -385,17 +384,24 @@ export function startServer(options: ServerOptions): Promise { // 5.1 GET /api/page api.get('/page', async (req, res) => { const url = req.query.url as string; - const snapshotParam = req.query.snapshot as string | undefined; + const snapshotParam = undefined; if (!url) { return res.status(400).json({ error: 'URL parameter is required' }); } + // URLs are stored as root-relative paths (e.g. '/stats'), but PageAnalysisUseCase + // needs an absolute URL to fetch/resolve. Reconstruct it from the site domain. + const urlForLookup = url; // path for DB normalized_url lookup + const urlForAnalysis = url.startsWith('/') + ? `https://${site!.domain}${url}` + : url; + try { // Use the same PageAnalysisUseCase as the CLI's `page` command const useCase = new PageAnalysisUseCase(); const result = await useCase.execute({ - url, + url: urlForAnalysis, snapshotId: snapshotParam ? parseInt(snapshotParam, 10) : undefined, seo: true, content: true, @@ -413,11 +419,11 @@ export function startServer(options: ServerOptions): Promise { // These are graph-level concerns not part of the page analysis use case const dbPage = db.prepare(` SELECT p.id, p.depth, - m.pagerank, m.pagerank_score, m.authority_score, m.hub_score + m.pagerank_score, m.auth_score, m.hub_score, m.heading_data FROM pages p LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? WHERE p.site_id = ? AND p.normalized_url = ? - `).get(targetSnapshotId, siteId, url) as any; + `).get(targetSnapshotId, siteId, urlForLookup) as any; let inlinks = 0, outlinks = 0; if (dbPage) { @@ -450,8 +456,7 @@ export function startServer(options: ServerOptions): Promise { }, metrics: { pageRank: dbPage?.pagerank_score || 0, - rawPageRank: dbPage?.pagerank || 0, - authority: dbPage?.authority_score || 0, + authority: dbPage?.auth_score || 0, hub: dbPage?.hub_score || 0, depth: dbPage?.depth || 0, inlinks, @@ -469,6 +474,7 @@ export function startServer(options: ServerOptions): Promise { images: page.images, links: page.links, structuredData: page.structuredData, + headingData: dbPage?.heading_data ? JSON.parse(dbPage.heading_data) : null, snapshotId: targetSnapshotId }); } catch (error: any) { @@ -480,12 +486,13 @@ export function startServer(options: ServerOptions): Promise { // 5.2 GET /api/page/inlinks api.get('/page/inlinks', validateSnapshot, (req, res) => { const currentSnapshotId = (req as any).snapshotId as number; - const url = req.query.url as string; + let url = req.query.url as string; const pageNum = parseInt(req.query.page as string || '1', 10); const pageSize = parseInt(req.query.pageSize as string || '50', 10); const offset = (pageNum - 1) * pageSize; if (!url) return res.status(400).json({ error: 'URL is required' }); + url = url.startsWith('/') ? `https://${site!.domain}${url}` : url; const page = db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as { id: number }; if (!page) return res.status(404).json({ error: 'Page not found' }); @@ -521,12 +528,13 @@ export function startServer(options: ServerOptions): Promise { // 5.3 GET /api/page/outlinks api.get('/page/outlinks', validateSnapshot, (req, res) => { const currentSnapshotId = (req as any).snapshotId as number; - const url = req.query.url as string; + let url = req.query.url as string; const pageNum = parseInt(req.query.page as string || '1', 10); const pageSize = parseInt(req.query.pageSize as string || '50', 10); const offset = (pageNum - 1) * pageSize; if (!url) return res.status(400).json({ error: 'URL is required' }); + url = url.startsWith('/') ? `https://${site!.domain}${url}` : url; const page = db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as { id: number }; if (!page) return res.status(404).json({ error: 'Page not found' }); @@ -561,9 +569,10 @@ export function startServer(options: ServerOptions): Promise { // 5.4 GET /api/page/cluster api.get('/page/cluster', validateSnapshot, (req, res) => { const currentSnapshotId = (req as any).snapshotId as number; - const url = req.query.url as string; + let url = req.query.url as string; if (!url) return res.status(400).json({ error: 'URL is required' }); + url = url.startsWith('/') ? `https://${site!.domain}${url}` : url; const page = db.prepare(` SELECT p.id, m.duplicate_cluster_id @@ -664,10 +673,53 @@ export function startServer(options: ServerOptions): Promise { }); }); - // 5.7 POST /api/page/crawl (Live crawl of single page) + // 5.7 GET /api/page/plugins + api.get('/page/plugins', validateSnapshot, (req, res) => { + const currentSnapshotId = (req as any).snapshotId as number; + let url = req.query.url as string; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const pageIdRow = db.prepare(`SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?`).get(siteId, url) as any; + if (!pageIdRow) return res.status(404).json({ error: 'Page not found' }); + const pageId = pageIdRow.id; + + const migrations = db.prepare('SELECT plugin_name FROM plugin_migrations').all() as any[]; + const pluginData: Record = {}; + + for (const migration of migrations) { + const pName = migration.plugin_name; + const tableName = `${pName.replace(/-/g, '_')}_plugin`; + try { + // Check if table exists + const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(tableName); + if (tableExists) { + const row = db.prepare(`SELECT * FROM ${tableName} WHERE snapshot_id = ? AND url_id = ? ORDER BY created_at DESC LIMIT 1`).get(currentSnapshotId, pageId); + if (row) { + const parsedRow: Record = { ...row }; + for (const key in parsedRow) { + if (typeof parsedRow[key] === 'string' && (parsedRow[key].startsWith('{') || parsedRow[key].startsWith('['))) { + try { + parsedRow[key] = JSON.parse(parsedRow[key] as string); + } catch { } + } + } + pluginData[pName] = parsedRow; + } + } + } catch (e) { + // Ignore + } + } + + res.json(pluginData); + }); + + // 5.8 POST /api/page/crawl (Live crawl of single page) api.post('/page/crawl', express.json(), strictRateLimiter, async (req, res) => { - const { url } = req.body; + let { url } = req.body; if (!url) return res.status(400).json({ error: 'URL is required' }); + url = url.startsWith('/') ? `https://${site!.domain}${url}` : url; try { console.log(chalk.cyan(` Live crawl requested: ${url}`)); diff --git a/packages/web/src/api.ts b/packages/web/src/api.ts index f643820..388f8a4 100644 --- a/packages/web/src/api.ts +++ b/packages/web/src/api.ts @@ -158,6 +158,7 @@ export interface PageDetails { valid: boolean; types: string[]; }; + headingData?: any; snapshotId: number; } @@ -284,6 +285,16 @@ export async function fetchPageDetails(url: string, snapshotId?: number): Promis return res.json(); } +export async function fetchPagePlugins(url: string, snapshotId?: number): Promise> { + const params = new URLSearchParams(); + params.append('url', url); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/plugins?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch page plugins'); + return res.json(); +} + export async function fetchPageInlinks(url: string, page: number = 1, snapshotId?: number): Promise { const params = new URLSearchParams(); params.append('url', url); diff --git a/packages/web/src/components/Tabs/ContentTab.tsx b/packages/web/src/components/Tabs/ContentTab.tsx index a73db0b..93b942f 100644 --- a/packages/web/src/components/Tabs/ContentTab.tsx +++ b/packages/web/src/components/Tabs/ContentTab.tsx @@ -59,6 +59,47 @@ export const ContentTab = ({ details }: { details: PageDetails }) => { /> + + {details.headingData && details.headingData.map && details.headingData.map.length > 0 && ( +
+
+
+

Heading Structure

+ + {details.headingData.status} Score: {details.headingData.score} + +
+ +
+ {details.headingData.map.map((node: any, idx: number) => ( +
+ H{node.level} + + {node.text} + +
+ ))} +
+ + {details.headingData.issues && details.headingData.issues.length > 0 && ( +
+

Structural Issues Found

+
    + {details.headingData.issues.map((issue: string, idx: number) => ( +
  • +
    + {issue} +
    • + ))} +
+
+ )} +
+
+ )} ); }; diff --git a/packages/web/src/components/Tabs/PerformanceTab.tsx b/packages/web/src/components/Tabs/PerformanceTab.tsx new file mode 100644 index 0000000..e267ac3 --- /dev/null +++ b/packages/web/src/components/Tabs/PerformanceTab.tsx @@ -0,0 +1,106 @@ +import { useEffect, useState } from 'react'; +import * as API from '../../api'; +import { Activity, Clock, Zap, AlertTriangle } from 'lucide-react'; + +export const PerformanceTab = ({ url, snapshotId }: { url: string, snapshotId: number }) => { + const [pluginsData, setPluginsData] = useState>({}); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const fetchPlugins = async () => { + setLoading(true); + try { + const data = await API.fetchPagePlugins(url, snapshotId); + setPluginsData(data); + setError(null); + } catch (e: any) { + setError(e.message || 'Failed to load plugin data.'); + } finally { + setLoading(false); + } + }; + + fetchPlugins(); + }, [url, snapshotId]); + + if (loading) { + return
Loading plugin data...
; + } + + if (error) { + return
Error: {error}
; + } + + const pagespeedData = pluginsData['pagespeed']; + + if (!pagespeedData) { + return ( +
+

No Performance Data

+

Run a live crawl with the --pagespeed flag to fetch Google PageSpeed Insights data.

+
+ ); + } + + return ( +
+

+ + PageSpeed Insights (Mobile) +

+ +
+ + + + +
+
+ ); +}; + +const getScoreColor = (score: number) => { + if (score >= 90) return 'green'; + if (score >= 50) return 'amber'; + return 'red'; +}; + +const MetricCard = ({ title, value, suffix = '', max, color }: any) => { + const colorClasses = { + green: 'text-green-600 dark:text-green-400', + amber: 'text-amber-600 dark:text-amber-400', + red: 'text-red-600 dark:text-red-400', + }; + + return ( +
+
+ {title} +
+
+ {value !== null && value !== undefined ? value : 'N/A'}{suffix} +
+
+ ); +}; diff --git a/packages/web/src/components/Tabs/SignalsTab.tsx b/packages/web/src/components/Tabs/SignalsTab.tsx new file mode 100644 index 0000000..bb3d9e3 --- /dev/null +++ b/packages/web/src/components/Tabs/SignalsTab.tsx @@ -0,0 +1,178 @@ +import { useEffect, useState } from 'react'; +import * as API from '../../api'; +import { Share2, Globe, Braces, AlertTriangle, CheckCircle, XCircle } from 'lucide-react'; + +export const SignalsTab = ({ url, snapshotId }: { url: string, snapshotId: number }) => { + const [pluginsData, setPluginsData] = useState>({}); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const fetchPlugins = async () => { + setLoading(true); + try { + const data = await API.fetchPagePlugins(url, snapshotId); + setPluginsData(data); + setError(null); + } catch (e: any) { + setError(e.message || 'Failed to load plugin data.'); + } finally { + setLoading(false); + } + }; + + fetchPlugins(); + }, [url, snapshotId]); + + if (loading) { + return
Loading plugin data...
; + } + + if (error) { + return
Error: {error}
; + } + + const signalsData = pluginsData['signals']; + + if (!signalsData) { + return ( +
+

No Signals Data

+

Run a live crawl with the --signals flag to fetch structured signals and metadata.

+
+ ); + } + + const parsedSignals = signalsData.signals_json; + + return ( +
+

+ + Signals & Metadata +

+ +
+ + + + +
+ +
+
+
+ +

Social Tags (OG & Twitter)

+
+
+ + + + + + + + +
+
+ +
+
+
+ +

Localization

+
+
+ + + + +
+
+ +
+
+ +

Structured Data

+
+
+ + 0 ? "text-red-500" : undefined} /> + + +
+
+
+
+
+ ); +}; + +const getScoreColor = (score: number) => { + if (score >= 90) return 'green'; + if (score >= 50) return 'amber'; + return 'red'; +}; + +const MetricCard = ({ title, value, suffix = '', max, color }: any) => { + const colorClasses = { + green: 'text-green-600 dark:text-green-400', + amber: 'text-amber-600 dark:text-amber-400', + red: 'text-red-600 dark:text-red-400', + }; + + return ( +
+
+ {title} +
+
+ {value !== null && value !== undefined ? value : 'N/A'}{suffix} +
+
+ ); +}; + +const StatusCard = ({ title, status }: { title: string, status: string }) => { + const isPass = status === 'PASS'; + const isWarning = status === 'WARNING'; + const Icon = isPass ? CheckCircle : isWarning ? AlertTriangle : XCircle; + const colorClass = isPass ? 'text-green-500' : isWarning ? 'text-amber-500' : 'text-red-500'; + + return ( +
+
+ {title} +
+
+ + {status} +
+
+ ); +}; + +const DataRow = ({ label, value, highlight }: { label: string, value: string | null | undefined, highlight?: string }) => ( +
+
{label}:
+
+ {value || None} +
+
+); diff --git a/packages/web/src/pages/SinglePage.tsx b/packages/web/src/pages/SinglePage.tsx index 8e5be58..3372e12 100644 --- a/packages/web/src/pages/SinglePage.tsx +++ b/packages/web/src/pages/SinglePage.tsx @@ -8,6 +8,8 @@ import { LinkingTab } from '../components/Tabs/LinkingTab'; import { ClusterTab } from '../components/Tabs/ClusterTab'; import { TechnicalTab } from '../components/Tabs/TechnicalTab'; import { GraphTab } from '../components/Tabs/GraphTab'; +import { PerformanceTab } from '../components/Tabs/PerformanceTab'; +import { SignalsTab } from '../components/Tabs/SignalsTab'; export const SinglePage = () => { const [searchParams] = useSearchParams(); @@ -204,7 +206,7 @@ export const SinglePage = () => { {/* Tabs Navigation */}
- {['content', 'linking', 'cluster', 'technical', 'graph'].map((tab) => ( + {['content', 'linking', 'cluster', 'technical', 'graph', 'performance', 'signals'].map((tab) => ( ))}
@@ -230,6 +234,8 @@ export const SinglePage = () => { {activeTab === 'cluster' && } {activeTab === 'technical' && } {activeTab === 'graph' && } + {activeTab === 'performance' && } + {activeTab === 'signals' && } ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6bc9a1..8fcf51b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 7.3.1 vite: specifier: 7.3.1 - version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) devDependencies: '@changesets/cli': specifier: ^2.29.8 @@ -38,7 +38,7 @@ importers: version: 8.56.1(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3) '@vitest/coverage-v8': specifier: ^4.0.18 - version: 4.0.18(vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7)) + version: 4.0.18(vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0)) eslint: specifier: ^10.0.0 version: 10.0.2(jiti@1.21.7) @@ -56,58 +56,22 @@ importers: version: 8.56.1(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3) vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7) + version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) packages/cli: dependencies: '@crawlith/core': specifier: workspace:* version: link:../core - '@crawlith/plugin-content-clustering': - specifier: workspace:* - version: link:../plugins/content-clustering - '@crawlith/plugin-crawl-policy': - specifier: workspace:* - version: link:../plugins/crawl-policy - '@crawlith/plugin-crawl-trap-analyzer': - specifier: workspace:* - version: link:../plugins/crawl-trap-analyzer - '@crawlith/plugin-duplicate-detection': - specifier: workspace:* - version: link:../plugins/duplicate-detection '@crawlith/plugin-exporter': specifier: workspace:* version: link:../plugins/exporter - '@crawlith/plugin-heading-health': - specifier: workspace:* - version: link:../plugins/heading-health - '@crawlith/plugin-health-score-engine': - specifier: workspace:* - version: link:../plugins/health-score-engine - '@crawlith/plugin-hits': - specifier: workspace:* - version: link:../plugins/hits - '@crawlith/plugin-orphan-intelligence': - specifier: workspace:* - version: link:../plugins/orphan-intelligence - '@crawlith/plugin-pagerank': - specifier: workspace:* - version: link:../plugins/pagerank '@crawlith/plugin-reporter': specifier: workspace:* version: link:../plugins/reporter '@crawlith/plugin-signals': specifier: workspace:* version: link:../plugins/signals - '@crawlith/plugin-simhash': - specifier: workspace:* - version: link:../plugins/simhash - '@crawlith/plugin-snapshot-diff': - specifier: workspace:* - version: link:../plugins/snapshot-diff - '@crawlith/plugin-soft404-detector': - specifier: workspace:* - version: link:../plugins/soft404-detector '@crawlith/server': specifier: workspace:* version: link:../server @@ -129,13 +93,13 @@ importers: version: 20.19.33 tsup: specifier: ^8.0.0 - version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(typescript@5.9.3) + version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3) typescript: specifier: ^5.4.5 version: 5.9.3 vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7) + version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) packages/core: dependencies: @@ -162,7 +126,7 @@ importers: version: 6.23.0 vite: specifier: 7.3.1 - version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) devDependencies: '@types/better-sqlite3': specifier: ^7.6.13 @@ -178,7 +142,7 @@ importers: version: 5.9.3 vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7) + version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) packages/infrastructure: dependencies: @@ -186,35 +150,27 @@ importers: specifier: workspace:* version: link:../core - packages/plugins/content-clustering: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - cheerio: - specifier: ^1.2.0 - version: 1.2.0 - - packages/plugins/crawl-policy: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - chalk: - specifier: ^5.6.2 - version: 5.6.2 - - packages/plugins/crawl-trap-analyzer: + packages/mcp: dependencies: - '@crawlith/core': + '@crawlith/cli': specifier: workspace:* - version: link:../../core - - packages/plugins/duplicate-detection: - dependencies: + version: link:../cli '@crawlith/core': specifier: workspace:* - version: link:../../core + version: link:../core + '@modelcontextprotocol/sdk': + specifier: ^1.17.5 + version: 1.27.1(zod@3.25.76) + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + tsx: + specifier: ^4.20.5 + version: 4.21.0 + typescript: + specifier: ^5.9.2 + version: 5.9.3 packages/plugins/exporter: dependencies: @@ -225,39 +181,6 @@ importers: specifier: ^5.6.2 version: 5.6.2 - packages/plugins/heading-health: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - - packages/plugins/health-score-engine: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - chalk: - specifier: ^5.6.2 - version: 5.6.2 - - packages/plugins/hits: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - - packages/plugins/orphan-intelligence: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - - packages/plugins/pagerank: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - packages/plugins/pagespeed: dependencies: '@crawlith/core': @@ -279,30 +202,6 @@ importers: specifier: workspace:* version: link:../../core - packages/plugins/simhash: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - - packages/plugins/snapshot-diff: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - chalk: - specifier: ^5.6.2 - version: 5.6.2 - - packages/plugins/soft404-detector: - dependencies: - '@crawlith/core': - specifier: workspace:* - version: link:../../core - cheerio: - specifier: ^1.2.0 - version: 1.2.0 - packages/server: dependencies: '@crawlith/core': @@ -338,7 +237,7 @@ importers: version: 7.13.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) tailwindcss: specifier: ^3.4.3 - version: 3.4.19 + version: 3.4.19(tsx@4.21.0) devDependencies: '@types/react': specifier: ^18.2.79 @@ -348,7 +247,7 @@ importers: version: 18.3.7(@types/react@18.3.28) '@vitejs/plugin-react': specifier: ^4.2.1 - version: 4.7.0(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)) + version: 4.7.0(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0)) autoprefixer: specifier: ^10.4.19 version: 10.4.24(postcss@8.5.6) @@ -360,7 +259,7 @@ importers: version: 5.9.3 vite: specifier: ^7.3.1 - version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + version: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) packages: @@ -713,6 +612,12 @@ packages: resolution: {integrity: sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + '@hono/node-server@1.19.10': + resolution: {integrity: sha512-hZ7nOssGqRgyV3FVVQdfi+U4q02uB23bpnYpdvNXkYTRRyWx84b7yf1ans+dnJ/7h41sGL3CeQTfO+ZGxuO+Iw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -767,6 +672,16 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} + '@modelcontextprotocol/sdk@1.27.1': + resolution: {integrity: sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1147,6 +1062,10 @@ packages: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1161,9 +1080,20 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + ajv@6.14.0: resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==} + ajv@8.18.0: + resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} + ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} @@ -1264,6 +1194,10 @@ packages: resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + body-parser@2.2.2: + resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} + engines: {node: '>=18'} + boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} @@ -1402,6 +1336,10 @@ packages: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} + content-disposition@1.0.1: + resolution: {integrity: sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==} + engines: {node: '>=18'} + content-type@1.0.5: resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} engines: {node: '>= 0.6'} @@ -1412,6 +1350,10 @@ packages: cookie-signature@1.0.7: resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + cookie@0.7.2: resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} @@ -1420,6 +1362,10 @@ packages: resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} engines: {node: '>=18'} + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -1666,6 +1612,14 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventsource-parser@3.0.6: + resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} @@ -1674,10 +1628,20 @@ packages: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + express-rate-limit@8.2.1: + resolution: {integrity: sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + express@4.22.1: resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} engines: {node: '>= 0.10.0'} + express@5.2.1: + resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} + engines: {node: '>= 18'} + extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} @@ -1694,6 +1658,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -1721,6 +1688,10 @@ packages: resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} engines: {node: '>= 0.8'} + finalhandler@2.1.1: + resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} + engines: {node: '>= 18.0.0'} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -1754,6 +1725,10 @@ packages: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -1789,6 +1764,9 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-tsconfig@4.13.6: + resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -1835,6 +1813,10 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hono@4.12.4: + resolution: {integrity: sha512-ooiZW1Xy8rQ4oELQ++otI2T9DsKpV0M6c6cO6JGx4RTfav9poFFLlet9UMXHZnoM1yG0HWGlQLswBGX3RZmHtg==} + engines: {node: '>=16.9.0'} + html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} @@ -1886,6 +1868,10 @@ packages: resolution: {integrity: sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + ip-address@10.0.1: + resolution: {integrity: sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==} + engines: {node: '>= 12'} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -1941,6 +1927,9 @@ packages: resolution: {integrity: sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==} engines: {node: '>=12'} + is-promise@4.0.0: + resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-subdir@1.2.0: resolution: {integrity: sha512-2AT6j+gXe/1ueqbW6fLZJiIw3F8iXGJtt0yDrZaBhAZEG1raiTxKWU+IPqMCzQAXOUCKdA4UDMgacKH25XG2Cw==} engines: {node: '>=4'} @@ -1975,6 +1964,9 @@ packages: resolution: {integrity: sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==} hasBin: true + jose@6.1.3: + resolution: {integrity: sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -2004,6 +1996,12 @@ packages: json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema-typed@8.0.2: + resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -2088,9 +2086,17 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + engines: {node: '>= 0.8'} + merge-descriptors@1.0.3: resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2107,10 +2113,18 @@ packages: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + mime@1.6.0: resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} engines: {node: '>=4'} @@ -2169,6 +2183,10 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + node-abi@3.87.0: resolution: {integrity: sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ==} engines: {node: '>=10'} @@ -2289,6 +2307,9 @@ packages: path-to-regexp@0.1.12: resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + path-to-regexp@8.3.0: + resolution: {integrity: sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==} + path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -2319,6 +2340,10 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} @@ -2420,6 +2445,10 @@ packages: resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} engines: {node: '>= 0.8'} + raw-body@3.0.2: + resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} + engines: {node: '>= 0.10'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -2481,10 +2510,17 @@ packages: resolution: {integrity: sha512-+crtS5QjFRqFCoQmvGduwYWEBng99ZvmFvF+cUJkGYF1L1BfU8C6Zp9T7f5vPAwyLkUExpvK+ANVZmGU49qi4Q==} engines: {node: '>=12'} + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.11: resolution: {integrity: sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==} engines: {node: '>= 0.4'} @@ -2507,6 +2543,10 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + router@2.2.0: + resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} + engines: {node: '>= 18'} + run-applescript@7.1.0: resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} engines: {node: '>=18'} @@ -2536,10 +2576,18 @@ packages: resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} engines: {node: '>= 0.8.0'} + send@1.2.1: + resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} + engines: {node: '>= 18'} + serve-static@1.16.3: resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} engines: {node: '>= 0.8.0'} + serve-static@2.2.1: + resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} + engines: {node: '>= 18'} + set-cookie-parser@2.7.2: resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} @@ -2756,6 +2804,11 @@ packages: typescript: optional: true + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} @@ -2775,6 +2828,10 @@ packages: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} + type-is@2.0.1: + resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} + engines: {node: '>= 0.6'} + typescript-eslint@8.56.1: resolution: {integrity: sha512-U4lM6pjmBX7J5wk4szltF7I1cGBHXZopnAXCMXb3+fZ3B/0Z3hq3wS/CCUB2NZBNAExK92mCU2tEohWuwVMsDQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2982,6 +3039,14 @@ packages: resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} engines: {node: '>=12.20'} + zod-to-json-schema@3.25.1: + resolution: {integrity: sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==} + peerDependencies: + zod: ^3.25 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + snapshots: '@alloc/quick-lru@5.2.0': {} @@ -3362,6 +3427,10 @@ snapshots: '@eslint/core': 1.1.0 levn: 0.4.1 + '@hono/node-server@1.19.10(hono@4.12.4)': + dependencies: + hono: 4.12.4 + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.7': @@ -3429,6 +3498,28 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@modelcontextprotocol/sdk@1.27.1(zod@3.25.76)': + dependencies: + '@hono/node-server': 1.19.10(hono@4.12.4) + ajv: 8.18.0 + ajv-formats: 3.0.1(ajv@8.18.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.6 + express: 5.2.1 + express-rate-limit: 8.2.1(express@5.2.1) + hono: 4.12.4 + jose: 6.1.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 3.25.76 + zod-to-json-schema: 3.25.1(zod@3.25.76) + transitivePeerDependencies: + - supports-color + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3746,7 +3837,7 @@ snapshots: '@typescript-eslint/types': 8.56.1 eslint-visitor-keys: 5.0.1 - '@vitejs/plugin-react@4.7.0(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7))': + '@vitejs/plugin-react@4.7.0(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0))': dependencies: '@babel/core': 7.29.0 '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) @@ -3754,11 +3845,11 @@ snapshots: '@rolldown/pluginutils': 1.0.0-beta.27 '@types/babel__core': 7.20.5 react-refresh: 0.17.0 - vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7))': + '@vitest/coverage-v8@4.0.18(vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0))': dependencies: '@bcoe/v8-coverage': 1.0.2 '@vitest/utils': 4.0.18 @@ -3770,7 +3861,7 @@ snapshots: obug: 2.1.1 std-env: 3.10.0 tinyrainbow: 3.0.3 - vitest: 4.0.18(@types/node@20.19.33)(jiti@1.21.7) + vitest: 4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) '@vitest/expect@4.0.18': dependencies: @@ -3781,13 +3872,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.18(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7))': + '@vitest/mocker@4.0.18(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0))': dependencies: '@vitest/spy': 4.0.18 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) '@vitest/pretty-format@4.0.18': dependencies: @@ -3816,6 +3907,11 @@ snapshots: mime-types: 2.1.35 negotiator: 0.6.3 + accepts@2.0.0: + dependencies: + mime-types: 3.0.2 + negotiator: 1.0.0 + acorn-jsx@5.3.2(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -3826,6 +3922,10 @@ snapshots: acorn@8.16.0: {} + ajv-formats@3.0.1(ajv@8.18.0): + optionalDependencies: + ajv: 8.18.0 + ajv@6.14.0: dependencies: fast-deep-equal: 3.1.3 @@ -3833,6 +3933,13 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ajv@8.18.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-align@3.0.1: dependencies: string-width: 4.2.3 @@ -3936,6 +4043,20 @@ snapshots: transitivePeerDependencies: - supports-color + body-parser@2.2.2: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 4.4.3 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + on-finished: 2.4.1 + qs: 6.14.2 + raw-body: 3.0.2 + type-is: 2.0.1 + transitivePeerDependencies: + - supports-color + boolbase@1.0.0: {} boxen@7.1.1: @@ -4093,16 +4214,25 @@ snapshots: dependencies: safe-buffer: 5.2.1 + content-disposition@1.0.1: {} + content-type@1.0.5: {} convert-source-map@2.0.0: {} cookie-signature@1.0.7: {} + cookie-signature@1.2.2: {} + cookie@0.7.2: {} cookie@1.1.1: {} + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + create-require@1.1.1: {} cross-spawn@7.0.6: @@ -4351,10 +4481,21 @@ snapshots: etag@1.8.1: {} + eventsource-parser@3.0.6: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.0.6 + expand-template@2.0.3: {} expect-type@1.3.0: {} + express-rate-limit@8.2.1(express@5.2.1): + dependencies: + express: 5.2.1 + ip-address: 10.0.1 + express@4.22.1: dependencies: accepts: 1.3.8 @@ -4391,6 +4532,39 @@ snapshots: transitivePeerDependencies: - supports-color + express@5.2.1: + dependencies: + accepts: 2.0.0 + body-parser: 2.2.2 + content-disposition: 1.0.1 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.2.2 + debug: 4.4.3 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.1.1 + fresh: 2.0.0 + http-errors: 2.0.1 + merge-descriptors: 2.0.0 + mime-types: 3.0.2 + on-finished: 2.4.1 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.7 + qs: 6.14.2 + range-parser: 1.2.1 + router: 2.2.0 + send: 1.2.1 + serve-static: 2.2.1 + statuses: 2.0.2 + type-is: 2.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + extendable-error@0.1.7: {} fast-deep-equal@3.1.3: {} @@ -4407,6 +4581,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-uri@3.1.0: {} + fastq@1.20.1: dependencies: reusify: 1.1.0 @@ -4437,6 +4613,17 @@ snapshots: transitivePeerDependencies: - supports-color + finalhandler@2.1.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -4471,6 +4658,8 @@ snapshots: fresh@0.5.2: {} + fresh@2.0.0: {} + fs-constants@1.0.0: {} fs-extra@7.0.1: @@ -4512,6 +4701,10 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + get-tsconfig@4.13.6: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -4558,6 +4751,8 @@ snapshots: dependencies: function-bind: 1.1.2 + hono@4.12.4: {} + html-escaper@2.0.2: {} htmlparser2@10.1.0: @@ -4603,6 +4798,8 @@ snapshots: ini@4.1.1: {} + ip-address@10.0.1: {} + ipaddr.js@1.9.1: {} is-binary-path@2.1.0: @@ -4640,6 +4837,8 @@ snapshots: is-path-inside@4.0.0: {} + is-promise@4.0.0: {} + is-subdir@1.2.0: dependencies: better-path-resolve: 1.0.0 @@ -4673,6 +4872,8 @@ snapshots: jiti@1.21.7: {} + jose@6.1.3: {} + joycon@3.1.1: {} js-tokens@10.0.0: {} @@ -4694,6 +4895,10 @@ snapshots: json-schema-traverse@0.4.1: {} + json-schema-traverse@1.0.0: {} + + json-schema-typed@8.0.2: {} + json-stable-stringify-without-jsonify@1.0.1: {} json5@2.2.3: {} @@ -4767,8 +4972,12 @@ snapshots: media-typer@0.3.0: {} + media-typer@1.1.0: {} + merge-descriptors@1.0.3: {} + merge-descriptors@2.0.0: {} + merge2@1.4.1: {} methods@1.1.2: {} @@ -4780,10 +4989,16 @@ snapshots: mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: dependencies: mime-db: 1.52.0 + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + mime@1.6.0: {} mimic-response@3.1.0: {} @@ -4829,6 +5044,8 @@ snapshots: negotiator@0.6.3: {} + negotiator@1.0.0: {} + node-abi@3.87.0: dependencies: semver: 7.7.4 @@ -4944,6 +5161,8 @@ snapshots: path-to-regexp@0.1.12: {} + path-to-regexp@8.3.0: {} + path-type@4.0.0: {} pathe@2.0.3: {} @@ -4960,6 +5179,8 @@ snapshots: pirates@4.0.7: {} + pkce-challenge@5.0.1: {} + pkg-types@1.3.1: dependencies: confbox: 0.1.8 @@ -4978,12 +5199,13 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.6 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.6): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.6 + tsx: 4.21.0 postcss-nested@6.2.0(postcss@8.5.6): dependencies: @@ -5057,6 +5279,13 @@ snapshots: iconv-lite: 0.4.24 unpipe: 1.0.0 + raw-body@3.0.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -5121,8 +5350,12 @@ snapshots: dependencies: rc: 1.2.8 + require-from-string@2.0.2: {} + resolve-from@5.0.0: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.11: dependencies: is-core-module: 2.16.1 @@ -5168,6 +5401,16 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.59.0 fsevents: 2.3.3 + router@2.2.0: + dependencies: + debug: 4.4.3 + depd: 2.0.0 + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.3.0 + transitivePeerDependencies: + - supports-color + run-applescript@7.1.0: {} run-parallel@1.2.0: @@ -5204,6 +5447,22 @@ snapshots: transitivePeerDependencies: - supports-color + send@1.2.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + serve-static@1.16.3: dependencies: encodeurl: 2.0.0 @@ -5213,6 +5472,15 @@ snapshots: transitivePeerDependencies: - supports-color + serve-static@2.2.1: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.2.1 + transitivePeerDependencies: + - supports-color + set-cookie-parser@2.7.2: {} setprototypeof@1.2.0: {} @@ -5338,7 +5606,7 @@ snapshots: supports-preserve-symlinks-flag@1.0.0: {} - tailwindcss@3.4.19: + tailwindcss@3.4.19(tsx@4.21.0): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -5357,7 +5625,7 @@ snapshots: postcss: 8.5.6 postcss-import: 15.1.0(postcss@8.5.6) postcss-js: 4.1.0(postcss@8.5.6) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.6) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0) postcss-nested: 6.2.0(postcss@8.5.6) postcss-selector-parser: 6.1.2 resolve: 1.22.11 @@ -5436,7 +5704,7 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - tsup@8.5.1(jiti@1.21.7)(postcss@8.5.6)(typescript@5.9.3): + tsup@8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3): dependencies: bundle-require: 5.1.0(esbuild@0.27.3) cac: 6.7.14 @@ -5447,7 +5715,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.6) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0) resolve-from: 5.0.0 rollup: 4.59.0 source-map: 0.7.6 @@ -5464,6 +5732,13 @@ snapshots: - tsx - yaml + tsx@4.21.0: + dependencies: + esbuild: 0.27.3 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + tunnel-agent@0.6.0: dependencies: safe-buffer: 5.2.1 @@ -5481,6 +5756,12 @@ snapshots: media-typer: 0.3.0 mime-types: 2.1.35 + type-is@2.0.1: + dependencies: + content-type: 1.0.5 + media-typer: 1.1.0 + mime-types: 3.0.2 + typescript-eslint@8.56.1(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3): dependencies: '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.1(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3))(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3) @@ -5537,7 +5818,7 @@ snapshots: vary@1.1.2: {} - vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7): + vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0): dependencies: esbuild: 0.27.3 fdir: 6.5.0(picomatch@4.0.3) @@ -5549,11 +5830,12 @@ snapshots: '@types/node': 20.19.33 fsevents: 2.3.3 jiti: 1.21.7 + tsx: 4.21.0 - vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7): + vitest@4.0.18(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0): dependencies: '@vitest/expect': 4.0.18 - '@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)) + '@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0)) '@vitest/pretty-format': 4.0.18 '@vitest/runner': 4.0.18 '@vitest/snapshot': 4.0.18 @@ -5570,7 +5852,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7) + vite: 7.3.1(@types/node@20.19.33)(jiti@1.21.7)(tsx@4.21.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.33 @@ -5647,3 +5929,9 @@ snapshots: yocto-queue@0.1.0: {} yocto-queue@1.2.2: {} + + zod-to-json-schema@3.25.1(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod@3.25.76: {}