diff --git a/docs/plugins/ai-agent-guide.md b/docs/plugins/ai-agent-guide.md index 217fd7b..5fb0a39 100644 --- a/docs/plugins/ai-agent-guide.md +++ b/docs/plugins/ai-agent-guide.md @@ -78,12 +78,41 @@ For data tied to specific pages/URLs. * `global`: Boolean. If `true`, looks across **all snapshots** instead of just the current one. * `ctx.db.data.all(): T[]` - Get all rows for your plugin in current snapshot. -**Example (24h Global Cache):** +* `ctx.db.data.all(): T[]` - Get all rows for your plugin in current snapshot. + +### 3. Declarative Storage (`scoreProvider` & `fetchMode`) +Plugins should use declarative storage to define their caching and scoring strategies natively: + ```typescript -const cached = ctx.db.data.find(url, { maxAge: '24h', global: true }); -if (cached) return cached.data; +export const MyPlugin: CrawlithPlugin = { + name: 'my-plugin', + scoreProvider: true, // Enables automatic weighting & aggregation in the core + storage: { + fetchMode: 'local', // 'network' | 'local' + perPage: { + columns: { + score: 'REAL', + weight: 'REAL', + reason: 'TEXT' + } + } + } +}; +``` +Note: `score` and `weight` are reserved columns managed automatically if omitted, but you can explicitly use them to define defaults. + +### 4. Smart Caching (`getOrFetch`) +Replaces `db.data.find()` and `db.data.save()`. The core handles the `--live` flag logic natively: + +**Example:** +```typescript +const row = await ctx.db.data.getOrFetch( + page.url, + async () => myService.compute(page.html) // Only executes if cache is stale or --live flag is passed +); ``` + --- ## πŸͺ Plugin Hooks Lifecycle diff --git a/packages/core/src/analysis/analyze.ts b/packages/core/src/analysis/analyze.ts index d4c83d1..476d37c 100644 --- a/packages/core/src/analysis/analyze.ts +++ b/packages/core/src/analysis/analyze.ts @@ -215,48 +215,10 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? crawledAt }; - if (options.plugins && options.plugins.length > 0) { - const { PluginRegistry } = await import('../plugin-system/plugin-registry.js'); - const { getCrawlithDB } = await import('../db/index.js'); - const registry = new PluginRegistry(options.plugins); - const pluginCtx = options.pluginContext || { command: 'page' }; - if (!pluginCtx.db) pluginCtx.db = getCrawlithDB(); - - await registry.runHook('onInit', pluginCtx); - await registry.runHook('onMetrics', pluginCtx, crawlData.graph); - - // Map graph node plugin data back to the results - for (const page of result.pages) { - const node = crawlData.graph.nodes.get(page.url); - if (node) { - const extra: Record = {}; - // Common node properties to exclude - const internalKeys = [ - 'url', 'status', 'html', 'depth', 'crawlStatus', 'metadata', - 'inLinks', 'outLinks', 'metrics', 'rank', 'hub', 'authority', - 'canonical', 'noindex', 'nofollow', 'brokenLinks', 'redirectChain', - 'incrementalStatus', 'etag', 'lastModified', 'contentHash', - 'isCollapsed', 'collapseInto', 'simhash', 'uniqueTokenRatio', - 'securityError', 'retries', 'bytesReceived', 'wordCount', - 'thinContentScore', 'externalLinkRatio', 'h1Count', 'h2Count', 'title' - ]; - for (const [key, value] of Object.entries(node)) { - if (value !== undefined && !internalKeys.includes(key)) { - extra[key] = value; - } - } - if (Object.keys(extra).length > 0) { - page.plugins = { ...(page.plugins || {}), ...extra }; - } - } - } - - await registry.runHook('onReport', pluginCtx, result); - } - return result; } + export function analyzePages(rootUrl: string, pages: Iterable | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] { const titleCounts = new Map(); const metaCounts = new Map(); diff --git a/packages/core/src/application/usecases.ts b/packages/core/src/application/usecases.ts index 1a9b913..ca60526 100644 --- a/packages/core/src/application/usecases.ts +++ b/packages/core/src/application/usecases.ts @@ -40,13 +40,16 @@ export interface SiteCrawlInput { export class CrawlSitegraph implements UseCase { async execute(input: SiteCrawlInput): Promise { - const ctx = input.context ?? { command: 'crawl' }; + const ctx = input.context ?? { command: 'crawl', scope: 'crawl' as const }; + ctx.scope = 'crawl'; ctx.db = getCrawlithDB(); const registry = new PluginRegistry(input.plugins ?? []); + await registry.applyStorage(ctx); await registry.runHook('onInit', ctx); if (ctx.terminate) return { snapshotId: 0, graph: undefined as any }; + await registry.runHook('onCrawlStart', ctx); if (ctx.terminate) return { snapshotId: 0, graph: undefined as any }; @@ -85,6 +88,10 @@ export class CrawlSitegraph implements UseCase 0) { const registry = new PluginRegistry(input.plugins); - const ctx = input.context ?? { command: 'analyze' }; - ctx.db = getCrawlithDB(); + const ctx: PluginContext = { + command: 'analyze', + scope: 'crawl', + ...(input.context || {}), + db: getCrawlithDB() + }; await registry.runHook('onInit', ctx); await registry.runHook('onReport', ctx, result); } @@ -143,13 +154,36 @@ export class PageAnalysisUseCase implements UseCase 0) { + const { PluginRegistry } = await import('../plugin-system/plugin-registry.js'); + const registry = new PluginRegistry(input.plugins); + + const pluginCtx: PluginContext = { + command: 'page', + scope: 'page', + targetUrl: input.url, + snapshotId: result.snapshotId, + live: input.live || !!(input.context?.flags?.live), + ...(input.context || {}), + db: getCrawlithDB(), + }; + + await registry.applyStorage(pluginCtx); + await registry.runHook('onInit', pluginCtx); + + // Fire onPage once per analyzed page (normally just 1 for the page command) + for (const page of result.pages) { + await registry.runHook('onPage', pluginCtx, { + url: page.url, + html: (page as any).html ?? '', + status: page.status, + }); + } + } + return result; } } diff --git a/packages/core/src/crawler/crawler.ts b/packages/core/src/crawler/crawler.ts index 65fc14f..60f2d9e 100644 --- a/packages/core/src/crawler/crawler.ts +++ b/packages/core/src/crawler/crawler.ts @@ -487,7 +487,6 @@ export class Crawler { nofollow: parseResult.nofollow ? 1 : 0, content_hash: parseResult.contentHash, simhash: parseResult.simhash, - soft404_score: parseResult.soft404Score, etag: res.etag, last_modified: res.lastModified, retries: res.retries diff --git a/packages/core/src/crawler/parser.ts b/packages/core/src/crawler/parser.ts index f7028f9..4ca9d38 100644 --- a/packages/core/src/crawler/parser.ts +++ b/packages/core/src/crawler/parser.ts @@ -17,8 +17,6 @@ export interface ParseResult { contentHash: string; simhash?: string; uniqueTokenRatio?: number; - soft404Score: number; - soft404Signals: string[]; } export class Parser { @@ -121,10 +119,6 @@ export class Parser { const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0; const simhash = SimHash.generate(tokens).toString(); - // 5. Soft 404 Detection (Migrated to Soft404DetectorPlugin) - const soft404Score = 0; - const soft404Signals: string[] = []; - return { links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })), html: html, // pass raw HTML for analysis @@ -133,9 +127,7 @@ export class Parser { nofollow, contentHash, simhash, - uniqueTokenRatio, - soft404Score, - soft404Signals + uniqueTokenRatio } } } diff --git a/packages/core/src/db/CrawlithDB.ts b/packages/core/src/db/CrawlithDB.ts index 6a40871..44df382 100644 --- a/packages/core/src/db/CrawlithDB.ts +++ b/packages/core/src/db/CrawlithDB.ts @@ -3,6 +3,7 @@ import { runBaseMigrations } from './migrations.js'; import { Statements } from './statements.js'; import { PluginRegistry } from './pluginRegistry.js'; import { normalizeUrl } from '../crawler/normalize.js'; +import type { CrawlithPlugin } from '../plugin-system/plugin-types.js'; export class CrawlithDB { private db: Database.Database; @@ -21,6 +22,10 @@ export class CrawlithDB { // Optional scoping properties private _pluginName?: string; private _snapshotId?: number | string; + /** Whether live fallback is allowed (from --live flag). Core-controlled. */ + private _live: boolean = false; + /** Whether this plugin makes network calls. Core-controlled via plugin.storage.fetchMode. */ + private _fetchMode: 'local' | 'network' = 'network'; constructor(dbPath: string) { this.db = new Database(dbPath); @@ -58,9 +63,22 @@ export class CrawlithDB { public get data() { return { save: (input: { url: string; data: T }) => this.insertPluginRow(input), + find: (url: string, options?: { maxAge?: string | number, global?: boolean }) => this.getPluginRow(url, undefined, undefined, options), - all: () => this.getPluginRows() + + all: () => this.getPluginRows(), + + /** + * Cache-first with live fallback. Core-enforced pattern: + * 1. If cached data exists β†’ return it (always, regardless of age) + * 2. If no cache + fetchMode='network' + live=false β†’ return null (skip) + * 3. If no cache + (fetchMode='local' OR live=true) β†’ call fetchFn, save, return + * + * Plugin authors NEVER touch ctx.live β€” the core injects it via scope(). + */ + getOrFetch: (url: string, fetchFn: () => Promise) => + this._getOrFetch(url, fetchFn), }; } @@ -69,7 +87,12 @@ export class CrawlithDB { */ public get report() { return { - save: (summary: any) => this.insertPluginReport({ summary }), + save: (summary: any, optionalScores?: { + totalScore?: number, + scoreCount?: number, + scoreWeightSum?: number, + scoreCalculatedAt?: string + }) => this.insertPluginReport({ summary, ...optionalScores }), find: () => this.getPluginReport() as T | null }; } @@ -80,14 +103,22 @@ export class CrawlithDB { /** * Create a scoped instance for a specific plugin. + * Also bakes in live + fetchMode so getOrFetch() can enforce the protocol + * without exposing those controls to the plugin author. */ - public scope(pluginName: string, snapshotId?: number | string): CrawlithDB { + public scope( + pluginName: string, + snapshotId?: number | string, + options: { live?: boolean; fetchMode?: 'local' | 'network' } = {} + ): CrawlithDB { if (this._pluginName && this._pluginName !== pluginName) { throw new Error(`Security Violation: Cannot re-scope a database instance already bound to "${this._pluginName}"`); } const scoped = Object.create(this); scoped._pluginName = pluginName; scoped._snapshotId = snapshotId; + scoped._live = options.live ?? false; + scoped._fetchMode = options.fetchMode ?? 'network'; return scoped; } @@ -103,10 +134,11 @@ export class CrawlithDB { if (!pluginName) throw new Error('Plugin name is required for registration (use unbound DB or scope() before calling)'); if (!columns) throw new Error('Columns definition is required'); - const tableName = `${pluginName}_plugin`; + // Sanitize: hyphens are invalid in unquoted SQLite identifiers (e.g. "heading-health" β†’ "heading_health_plugin") + const tableName = this._toTableName(pluginName); // Validate columns - const reserved = ['id', 'snapshot_id', 'url_id', 'created_at']; + const reserved = ['id', 'snapshot_id', 'url_id', 'score', 'weight', 'created_at']; for (const col of Object.keys(columns)) { if (reserved.includes(col.toLowerCase())) { throw new Error(`Plugin "${pluginName}" cannot define reserved column "${col}". Reserved: ${reserved.join(', ')}`); @@ -124,6 +156,8 @@ export class CrawlithDB { 'snapshot_id INTEGER NOT NULL', 'url_id INTEGER NOT NULL', ...Object.entries(columns).map(([col, type]) => `${col} ${type}`), + "score REAL", + "weight REAL DEFAULT 1.0", "created_at TEXT DEFAULT (datetime('now'))", 'FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE', 'FOREIGN KEY(url_id) REFERENCES pages(id) ON DELETE CASCADE' @@ -166,9 +200,13 @@ export class CrawlithDB { } public insertPluginReport(input: { - snapshotId?: number | string - pluginName?: string - summary: unknown + snapshotId?: number | string; + pluginName?: string; + summary: unknown; + totalScore?: number; + scoreCount?: number; + scoreWeightSum?: number; + scoreCalculatedAt?: string; }): void { const snapshotId = input.snapshotId || this._snapshotId; const pluginName = input.pluginName || this._pluginName; @@ -179,7 +217,15 @@ export class CrawlithDB { this._assertSnapshotExists(snapshotId); const data = JSON.stringify(input.summary); - this.statements.insertPluginReport.run(snapshotId, pluginName, data); + this.statements.insertPluginReport.run( + snapshotId, + pluginName, + data, + input.totalScore ?? null, + input.scoreCount ?? null, + input.scoreWeightSum ?? null, + input.scoreCalculatedAt ?? null + ); } public insertPluginRow(input: { @@ -234,7 +280,7 @@ export class CrawlithDB { } public getPluginRows(tableName?: string, snapshotId?: number | string): T[] { - const targetTable = tableName || (this._pluginName ? `${this._pluginName}_plugin` : undefined); + const targetTable = tableName || (this._pluginName ? this._toTableName(this._pluginName) : undefined); const sid = snapshotId || this._snapshotId; if (!targetTable || !sid) throw new Error('Table name and snapshotId are required'); @@ -283,10 +329,10 @@ export class CrawlithDB { const params: any[] = []; if (options.global) { - // Join with pages to find the URL globally - query += ` JOIN pages p ON t.url_id = p.id WHERE p.url = ?`; - query += ` AND p.snapshot_id = t.snapshot_id`; // Sanity check + // Join with pages to find the URL globally across all snapshots + query += ` JOIN pages p ON t.url_id = p.id WHERE p.normalized_url = ?`; params.push(targetUrl); + } else { const urlId = this.getPageIdByUrl(sid!, targetUrl); if (!urlId) return null; @@ -343,6 +389,109 @@ export class CrawlithDB { }); } + private async _getOrFetch(url: string, fetchFn: () => Promise): Promise { + // 1. Check cache (global across snapshots) + const cached = this.getPluginRow(url, undefined, undefined, { global: true }); + if (cached !== null) { + return cached; // Always use cache when it exists + } + + // 2. No cache. Can we fetch live? + const canFetchFresh = this._fetchMode === 'local' || (this._fetchMode === 'network' && this._live); + + if (!canFetchFresh) { + return null; // Silent skip, no data to return + } + + // 3. Compute/Fetch fresh data + const freshData = await fetchFn(); + + // 4. Save to DB + this.insertPluginRow({ url, data: freshData }); + + return freshData; + } + + public aggregateScoreProviders(snapshotId: number | string, plugins: CrawlithPlugin[]): void { + this._assertSnapshotExists(snapshotId); + + for (const plugin of plugins) { + if (!plugin.scoreProvider || !plugin.storage?.perPage?.columns) continue; + + const tableName = this._toTableName(plugin.name); + if (!this.registry.isTableRegistered(tableName)) continue; + + try { + // Ensure the table schema strictly conforms by ignoring plugins that might have bypassed registry constraints + const aggregate = this.db.prepare(` + SELECT + SUM(score * weight) as total_score, + SUM(weight) as score_weight_sum, + COUNT(score) as score_count + FROM ${tableName} + WHERE snapshot_id = ? AND score IS NOT NULL + `).get(snapshotId) as any; + + if (!aggregate || aggregate.score_count === 0) continue; + + const { total_score, score_weight_sum, score_count } = aggregate; + + const recentReport = this.db.prepare(` + SELECT id FROM plugin_reports + WHERE snapshot_id = ? AND plugin_name = ? + ORDER BY created_at DESC LIMIT 1 + `).get(snapshotId, plugin.name) as { id: number } | undefined; + + if (recentReport) { + this.db.prepare(` + UPDATE plugin_reports + SET total_score = ?, score_weight_sum = ?, score_count = ?, score_calculated_at = datetime('now') + WHERE id = ? + `).run(total_score, score_weight_sum, score_count, recentReport.id); + } else { + this.db.prepare(` + INSERT INTO plugin_reports + (snapshot_id, plugin_name, data, total_score, score_weight_sum, score_count, score_calculated_at) + VALUES (?, ?, '{}', ?, ?, ?, datetime('now')) + `).run(snapshotId, plugin.name, total_score, score_weight_sum, score_count); + } + } catch (err) { + console.error(`[CrawlithDB.Aggregation] Failed to aggregate scores for plugin: ${plugin.name} - ${(err as Error).message}`); + } + } + + try { + // After all plugins are aggregated, tally up the snapshot-level totals + const snapshotAggregate = this.db.prepare(` + SELECT + SUM(total_score) as overall_total_score, + SUM(score_weight_sum) as overall_weight_sum, + SUM(score_count) as overall_score_count + FROM plugin_reports + WHERE snapshot_id = ? AND total_score IS NOT NULL + `).get(snapshotId) as any; + + if (snapshotAggregate && snapshotAggregate.overall_score_count > 0) { + this.db.prepare(` + UPDATE snapshots + SET + total_score = ?, + score_weight_sum = ?, + score_count = ?, + score_calculated_at = datetime('now') + WHERE id = ? + `).run( + snapshotAggregate.overall_total_score, + snapshotAggregate.overall_weight_sum, + snapshotAggregate.overall_score_count, + snapshotId + ); + } + } catch (err) { + console.error(`[CrawlithDB.Aggregation] Failed to aggregate snapshot-level scores: ${(err as Error).message}`); + } + } + public runInTransaction(fn: () => void): void { const tx = this.db.transaction(fn); tx(); @@ -350,11 +499,16 @@ export class CrawlithDB { private _resolveTableName(name: string): string { if (this.registry.isTableRegistered(name)) return name; - const pluginTable = `${name}_plugin`; + const pluginTable = this._toTableName(name); if (this.registry.isTableRegistered(pluginTable)) return pluginTable; return name; // Will fail assertion later } + /** Converts a plugin name to its canonical SQLite table name, sanitizing invalid characters. */ + private _toTableName(pluginName: string): string { + return `${pluginName.replace(/-/g, '_')}_plugin`; + } + public close(): void { this.db.close(); } diff --git a/packages/core/src/db/graphLoader.ts b/packages/core/src/db/graphLoader.ts index ec9f5ee..d02a685 100644 --- a/packages/core/src/db/graphLoader.ts +++ b/packages/core/src/db/graphLoader.ts @@ -64,7 +64,6 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph { etag: p.etag || undefined, lastModified: p.last_modified || undefined, html: p.html || undefined, - soft404Score: p.soft404_score || undefined, noindex: !!p.noindex, nofollow: !!p.nofollow, incrementalStatus, diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts index cff489c..dead079 100644 --- a/packages/core/src/db/index.ts +++ b/packages/core/src/db/index.ts @@ -43,7 +43,6 @@ export function getCrawlithDB(): CrawlithDB { // Migrations for existing tables try { dbInstance.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`); } catch (_e) { /* ignore */ } - try { dbInstance.exec(`ALTER TABLE pages ADD COLUMN soft404_score REAL;`); } catch (_e) { /* ignore */ } // Security controls: Ensure file permissions are 600 (user read/write only) if (dbPath !== ':memory:') { diff --git a/packages/core/src/db/migrations.ts b/packages/core/src/db/migrations.ts index ff3811a..4dfbc09 100644 --- a/packages/core/src/db/migrations.ts +++ b/packages/core/src/db/migrations.ts @@ -26,6 +26,10 @@ export function runBaseMigrations(db: Database) { health_score REAL, orphan_count INTEGER, thin_content_count INTEGER, + total_score REAL, + score_count INTEGER, + score_weight_sum REAL, + score_calculated_at TEXT, FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE ); `); @@ -45,7 +49,6 @@ export function runBaseMigrations(db: Database) { etag TEXT, last_modified TEXT, html TEXT, - soft404_score REAL, noindex INTEGER DEFAULT 0, nofollow INTEGER DEFAULT 0, security_error TEXT, @@ -68,7 +71,6 @@ export function runBaseMigrations(db: Database) { // Migrations for existing tables try { db.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`); } catch (_e) { /* ignore */ } - try { db.exec(`ALTER TABLE pages ADD COLUMN soft404_score REAL;`); } catch (_e) { /* ignore */ } db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`); @@ -157,6 +159,10 @@ export function runBaseMigrations(db: Database) { snapshot_id INTEGER NOT NULL, plugin_name TEXT NOT NULL, data TEXT NOT NULL, + total_score REAL, + score_count INTEGER, + score_weight_sum REAL, + score_calculated_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE ); diff --git a/packages/core/src/db/repositories/PageRepository.ts b/packages/core/src/db/repositories/PageRepository.ts index 233e3dc..fe82bf6 100644 --- a/packages/core/src/db/repositories/PageRepository.ts +++ b/packages/core/src/db/repositories/PageRepository.ts @@ -13,7 +13,6 @@ export interface Page { etag: string | null; last_modified: string | null; html: string | null; - soft404_score: number | null; noindex: number; nofollow: number; security_error: string | null; @@ -38,13 +37,13 @@ export class PageRepository { INSERT INTO pages ( site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id, http_status, canonical_url, content_hash, simhash, etag, last_modified, html, - soft404_score, noindex, nofollow, security_error, retries, depth, + noindex, nofollow, security_error, retries, depth, discovered_via_sitemap, redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type, updated_at ) VALUES ( @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id, @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html, - @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth, + @noindex, @nofollow, @security_error, @retries, @depth, @discovered_via_sitemap, @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type, datetime('now') ) @@ -57,7 +56,6 @@ export class PageRepository { etag = COALESCE(excluded.etag, pages.etag), last_modified = COALESCE(excluded.last_modified, pages.last_modified), html = COALESCE(excluded.html, pages.html), - soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score), noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END, nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END, security_error = COALESCE(excluded.security_error, pages.security_error), @@ -88,7 +86,6 @@ export class PageRepository { etag: page.etag ?? null, last_modified: page.last_modified ?? null, html: page.html ?? null, - soft404_score: page.soft404_score ?? null, noindex: page.noindex ?? 0, nofollow: page.nofollow ?? 0, security_error: page.security_error ?? null, @@ -142,13 +139,13 @@ export class PageRepository { INSERT INTO pages ( site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id, http_status, canonical_url, content_hash, simhash, etag, last_modified, html, - soft404_score, noindex, nofollow, security_error, retries, depth, + noindex, nofollow, security_error, retries, depth, discovered_via_sitemap, redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type, updated_at ) VALUES ( @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id, @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html, - @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth, + @noindex, @nofollow, @security_error, @retries, @depth, @discovered_via_sitemap, @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type, datetime('now') ) @@ -161,7 +158,6 @@ export class PageRepository { etag = COALESCE(excluded.etag, pages.etag), last_modified = COALESCE(excluded.last_modified, pages.last_modified), html = COALESCE(excluded.html, pages.html), - soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score), noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END, nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END, security_error = COALESCE(excluded.security_error, pages.security_error), @@ -192,7 +188,6 @@ export class PageRepository { etag: page.etag ?? null, last_modified: page.last_modified ?? null, html: page.html ?? null, - soft404_score: page.soft404_score ?? null, noindex: page.noindex ?? 0, nofollow: page.nofollow ?? 0, security_error: page.security_error ?? null, diff --git a/packages/core/src/db/schema.ts b/packages/core/src/db/schema.ts index 8a2b8aa..36a7b63 100644 --- a/packages/core/src/db/schema.ts +++ b/packages/core/src/db/schema.ts @@ -45,7 +45,6 @@ export function initSchema(db: Database) { etag TEXT, last_modified TEXT, html TEXT, - soft404_score REAL, noindex INTEGER DEFAULT 0, nofollow INTEGER DEFAULT 0, security_error TEXT, diff --git a/packages/core/src/db/statements.ts b/packages/core/src/db/statements.ts index 779c736..6306a96 100644 --- a/packages/core/src/db/statements.ts +++ b/packages/core/src/db/statements.ts @@ -1,46 +1,47 @@ import { Database, Statement } from 'better-sqlite3'; export class Statements { - public getPageIdByUrl: Statement; - public insertPluginReport: Statement; - public getPluginReport: Statement; - public deleteSnapshotPlugins: Statement; - public getSnapshot: Statement; - public getMigration: Statement; - public insertMigration: Statement; - - constructor(private db: Database) { - this.getPageIdByUrl = this.db.prepare(` + public getPageIdByUrl: Statement; + public insertPluginReport: Statement; + public getPluginReport: Statement; + public deleteSnapshotPlugins: Statement; + public getSnapshot: Statement; + public getMigration: Statement; + public insertMigration: Statement; + + constructor(private db: Database) { + this.getPageIdByUrl = this.db.prepare(` SELECT id FROM pages WHERE site_id = (SELECT site_id FROM snapshots WHERE id = ?) AND normalized_url = ? `); - this.insertPluginReport = this.db.prepare(` - INSERT OR REPLACE INTO plugin_reports (snapshot_id, plugin_name, data) - VALUES (?, ?, ?) + this.insertPluginReport = this.db.prepare(` + INSERT OR REPLACE INTO plugin_reports + (snapshot_id, plugin_name, data, total_score, score_count, score_weight_sum, score_calculated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) `); - this.getPluginReport = this.db.prepare(` + this.getPluginReport = this.db.prepare(` SELECT data FROM plugin_reports WHERE snapshot_id = ? AND plugin_name = ? ORDER BY created_at DESC LIMIT 1 `); - this.deleteSnapshotPlugins = this.db.prepare(` + this.deleteSnapshotPlugins = this.db.prepare(` DELETE FROM plugin_reports WHERE snapshot_id = ? `); - this.getSnapshot = this.db.prepare(` + this.getSnapshot = this.db.prepare(` SELECT id FROM snapshots WHERE id = ? `); - this.getMigration = this.db.prepare(` + this.getMigration = this.db.prepare(` SELECT plugin_name FROM plugin_migrations WHERE plugin_name = ? `); - this.insertMigration = this.db.prepare(` + this.insertMigration = this.db.prepare(` INSERT INTO plugin_migrations (plugin_name) VALUES (?) `); - } + } } diff --git a/packages/core/src/plugin-system/plugin-registry.ts b/packages/core/src/plugin-system/plugin-registry.ts index 5a05e77..3382c40 100644 --- a/packages/core/src/plugin-system/plugin-registry.ts +++ b/packages/core/src/plugin-system/plugin-registry.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import { CrawlithPlugin, PluginContext } from './plugin-types.js'; +import { CrawlithPlugin, PluginContext, CLIWriter } from './plugin-types.js'; import { PluginConfig } from './plugin-config.js'; export class PluginRegistry { @@ -9,8 +9,16 @@ export class PluginRegistry { this.plugins = plugins; } + public get pluginsList(): CrawlithPlugin[] { + return this.plugins; + } + private registeredCommands = new WeakSet(); + /** + * Registers all plugin CLI flags on the given command. + * Handles both declarative `cli` config and legacy `register(cmd)` callbacks. + */ registerPlugins(program: any) { if (!(program instanceof Command)) return; @@ -18,33 +26,109 @@ export class PluginRegistry { if (this.registeredCommands.has(cmd)) return; this.registeredCommands.add(cmd); + const cmdName = cmd.name() as 'page' | 'crawl'; + for (const plugin of this.plugins) { + // Declarative cli registration (preferred) + if (plugin.cli) { + const targets = plugin.cli.for ?? ['page', 'crawl']; + if (targets.includes(cmdName)) { + cmd.option(plugin.cli.flag, plugin.cli.description); + for (const opt of plugin.cli.options ?? []) { + const dv = opt.defaultValue; + if (dv !== undefined && dv !== null) { + cmd.option(opt.flag, opt.description, dv as string | boolean | string[]); + } else { + cmd.option(opt.flag, opt.description); + } + } + } + } + + // Legacy imperative registration (backwards compat) if (typeof plugin.register === 'function') { plugin.register(cmd); } } + for (const sub of cmd.commands) { traverse(sub); } }; + traverse(program); } + /** + * Applies declarative `storage` schemas for all plugins. + * Called by the core after `getCrawlithDB()` is available, before any hooks run. + */ + applyStorage(context: PluginContext): void { + for (const plugin of this.plugins) { + if (!plugin.storage?.perPage?.columns) continue; + if (!context.db) continue; + + try { + const scopedDb = context.db.scope(plugin.name, context.snapshotId, { + live: context.live, + fetchMode: plugin.storage.fetchMode + }); + scopedDb.schema.define(plugin.storage.perPage.columns as any); + } catch (err) { + context.logger?.error( + `[plugin:${plugin.name}] Storage schema failed: ${(err as Error).message}` + ); + } + } + } + + // ─── Scope guards ───────────────────────────────────────────────────────── + + /** Hooks that only make sense during a full site crawl. */ + private static readonly CRAWL_ONLY_HOOKS = new Set([ + 'onCrawlStart', 'onPageParsed', 'onGraphBuilt', 'onMetrics', 'onReport', 'shouldEnqueueUrl' + ]); + + /** Hooks that only make sense during a single-page analysis. */ + private static readonly PAGE_ONLY_HOOKS = new Set([ + 'onPage' + ]); + + // ─── Hook runners ───────────────────────────────────────────────────────── + async runHook(hookName: string, context: PluginContext, payload?: any) { + // Enforce scope: silently skip hooks that don't belong to this execution context. + // Undefined scope (legacy/test) is treated as permissive. + if (context.scope === 'page' && PluginRegistry.CRAWL_ONLY_HOOKS.has(hookName)) return; + if (context.scope === 'crawl' && PluginRegistry.PAGE_ONLY_HOOKS.has(hookName)) return; + for (const plugin of this.plugins) { const hooks = plugin.hooks as any; if (hooks && typeof hooks[hookName] === 'function') { - const scopedDb = context.db?.scope(plugin.name, context.snapshotId || (payload?.snapshotId)); + const scopedDb = context.db?.scope(plugin.name, context.snapshotId || payload?.snapshotId, { + live: context.live, + fetchMode: plugin.storage?.fetchMode + }); const scopedConfig = new PluginConfig(plugin.name); // Resolve targetUrl from payload if available (standard result object) const targetUrl = context.targetUrl || payload?.pages?.[0]?.url; - const scopedContext = { + // Build a CLIWriter that prefixes plugin name β€” satisfies both cli and logger + const cliWriter: CLIWriter = { + info: (m) => context.logger?.info(m), + warn: (m) => context.logger?.warn(m), + error: (m) => context.logger?.error(m), + debug: (m) => context.logger?.debug(m), + }; + + const scopedContext: PluginContext = { ...context, db: scopedDb, config: scopedConfig, - targetUrl + targetUrl, + cli: cliWriter, + logger: cliWriter, // keep logger alias in sync }; try { @@ -54,7 +138,9 @@ export class PluginRegistry { await hooks[hookName](scopedContext); } } catch (err) { - context.logger?.error(`[plugin:${plugin.name}] Hook ${hookName} failed: ${(err as Error).message}`); + context.logger?.error( + `[plugin:${plugin.name}] Hook ${hookName} failed: ${(err as Error).message}` + ); } } } @@ -64,15 +150,20 @@ export class PluginRegistry { for (const plugin of this.plugins) { const hooks = plugin.hooks as any; if (hooks && typeof hooks[hookName] === 'function') { - const scopedDb = context.db?.scope(plugin.name, context.snapshotId); + const scopedDb = context.db?.scope(plugin.name, context.snapshotId, { + live: context.live, + fetchMode: plugin.storage?.fetchMode + }); const scopedConfig = new PluginConfig(plugin.name); - const scopedContext = { ...context, db: scopedDb, config: scopedConfig }; + const scopedContext: PluginContext = { ...context, db: scopedDb, config: scopedConfig }; try { const result = hooks[hookName](scopedContext, ...args); if (result !== undefined) return result; } catch (err) { - context.logger?.error(`[plugin:${plugin.name}] Sync bail hook ${hookName} failed: ${(err as Error).message}`); + context.logger?.error( + `[plugin:${plugin.name}] Sync bail hook ${hookName} failed: ${(err as Error).message}` + ); } } } diff --git a/packages/core/src/plugin-system/plugin-types.ts b/packages/core/src/plugin-system/plugin-types.ts index 50ba0cd..1f55ac1 100644 --- a/packages/core/src/plugin-system/plugin-types.ts +++ b/packages/core/src/plugin-system/plugin-types.ts @@ -1,8 +1,56 @@ -import { Command } from 'commander'; import type { CrawlithDB } from '../db/CrawlithDB.js'; +// ─── Scope ─────────────────────────────────────────────────────────────────── + +/** + * Execution scope β€” set by the core before any hooks run. + * - `page` β†’ single-URL analysis (crawlith page …) + * - `crawl` β†’ full site crawl (crawlith crawl …) + */ +export type PluginScope = 'page' | 'crawl'; + +// ─── Output writers ─────────────────────────────────────────────────────────── + +/** Injected into hook contexts to let plugins emit structured CLI output. */ +export interface CLIWriter { + info(message: string): void; + warn(message: string): void; + error(message: string): void; + debug(message: string): void; +} + +/** + * Injected into report-phase hook contexts. + * Plugins contribute structured data that the core aggregates into the final output. + */ +export interface ReportWriter { + /** Attach a named data section to the report output. */ + addSection(pluginName: string, data: unknown): void; + /** Optionally contribute a weighted score component. */ + contributeScore?(input: { label: string; score: number; weight: number }): void; +} + +// ─── Per-page input ─────────────────────────────────────────────────────────── + +/** Passed to `onPage` β€” the single URL being analyzed. */ +export interface PageInput { + /** Absolute URL of the page being analyzed. */ + url: string; + /** Raw HTML content of the page. */ + html: string; + /** HTTP status code. */ + status: number; +} + +// ─── Scoped hook contexts ───────────────────────────────────────────────────── + +/** Base context available in every hook. */ export interface PluginContext { + /** Execution scope. Undefined only in legacy / test contexts β€” treated as permissive. */ + scope?: PluginScope; command?: string; + /** Whether live fallback is allowed (from --live flag). Core-controlled. */ + live?: boolean; flags?: Record; snapshotId?: number; targetUrl?: string; @@ -12,29 +60,136 @@ export interface PluginContext { require(key?: string): string; set(value: string): void; }; - logger?: { - info(msg: string): void; - warn(msg: string): void; - error(msg: string): void; - debug(msg: string): void; - }; + /** CLI writer β€” populated by the registry before each hook call. */ + cli?: CLIWriter; + /** Legacy logger alias β€” kept for backwards compatibility. */ + logger?: CLIWriter; + metadata?: Record; [key: string]: any; } +// ─── CLI option declaration ─────────────────────────────────────────────────── + +export interface PluginCliOption { + /** e.g. '--my-flag ' */ + flag: string; + description: string; + defaultValue?: unknown; +} + +// ─── Plugin storage declaration ─────────────────────────────────────────────── + +export type PluginColumnType = 'INTEGER' | 'REAL' | 'TEXT'; + +export interface PluginStorage { + /** Whether this plugin fetches data from a network or computes locally. Defaults to 'network'. */ + fetchMode?: 'local' | 'network'; + /** + * Per-URL columns to add to the plugin's scoped data table. + * The core creates the table automatically before `onInit` runs, + * so plugins never need to call `ctx.db.schema.define()`. + */ + perPage?: { + columns: Record; + }; +} + +// ─── Plugin interface ───────────────────────────────────────────────────────── + export interface CrawlithPlugin { name: string; version?: string; description?: string; - register?: (cli: Command) => void; + /** + * Declarative CLI registration. + * The core registers these options on the appropriate commands β€” + * no need to interact with Commander directly. + * + * `for` controls which commands expose the options: + * - `['page', 'crawl']` (default when omitted) β€” both commands + * - `['page']` β€” page command only + * - `['crawl']` β€” crawl command only + */ + cli?: { + flag: string; + description: string; + for?: ('page' | 'crawl')[]; + options?: PluginCliOption[]; + }; + + /** + * Declarative storage schema. + * The core creates the plugin's scoped table before `onInit` runs. + * Plugins that don't persist data can omit this entirely. + */ + storage?: PluginStorage; + + /** + * Set to true to declare this plugin as a Score Provider. + * The core will automatically aggregate the `score` and `weight` columns + * from this plugin's storage table during snapshot aggregation. + */ + scoreProvider?: boolean; + + /** + * Legacy imperative CLI registration β€” kept for backwards compatibility. + * Prefer `cli` for new plugins. + * @deprecated Use `cli` instead. + */ + register?: (cli: any) => void; hooks?: { + /** + * Runs on both `page` and `crawl` scopes. + * Use for any setup that doesn't depend on the scope β€” + * e.g. initialising in-memory state, reading config. + * DB schema is already created by the time this runs (via `storage`). + */ onInit?: (ctx: PluginContext) => void | Promise; + + /** + * Single-page hook β€” `page` scope only. + * Receives the target URL, its raw HTML, and HTTP status. + * Use this for URL-scoped plugins (PageSpeed, heading-health, etc.). + */ + onPage?: (ctx: PluginContext, page: PageInput) => void | Promise; + + /** + * Fired at the very start of a crawl β€” `crawl` scope only. + * Use to initialise crawl-wide state or validate config. + */ onCrawlStart?: (ctx: PluginContext) => void | Promise; + + /** + * URL enqueue filter β€” `crawl` scope only. + * Return `false` to prevent a URL from being crawled. + */ shouldEnqueueUrl?: (ctx: PluginContext, url: string, depth: number) => boolean; + + /** + * Fired after each page is fetched and parsed β€” `crawl` scope only. + * Use for real-time per-page processing without waiting for the full graph. + */ onPageParsed?: (ctx: PluginContext, page: any) => void | Promise; + + /** + * Fired after the full link graph is built β€” `crawl` scope only. + * Graph structure is complete; metrics have not been computed yet. + */ onGraphBuilt?: (ctx: PluginContext, graph: any) => void | Promise; - onMetrics?: (ctx: PluginContext, metrics: any) => void | Promise; + + /** + * Graph-level metrics phase β€” `crawl` scope only. + * All pages are available; use for cross-page analysis (duplicate + * detection, PageRank, heading structure across the site, etc.). + */ + onMetrics?: (ctx: PluginContext, graph: any) => void | Promise; + + /** + * Final report phase β€” `crawl` scope only. + * Attach snapshot-level summary data to the result object. + */ onReport?: (ctx: PluginContext, report: any) => void | Promise; }; } diff --git a/packages/plugins/heading-health/README.md b/packages/plugins/heading-health/README.md index 38a4876..58de142 100644 --- a/packages/plugins/heading-health/README.md +++ b/packages/plugins/heading-health/README.md @@ -1,9 +1,124 @@ -# Heading Health Plugin +# `@crawlith/plugin-heading-health` -Analyzes heading structure, hierarchy health, and content distribution across pages. +Analyzes the heading structure, hierarchy health, and content distribution of every page in a crawl or single-page analysis. -## Features -- **Hierarchy Audit:** Detects missing H1s, multiple H1s, and hierarchy skips (e.g., H2 to H4). -- **Structural Entropy:** Measures the diversity and balance of heading levels. -- **Content Mapping:** Visualizes the document outline with parent-child relationships. -- **Divergence Check:** Flags when H1 text significantly differs from the page title. +--- + +## What It Does + +The heading-health plugin parses the raw HTML of each crawled page, extracts all `

` through `

` tags, and runs a suite of structural checks. It scores each page from **0–100**, assigns a status (`Healthy`, `Moderate`, `Poor`), and surfaces actionable issues. A snapshot-level summary is also produced, giving you a site-wide view of heading quality. + +--- + +## How It Works + +The plugin runs across three lifecycle hooks: + +### `onInit` +Registers the plugin's database schema columns (`score`, `status`, `analysis_json`) so per-URL results can be persisted across runs. + +### `onMetrics` +Triggered after crawling, once the full page graph is available. For each page node in the graph: + +1. **Extracts headings** from raw HTML using regex-based parsing, preserving heading level and text. +2. **Builds a tree** tracking parent–child relationships between headings. +3. **Segments content** between headings to measure word count and keyword concentration per section. +4. **Computes structural metrics** (see below). +5. **Detects site-wide duplicates** by comparing H1 norms, H2 set hashes, and structural pattern hashes across all pages (cross-page duplicate risk for sections). +6. **Scores and statuses** each page, attaches the payload to the graph node, and persists data to the database. +7. Emits a **snapshot-level summary** to the context. + +### `onReport` +Attaches the snapshot-level `HeadingHealthSummary` to the final result object under `result.plugins.headingHealth`. + +--- + +## CLI Usage + +Add `--heading` to the `crawl` or `page` commands to activate the plugin: + +```bash +# Single page +crawlith page https://example.com --heading + +# Full crawl +crawlith crawl https://example.com --heading + +# Force a fresh recompute (bypass 24h cache) +crawlith crawl https://example.com --heading --heading-force-refresh +``` + +--- + +## Metrics Explained + +Each page receives a `HeadingHealthPayload` with the following fields: + +| Field | Description | +| :--- | :--- | +| `score` | Overall heading health score, 0–100 | +| `status` | `Healthy` (β‰₯70), `Moderate` (β‰₯40), or `Poor` (<40) | +| `issues` | List of human-readable issue strings detected | +| `map` | Ordered array of all heading nodes with level, text, index, and parent index | +| `missing_h1` | `1` if no H1 exists on the page, else `0` | +| `multiple_h1` | `1` if more than one H1 exists, else `0` | +| `entropy` | Shannon entropy of heading level distribution β€” higher = more chaotic structure | +| `max_depth` | Deepest heading level used (e.g., `4` means H4 is the deepest) | +| `avg_depth` | Average heading level across all headings | +| `heading_density` | Ratio of heading count to total word count | +| `fragmentation` | Proportion of headings that are H1 or H2 β€” high fragmentation = too many top-level headings | +| `volatility` | Average level-jump between consecutive headings β€” high = erratic structure | +| `hierarchy_skips` | Number of times heading level jumps by more than 1 (e.g., H2 β†’ H4) | +| `reverse_jumps` | Number of times heading level drops by more than 1 (e.g., H4 β†’ H2) | +| `thin_sections` | Number of content sections containing fewer than 80 words | +| `duplicate_h1_group` | Number of other pages sharing the same normalized H1 text | +| `similar_h1_group` | Number of pages with a Jaccard similarity > threshold to this page's H1 | +| `identical_h2_set_group` | Number of pages sharing the exact same ordered set of H2 text | +| `duplicate_pattern_group` | Number of pages with the same heading level pattern (e.g., `1>2>2>3`) | +| `template_risk` | Combined risk score indicating the page may use a templated heading structure | + +--- + +## Site-Wide Summary (`HeadingHealthSummary`) + +Attached to `result.plugins.headingHealth` after a crawl: + +| Field | Description | +| :--- | :--- | +| `avgScore` | Mean heading health score across all evaluated pages | +| `evaluatedPages` | Number of pages that had headings and were scored | +| `totalMissing` | Total pages missing an H1 | +| `totalMultiple` | Total pages with multiple H1 tags | +| `totalSkips` | Sum of hierarchy skips across all pages | +| `totalReverseJumps` | Sum of reverse hierarchy jumps across all pages | +| `totalThinSections` | Total count of thin content sections across the site | +| `avgEntropy` | Mean structural entropy β€” a site-wide indicator of heading consistency | +| `poorPages` | Number of pages rated `Poor` | + +--- + +## Detected Issues + +The plugin surfaces plain-language issues on each page: + +- `Missing H1` β€” No H1 tag found +- `Multiple H1 found` β€” More than one H1 on the page +- `Empty or near-empty H1` β€” H1 text is fewer than 6 characters +- `H1 diverges from ` β€” H1 and page title have low Jaccard similarity (< 0.3) +- `N hierarchy skips detected` β€” Heading level jumps more than 1 step forward +- `N reverse hierarchy jumps detected` β€” Heading level drops more than 1 step +- `Thin section under "..."` β€” A content section has fewer than 80 words +- `High structural entropy` β€” Entropy score exceeds 2.1 +- `Section fragmentation is high` β€” More than 65% of headings are H1/H2 + +--- + +## Data Storage + +Per-URL results are persisted in the plugin's scoped database table. The stored columns are: + +- `score` β€” Integer health score +- `status` β€” `Healthy`, `Moderate`, or `Poor` +- `analysis_json` β€” Full `HeadingHealthPayload` serialized as JSON + +This data is available for future queries via `ctx.db.data.find()` in other plugins or custom tooling. diff --git a/packages/plugins/heading-health/index.test.ts b/packages/plugins/heading-health/index.test.ts index a06a975..80f571b 100644 --- a/packages/plugins/heading-health/index.test.ts +++ b/packages/plugins/heading-health/index.test.ts @@ -1,102 +1,83 @@ -import { describe, expect, it } from 'vitest'; -import HeadingHealthPlugin from './index.js'; +import { describe, expect, it, vi } from 'vitest'; import type { PluginContext } from '@crawlith/core'; +import HeadingHealthPlugin from './index.js'; describe('heading-health plugin', () => { - it('computes detailed per-page heading map data and summary metrics', async () => { - const rawNodes = [ - { - url: 'https://example.com/a', - status: 200, - title: 'Product A Overview', - html: ` - <html><body> - <h1>Product A</h1> - <p>${'intro '.repeat(50)}</p> - <h3>Features</h3> - <p>${'feature '.repeat(20)}</p> - <h2>FAQ</h2> - <p>${'faq '.repeat(90)}</p> - </body></html> - ` - }, - { - url: 'https://example.com/b', - status: 200, - title: 'Product A Details', - html: ` - <html><body> - <h1>Product A</h1> - <h2>Specs</h2> - <p>${'spec '.repeat(30)}</p> - </body></html> - ` - } - ]; + it('computes per-page heading payload and summary metrics', async () => { + const rawNodes = [ + { + url: 'https://example.com/a', + status: 200, + title: 'Product A Overview', + html: ` + <html><body> + <h1>Product A</h1> + <p>${'intro '.repeat(50)}</p> + <h3>Features</h3> + <p>${'feature '.repeat(20)}</p> + <h2>FAQ</h2> + <p>${'faq '.repeat(90)}</p> + </body></html> + ` + }, + { + url: 'https://example.com/b', + status: 200, + title: 'Product A Details', + html: ` + <html><body> + <h1>Product A</h1> + <h2>Specs</h2> + <p>${'spec '.repeat(30)}</p> + </body></html> + ` + } + ]; + + const graph = { getNodes: () => rawNodes }; - const graph = { - getNodes() { - return rawNodes; - } - }; + const getOrFetch = vi.fn(async (url, fetchFn) => fetchFn()); - const ctx: PluginContext = { flags: { heading: true } }; + const ctx: PluginContext = { + flags: { heading: true }, + db: { + data: { getOrFetch } + } as any + }; - if (HeadingHealthPlugin.hooks?.onMetrics) { - await HeadingHealthPlugin.hooks.onMetrics(ctx, graph); - } + await HeadingHealthPlugin.hooks?.onMetrics?.(ctx, graph); - const nodes = graph.getNodes(); - const pageA = (nodes.find((n: any) => n.url === 'https://example.com/a') as any)?.headingHealth; - expect(pageA).toBeDefined(); - expect(Array.isArray(pageA.map)).toBe(true); - expect(Array.isArray(pageA.issues)).toBe(true); - expect(pageA.hierarchy_skips).toBe(1); - expect(pageA.reverse_jumps).toBe(0); - expect(pageA.thin_sections).toBeGreaterThan(0); - expect(pageA.score).toBeLessThan(100); + const pageA = (rawNodes.find((node) => node.url === 'https://example.com/a') as any)?.headingHealth; + expect(pageA).toBeDefined(); + expect(Array.isArray(pageA.map)).toBe(true); + expect(Array.isArray(pageA.issues)).toBe(true); + expect(pageA.hierarchy_skips).toBe(1); + expect(pageA.reverse_jumps).toBe(0); + expect(pageA.thin_sections).toBeGreaterThan(0); + expect(pageA.score).toBeLessThan(100); - const summary = ctx.metadata?.headingHealthSummary; - expect(summary).toBeDefined(); - expect(summary.evaluatedPages).toBe(2); - expect(summary.totalMissing).toBe(0); - expect(summary.totalSkips).toBeGreaterThan(0); - }); + const summary = ctx.metadata?.headingHealthSummary; + expect(summary).toBeDefined(); + expect(summary.evaluatedPages).toBe(2); + expect(summary.totalMissing).toBe(0); + expect(summary.totalSkips).toBeGreaterThan(0); - it('attaches rich analysis payload on report output', async () => { - const ctx: PluginContext = { - flags: { heading: true }, - metadata: { - headingHealthSummary: { avgScore: 50 }, - headingHealth: { - 'https://example.com/no-h1': { - issues: ['Missing H1'], map: [{ level: 2, text: 'Section', index: 0 }], - hierarchy_skips: 0 - } - } - } - }; + expect(getOrFetch).toHaveBeenCalledTimes(2); + }); - const result: any = { - pages: [ - { - url: 'https://example.com/no-h1', - title: 'No Heading Page', - plugins: { - headingHealth: ctx.metadata.headingHealth['https://example.com/no-h1'] - } - } - ] - }; + it('attaches summary payload on report output', async () => { + const ctx: PluginContext = { + flags: { heading: true }, + metadata: { + headingHealthSummary: { avgScore: 50, evaluatedPages: 1, poorPages: 1 } + } + }; - if (HeadingHealthPlugin.hooks?.onReport) { - await HeadingHealthPlugin.hooks.onReport(ctx, result); - } + const result: any = { pages: [] }; - expect(result.plugins.headingHealth).toBeDefined(); - expect(result.plugins.headingHealth.avgScore).toBe(50); + await HeadingHealthPlugin.hooks?.onReport?.(ctx, result); - expect(result.pages[0].plugins.headingHealth).toBeDefined(); - expect(result.pages[0].plugins.headingHealth.issues).toContain('Missing H1'); - }); + expect(result.plugins.headingHealth).toBeDefined(); + expect(result.plugins.headingHealth.avgScore).toBe(50); + }); }); diff --git a/packages/plugins/heading-health/index.ts b/packages/plugins/heading-health/index.ts index f9bf78f..74183be 100644 --- a/packages/plugins/heading-health/index.ts +++ b/packages/plugins/heading-health/index.ts @@ -1,154 +1,43 @@ -import { Command, CrawlithPlugin, PluginContext } from '@crawlith/core'; -import { analyzeHeadingHealth, enrichDuplicateRisk, jaccardSimilarity } from './src/analyzer.js'; -import { LocalPageAnalysis } from './src/types.js'; - -function scoreHealth(input: { - metrics: LocalPageAnalysis['metrics']; - thinSectionCount: number; - duplicateH1GroupSize: number; - similarH1GroupSize: number; - identicalH2SetGroupSize: number; - duplicatePatternGroupSize: number; - templateRisk: number; - issues: string[]; -}) { - let score = 100; - const m = input.metrics; - if (m.missingH1) score -= 20; - if (m.multipleH1) score -= 6; - score -= m.hierarchySkips * 8; - score -= m.reverseJumps * 6; - score -= Math.round(m.entropy * 7); - score -= Math.round(m.fragmentation * 20); - score -= Math.round(m.levelVolatility * 6); - score -= input.thinSectionCount * 4; - if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3); - if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2); - if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2); - if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2); - score -= Math.round(input.templateRisk * 12); - score = Math.max(0, Math.min(100, score)); - return { score, status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor', issues: input.issues }; -} - -const computeTemplateRisk = (similar: number, h2set: number, pattern: number) => - Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3)); - +import type { CrawlithPlugin } from '@crawlith/core'; +import { HeadingHealthHooks } from './src/plugin.js'; + +/** + * Heading Health Plugin. + * + * Analyzes heading hierarchy quality, content thinness, and template duplication risk. + * Persists URL-scoped analysis records via ctx.db for 24h cache reuse. + */ export const HeadingHealthPlugin: CrawlithPlugin = { - name: 'heading-health', - description: 'Analyzes heading structure, hierarchy health, and content distribution', - register: (cli: Command) => { - if (cli.name() === 'crawl' || cli.name() === 'page') { - cli.option('--heading', 'Analyze heading structure and hierarchy health'); - } - }, - hooks: { - onMetrics: async (ctx: PluginContext, graph: any) => { - if (!ctx.flags?.heading) return; - const nodes = graph.getNodes(); - const analyzedPages: LocalPageAnalysis[] = []; - - for (const node of nodes) { - if (node.status < 200 || node.status >= 300 || !node.html) continue; - const analysis = analyzeHeadingHealth(node.html, node.title || (node as any).rawTitle); - analysis.url = node.url; - analyzedPages.push(analysis); - } - - enrichDuplicateRisk(analyzedPages); + name: 'heading-health', + description: 'Analyzes heading structure, hierarchy health, and content distribution', + + cli: { + flag: '--heading', + description: 'Analyze heading structure and hierarchy health', + for: ['page', 'crawl'], + options: [ + { flag: '--heading-force-refresh', description: 'Bypass the 24h heading-health cache and recompute' } + ] + }, - const exactH1Buckets = new Map<string, string[]>(); - const h2SetBuckets = new Map<string, string[]>(); - const patternBuckets = new Map<string, string[]>(); - for (const page of analyzedPages) { - if (page.h1Norm) { - const bucket = exactH1Buckets.get(page.h1Norm) || []; - bucket.push(page.url); - exactH1Buckets.set(page.h1Norm, bucket); - } - h2SetBuckets.set(page.h2SetHash, [...(h2SetBuckets.get(page.h2SetHash) || []), page.url]); - patternBuckets.set(page.patternHash, [...(patternBuckets.get(page.patternHash) || []), page.url]); - } + scoreProvider: true, - const similarH1GroupSizes = new Map<string, number>(); - const uniqueH1 = Array.from(new Set(analyzedPages.map(p => p.h1Norm).filter(Boolean))); - const similarBuckets = new Map<string, Set<string>>(); - for (const h1 of uniqueH1) similarBuckets.set(h1, new Set([h1])); + storage: { + fetchMode: 'local', + perPage: { - for (let i = 0; i < uniqueH1.length; i++) { - for (let j = i + 1; j < uniqueH1.length; j++) { - const a = uniqueH1[i]; - const b = uniqueH1[j]; - if (jaccardSimilarity(a, b) >= 0.7) { - similarBuckets.get(a)?.add(b); - similarBuckets.get(b)?.add(a); - } + columns: { + status: 'TEXT', + analysis_json: 'TEXT' + } } - } - - for (const page of analyzedPages) { - similarH1GroupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0)); - } - - let totalScore = 0, totalMissing = 0, totalMultiple = 0, totalSkips = 0, totalReverseJumps = 0, totalThinSections = 0, totalEntropy = 0, poorPages = 0; - - for (const node of nodes) { - const page = analyzedPages.find(p => p.url === node.url); - if (!page) continue; - - const duplicateH1GroupSize = page.h1Norm ? (exactH1Buckets.get(page.h1Norm)?.length || 1) : 0; - const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0; - const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1; - const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1; - const templateRisk = computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize); - const thinSectionCount = page.sections.filter(s => s.thin).length; - - const health = scoreHealth({ - metrics: page.metrics, - thinSectionCount, - duplicateH1GroupSize, - similarH1GroupSize, - identicalH2SetGroupSize, - duplicatePatternGroupSize, - templateRisk, - issues: page.issues - }); - - if (health.status === 'Poor') poorPages++; - totalScore += health.score; totalMissing += page.metrics.missingH1; totalMultiple += page.metrics.multipleH1; - totalSkips += page.metrics.hierarchySkips; totalReverseJumps += page.metrics.reverseJumps; totalThinSections += thinSectionCount; - totalEntropy += page.metrics.entropy; - - (node as any).headingHealth = { - score: health.score, status: health.status, issues: health.issues, map: page.headingNodes, - missing_h1: page.metrics.missingH1, multiple_h1: page.metrics.multipleH1, - entropy: page.metrics.entropy, max_depth: page.metrics.max_depth, avg_depth: page.metrics.avgDepth, - heading_density: page.metrics.headingDensity, fragmentation: page.metrics.fragmentation, - volatility: page.metrics.levelVolatility, hierarchy_skips: page.metrics.hierarchySkips, - reverse_jumps: page.metrics.reverseJumps, thin_sections: thinSectionCount, - duplicate_h1_group: duplicateH1GroupSize, similar_h1_group: similarH1GroupSize, - identical_h2_set_group: identicalH2SetGroupSize, duplicate_pattern_group: duplicatePatternGroupSize, - template_risk: templateRisk - }; - } - - const evaluatedPages = analyzedPages.length; - ctx.metadata = ctx.metadata || {}; - ctx.metadata.headingHealthSummary = { - avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0, - evaluatedPages, totalMissing, totalMultiple, totalSkips, totalReverseJumps, totalThinSections, - avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0, - poorPages - }; }, - onReport: async (ctx: PluginContext, result: any) => { - if (!ctx.flags?.heading) return; - if (ctx.metadata?.headingHealthSummary) { - if (!result.plugins) result.plugins = {}; - result.plugins.headingHealth = ctx.metadata.headingHealthSummary; - } + + hooks: { + onPage: HeadingHealthHooks.onPage, + onMetrics: HeadingHealthHooks.onMetrics, + onReport: HeadingHealthHooks.onReport } - } }; export default HeadingHealthPlugin; diff --git a/packages/plugins/heading-health/src/HeadingHealthService.ts b/packages/plugins/heading-health/src/HeadingHealthService.ts new file mode 100644 index 0000000..4c11812 --- /dev/null +++ b/packages/plugins/heading-health/src/HeadingHealthService.ts @@ -0,0 +1,245 @@ +import { analyzeHeadingHealth, enrichDuplicateRisk, jaccardSimilarity } from './analyzer.js'; +import type { HeadingHealthPayload, HeadingHealthSummary, LocalPageAnalysis } from './types.js'; + +/** + * Coordinates heading analysis across graph nodes and builds report-safe payloads. + */ +export class HeadingHealthService { + /** + * Builds page-level payloads plus a snapshot summary for every eligible node. + */ + public evaluateNodes(nodes: Array<Record<string, any>>): { + payloadsByUrl: Map<string, HeadingHealthPayload>; + summary: HeadingHealthSummary; + } { + const analyzedPages = this.collectAnalyses(nodes); + enrichDuplicateRisk(analyzedPages); + + const exactH1Buckets = this.buildBuckets(analyzedPages, (page) => page.h1Norm); + const h2SetBuckets = this.buildBuckets(analyzedPages, (page) => page.h2SetHash); + const patternBuckets = this.buildBuckets(analyzedPages, (page) => page.patternHash); + const similarH1GroupSizes = this.computeSimilarH1GroupSizes(analyzedPages); + + const payloadsByUrl = new Map<string, HeadingHealthPayload>(); + + let totalScore = 0; + let totalMissing = 0; + let totalMultiple = 0; + let totalSkips = 0; + let totalReverseJumps = 0; + let totalThinSections = 0; + let totalEntropy = 0; + let poorPages = 0; + + for (const page of analyzedPages) { + const duplicateH1GroupSize = page.h1Norm ? exactH1Buckets.get(page.h1Norm)?.length || 1 : 0; + const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0; + const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1; + const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1; + const templateRisk = computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize); + const thinSectionCount = page.sections.filter((section) => section.thin).length; + + const health = scoreHealth({ + metrics: page.metrics, + thinSectionCount, + duplicateH1GroupSize, + similarH1GroupSize, + identicalH2SetGroupSize, + duplicatePatternGroupSize, + templateRisk, + issues: page.issues + }); + + if (health.status === 'Poor') { + poorPages += 1; + } + + totalScore += health.score; + totalMissing += page.metrics.missingH1; + totalMultiple += page.metrics.multipleH1; + totalSkips += page.metrics.hierarchySkips; + totalReverseJumps += page.metrics.reverseJumps; + totalThinSections += thinSectionCount; + totalEntropy += page.metrics.entropy; + + payloadsByUrl.set(page.url, { + score: health.score, + status: health.status, + issues: health.issues, + map: page.headingNodes, + missing_h1: page.metrics.missingH1, + multiple_h1: page.metrics.multipleH1, + entropy: page.metrics.entropy, + max_depth: page.metrics.maxDepth, + avg_depth: page.metrics.avgDepth, + heading_density: page.metrics.headingDensity, + fragmentation: page.metrics.fragmentation, + volatility: page.metrics.levelVolatility, + hierarchy_skips: page.metrics.hierarchySkips, + reverse_jumps: page.metrics.reverseJumps, + thin_sections: thinSectionCount, + duplicate_h1_group: duplicateH1GroupSize, + similar_h1_group: similarH1GroupSize, + identical_h2_set_group: identicalH2SetGroupSize, + duplicate_pattern_group: duplicatePatternGroupSize, + template_risk: templateRisk + }); + } + + const evaluatedPages = analyzedPages.length; + const summary: HeadingHealthSummary = { + avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0, + evaluatedPages, + totalMissing, + totalMultiple, + totalSkips, + totalReverseJumps, + totalThinSections, + avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0, + poorPages + }; + + return { payloadsByUrl, summary }; + } + + /** + * Evaluates a single page's heading health directly from raw HTML. + * Used by the `onPage` hook (page command). Cross-page duplicate signals are omitted + * since there is only one page to compare against. + */ + public evaluateSinglePage(url: string, html: string): HeadingHealthPayload { + const analysis = analyzeHeadingHealth(html); + analysis.url = url; + + const thinSectionCount = analysis.sections.filter((section) => section.thin).length; + const health = scoreHealth({ + metrics: analysis.metrics, + thinSectionCount, + duplicateH1GroupSize: 1, + similarH1GroupSize: 0, + identicalH2SetGroupSize: 1, + duplicatePatternGroupSize: 1, + templateRisk: 0, + issues: analysis.issues + }); + + return { + score: health.score, + status: health.status, + issues: health.issues, + map: analysis.headingNodes, + missing_h1: analysis.metrics.missingH1, + multiple_h1: analysis.metrics.multipleH1, + entropy: analysis.metrics.entropy, + max_depth: analysis.metrics.maxDepth, + avg_depth: analysis.metrics.avgDepth, + heading_density: analysis.metrics.headingDensity, + fragmentation: analysis.metrics.fragmentation, + volatility: analysis.metrics.levelVolatility, + hierarchy_skips: analysis.metrics.hierarchySkips, + reverse_jumps: analysis.metrics.reverseJumps, + thin_sections: thinSectionCount, + duplicate_h1_group: 1, + similar_h1_group: 0, + identical_h2_set_group: 1, + duplicate_pattern_group: 1, + template_risk: 0 + }; + } + + private collectAnalyses(nodes: Array<Record<string, any>>): LocalPageAnalysis[] { + const analyses: LocalPageAnalysis[] = []; + + for (const node of nodes) { + if (node.status < 200 || node.status >= 300 || !node.html || !node.url) { + continue; + } + + const analysis = analyzeHeadingHealth(node.html, node.title || node.rawTitle); + analysis.url = node.url; + analyses.push(analysis); + } + + return analyses; + } + + private buildBuckets(pages: LocalPageAnalysis[], selector: (page: LocalPageAnalysis) => string): Map<string, string[]> { + const buckets = new Map<string, string[]>(); + for (const page of pages) { + const key = selector(page); + if (!key) { + continue; + } + buckets.set(key, [...(buckets.get(key) || []), page.url]); + } + return buckets; + } + + private computeSimilarH1GroupSizes(pages: LocalPageAnalysis[]): Map<string, number> { + const uniqueH1 = Array.from(new Set(pages.map((page) => page.h1Norm).filter(Boolean))); + const similarBuckets = new Map<string, Set<string>>(); + + for (const h1 of uniqueH1) { + similarBuckets.set(h1, new Set([h1])); + } + + for (let i = 0; i < uniqueH1.length; i += 1) { + for (let j = i + 1; j < uniqueH1.length; j += 1) { + const a = uniqueH1[i]; + const b = uniqueH1[j]; + if (jaccardSimilarity(a, b) >= 0.7) { + similarBuckets.get(a)?.add(b); + similarBuckets.get(b)?.add(a); + } + } + } + + const groupSizes = new Map<string, number>(); + for (const page of pages) { + groupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0)); + } + + return groupSizes; + } +} + +function scoreHealth(input: { + metrics: LocalPageAnalysis['metrics']; + thinSectionCount: number; + duplicateH1GroupSize: number; + similarH1GroupSize: number; + identicalH2SetGroupSize: number; + duplicatePatternGroupSize: number; + templateRisk: number; + issues: string[]; +}): { score: number; status: 'Healthy' | 'Moderate' | 'Poor'; issues: string[] } { + let score = 100; + const metrics = input.metrics; + + if (metrics.missingH1) score -= 20; + if (metrics.multipleH1) score -= 6; + score -= metrics.hierarchySkips * 8; + score -= metrics.reverseJumps * 6; + score -= Math.round(metrics.entropy * 7); + score -= Math.round(metrics.fragmentation * 20); + score -= Math.round(metrics.levelVolatility * 6); + score -= input.thinSectionCount * 4; + + if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3); + if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2); + if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2); + if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2); + score -= Math.round(input.templateRisk * 12); + + score = Math.max(0, Math.min(100, score)); + + return { + score, + status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor', + issues: input.issues + }; +} + +function computeTemplateRisk(similar: number, h2set: number, pattern: number): number { + return Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3)); +} diff --git a/packages/plugins/heading-health/src/Output.ts b/packages/plugins/heading-health/src/Output.ts new file mode 100644 index 0000000..b60707e --- /dev/null +++ b/packages/plugins/heading-health/src/Output.ts @@ -0,0 +1,26 @@ +import type { PluginContext } from '@crawlith/core'; +import type { HeadingHealthPayload, HeadingHealthSummary } from './types.js'; + +/** + * Responsible for plugin logging output. + */ +export class HeadingHealthOutput { + /** + * Emits a single-line result for a single-page heading-health analysis (page command). + */ + public static emitSingle(ctx: PluginContext, payload: HeadingHealthPayload): void { + ctx.logger?.info( + `[plugin:heading-health] score=${payload.score} (${payload.status})` + + (payload.issues.length ? ` β€” ${payload.issues.join(', ')}` : '') + ); + } + + /** + * Emits a single-line summary for heading-health metrics (crawl command). + */ + public static emitSummary(ctx: PluginContext, summary: HeadingHealthSummary): void { + ctx.logger?.info( + `[plugin:heading-health] analyzed=${summary.evaluatedPages}, avgScore=${summary.avgScore}, poorPages=${summary.poorPages}` + ); + } +} diff --git a/packages/plugins/heading-health/src/analyzer.ts b/packages/plugins/heading-health/src/analyzer.ts index 5bf235b..5535476 100644 --- a/packages/plugins/heading-health/src/analyzer.ts +++ b/packages/plugins/heading-health/src/analyzer.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { HeadingLevel, HeadingNode, SectionMetrics, LocalPageAnalysis } from './types.js'; +import type { HeadingLevel, HeadingNode, LocalPageAnalysis, SectionMetrics } from './types.js'; const STOPWORDS = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'your', 'about', 'into', 'over', 'under', 'are', 'was', 'were', 'can', 'has', 'have', 'had', 'you', 'our', 'out', 'all']); const THIN_SECTION_WORDS = 80; @@ -8,42 +8,55 @@ const TITLE_PATTERN = /<title\b[^<>]*>([\s\S]*?)<\/title>/i; const normalizeText = (input: string) => input.replace(/<[^<>]*>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/\s+/g, ' ').trim(); const normalizeComparable = (input: string) => normalizeText(input).toLowerCase(); -const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((t) => t.length > 2 && !STOPWORDS.has(t)); +const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((token) => token.length > 2 && !STOPWORDS.has(token)); const stableHash = (input: string) => createHash('sha1').update(input).digest('hex').slice(0, 16); +/** + * Calculates token-level Jaccard similarity between two text values. + */ export function jaccardSimilarity(a: string, b: string): number { const aSet = new Set(tokenize(a)); const bSet = new Set(tokenize(b)); - if (!aSet.size || !bSet.size) return 0; - let intersection = 0; - for (const token of aSet) if (bSet.has(token)) intersection++; - return intersection / (aSet.size + bSet.size - intersection); -} -function entropy(values: number[]): number { - const total = values.reduce((a, b) => a + b, 0); - if (!total) return 0; - return values.reduce((sum, value) => value === 0 ? sum : sum - (value / total) * Math.log2(value / total), 0); -} + if (!aSet.size || !bSet.size) { + return 0; + } -function getTitleFromHtml(html: string): string { - const match = html.match(TITLE_PATTERN); - return match ? normalizeText(match[1]) : ''; + let intersection = 0; + for (const token of aSet) { + if (bSet.has(token)) { + intersection += 1; + } + } + + return intersection / (aSet.size + bSet.size - intersection); } +/** + * Performs per-page heading extraction and structural scoring signal generation. + */ export function analyzeHeadingHealth(html: string, fallbackTitle?: string): LocalPageAnalysis { const segments: Array<{ level: HeadingLevel; text: string; start: number; end: number }> = []; for (const match of html.matchAll(HEADING_PATTERN)) { const level = Number(match[1]) as HeadingLevel; - segments.push({ level, text: normalizeText(match[2] || ''), start: match.index || 0, end: (match.index || 0) + match[0].length }); + segments.push({ + level, + text: normalizeText(match[2] || ''), + start: match.index || 0, + end: (match.index || 0) + match[0].length + }); } const headingNodes: HeadingNode[] = []; const stack: HeadingNode[] = []; segments.forEach((segment, index) => { const node: HeadingNode = { level: segment.level, text: segment.text, index }; - while (stack.length > 0 && stack[stack.length - 1].level >= node.level) stack.pop(); - if (stack.length > 0) node.parentIndex = stack[stack.length - 1].index; + while (stack.length > 0 && stack[stack.length - 1].level >= node.level) { + stack.pop(); + } + if (stack.length > 0) { + node.parentIndex = stack[stack.length - 1].index; + } stack.push(node); headingNodes.push(node); }); @@ -51,15 +64,18 @@ export function analyzeHeadingHealth(html: string, fallbackTitle?: string): Loca const sections: SectionMetrics[] = []; const pageWords = tokenize(html); const frequency = new Map<string, number>(); - for (const word of pageWords) frequency.set(word, (frequency.get(word) || 0) + 1); + for (const word of pageWords) { + frequency.set(word, (frequency.get(word) || 0) + 1); + } - for (let i = 0; i < segments.length; i++) { + for (let i = 0; i < segments.length; i += 1) { const current = segments[i]; const next = segments[i + 1]; const textChunk = html.slice(current.end, next ? next.start : html.length); const words = tokenize(textChunk); const headingTokens = tokenize(current.text); const concentration = headingTokens.reduce((sum, token) => sum + (frequency.get(token) || 0), 0); + sections.push({ headingIndex: headingNodes[i]?.index ?? i, headingText: current.text, @@ -71,57 +87,127 @@ export function analyzeHeadingHealth(html: string, fallbackTitle?: string): Loca } const levelCounts = [0, 0, 0, 0, 0, 0]; - headingNodes.forEach(n => levelCounts[n.level - 1]++); + headingNodes.forEach((node) => { + levelCounts[node.level - 1] += 1; + }); - const entropyScore = Number(entropy(levelCounts).toFixed(3)); + const entropyScore = Number(calculateEntropy(levelCounts).toFixed(3)); const missingH1 = levelCounts[0] === 0 ? 1 : 0; const multipleH1 = levelCounts[0] > 1 ? 1 : 0; - let hierarchySkips = 0; let reverseJumps = 0; let volatilitySum = 0; - for (let i = 1; i < headingNodes.length; i++) { + let hierarchySkips = 0; + let reverseJumps = 0; + let volatilitySum = 0; + for (let i = 1; i < headingNodes.length; i += 1) { const delta = headingNodes[i].level - headingNodes[i - 1].level; volatilitySum += Math.abs(delta); - if (delta > 1) hierarchySkips++; - if (delta < -1) reverseJumps++; + if (delta > 1) { + hierarchySkips += 1; + } + if (delta < -1) { + reverseJumps += 1; + } } - const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((n) => n.level)) : 0; + const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((node) => node.level)) : 0; const avgDepth = headingNodes.length ? Number((headingNodes.reduce((sum, node) => sum + node.level, 0) / headingNodes.length).toFixed(2)) : 0; const headingDensity = pageWords.length ? Number((headingNodes.length / pageWords.length).toFixed(4)) : 0; - const fragmentation = headingNodes.length ? Number((headingNodes.filter((n) => n.level <= 2).length / headingNodes.length).toFixed(3)) : 0; + const fragmentation = headingNodes.length ? Number((headingNodes.filter((node) => node.level <= 2).length / headingNodes.length).toFixed(3)) : 0; const levelVolatility = headingNodes.length > 1 ? Number((volatilitySum / (headingNodes.length - 1)).toFixed(3)) : 0; - const issues: string[] = []; const h1Nodes = headingNodes.filter((node) => node.level === 1); + const issues: string[] = []; if (missingH1) issues.push('Missing H1'); if (multipleH1) issues.push('Multiple H1 found'); if (h1Nodes.some((node) => node.text.length < 6)) issues.push('Empty or near-empty H1'); + const title = fallbackTitle || getTitleFromHtml(html); - if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) issues.push('H1 diverges from <title>'); + if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) { + issues.push('H1 diverges from <title>'); + } if (hierarchySkips > 0) issues.push(`${hierarchySkips} hierarchy skips detected`); if (reverseJumps > 0) issues.push(`${reverseJumps} reverse hierarchy jumps detected`); - for (const thin of sections.filter((s) => s.thin).slice(0, 2)) issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`); + for (const thin of sections.filter((section) => section.thin).slice(0, 2)) { + issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`); + } if (entropyScore > 2.1) issues.push('High structural entropy'); if (fragmentation > 0.65) issues.push('Section fragmentation is high'); const h1Norm = normalizeComparable(h1Nodes[0]?.text || ''); - const h2SetHash = stableHash(headingNodes.filter((n) => n.level === 2).map((n) => normalizeComparable(n.text)).filter(Boolean).sort().join('|')); - const patternHash = stableHash(headingNodes.map((n) => n.level).join('>')); - - return { url: '', headingNodes, sections, h1Norm, h2SetHash, patternHash, issues, metrics: { entropy: entropyScore, maxDepth, avgDepth, headingDensity, fragmentation, levelVolatility, hierarchySkips, reverseJumps, missingH1, multipleH1 } }; + const h2SetHash = stableHash( + headingNodes + .filter((node) => node.level === 2) + .map((node) => normalizeComparable(node.text)) + .filter(Boolean) + .sort() + .join('|') + ); + const patternHash = stableHash(headingNodes.map((node) => node.level).join('>')); + + return { + url: '', + headingNodes, + sections, + h1Norm, + h2SetHash, + patternHash, + issues, + metrics: { + entropy: entropyScore, + maxDepth, + avgDepth, + headingDensity, + fragmentation, + levelVolatility, + hierarchySkips, + reverseJumps, + missingH1, + multipleH1 + } + }; } +/** + * Enriches section-level duplicate risk by comparing normalized section signatures across pages. + */ export function enrichDuplicateRisk(pages: LocalPageAnalysis[]): void { const buckets = new Map<string, string[]>(); - for (const page of pages) for (const section of page.sections) { - const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); - const bucket = buckets.get(key) || []; - bucket.push(page.url); - buckets.set(key, bucket); + + for (const page of pages) { + for (const section of page.sections) { + const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); + const bucket = buckets.get(key) || []; + bucket.push(page.url); + buckets.set(key, bucket); + } + } + + for (const page of pages) { + for (const section of page.sections) { + const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); + const size = (buckets.get(key) || []).length; + section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3)); + } } - for (const page of pages) for (const section of page.sections) { - const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`); - const size = (buckets.get(key) || []).length; - section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3)); +} + +function calculateEntropy(values: number[]): number { + const total = values.reduce((a, b) => a + b, 0); + if (!total) { + return 0; } + + return values.reduce((sum, value) => { + if (value === 0) { + return sum; + } + + const probability = value / total; + return sum - probability * Math.log2(probability); + }, 0); +} + +function getTitleFromHtml(html: string): string { + const match = html.match(TITLE_PATTERN); + return match ? normalizeText(match[1]) : ''; } diff --git a/packages/plugins/heading-health/src/cli.ts b/packages/plugins/heading-health/src/cli.ts new file mode 100644 index 0000000..9e78d3b --- /dev/null +++ b/packages/plugins/heading-health/src/cli.ts @@ -0,0 +1,11 @@ +import type { Command } from '@crawlith/core'; + +/** + * Registers CLI options exposed by the heading-health plugin. + */ +export function registerHeadingHealthCli(cli: Command): void { + if (cli.name() === 'crawl' || cli.name() === 'page') { + cli.option('--heading', 'Analyze heading structure and hierarchy health'); + cli.option('--heading-force-refresh', 'Bypass the 24h heading-health cache and recompute'); + } +} diff --git a/packages/plugins/heading-health/src/plugin.ts b/packages/plugins/heading-health/src/plugin.ts new file mode 100644 index 0000000..26ec749 --- /dev/null +++ b/packages/plugins/heading-health/src/plugin.ts @@ -0,0 +1,97 @@ +import type { PluginContext, PageInput } from '@crawlith/core'; +import { HeadingHealthOutput } from './Output.js'; +import { HeadingHealthService } from './HeadingHealthService.js'; +import type { HeadingHealthPayload, HeadingHealthRow, HeadingHealthSummary } from './types.js'; + +/** + * Lifecycle hooks for the heading-health plugin. + * Schema registration is handled declaratively via plugin.storage β€” no onInit needed. + */ +export const HeadingHealthHooks = { + /** + * Single-page hook (page command). Analyzes one URL's HTML directly. + * No cross-page duplicate signals since only one page is available. + */ + onPage: async (ctx: PluginContext, page: PageInput): Promise<void> => { + if (!ctx.flags?.heading || !ctx.db) return; + + const service = new HeadingHealthService(); + + const row = await ctx.db.data.getOrFetch<HeadingHealthRow>( + page.url, + async () => { + const payload = service.evaluateSinglePage(page.url, page.html); + return { + score: payload.score, + status: payload.status, + // Store as string, but getOrFetch/find will auto-parse to object later + analysis_json: JSON.stringify(payload) + } as unknown as HeadingHealthRow; + } + ); + + if (!row) return; + + // analysis_json gets auto-parsed by CrawlithDB._parseRow + const payload = (typeof row.analysis_json === 'string' + ? JSON.parse(row.analysis_json) + : row.analysis_json) as HeadingHealthPayload; + + HeadingHealthOutput.emitSingle(ctx, payload); + }, + + /** + * Graph-level hook (crawl command). Evaluates all nodes with cross-page duplicate detection. + */ + onMetrics: async (ctx: PluginContext, graph: any): Promise<void> => { + if (!ctx.flags?.heading || !ctx.db) return; + + const nodes = graph?.getNodes?.(); + if (!Array.isArray(nodes)) return; + + const service = new HeadingHealthService(); + const { payloadsByUrl, summary } = service.evaluateNodes(nodes); + + for (const node of nodes) { + const url = node?.url; + if (!url) continue; + + const livePayload = payloadsByUrl.get(url); + if (!livePayload) continue; + + const row = await ctx.db.data.getOrFetch<HeadingHealthRow>( + url, + async () => ({ + score: livePayload.score, + status: livePayload.status, + analysis_json: JSON.stringify(livePayload) + } as unknown as HeadingHealthRow) + ); + + if (row) { + const payload = (typeof row.analysis_json === 'string' + ? JSON.parse(row.analysis_json) + : row.analysis_json) as HeadingHealthPayload; + + (node as any).headingHealth = payload; + } + } + + ctx.metadata = ctx.metadata || {}; + ctx.metadata.headingHealthSummary = summary; + HeadingHealthOutput.emitSummary(ctx, summary); + }, + + /** + * Attaches snapshot-level heading-health summary to the final report object. + */ + onReport: async (ctx: PluginContext, result: any): Promise<void> => { + if (!ctx.flags?.heading) return; + + const summary = ctx.metadata?.headingHealthSummary as HeadingHealthSummary | undefined; + if (!summary) return; + + result.plugins = result.plugins || {}; + result.plugins.headingHealth = summary; + } +}; diff --git a/packages/plugins/heading-health/src/types.ts b/packages/plugins/heading-health/src/types.ts index 088fa77..4ab8998 100644 --- a/packages/plugins/heading-health/src/types.ts +++ b/packages/plugins/heading-health/src/types.ts @@ -1,5 +1,11 @@ +/** + * Supported heading levels within HTML content. + */ export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6; +/** + * Represents a normalized heading node extracted from the DOM. + */ export interface HeadingNode { level: HeadingLevel; text: string; @@ -7,6 +13,9 @@ export interface HeadingNode { parentIndex?: number; } +/** + * Represents content statistics for a section under a heading. + */ export interface SectionMetrics { headingIndex: number; headingText: string; @@ -16,6 +25,9 @@ export interface SectionMetrics { duplicateRisk: number; } +/** + * Raw heading analysis generated for a single URL. + */ export interface LocalPageAnalysis { url: string; headingNodes: HeadingNode[]; @@ -37,3 +49,53 @@ export interface LocalPageAnalysis { multipleH1: number; }; } + +/** + * Final heading-health payload attached to a page node. + */ +export interface HeadingHealthPayload { + score: number; + status: 'Healthy' | 'Moderate' | 'Poor'; + issues: string[]; + map: HeadingNode[]; + missing_h1: number; + multiple_h1: number; + entropy: number; + max_depth: number; + avg_depth: number; + heading_density: number; + fragmentation: number; + volatility: number; + hierarchy_skips: number; + reverse_jumps: number; + thin_sections: number; + duplicate_h1_group: number; + similar_h1_group: number; + identical_h2_set_group: number; + duplicate_pattern_group: number; + template_risk: number; +} + +/** + * Snapshot-level summary emitted by the plugin. + */ +export interface HeadingHealthSummary { + avgScore: number; + evaluatedPages: number; + totalMissing: number; + totalMultiple: number; + totalSkips: number; + totalReverseJumps: number; + totalThinSections: number; + avgEntropy: number; + poorPages: number; +} + +/** + * Persisted row shape used by ctx.db.data.find/save. + */ +export interface HeadingHealthRow { + analysis_json: string; + score: number; + status: 'Healthy' | 'Moderate' | 'Poor'; +} diff --git a/packages/plugins/pagespeed/index.ts b/packages/plugins/pagespeed/index.ts index 1615cc1..298e9ce 100644 --- a/packages/plugins/pagespeed/index.ts +++ b/packages/plugins/pagespeed/index.ts @@ -4,7 +4,7 @@ import { PageSpeedHooks } from './src/plugin.js'; /** * PageSpeed Insights Plugin for Crawlith. - * + * * Provides automated performance auditing for scanned URLs using the Google PageSpeed Insights API. * Features: * - 24h Global Persistent Caching (via ctx.db) @@ -13,22 +13,37 @@ import { PageSpeedHooks } from './src/plugin.js'; export const PageSpeedPlugin: CrawlithPlugin = { name: 'pagespeed', - /** - * Registers CLI commands and options for the PageSpeed plugin. - * This acts as the visual manifest of what the plugin adds to the CLI. - */ - register: (cli) => { - registerConfigCommand(cli); + cli: { + flag: '--pagespeed', + description: 'Attach Google PageSpeed Insights report (mobile strategy)', + for: ['page'], + options: [ + { flag: '--force', description: 'Force a fresh PageSpeed API request and bypass 24h cache' } + ] + }, - if (cli.name() === 'page') { - cli.option('--pagespeed', 'Attach Google PageSpeed Insights report (mobile strategy)'); - cli.option('--force', 'Force a fresh PageSpeed API request and bypass 24h cache'); + scoreProvider: true, + + storage: { + perPage: { + columns: { + strategy: 'TEXT', + performance_score: 'INTEGER', + lcp: 'REAL', + cls: 'REAL', + tbt: 'REAL', + raw_json: 'TEXT' + } } }, /** - * Internal logic implementation is delegated to src/plugin.js + * Keeps the `pagespeed config` subcommand β€” not replaceable by declarative cli. */ + register: (cli) => { + registerConfigCommand(cli); + }, + hooks: PageSpeedHooks }; diff --git a/packages/plugins/pagespeed/src/plugin.ts b/packages/plugins/pagespeed/src/plugin.ts index 4485fa5..45164ba 100644 --- a/packages/plugins/pagespeed/src/plugin.ts +++ b/packages/plugins/pagespeed/src/plugin.ts @@ -1,87 +1,62 @@ import { PageSpeedService } from './PageSpeedService.js'; import { PageSpeedOutput } from './PageSpeedOutput.js'; import { PageSpeedRow } from './types.js'; -import type { PluginContext } from '@crawlith/core'; +import type { PluginContext, PageInput } from '@crawlith/core'; const STRATEGY = 'mobile'; /** * PageSpeed Plugin Hooks implementation. + * Schema registration is handled declaratively via plugin.storage β€” no onInit needed. */ export const PageSpeedHooks = { /** - * Initialization hook: Registers the persistent database schema for PageSpeed metrics. - */ - onInit: async (ctx: PluginContext) => { - if (!ctx.db) return; - - ctx.db.schema.define({ - strategy: "TEXT CHECK(strategy IN ('mobile', 'desktop')) NOT NULL", - performance_score: "INTEGER", - lcp: "REAL", - cls: "REAL", - tbt: "REAL", - raw_json: "TEXT NOT NULL" - }); - }, - - /** - * Report hook: Executes after a page analysis to perform performance auditing. + * Page hook: Executes for a single URL to perform performance auditing. * Implements smart caching and persists results to the database. */ - onReport: async (ctx: PluginContext, result: any) => { + onPage: async (ctx: PluginContext, page: PageInput) => { const flags = ctx.flags || {}; - if (ctx.command !== 'page' || !flags.pagespeed) return; + if (!flags.pagespeed) return; if (!ctx.db || !ctx.config) return; - result.plugins = result.plugins || {}; const service = new PageSpeedService(); try { - const pageUrl = ctx.targetUrl; + const pageUrl = page.url; if (!pageUrl || !URL.canParse(pageUrl)) { throw new Error('PageSpeed requires an absolute target URL.'); } const apiKey = ctx.config.require(); - // Smart Caching: Look globally across snapshots for this URL within last 24h - let raw: any = null; - if (!flags.force) { - const cached = ctx.db.data.find<PageSpeedRow>(pageUrl, { - maxAge: '24h', - global: true - }); - if (cached) raw = cached.raw_json; - } - - let summary; - - if (raw) { - summary = service.summarize(raw, 'cache', STRATEGY); - } else { - raw = await service.fetch(pageUrl, apiKey, STRATEGY); - summary = service.summarize(raw, 'api', STRATEGY); - - ctx.db.data.save({ - url: pageUrl, - data: { + const row = await ctx.db.data.getOrFetch<PageSpeedRow>( + pageUrl, + async () => { + const freshRaw = await service.fetch(pageUrl, apiKey, STRATEGY); + const freshSummary = service.summarize(freshRaw, 'api', STRATEGY); + return { strategy: STRATEGY, - performance_score: summary.score, - lcp: summary.lcp, - cls: summary.cls, - tbt: summary.tbt, - raw_json: raw - } - }); + performance_score: freshSummary.score, + lcp: freshSummary.lcp, + cls: freshSummary.cls, + tbt: freshSummary.tbt, + raw_json: freshRaw + } as PageSpeedRow; + } + ); + + if (!row) { + ctx.cli?.info(`[plugin:pagespeed] No local cache for ${pageUrl}. Run with --live to fetch from API.`); + return; } - // Only attach summary to the result object (prevents raw JSON leakage) - result.plugins.pagespeed = { summary }; + // Summarize the data that was retrieved (either cached or freshly fetched above) + const summary = service.summarize(row.raw_json, row.created_at ? 'cache' : 'api', STRATEGY); + PageSpeedOutput.emit(ctx, { summary }); + } catch (error) { const message = (error as Error).message; - result.plugins.pagespeed = { error: message }; PageSpeedOutput.emit(ctx, { error: message }); ctx.logger?.error(`[plugin:pagespeed] ${message}`); } diff --git a/packages/plugins/signals/index.ts b/packages/plugins/signals/index.ts index dc7b962..b086d39 100644 --- a/packages/plugins/signals/index.ts +++ b/packages/plugins/signals/index.ts @@ -208,6 +208,7 @@ const SignalsPlugin: CrawlithPlugin = { }); insertTx(buffer); + ctx.metadata = ctx.metadata ?? {}; ctx.metadata.signalsReport = computeSignalsReport(snapshotId); ctx.metadata.signalsBuffer = []; }, @@ -216,8 +217,8 @@ const SignalsPlugin: CrawlithPlugin = { const snapshotId = report?.snapshotId || ctx.snapshotId; let signalsReport = ctx.metadata?.signalsReport; - if (!signalsReport && ctx.metadata?.signalsBuffer?.length > 0) { - const buffer = ctx.metadata.signalsBuffer as BufferedSignal[]; + if (!signalsReport && (ctx.metadata?.signalsBuffer?.length ?? 0) > 0) { + const buffer = (ctx.metadata?.signalsBuffer ?? []) as BufferedSignal[]; const coverage = { og: buffer.every(s => s.hasOg) ? 100 : (buffer.some(s => s.hasOg) ? 50 : 0), jsonld: buffer.every(s => s.hasJsonld) ? 100 : (buffer.some(s => s.hasJsonld) ? 50 : 0), diff --git a/packages/plugins/soft404-detector/README.md b/packages/plugins/soft404-detector/README.md index 60b6e73..fa8c169 100644 --- a/packages/plugins/soft404-detector/README.md +++ b/packages/plugins/soft404-detector/README.md @@ -1,9 +1,63 @@ -# Soft404 Detector Plugin +# πŸ•΅οΈ Soft 404 Detector Plugin -Crawlith plugin for soft404 detector +The **Soft 404 Detector** plugin automatically analyzes site pages to detect "Soft 404" responsesβ€”pages that return a HTTP `200 OK` status code but conceptually behave like an error page (e.g., stating "Page Not Found", lacking content, or providing no outbound navigation). -## Installation -This plugin is built-in. +It scores pages between `0.0` and `1.0` and outputs detailed reasoning regarding *why* the page triggered an alert. As a registered **Score Provider** natively mapped into Crawlith's core, its analyses are aggregated continuously across the crawl footprint. + +## Features + +- **Text Pattern Matching:** Flags `<title>` tags and `<h1>` headers containing error phrases (e.g., `not found`, `error`, `unavailable`). +- **Body Context Extraction:** Detects explicit in-text error declarations (e.g., "page not found"). +- **Content Density Checking:** Severely low word counts (< 50 words) heavily increase the soft 404 probability. +- **Link Isolation:** Flags pages acting as dead-ends, lacking any outbound links entirely. +- **Smart Caching (`fetchMode: 'local'`):** Computes entirely on the local device, safely executing against previously downloaded `html` payloads, supporting instant execution during `page` and `crawl` snapshot pipelines without re-triggering remote requests unless bypassing cache thresholds (`--live`). ## Usage -Include it in your Crawlith configuration or CLI usage. + +Enable the plugin via the CLI using the `--detect-soft404` flag. + +### Command Line Interface + +```bash +# Analyze a single page locally for Soft 404 violations +crawlith page --detect-soft404 https://example.com/missing-product + +# Deep crawl a fully site, scanning all discovered URLs for Soft 404 structures +crawlith crawl --detect-soft404 https://example.com +``` + +## Output & Interpretation + +### Single Page (`page` command) +For specific URL lookups, the plugin outputs a simple terminal read-out on execution: +```plaintext +[Soft404] https://example.com/broken-page | Score: 1.0 | Reason: title_contains_not found, h1_contains_404, very_low_word_count +``` + +### Full Site Crawl JSON Output (`crawl` command) +During extensive crawls, nodes heavily penalized map into the JSON reporter object (`--format json` or `--export json`): + +```json +{ + "plugins": { + "soft404": { + "totalDetected": 3, + "topSample": [ + { + "url": "https://example.com/missing", + "score": 0.8 + } + ] + } + } +} +``` + +The plugin also explicitly injects `soft404Score` directly onto graph `Node` objects, enabling graph algorithms to dynamically down-weight links originating from dead-end error patterns. + +## Architecture Guidelines + +This plugin strictly implements the [AI Agent Guide's](../../../docs/plugins/ai-agent-guide.md) architectural standards: +- All business logic lives inside a standalone `Soft404Service` object (`src/Service.ts`). +- Storage leverages the `ctx.db.data.getOrFetch` fluent API. +- Implements `scoreProvider: true` natively inside its manifest for top-level aggregation logic. diff --git a/packages/plugins/soft404-detector/index.ts b/packages/plugins/soft404-detector/index.ts index f03fadc..9776790 100644 --- a/packages/plugins/soft404-detector/index.ts +++ b/packages/plugins/soft404-detector/index.ts @@ -1,109 +1,30 @@ - -import { CrawlithPlugin, PluginContext } from '@crawlith/core'; -import * as cheerio from 'cheerio'; -import { Command } from '@crawlith/core'; +import { CrawlithPlugin } from '@crawlith/core'; +import { Soft404Hooks } from './src/plugin.js'; /** - * Soft404 Detector Plugin - * Crawlith plugin for soft404 detector + * Description: Soft404 Detector Plugin calculates the likelihood of a Soft 404 response. + * @requirements None. */ export const Soft404DetectorPlugin: CrawlithPlugin = { name: 'soft404-detector', - register: (cli: Command) => { - if (cli.name() === 'crawl' || cli.name() === 'page') { - cli.option('--detect-soft404', 'Detect soft 404 pages'); - } - }, - - hooks: { - onMetrics: async (ctx: PluginContext, graph: any) => { - const flags = ctx.flags || {}; - - if (!flags.detectSoft404) { - return; - } - - ctx.logger?.info?.('πŸ•΅οΈ Detecting soft 404 pages...'); - - const nodes = graph.getNodes(); - - for (const node of nodes) { - if (node.status === 200 && node.html) { - let score = 0; - const signals: string[] = []; - - const $ = cheerio.load(node.html); - $('script, style, noscript, iframe').remove(); - - const cleanText = $('body').text().replace(/\s+/g, ' ').trim(); - const title = $('title').text().toLowerCase(); - const h1Text = $('h1').first().text().toLowerCase(); - const bodyText = cleanText.toLowerCase(); - const errorPatterns = ['404', 'not found', 'error', "doesn't exist", 'unavailable', 'invalid']; - - for (const pattern of errorPatterns) { - if (title.includes(pattern)) { - score += 0.4; - signals.push(`title_contains_${pattern}`); - break; - } - } - - for (const pattern of errorPatterns) { - if (h1Text.includes(pattern)) { - score += 0.3; - signals.push(`h1_contains_${pattern}`); - break; - } - } - - if (bodyText.includes('page not found') || bodyText.includes('404 error')) { - score += 0.2; - signals.push('body_error_phrase'); - } - - const words = cleanText.split(/\s+/).filter(w => w.length > 0); - if (words.length < 50) { - score += 0.3; - signals.push('very_low_word_count'); - } else if (words.length < 150) { - score += 0.1; - signals.push('low_word_count'); - } - - if (node.outLinks === 0) { - score += 0.2; - signals.push('no_outbound_links'); - } - - score = Math.min(1.0, score); - - if (score > 0) { - (node as any).soft404 = { - score: Number(score.toFixed(2)), - reason: signals.join(', ') - }; - } - } - } + cli: { + flag: '--detect-soft404', + description: 'Detect soft 404 pages' + }, - ctx.logger?.info?.(`πŸ•΅οΈ Soft 404 detection complete.`); - }, - onReport: async (ctx: PluginContext, result: any) => { - const flags = ctx.flags || {}; - if (!flags.detectSoft404) return; + scoreProvider: true, - const soft404Pages = result.pages.filter((p: any) => p.plugins?.soft404?.score > 0.5); - if (soft404Pages.length > 0) { - if (!result.plugins) result.plugins = {}; - result.plugins.soft404 = { - totalDetected: soft404Pages.length, - topSample: soft404Pages.slice(0, 5).map((p: any) => ({ url: p.url, score: p.plugins.soft404.score })) - }; + storage: { + fetchMode: 'local', + perPage: { + columns: { + reason: 'TEXT' } } - } + }, + + hooks: Soft404Hooks }; export default Soft404DetectorPlugin; diff --git a/packages/plugins/soft404-detector/src/Output.ts b/packages/plugins/soft404-detector/src/Output.ts new file mode 100644 index 0000000..9fab8b3 --- /dev/null +++ b/packages/plugins/soft404-detector/src/Output.ts @@ -0,0 +1,23 @@ +import { PluginContext } from '@crawlith/core'; +import { Soft404Result } from './types.js'; + +/** + * Handles presentation and standardized logging for the Soft404 plugin. + */ +export class Soft404Output { + /** + * Emits logging output for a single evaluated page. + * @param {PluginContext} ctx - Plugin context for logging access. + * @param {string} url - Target URL. + * @param {Soft404Result} result - The evaluation payload. + */ + public static emitSingle(ctx: PluginContext, url: string, result: Soft404Result): void { + if (!ctx.logger) return; + + if (result.score === 0) { + ctx.logger.info(`[Soft404] ${url} | Score: 0 (No issues detected)`); + } else { + ctx.logger.warn(`[Soft404] ${url} | Score: ${result.score} | Reason: ${result.reason}`); + } + } +} diff --git a/packages/plugins/soft404-detector/src/Service.ts b/packages/plugins/soft404-detector/src/Service.ts new file mode 100644 index 0000000..4c2e8ee --- /dev/null +++ b/packages/plugins/soft404-detector/src/Service.ts @@ -0,0 +1,73 @@ +import * as cheerio from 'cheerio'; +import { Soft404Result } from './types.js'; + +/** + * Service to analyze HTML content for soft 404 signals. + * Extracts signals from title, H1, body content length, and outlinks. + */ +export class Soft404Service { + /** + * Analyzes HTML string to determine probability of being a soft 404 page. + * @param {string | undefined} html - Raw HTML source code. + * @param {number} outLinks - Total number of outbound links extracted during parsing. + * @returns {Soft404Result} A calculated score between 0.0 and 1.0, and the matched reasons. + */ + public analyze(html: string | undefined, outLinks: number): Soft404Result { + if (!html) return { score: 0, reason: '' }; + + let score = 0; + const signals: string[] = []; + + const $ = cheerio.load(html); + $('script, style, noscript, iframe').remove(); + + const cleanText = $('body').text().replace(/\s+/g, ' ').trim(); + const title = $('title').text().toLowerCase(); + const h1Text = $('h1').first().text().toLowerCase(); + const bodyText = cleanText.toLowerCase(); + + const errorPatterns = ['404', 'not found', 'error', "doesn't exist", 'unavailable', 'invalid']; + + for (const pattern of errorPatterns) { + if (title.includes(pattern)) { + score += 0.4; + signals.push(`title_contains_${pattern}`); + break; + } + } + + for (const pattern of errorPatterns) { + if (h1Text.includes(pattern)) { + score += 0.3; + signals.push(`h1_contains_${pattern}`); + break; + } + } + + if (bodyText.includes('page not found') || bodyText.includes('404 error')) { + score += 0.2; + signals.push('body_error_phrase'); + } + + const words = cleanText.split(/\s+/).filter(w => w.length > 0); + if (words.length < 50) { + score += 0.3; + signals.push('very_low_word_count'); + } else if (words.length < 150) { + score += 0.1; + signals.push('low_word_count'); + } + + if (outLinks === 0) { + score += 0.2; + signals.push('no_outbound_links'); + } + + score = Math.min(1.0, score); + + return { + score: Number(score.toFixed(2)), + reason: signals.join(', ') + }; + } +} diff --git a/packages/plugins/soft404-detector/src/plugin.ts b/packages/plugins/soft404-detector/src/plugin.ts new file mode 100644 index 0000000..d0e3014 --- /dev/null +++ b/packages/plugins/soft404-detector/src/plugin.ts @@ -0,0 +1,92 @@ +import { PluginContext, PageInput } from '@crawlith/core'; +import { Soft404Service } from './Service.js'; +import { Soft404Row } from './types.js'; + +export const Soft404Hooks = { + /** + * Called during `crawlith page` command to analyze the single page immediately. + */ + onPage: async (ctx: PluginContext, page: PageInput) => { + if (!ctx.flags?.detectSoft404 || !ctx.db) return; + + const service = new Soft404Service(); + // Cache the soft404 result natively inside SQLite via getOrFetch. + const row = await ctx.db.data.getOrFetch<Soft404Row>( + page.url, + async () => service.analyze(page.html, 0 /* onPage command doesn't provide fully built links graph yet */) + ); + + if (row && ctx.logger) { + if (row.score > 0) { + ctx.logger.warn(`[soft404-detector] Score for ${page.url}: ${row.score} (${row.reason})`); + } else { + ctx.logger.info(`[soft404-detector] Score for ${page.url}: 0 (No issues detected)`); + } + } + }, + + /** + * Called on the fully built graph when a `crawl` evaluates metrics across the snapshot. + */ + onMetrics: async (ctx: PluginContext, graph: any) => { + const flags = ctx.flags || {}; + if (!flags.detectSoft404 || !ctx.db) return; + + ctx.logger?.info?.('πŸ•΅οΈ Detecting soft 404 pages...'); + + const service = new Soft404Service(); + const nodes = graph.getNodes(); + + let issueCount = 0; + + for (const node of nodes) { + if (node.status === 200) { + // Caches internally inside 'soft404_detector_plugin' SQLite table. + const row = await ctx.db.data.getOrFetch<Soft404Row>( + node.url, + async () => service.analyze(node.html, node.outLinks || 0) + ); + + if (row && row.score > 0) { + (node as any).soft404 = { + score: row.score, + reason: row.reason + }; + // Map back to node so pagerank/crawlers can read it downstream. + node.soft404Score = row.score; + issueCount++; + } + } + } + + if (issueCount > 0) { + ctx.logger?.warn(`πŸ•΅οΈ Soft 404 detection complete. Found ${issueCount} issues!`); + } else { + ctx.logger?.info(`πŸ•΅οΈ Soft 404 detection complete.`); + } + }, + + /** + * Evaluates the active snapshot results and attaches them statically mapped onto the final JSON reporter. + */ + onReport: async (ctx: PluginContext, result: any) => { + const flags = ctx.flags || {}; + if (!flags.detectSoft404 || !ctx.db) return; + + // Pull directly from our scoped SQLite plugin table + const allRows = ctx.db.data.all<{ url_id: number; score: number; reason: string }>(); + const soft404Pages = allRows.filter(r => r.score > 0.5); + + if (soft404Pages.length > 0) { + if (!result.plugins) result.plugins = {}; + + const mappedPages = (result.pages || []).filter((p: any) => p.soft404Score && p.soft404Score > 0.5) + .map((p: any) => ({ url: p.url, score: p.soft404Score })); + + result.plugins.soft404 = { + totalDetected: mappedPages.length > 0 ? mappedPages.length : soft404Pages.length, + topSample: mappedPages.slice(0, 5) + }; + } + } +}; diff --git a/packages/plugins/soft404-detector/src/types.ts b/packages/plugins/soft404-detector/src/types.ts new file mode 100644 index 0000000..0ece4fe --- /dev/null +++ b/packages/plugins/soft404-detector/src/types.ts @@ -0,0 +1,9 @@ +export interface Soft404Result { + score: number; + reason: string; +} + +export interface Soft404Row { + score: number; + reason: string; +} diff --git a/packages/plugins/soft404-detector/tests/soft404.test.ts b/packages/plugins/soft404-detector/tests/soft404.test.ts index 3fb9833..fb2adc9 100644 --- a/packages/plugins/soft404-detector/tests/soft404.test.ts +++ b/packages/plugins/soft404-detector/tests/soft404.test.ts @@ -12,7 +12,18 @@ describe('Soft 404 Detection Plugin', () => { node.html = html; node.outLinks = outLinks; - await Soft404DetectorPlugin.hooks!.onMetrics!({ flags: { detectSoft404: true } } as any, graph); + const ctx: any = { + flags: { detectSoft404: true }, + db: { + data: { + getOrFetch: async (url: string, fetchFn: Function) => { + return await fetchFn(); + } + } + } + }; + + await Soft404DetectorPlugin.hooks!.onMetrics!(ctx, graph); return node as any; };