From 76741455a8388c17185b70b75e6228e410690a05 Mon Sep 17 00:00:00 2001
From: Saurabh <41580629+saurabhsharma2u@users.noreply.github.com>
Date: Tue, 3 Mar 2026 09:12:29 -0800
Subject: [PATCH 1/3] Redesign heading-health plugin architecture and hooks
---
packages/plugins/heading-health/index.test.ts | 160 +++++++-------
packages/plugins/heading-health/index.ts | 164 ++------------
.../src/HeadingHealthService.ts | 200 ++++++++++++++++++
packages/plugins/heading-health/src/Output.ts | 16 ++
.../plugins/heading-health/src/analyzer.ts | 174 +++++++++++----
packages/plugins/heading-health/src/cli.ts | 11 +
packages/plugins/heading-health/src/plugin.ts | 111 ++++++++++
packages/plugins/heading-health/src/types.ts | 62 ++++++
8 files changed, 617 insertions(+), 281 deletions(-)
create mode 100644 packages/plugins/heading-health/src/HeadingHealthService.ts
create mode 100644 packages/plugins/heading-health/src/Output.ts
create mode 100644 packages/plugins/heading-health/src/cli.ts
create mode 100644 packages/plugins/heading-health/src/plugin.ts
diff --git a/packages/plugins/heading-health/index.test.ts b/packages/plugins/heading-health/index.test.ts
index a06a975..6bccdd0 100644
--- a/packages/plugins/heading-health/index.test.ts
+++ b/packages/plugins/heading-health/index.test.ts
@@ -1,102 +1,86 @@
-import { describe, expect, it } from 'vitest';
-import HeadingHealthPlugin from './index.js';
+import { describe, expect, it, vi } from 'vitest';
import type { PluginContext } from '@crawlith/core';
+import HeadingHealthPlugin from './index.js';
describe('heading-health plugin', () => {
- it('computes detailed per-page heading map data and summary metrics', async () => {
- const rawNodes = [
- {
- url: 'https://example.com/a',
- status: 200,
- title: 'Product A Overview',
- html: `
-
- Product A
- ${'intro '.repeat(50)}
- Features
- ${'feature '.repeat(20)}
- FAQ
- ${'faq '.repeat(90)}
-
- `
- },
- {
- url: 'https://example.com/b',
- status: 200,
- title: 'Product A Details',
- html: `
-
- Product A
- Specs
- ${'spec '.repeat(30)}
-
- `
- }
- ];
+ it('computes per-page heading payload and summary metrics', async () => {
+ const rawNodes = [
+ {
+ url: 'https://example.com/a',
+ status: 200,
+ title: 'Product A Overview',
+ html: `
+
+ Product A
+ ${'intro '.repeat(50)}
+ Features
+ ${'feature '.repeat(20)}
+ FAQ
+ ${'faq '.repeat(90)}
+
+ `
+ },
+ {
+ url: 'https://example.com/b',
+ status: 200,
+ title: 'Product A Details',
+ html: `
+
+ Product A
+ Specs
+ ${'spec '.repeat(30)}
+
+ `
+ }
+ ];
+
+ const graph = { getNodes: () => rawNodes };
- const graph = {
- getNodes() {
- return rawNodes;
- }
- };
+ const save = vi.fn();
+ const find = vi.fn(() => null);
- const ctx: PluginContext = { flags: { heading: true } };
+ const ctx: PluginContext = {
+ flags: { heading: true },
+ db: {
+ schema: { define: vi.fn() },
+ data: { save, find }
+ } as any
+ };
- if (HeadingHealthPlugin.hooks?.onMetrics) {
- await HeadingHealthPlugin.hooks.onMetrics(ctx, graph);
- }
+ await HeadingHealthPlugin.hooks?.onMetrics?.(ctx, graph);
- const nodes = graph.getNodes();
- const pageA = (nodes.find((n: any) => n.url === 'https://example.com/a') as any)?.headingHealth;
- expect(pageA).toBeDefined();
- expect(Array.isArray(pageA.map)).toBe(true);
- expect(Array.isArray(pageA.issues)).toBe(true);
- expect(pageA.hierarchy_skips).toBe(1);
- expect(pageA.reverse_jumps).toBe(0);
- expect(pageA.thin_sections).toBeGreaterThan(0);
- expect(pageA.score).toBeLessThan(100);
+ const pageA = (rawNodes.find((node) => node.url === 'https://example.com/a') as any)?.headingHealth;
+ expect(pageA).toBeDefined();
+ expect(Array.isArray(pageA.map)).toBe(true);
+ expect(Array.isArray(pageA.issues)).toBe(true);
+ expect(pageA.hierarchy_skips).toBe(1);
+ expect(pageA.reverse_jumps).toBe(0);
+ expect(pageA.thin_sections).toBeGreaterThan(0);
+ expect(pageA.score).toBeLessThan(100);
- const summary = ctx.metadata?.headingHealthSummary;
- expect(summary).toBeDefined();
- expect(summary.evaluatedPages).toBe(2);
- expect(summary.totalMissing).toBe(0);
- expect(summary.totalSkips).toBeGreaterThan(0);
- });
+ const summary = ctx.metadata?.headingHealthSummary;
+ expect(summary).toBeDefined();
+ expect(summary.evaluatedPages).toBe(2);
+ expect(summary.totalMissing).toBe(0);
+ expect(summary.totalSkips).toBeGreaterThan(0);
- it('attaches rich analysis payload on report output', async () => {
- const ctx: PluginContext = {
- flags: { heading: true },
- metadata: {
- headingHealthSummary: { avgScore: 50 },
- headingHealth: {
- 'https://example.com/no-h1': {
- issues: ['Missing H1'], map: [{ level: 2, text: 'Section', index: 0 }],
- hierarchy_skips: 0
- }
- }
- }
- };
+ expect(find).toHaveBeenCalledTimes(2);
+ expect(save).toHaveBeenCalledTimes(2);
+ });
- const result: any = {
- pages: [
- {
- url: 'https://example.com/no-h1',
- title: 'No Heading Page',
- plugins: {
- headingHealth: ctx.metadata.headingHealth['https://example.com/no-h1']
- }
- }
- ]
- };
+ it('attaches summary payload on report output', async () => {
+ const ctx: PluginContext = {
+ flags: { heading: true },
+ metadata: {
+ headingHealthSummary: { avgScore: 50, evaluatedPages: 1, poorPages: 1 }
+ }
+ };
- if (HeadingHealthPlugin.hooks?.onReport) {
- await HeadingHealthPlugin.hooks.onReport(ctx, result);
- }
+ const result: any = { pages: [] };
- expect(result.plugins.headingHealth).toBeDefined();
- expect(result.plugins.headingHealth.avgScore).toBe(50);
+ await HeadingHealthPlugin.hooks?.onReport?.(ctx, result);
- expect(result.pages[0].plugins.headingHealth).toBeDefined();
- expect(result.pages[0].plugins.headingHealth.issues).toContain('Missing H1');
- });
+ expect(result.plugins.headingHealth).toBeDefined();
+ expect(result.plugins.headingHealth.avgScore).toBe(50);
+ });
});
diff --git a/packages/plugins/heading-health/index.ts b/packages/plugins/heading-health/index.ts
index f9bf78f..f99f355 100644
--- a/packages/plugins/heading-health/index.ts
+++ b/packages/plugins/heading-health/index.ts
@@ -1,154 +1,20 @@
-import { Command, CrawlithPlugin, PluginContext } from '@crawlith/core';
-import { analyzeHeadingHealth, enrichDuplicateRisk, jaccardSimilarity } from './src/analyzer.js';
-import { LocalPageAnalysis } from './src/types.js';
-
-function scoreHealth(input: {
- metrics: LocalPageAnalysis['metrics'];
- thinSectionCount: number;
- duplicateH1GroupSize: number;
- similarH1GroupSize: number;
- identicalH2SetGroupSize: number;
- duplicatePatternGroupSize: number;
- templateRisk: number;
- issues: string[];
-}) {
- let score = 100;
- const m = input.metrics;
- if (m.missingH1) score -= 20;
- if (m.multipleH1) score -= 6;
- score -= m.hierarchySkips * 8;
- score -= m.reverseJumps * 6;
- score -= Math.round(m.entropy * 7);
- score -= Math.round(m.fragmentation * 20);
- score -= Math.round(m.levelVolatility * 6);
- score -= input.thinSectionCount * 4;
- if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3);
- if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2);
- if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2);
- if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2);
- score -= Math.round(input.templateRisk * 12);
- score = Math.max(0, Math.min(100, score));
- return { score, status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor', issues: input.issues };
-}
-
-const computeTemplateRisk = (similar: number, h2set: number, pattern: number) =>
- Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3));
-
+import type { CrawlithPlugin } from '@crawlith/core';
+import { registerHeadingHealthCli } from './src/cli.js';
+import { HeadingHealthHooks } from './src/plugin.js';
+
+/**
+ * Heading Health Plugin.
+ *
+ * Analyzes heading hierarchy quality, content thinness, and template duplication risk.
+ * Persists URL-scoped analysis records via ctx.db for 24h cache reuse.
+ */
export const HeadingHealthPlugin: CrawlithPlugin = {
- name: 'heading-health',
- description: 'Analyzes heading structure, hierarchy health, and content distribution',
- register: (cli: Command) => {
- if (cli.name() === 'crawl' || cli.name() === 'page') {
- cli.option('--heading', 'Analyze heading structure and hierarchy health');
- }
- },
- hooks: {
- onMetrics: async (ctx: PluginContext, graph: any) => {
- if (!ctx.flags?.heading) return;
- const nodes = graph.getNodes();
- const analyzedPages: LocalPageAnalysis[] = [];
-
- for (const node of nodes) {
- if (node.status < 200 || node.status >= 300 || !node.html) continue;
- const analysis = analyzeHeadingHealth(node.html, node.title || (node as any).rawTitle);
- analysis.url = node.url;
- analyzedPages.push(analysis);
- }
-
- enrichDuplicateRisk(analyzedPages);
-
- const exactH1Buckets = new Map();
- const h2SetBuckets = new Map();
- const patternBuckets = new Map();
- for (const page of analyzedPages) {
- if (page.h1Norm) {
- const bucket = exactH1Buckets.get(page.h1Norm) || [];
- bucket.push(page.url);
- exactH1Buckets.set(page.h1Norm, bucket);
- }
- h2SetBuckets.set(page.h2SetHash, [...(h2SetBuckets.get(page.h2SetHash) || []), page.url]);
- patternBuckets.set(page.patternHash, [...(patternBuckets.get(page.patternHash) || []), page.url]);
- }
-
- const similarH1GroupSizes = new Map();
- const uniqueH1 = Array.from(new Set(analyzedPages.map(p => p.h1Norm).filter(Boolean)));
- const similarBuckets = new Map>();
- for (const h1 of uniqueH1) similarBuckets.set(h1, new Set([h1]));
-
- for (let i = 0; i < uniqueH1.length; i++) {
- for (let j = i + 1; j < uniqueH1.length; j++) {
- const a = uniqueH1[i];
- const b = uniqueH1[j];
- if (jaccardSimilarity(a, b) >= 0.7) {
- similarBuckets.get(a)?.add(b);
- similarBuckets.get(b)?.add(a);
- }
- }
- }
-
- for (const page of analyzedPages) {
- similarH1GroupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0));
- }
-
- let totalScore = 0, totalMissing = 0, totalMultiple = 0, totalSkips = 0, totalReverseJumps = 0, totalThinSections = 0, totalEntropy = 0, poorPages = 0;
-
- for (const node of nodes) {
- const page = analyzedPages.find(p => p.url === node.url);
- if (!page) continue;
-
- const duplicateH1GroupSize = page.h1Norm ? (exactH1Buckets.get(page.h1Norm)?.length || 1) : 0;
- const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0;
- const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1;
- const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1;
- const templateRisk = computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize);
- const thinSectionCount = page.sections.filter(s => s.thin).length;
-
- const health = scoreHealth({
- metrics: page.metrics,
- thinSectionCount,
- duplicateH1GroupSize,
- similarH1GroupSize,
- identicalH2SetGroupSize,
- duplicatePatternGroupSize,
- templateRisk,
- issues: page.issues
- });
-
- if (health.status === 'Poor') poorPages++;
- totalScore += health.score; totalMissing += page.metrics.missingH1; totalMultiple += page.metrics.multipleH1;
- totalSkips += page.metrics.hierarchySkips; totalReverseJumps += page.metrics.reverseJumps; totalThinSections += thinSectionCount;
- totalEntropy += page.metrics.entropy;
-
- (node as any).headingHealth = {
- score: health.score, status: health.status, issues: health.issues, map: page.headingNodes,
- missing_h1: page.metrics.missingH1, multiple_h1: page.metrics.multipleH1,
- entropy: page.metrics.entropy, max_depth: page.metrics.max_depth, avg_depth: page.metrics.avgDepth,
- heading_density: page.metrics.headingDensity, fragmentation: page.metrics.fragmentation,
- volatility: page.metrics.levelVolatility, hierarchy_skips: page.metrics.hierarchySkips,
- reverse_jumps: page.metrics.reverseJumps, thin_sections: thinSectionCount,
- duplicate_h1_group: duplicateH1GroupSize, similar_h1_group: similarH1GroupSize,
- identical_h2_set_group: identicalH2SetGroupSize, duplicate_pattern_group: duplicatePatternGroupSize,
- template_risk: templateRisk
- };
- }
-
- const evaluatedPages = analyzedPages.length;
- ctx.metadata = ctx.metadata || {};
- ctx.metadata.headingHealthSummary = {
- avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0,
- evaluatedPages, totalMissing, totalMultiple, totalSkips, totalReverseJumps, totalThinSections,
- avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0,
- poorPages
- };
+ name: 'heading-health',
+ description: 'Analyzes heading structure, hierarchy health, and content distribution',
+ register: (cli) => {
+ registerHeadingHealthCli(cli);
},
- onReport: async (ctx: PluginContext, result: any) => {
- if (!ctx.flags?.heading) return;
- if (ctx.metadata?.headingHealthSummary) {
- if (!result.plugins) result.plugins = {};
- result.plugins.headingHealth = ctx.metadata.headingHealthSummary;
- }
- }
- }
+ hooks: HeadingHealthHooks
};
export default HeadingHealthPlugin;
diff --git a/packages/plugins/heading-health/src/HeadingHealthService.ts b/packages/plugins/heading-health/src/HeadingHealthService.ts
new file mode 100644
index 0000000..dd54f80
--- /dev/null
+++ b/packages/plugins/heading-health/src/HeadingHealthService.ts
@@ -0,0 +1,200 @@
+import { analyzeHeadingHealth, enrichDuplicateRisk, jaccardSimilarity } from './analyzer.js';
+import type { HeadingHealthPayload, HeadingHealthSummary, LocalPageAnalysis } from './types.js';
+
+/**
+ * Coordinates heading analysis across graph nodes and builds report-safe payloads.
+ */
+export class HeadingHealthService {
+ /**
+ * Builds page-level payloads plus a snapshot summary for every eligible node.
+ */
+ public evaluateNodes(nodes: Array>): {
+ payloadsByUrl: Map;
+ summary: HeadingHealthSummary;
+ } {
+ const analyzedPages = this.collectAnalyses(nodes);
+ enrichDuplicateRisk(analyzedPages);
+
+ const exactH1Buckets = this.buildBuckets(analyzedPages, (page) => page.h1Norm);
+ const h2SetBuckets = this.buildBuckets(analyzedPages, (page) => page.h2SetHash);
+ const patternBuckets = this.buildBuckets(analyzedPages, (page) => page.patternHash);
+ const similarH1GroupSizes = this.computeSimilarH1GroupSizes(analyzedPages);
+
+ const payloadsByUrl = new Map();
+
+ let totalScore = 0;
+ let totalMissing = 0;
+ let totalMultiple = 0;
+ let totalSkips = 0;
+ let totalReverseJumps = 0;
+ let totalThinSections = 0;
+ let totalEntropy = 0;
+ let poorPages = 0;
+
+ for (const page of analyzedPages) {
+ const duplicateH1GroupSize = page.h1Norm ? exactH1Buckets.get(page.h1Norm)?.length || 1 : 0;
+ const similarH1GroupSize = similarH1GroupSizes.get(page.url) || 0;
+ const identicalH2SetGroupSize = h2SetBuckets.get(page.h2SetHash)?.length || 1;
+ const duplicatePatternGroupSize = patternBuckets.get(page.patternHash)?.length || 1;
+ const templateRisk = computeTemplateRisk(similarH1GroupSize, identicalH2SetGroupSize, duplicatePatternGroupSize);
+ const thinSectionCount = page.sections.filter((section) => section.thin).length;
+
+ const health = scoreHealth({
+ metrics: page.metrics,
+ thinSectionCount,
+ duplicateH1GroupSize,
+ similarH1GroupSize,
+ identicalH2SetGroupSize,
+ duplicatePatternGroupSize,
+ templateRisk,
+ issues: page.issues
+ });
+
+ if (health.status === 'Poor') {
+ poorPages += 1;
+ }
+
+ totalScore += health.score;
+ totalMissing += page.metrics.missingH1;
+ totalMultiple += page.metrics.multipleH1;
+ totalSkips += page.metrics.hierarchySkips;
+ totalReverseJumps += page.metrics.reverseJumps;
+ totalThinSections += thinSectionCount;
+ totalEntropy += page.metrics.entropy;
+
+ payloadsByUrl.set(page.url, {
+ score: health.score,
+ status: health.status,
+ issues: health.issues,
+ map: page.headingNodes,
+ missing_h1: page.metrics.missingH1,
+ multiple_h1: page.metrics.multipleH1,
+ entropy: page.metrics.entropy,
+ max_depth: page.metrics.maxDepth,
+ avg_depth: page.metrics.avgDepth,
+ heading_density: page.metrics.headingDensity,
+ fragmentation: page.metrics.fragmentation,
+ volatility: page.metrics.levelVolatility,
+ hierarchy_skips: page.metrics.hierarchySkips,
+ reverse_jumps: page.metrics.reverseJumps,
+ thin_sections: thinSectionCount,
+ duplicate_h1_group: duplicateH1GroupSize,
+ similar_h1_group: similarH1GroupSize,
+ identical_h2_set_group: identicalH2SetGroupSize,
+ duplicate_pattern_group: duplicatePatternGroupSize,
+ template_risk: templateRisk
+ });
+ }
+
+ const evaluatedPages = analyzedPages.length;
+ const summary: HeadingHealthSummary = {
+ avgScore: evaluatedPages ? Math.round(totalScore / evaluatedPages) : 0,
+ evaluatedPages,
+ totalMissing,
+ totalMultiple,
+ totalSkips,
+ totalReverseJumps,
+ totalThinSections,
+ avgEntropy: evaluatedPages ? Number((totalEntropy / evaluatedPages).toFixed(3)) : 0,
+ poorPages
+ };
+
+ return { payloadsByUrl, summary };
+ }
+
+ private collectAnalyses(nodes: Array>): LocalPageAnalysis[] {
+ const analyses: LocalPageAnalysis[] = [];
+
+ for (const node of nodes) {
+ if (node.status < 200 || node.status >= 300 || !node.html || !node.url) {
+ continue;
+ }
+
+ const analysis = analyzeHeadingHealth(node.html, node.title || node.rawTitle);
+ analysis.url = node.url;
+ analyses.push(analysis);
+ }
+
+ return analyses;
+ }
+
+ private buildBuckets(pages: LocalPageAnalysis[], selector: (page: LocalPageAnalysis) => string): Map {
+ const buckets = new Map();
+ for (const page of pages) {
+ const key = selector(page);
+ if (!key) {
+ continue;
+ }
+ buckets.set(key, [...(buckets.get(key) || []), page.url]);
+ }
+ return buckets;
+ }
+
+ private computeSimilarH1GroupSizes(pages: LocalPageAnalysis[]): Map {
+ const uniqueH1 = Array.from(new Set(pages.map((page) => page.h1Norm).filter(Boolean)));
+ const similarBuckets = new Map>();
+
+ for (const h1 of uniqueH1) {
+ similarBuckets.set(h1, new Set([h1]));
+ }
+
+ for (let i = 0; i < uniqueH1.length; i += 1) {
+ for (let j = i + 1; j < uniqueH1.length; j += 1) {
+ const a = uniqueH1[i];
+ const b = uniqueH1[j];
+ if (jaccardSimilarity(a, b) >= 0.7) {
+ similarBuckets.get(a)?.add(b);
+ similarBuckets.get(b)?.add(a);
+ }
+ }
+ }
+
+ const groupSizes = new Map();
+ for (const page of pages) {
+ groupSizes.set(page.url, similarBuckets.get(page.h1Norm)?.size || (page.h1Norm ? 1 : 0));
+ }
+
+ return groupSizes;
+ }
+}
+
+function scoreHealth(input: {
+ metrics: LocalPageAnalysis['metrics'];
+ thinSectionCount: number;
+ duplicateH1GroupSize: number;
+ similarH1GroupSize: number;
+ identicalH2SetGroupSize: number;
+ duplicatePatternGroupSize: number;
+ templateRisk: number;
+ issues: string[];
+}): { score: number; status: 'Healthy' | 'Moderate' | 'Poor'; issues: string[] } {
+ let score = 100;
+ const metrics = input.metrics;
+
+ if (metrics.missingH1) score -= 20;
+ if (metrics.multipleH1) score -= 6;
+ score -= metrics.hierarchySkips * 8;
+ score -= metrics.reverseJumps * 6;
+ score -= Math.round(metrics.entropy * 7);
+ score -= Math.round(metrics.fragmentation * 20);
+ score -= Math.round(metrics.levelVolatility * 6);
+ score -= input.thinSectionCount * 4;
+
+ if (input.duplicateH1GroupSize > 1) score -= Math.min(16, (input.duplicateH1GroupSize - 1) * 3);
+ if (input.similarH1GroupSize > 1) score -= Math.min(8, (input.similarH1GroupSize - 1) * 2);
+ if (input.identicalH2SetGroupSize > 1) score -= Math.min(10, (input.identicalH2SetGroupSize - 1) * 2);
+ if (input.duplicatePatternGroupSize > 1) score -= Math.min(12, (input.duplicatePatternGroupSize - 1) * 2);
+ score -= Math.round(input.templateRisk * 12);
+
+ score = Math.max(0, Math.min(100, score));
+
+ return {
+ score,
+ status: score >= 80 ? 'Healthy' : score >= 55 ? 'Moderate' : 'Poor',
+ issues: input.issues
+ };
+}
+
+function computeTemplateRisk(similar: number, h2set: number, pattern: number): number {
+ return Number(Math.max(0, Math.min(1, ((similar - 1) * 0.15) + ((h2set - 1) * 0.2) + ((pattern - 1) * 0.2))).toFixed(3));
+}
diff --git a/packages/plugins/heading-health/src/Output.ts b/packages/plugins/heading-health/src/Output.ts
new file mode 100644
index 0000000..c8cdfbe
--- /dev/null
+++ b/packages/plugins/heading-health/src/Output.ts
@@ -0,0 +1,16 @@
+import type { PluginContext } from '@crawlith/core';
+import type { HeadingHealthSummary } from './types.js';
+
+/**
+ * Responsible for plugin logging output.
+ */
+export class HeadingHealthOutput {
+ /**
+ * Emits a single-line summary for heading-health metrics.
+ */
+ public static emitSummary(ctx: PluginContext, summary: HeadingHealthSummary): void {
+ ctx.logger?.info(
+ `[plugin:heading-health] analyzed=${summary.evaluatedPages}, avgScore=${summary.avgScore}, poorPages=${summary.poorPages}`
+ );
+ }
+}
diff --git a/packages/plugins/heading-health/src/analyzer.ts b/packages/plugins/heading-health/src/analyzer.ts
index 5bf235b..5535476 100644
--- a/packages/plugins/heading-health/src/analyzer.ts
+++ b/packages/plugins/heading-health/src/analyzer.ts
@@ -1,5 +1,5 @@
import { createHash } from 'node:crypto';
-import { HeadingLevel, HeadingNode, SectionMetrics, LocalPageAnalysis } from './types.js';
+import type { HeadingLevel, HeadingNode, LocalPageAnalysis, SectionMetrics } from './types.js';
const STOPWORDS = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'your', 'about', 'into', 'over', 'under', 'are', 'was', 'were', 'can', 'has', 'have', 'had', 'you', 'our', 'out', 'all']);
const THIN_SECTION_WORDS = 80;
@@ -8,42 +8,55 @@ const TITLE_PATTERN = /]*>([\s\S]*?)<\/title>/i;
const normalizeText = (input: string) => input.replace(/<[^<>]*>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/\s+/g, ' ').trim();
const normalizeComparable = (input: string) => normalizeText(input).toLowerCase();
-const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((t) => t.length > 2 && !STOPWORDS.has(t));
+const tokenize = (input: string) => normalizeComparable(input).replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((token) => token.length > 2 && !STOPWORDS.has(token));
const stableHash = (input: string) => createHash('sha1').update(input).digest('hex').slice(0, 16);
+/**
+ * Calculates token-level Jaccard similarity between two text values.
+ */
export function jaccardSimilarity(a: string, b: string): number {
const aSet = new Set(tokenize(a));
const bSet = new Set(tokenize(b));
- if (!aSet.size || !bSet.size) return 0;
- let intersection = 0;
- for (const token of aSet) if (bSet.has(token)) intersection++;
- return intersection / (aSet.size + bSet.size - intersection);
-}
-function entropy(values: number[]): number {
- const total = values.reduce((a, b) => a + b, 0);
- if (!total) return 0;
- return values.reduce((sum, value) => value === 0 ? sum : sum - (value / total) * Math.log2(value / total), 0);
-}
+ if (!aSet.size || !bSet.size) {
+ return 0;
+ }
-function getTitleFromHtml(html: string): string {
- const match = html.match(TITLE_PATTERN);
- return match ? normalizeText(match[1]) : '';
+ let intersection = 0;
+ for (const token of aSet) {
+ if (bSet.has(token)) {
+ intersection += 1;
+ }
+ }
+
+ return intersection / (aSet.size + bSet.size - intersection);
}
+/**
+ * Performs per-page heading extraction and structural scoring signal generation.
+ */
export function analyzeHeadingHealth(html: string, fallbackTitle?: string): LocalPageAnalysis {
const segments: Array<{ level: HeadingLevel; text: string; start: number; end: number }> = [];
for (const match of html.matchAll(HEADING_PATTERN)) {
const level = Number(match[1]) as HeadingLevel;
- segments.push({ level, text: normalizeText(match[2] || ''), start: match.index || 0, end: (match.index || 0) + match[0].length });
+ segments.push({
+ level,
+ text: normalizeText(match[2] || ''),
+ start: match.index || 0,
+ end: (match.index || 0) + match[0].length
+ });
}
const headingNodes: HeadingNode[] = [];
const stack: HeadingNode[] = [];
segments.forEach((segment, index) => {
const node: HeadingNode = { level: segment.level, text: segment.text, index };
- while (stack.length > 0 && stack[stack.length - 1].level >= node.level) stack.pop();
- if (stack.length > 0) node.parentIndex = stack[stack.length - 1].index;
+ while (stack.length > 0 && stack[stack.length - 1].level >= node.level) {
+ stack.pop();
+ }
+ if (stack.length > 0) {
+ node.parentIndex = stack[stack.length - 1].index;
+ }
stack.push(node);
headingNodes.push(node);
});
@@ -51,15 +64,18 @@ export function analyzeHeadingHealth(html: string, fallbackTitle?: string): Loca
const sections: SectionMetrics[] = [];
const pageWords = tokenize(html);
const frequency = new Map();
- for (const word of pageWords) frequency.set(word, (frequency.get(word) || 0) + 1);
+ for (const word of pageWords) {
+ frequency.set(word, (frequency.get(word) || 0) + 1);
+ }
- for (let i = 0; i < segments.length; i++) {
+ for (let i = 0; i < segments.length; i += 1) {
const current = segments[i];
const next = segments[i + 1];
const textChunk = html.slice(current.end, next ? next.start : html.length);
const words = tokenize(textChunk);
const headingTokens = tokenize(current.text);
const concentration = headingTokens.reduce((sum, token) => sum + (frequency.get(token) || 0), 0);
+
sections.push({
headingIndex: headingNodes[i]?.index ?? i,
headingText: current.text,
@@ -71,57 +87,127 @@ export function analyzeHeadingHealth(html: string, fallbackTitle?: string): Loca
}
const levelCounts = [0, 0, 0, 0, 0, 0];
- headingNodes.forEach(n => levelCounts[n.level - 1]++);
+ headingNodes.forEach((node) => {
+ levelCounts[node.level - 1] += 1;
+ });
- const entropyScore = Number(entropy(levelCounts).toFixed(3));
+ const entropyScore = Number(calculateEntropy(levelCounts).toFixed(3));
const missingH1 = levelCounts[0] === 0 ? 1 : 0;
const multipleH1 = levelCounts[0] > 1 ? 1 : 0;
- let hierarchySkips = 0; let reverseJumps = 0; let volatilitySum = 0;
- for (let i = 1; i < headingNodes.length; i++) {
+ let hierarchySkips = 0;
+ let reverseJumps = 0;
+ let volatilitySum = 0;
+ for (let i = 1; i < headingNodes.length; i += 1) {
const delta = headingNodes[i].level - headingNodes[i - 1].level;
volatilitySum += Math.abs(delta);
- if (delta > 1) hierarchySkips++;
- if (delta < -1) reverseJumps++;
+ if (delta > 1) {
+ hierarchySkips += 1;
+ }
+ if (delta < -1) {
+ reverseJumps += 1;
+ }
}
- const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((n) => n.level)) : 0;
+ const maxDepth = headingNodes.length ? Math.max(...headingNodes.map((node) => node.level)) : 0;
const avgDepth = headingNodes.length ? Number((headingNodes.reduce((sum, node) => sum + node.level, 0) / headingNodes.length).toFixed(2)) : 0;
const headingDensity = pageWords.length ? Number((headingNodes.length / pageWords.length).toFixed(4)) : 0;
- const fragmentation = headingNodes.length ? Number((headingNodes.filter((n) => n.level <= 2).length / headingNodes.length).toFixed(3)) : 0;
+ const fragmentation = headingNodes.length ? Number((headingNodes.filter((node) => node.level <= 2).length / headingNodes.length).toFixed(3)) : 0;
const levelVolatility = headingNodes.length > 1 ? Number((volatilitySum / (headingNodes.length - 1)).toFixed(3)) : 0;
- const issues: string[] = [];
const h1Nodes = headingNodes.filter((node) => node.level === 1);
+ const issues: string[] = [];
if (missingH1) issues.push('Missing H1');
if (multipleH1) issues.push('Multiple H1 found');
if (h1Nodes.some((node) => node.text.length < 6)) issues.push('Empty or near-empty H1');
+
const title = fallbackTitle || getTitleFromHtml(html);
- if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) issues.push('H1 diverges from ');
+ if (title && h1Nodes[0] && jaccardSimilarity(title, h1Nodes[0].text) < 0.3) {
+ issues.push('H1 diverges from <title>');
+ }
if (hierarchySkips > 0) issues.push(`${hierarchySkips} hierarchy skips detected`);
if (reverseJumps > 0) issues.push(`${reverseJumps} reverse hierarchy jumps detected`);
- for (const thin of sections.filter((s) => s.thin).slice(0, 2)) issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`);
+ for (const thin of sections.filter((section) => section.thin).slice(0, 2)) {
+ issues.push(`Thin section under "${thin.headingText || 'Untitled heading'}"`);
+ }
if (entropyScore > 2.1) issues.push('High structural entropy');
if (fragmentation > 0.65) issues.push('Section fragmentation is high');
const h1Norm = normalizeComparable(h1Nodes[0]?.text || '');
- const h2SetHash = stableHash(headingNodes.filter((n) => n.level === 2).map((n) => normalizeComparable(n.text)).filter(Boolean).sort().join('|'));
- const patternHash = stableHash(headingNodes.map((n) => n.level).join('>'));
-
- return { url: '', headingNodes, sections, h1Norm, h2SetHash, patternHash, issues, metrics: { entropy: entropyScore, maxDepth, avgDepth, headingDensity, fragmentation, levelVolatility, hierarchySkips, reverseJumps, missingH1, multipleH1 } };
+ const h2SetHash = stableHash(
+ headingNodes
+ .filter((node) => node.level === 2)
+ .map((node) => normalizeComparable(node.text))
+ .filter(Boolean)
+ .sort()
+ .join('|')
+ );
+ const patternHash = stableHash(headingNodes.map((node) => node.level).join('>'));
+
+ return {
+ url: '',
+ headingNodes,
+ sections,
+ h1Norm,
+ h2SetHash,
+ patternHash,
+ issues,
+ metrics: {
+ entropy: entropyScore,
+ maxDepth,
+ avgDepth,
+ headingDensity,
+ fragmentation,
+ levelVolatility,
+ hierarchySkips,
+ reverseJumps,
+ missingH1,
+ multipleH1
+ }
+ };
}
+/**
+ * Enriches section-level duplicate risk by comparing normalized section signatures across pages.
+ */
export function enrichDuplicateRisk(pages: LocalPageAnalysis[]): void {
const buckets = new Map<string, string[]>();
- for (const page of pages) for (const section of page.sections) {
- const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`);
- const bucket = buckets.get(key) || [];
- bucket.push(page.url);
- buckets.set(key, bucket);
+
+ for (const page of pages) {
+ for (const section of page.sections) {
+ const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`);
+ const bucket = buckets.get(key) || [];
+ bucket.push(page.url);
+ buckets.set(key, bucket);
+ }
+ }
+
+ for (const page of pages) {
+ for (const section of page.sections) {
+ const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`);
+ const size = (buckets.get(key) || []).length;
+ section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3));
+ }
}
- for (const page of pages) for (const section of page.sections) {
- const key = stableHash(`${normalizeComparable(section.headingText)}:${section.words}`);
- const size = (buckets.get(key) || []).length;
- section.duplicateRisk = Number(Math.min(1, (size - 1) / 5).toFixed(3));
+}
+
+function calculateEntropy(values: number[]): number {
+ const total = values.reduce((a, b) => a + b, 0);
+ if (!total) {
+ return 0;
}
+
+ return values.reduce((sum, value) => {
+ if (value === 0) {
+ return sum;
+ }
+
+ const probability = value / total;
+ return sum - probability * Math.log2(probability);
+ }, 0);
+}
+
+function getTitleFromHtml(html: string): string {
+ const match = html.match(TITLE_PATTERN);
+ return match ? normalizeText(match[1]) : '';
}
diff --git a/packages/plugins/heading-health/src/cli.ts b/packages/plugins/heading-health/src/cli.ts
new file mode 100644
index 0000000..9e78d3b
--- /dev/null
+++ b/packages/plugins/heading-health/src/cli.ts
@@ -0,0 +1,11 @@
+import type { Command } from '@crawlith/core';
+
+/**
+ * Registers CLI options exposed by the heading-health plugin.
+ */
+export function registerHeadingHealthCli(cli: Command): void {
+ if (cli.name() === 'crawl' || cli.name() === 'page') {
+ cli.option('--heading', 'Analyze heading structure and hierarchy health');
+ cli.option('--heading-force-refresh', 'Bypass the 24h heading-health cache and recompute');
+ }
+}
diff --git a/packages/plugins/heading-health/src/plugin.ts b/packages/plugins/heading-health/src/plugin.ts
new file mode 100644
index 0000000..cfc491b
--- /dev/null
+++ b/packages/plugins/heading-health/src/plugin.ts
@@ -0,0 +1,111 @@
+import type { PluginContext } from '@crawlith/core';
+import { HeadingHealthOutput } from './Output.js';
+import { HeadingHealthService } from './HeadingHealthService.js';
+import type { HeadingHealthPayload, HeadingHealthRow, HeadingHealthSummary } from './types.js';
+
+/**
+ * Lifecycle hooks for the heading-health plugin.
+ */
+export const HeadingHealthHooks = {
+ /**
+ * Registers schema columns used for URL-scoped heading-health persistence.
+ */
+ onInit: async (ctx: PluginContext): Promise<void> => {
+ if (!ctx.db) {
+ return;
+ }
+
+ ctx.db.schema.define({
+ score: 'INTEGER NOT NULL',
+ status: "TEXT CHECK(status IN ('Healthy', 'Moderate', 'Poor')) NOT NULL",
+ analysis_json: 'TEXT NOT NULL'
+ });
+ },
+
+ /**
+ * Evaluates heading health for each eligible graph node, persisting URL-scoped data with 24h cache reads.
+ */
+ onMetrics: async (ctx: PluginContext, graph: any): Promise<void> => {
+ if (!ctx.flags?.heading) {
+ return;
+ }
+
+ const nodes = graph?.getNodes?.();
+ if (!Array.isArray(nodes)) {
+ return;
+ }
+
+ const service = new HeadingHealthService();
+ const { payloadsByUrl, summary } = service.evaluateNodes(nodes);
+
+ for (const node of nodes) {
+ const url = node?.url;
+ if (!url) {
+ continue;
+ }
+
+ const livePayload = payloadsByUrl.get(url);
+ if (!livePayload) {
+ continue;
+ }
+
+ const payload = readCachedPayload(ctx, url, livePayload);
+ (node as any).headingHealth = payload;
+ persistPayload(ctx, url, payload);
+ }
+
+ ctx.metadata = ctx.metadata || {};
+ ctx.metadata.headingHealthSummary = summary;
+ HeadingHealthOutput.emitSummary(ctx, summary);
+ },
+
+ /**
+ * Attaches snapshot-level heading-health summary to the final report object.
+ */
+ onReport: async (ctx: PluginContext, result: any): Promise<void> => {
+ if (!ctx.flags?.heading) {
+ return;
+ }
+
+ const summary = ctx.metadata?.headingHealthSummary as HeadingHealthSummary | undefined;
+ if (!summary) {
+ return;
+ }
+
+ result.plugins = result.plugins || {};
+ result.plugins.headingHealth = summary;
+ }
+};
+
+function readCachedPayload(ctx: PluginContext, url: string, fallback: HeadingHealthPayload): HeadingHealthPayload {
+ if (!ctx.db || ctx.flags?.headingForceRefresh) {
+ return fallback;
+ }
+
+ const cached = ctx.db.data.find<HeadingHealthRow>(url, { maxAge: '24h', global: true });
+ if (!cached?.analysis_json) {
+ return fallback;
+ }
+
+ try {
+ return JSON.parse(cached.analysis_json) as HeadingHealthPayload;
+ } catch {
+ ctx.logger?.warn(`[plugin:heading-health] invalid cached JSON for ${url}, recomputing`);
+ return fallback;
+ }
+}
+
+function persistPayload(ctx: PluginContext, url: string, payload: HeadingHealthPayload): void {
+ if (!ctx.db) {
+ return;
+ }
+
+ ctx.db.data.save({
+ url,
+ data: {
+ score: payload.score,
+ status: payload.status,
+ analysis_json: JSON.stringify(payload)
+ }
+ });
+}
diff --git a/packages/plugins/heading-health/src/types.ts b/packages/plugins/heading-health/src/types.ts
index 088fa77..4ab8998 100644
--- a/packages/plugins/heading-health/src/types.ts
+++ b/packages/plugins/heading-health/src/types.ts
@@ -1,5 +1,11 @@
+/**
+ * Supported heading levels within HTML content.
+ */
export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6;
+/**
+ * Represents a normalized heading node extracted from the DOM.
+ */
export interface HeadingNode {
level: HeadingLevel;
text: string;
@@ -7,6 +13,9 @@ export interface HeadingNode {
parentIndex?: number;
}
+/**
+ * Represents content statistics for a section under a heading.
+ */
export interface SectionMetrics {
headingIndex: number;
headingText: string;
@@ -16,6 +25,9 @@ export interface SectionMetrics {
duplicateRisk: number;
}
+/**
+ * Raw heading analysis generated for a single URL.
+ */
export interface LocalPageAnalysis {
url: string;
headingNodes: HeadingNode[];
@@ -37,3 +49,53 @@ export interface LocalPageAnalysis {
multipleH1: number;
};
}
+
+/**
+ * Final heading-health payload attached to a page node.
+ */
+export interface HeadingHealthPayload {
+ score: number;
+ status: 'Healthy' | 'Moderate' | 'Poor';
+ issues: string[];
+ map: HeadingNode[];
+ missing_h1: number;
+ multiple_h1: number;
+ entropy: number;
+ max_depth: number;
+ avg_depth: number;
+ heading_density: number;
+ fragmentation: number;
+ volatility: number;
+ hierarchy_skips: number;
+ reverse_jumps: number;
+ thin_sections: number;
+ duplicate_h1_group: number;
+ similar_h1_group: number;
+ identical_h2_set_group: number;
+ duplicate_pattern_group: number;
+ template_risk: number;
+}
+
+/**
+ * Snapshot-level summary emitted by the plugin.
+ */
+export interface HeadingHealthSummary {
+ avgScore: number;
+ evaluatedPages: number;
+ totalMissing: number;
+ totalMultiple: number;
+ totalSkips: number;
+ totalReverseJumps: number;
+ totalThinSections: number;
+ avgEntropy: number;
+ poorPages: number;
+}
+
+/**
+ * Persisted row shape used by ctx.db.data.find/save.
+ */
+export interface HeadingHealthRow {
+ analysis_json: string;
+ score: number;
+ status: 'Healthy' | 'Moderate' | 'Poor';
+}
From 57c9a5f51b281008e1418e537eb138487fbdbc34 Mon Sep 17 00:00:00 2001
From: Saurabh <saurabhsharma2u@gmail.com>
Date: Tue, 3 Mar 2026 10:52:31 -0800
Subject: [PATCH 2/3] feat(core): implement ScoreProvider architecture with
automatic aggregation
---
packages/core/src/analysis/analyze.ts | 40 +---
packages/core/src/application/usecases.ts | 50 ++++-
packages/core/src/db/CrawlithDB.ts | 182 ++++++++++++++++--
packages/core/src/db/migrations.ts | 8 +
packages/core/src/db/statements.ts | 39 ++--
.../core/src/plugin-system/plugin-registry.ts | 107 +++++++++-
.../core/src/plugin-system/plugin-types.ts | 173 ++++++++++++++++-
packages/plugins/heading-health/README.md | 129 ++++++++++++-
packages/plugins/heading-health/index.test.ts | 9 +-
packages/plugins/heading-health/index.ts | 31 ++-
.../src/HeadingHealthService.ts | 45 +++++
packages/plugins/heading-health/src/Output.ts | 14 +-
packages/plugins/heading-health/src/plugin.ts | 118 +++++-------
packages/plugins/pagespeed/index.ts | 37 ++--
packages/plugins/pagespeed/src/plugin.ts | 81 +++-----
packages/plugins/signals/index.ts | 5 +-
16 files changed, 820 insertions(+), 248 deletions(-)
diff --git a/packages/core/src/analysis/analyze.ts b/packages/core/src/analysis/analyze.ts
index d4c83d1..476d37c 100644
--- a/packages/core/src/analysis/analyze.ts
+++ b/packages/core/src/analysis/analyze.ts
@@ -215,48 +215,10 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context?
crawledAt
};
- if (options.plugins && options.plugins.length > 0) {
- const { PluginRegistry } = await import('../plugin-system/plugin-registry.js');
- const { getCrawlithDB } = await import('../db/index.js');
- const registry = new PluginRegistry(options.plugins);
- const pluginCtx = options.pluginContext || { command: 'page' };
- if (!pluginCtx.db) pluginCtx.db = getCrawlithDB();
-
- await registry.runHook('onInit', pluginCtx);
- await registry.runHook('onMetrics', pluginCtx, crawlData.graph);
-
- // Map graph node plugin data back to the results
- for (const page of result.pages) {
- const node = crawlData.graph.nodes.get(page.url);
- if (node) {
- const extra: Record<string, any> = {};
- // Common node properties to exclude
- const internalKeys = [
- 'url', 'status', 'html', 'depth', 'crawlStatus', 'metadata',
- 'inLinks', 'outLinks', 'metrics', 'rank', 'hub', 'authority',
- 'canonical', 'noindex', 'nofollow', 'brokenLinks', 'redirectChain',
- 'incrementalStatus', 'etag', 'lastModified', 'contentHash',
- 'isCollapsed', 'collapseInto', 'simhash', 'uniqueTokenRatio',
- 'securityError', 'retries', 'bytesReceived', 'wordCount',
- 'thinContentScore', 'externalLinkRatio', 'h1Count', 'h2Count', 'title'
- ];
- for (const [key, value] of Object.entries(node)) {
- if (value !== undefined && !internalKeys.includes(key)) {
- extra[key] = value;
- }
- }
- if (Object.keys(extra).length > 0) {
- page.plugins = { ...(page.plugins || {}), ...extra };
- }
- }
- }
-
- await registry.runHook('onReport', pluginCtx, result);
- }
-
return result;
}
+
export function analyzePages(rootUrl: string, pages: Iterable<CrawlPage> | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] {
const titleCounts = new Map<string, number>();
const metaCounts = new Map<string, number>();
diff --git a/packages/core/src/application/usecases.ts b/packages/core/src/application/usecases.ts
index 1a9b913..ca60526 100644
--- a/packages/core/src/application/usecases.ts
+++ b/packages/core/src/application/usecases.ts
@@ -40,13 +40,16 @@ export interface SiteCrawlInput {
export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphResult> {
async execute(input: SiteCrawlInput): Promise<CrawlSitegraphResult> {
- const ctx = input.context ?? { command: 'crawl' };
+ const ctx = input.context ?? { command: 'crawl', scope: 'crawl' as const };
+ ctx.scope = 'crawl';
ctx.db = getCrawlithDB();
const registry = new PluginRegistry(input.plugins ?? []);
+ await registry.applyStorage(ctx);
await registry.runHook('onInit', ctx);
if (ctx.terminate) return { snapshotId: 0, graph: undefined as any };
+
await registry.runHook('onCrawlStart', ctx);
if (ctx.terminate) return { snapshotId: 0, graph: undefined as any };
@@ -85,6 +88,10 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes
computeHITS: false
});
+ if (ctx.db) {
+ ctx.db.aggregateScoreProviders(snapshotId, registry.pluginsList);
+ }
+
await registry.runHook('onReport', ctx, { snapshotId, graph });
return { snapshotId, graph };
}
@@ -96,8 +103,12 @@ export class AnalyzeSnapshot implements UseCase<{ url: string; options: AnalyzeO
if (input.plugins && input.plugins.length > 0) {
const registry = new PluginRegistry(input.plugins);
- const ctx = input.context ?? { command: 'analyze' };
- ctx.db = getCrawlithDB();
+ const ctx: PluginContext = {
+ command: 'analyze',
+ scope: 'crawl',
+ ...(input.context || {}),
+ db: getCrawlithDB()
+ };
await registry.runHook('onInit', ctx);
await registry.runHook('onReport', ctx, result);
}
@@ -143,13 +154,36 @@ export class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisR
minClusterSize: input.minClusterSize,
debug: input.debug,
allPages: input.allPages,
- plugins: input.plugins,
- pluginContext: {
- ...(input.context || { command: 'page' }),
- db: getCrawlithDB()
- }
}, this.context);
+ // Run plugins with page scope — only onInit + onPage are called
+ if (input.plugins && input.plugins.length > 0) {
+ const { PluginRegistry } = await import('../plugin-system/plugin-registry.js');
+ const registry = new PluginRegistry(input.plugins);
+
+ const pluginCtx: PluginContext = {
+ command: 'page',
+ scope: 'page',
+ targetUrl: input.url,
+ snapshotId: result.snapshotId,
+ live: input.live || !!(input.context?.flags?.live),
+ ...(input.context || {}),
+ db: getCrawlithDB(),
+ };
+
+ await registry.applyStorage(pluginCtx);
+ await registry.runHook('onInit', pluginCtx);
+
+ // Fire onPage once per analyzed page (normally just 1 for the page command)
+ for (const page of result.pages) {
+ await registry.runHook('onPage', pluginCtx, {
+ url: page.url,
+ html: (page as any).html ?? '',
+ status: page.status,
+ });
+ }
+ }
+
return result;
}
}
diff --git a/packages/core/src/db/CrawlithDB.ts b/packages/core/src/db/CrawlithDB.ts
index 6a40871..44df382 100644
--- a/packages/core/src/db/CrawlithDB.ts
+++ b/packages/core/src/db/CrawlithDB.ts
@@ -3,6 +3,7 @@ import { runBaseMigrations } from './migrations.js';
import { Statements } from './statements.js';
import { PluginRegistry } from './pluginRegistry.js';
import { normalizeUrl } from '../crawler/normalize.js';
+import type { CrawlithPlugin } from '../plugin-system/plugin-types.js';
export class CrawlithDB {
private db: Database.Database;
@@ -21,6 +22,10 @@ export class CrawlithDB {
// Optional scoping properties
private _pluginName?: string;
private _snapshotId?: number | string;
+ /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+ private _live: boolean = false;
+ /** Whether this plugin makes network calls. Core-controlled via plugin.storage.fetchMode. */
+ private _fetchMode: 'local' | 'network' = 'network';
constructor(dbPath: string) {
this.db = new Database(dbPath);
@@ -58,9 +63,22 @@ export class CrawlithDB {
public get data() {
return {
save: <T>(input: { url: string; data: T }) => this.insertPluginRow(input),
+
find: <T>(url: string, options?: { maxAge?: string | number, global?: boolean }) =>
this.getPluginRow<T>(url, undefined, undefined, options),
- all: <T>() => this.getPluginRows<T>()
+
+ all: <T>() => this.getPluginRows<T>(),
+
+ /**
+ * Cache-first with live fallback. Core-enforced pattern:
+ * 1. If cached data exists → return it (always, regardless of age)
+ * 2. If no cache + fetchMode='network' + live=false → return null (skip)
+ * 3. If no cache + (fetchMode='local' OR live=true) → call fetchFn, save, return
+ *
+ * Plugin authors NEVER touch ctx.live — the core injects it via scope().
+ */
+ getOrFetch: <T>(url: string, fetchFn: () => Promise<T>) =>
+ this._getOrFetch<T>(url, fetchFn),
};
}
@@ -69,7 +87,12 @@ export class CrawlithDB {
*/
public get report() {
return {
- save: (summary: any) => this.insertPluginReport({ summary }),
+ save: (summary: any, optionalScores?: {
+ totalScore?: number,
+ scoreCount?: number,
+ scoreWeightSum?: number,
+ scoreCalculatedAt?: string
+ }) => this.insertPluginReport({ summary, ...optionalScores }),
find: <T>() => this.getPluginReport() as T | null
};
}
@@ -80,14 +103,22 @@ export class CrawlithDB {
/**
* Create a scoped instance for a specific plugin.
+ * Also bakes in live + fetchMode so getOrFetch() can enforce the protocol
+ * without exposing those controls to the plugin author.
*/
- public scope(pluginName: string, snapshotId?: number | string): CrawlithDB {
+ public scope(
+ pluginName: string,
+ snapshotId?: number | string,
+ options: { live?: boolean; fetchMode?: 'local' | 'network' } = {}
+ ): CrawlithDB {
if (this._pluginName && this._pluginName !== pluginName) {
throw new Error(`Security Violation: Cannot re-scope a database instance already bound to "${this._pluginName}"`);
}
const scoped = Object.create(this);
scoped._pluginName = pluginName;
scoped._snapshotId = snapshotId;
+ scoped._live = options.live ?? false;
+ scoped._fetchMode = options.fetchMode ?? 'network';
return scoped;
}
@@ -103,10 +134,11 @@ export class CrawlithDB {
if (!pluginName) throw new Error('Plugin name is required for registration (use unbound DB or scope() before calling)');
if (!columns) throw new Error('Columns definition is required');
- const tableName = `${pluginName}_plugin`;
+ // Sanitize: hyphens are invalid in unquoted SQLite identifiers (e.g. "heading-health" → "heading_health_plugin")
+ const tableName = this._toTableName(pluginName);
// Validate columns
- const reserved = ['id', 'snapshot_id', 'url_id', 'created_at'];
+ const reserved = ['id', 'snapshot_id', 'url_id', 'score', 'weight', 'created_at'];
for (const col of Object.keys(columns)) {
if (reserved.includes(col.toLowerCase())) {
throw new Error(`Plugin "${pluginName}" cannot define reserved column "${col}". Reserved: ${reserved.join(', ')}`);
@@ -124,6 +156,8 @@ export class CrawlithDB {
'snapshot_id INTEGER NOT NULL',
'url_id INTEGER NOT NULL',
...Object.entries(columns).map(([col, type]) => `${col} ${type}`),
+ "score REAL",
+ "weight REAL DEFAULT 1.0",
"created_at TEXT DEFAULT (datetime('now'))",
'FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE',
'FOREIGN KEY(url_id) REFERENCES pages(id) ON DELETE CASCADE'
@@ -166,9 +200,13 @@ export class CrawlithDB {
}
public insertPluginReport(input: {
- snapshotId?: number | string
- pluginName?: string
- summary: unknown
+ snapshotId?: number | string;
+ pluginName?: string;
+ summary: unknown;
+ totalScore?: number;
+ scoreCount?: number;
+ scoreWeightSum?: number;
+ scoreCalculatedAt?: string;
}): void {
const snapshotId = input.snapshotId || this._snapshotId;
const pluginName = input.pluginName || this._pluginName;
@@ -179,7 +217,15 @@ export class CrawlithDB {
this._assertSnapshotExists(snapshotId);
const data = JSON.stringify(input.summary);
- this.statements.insertPluginReport.run(snapshotId, pluginName, data);
+ this.statements.insertPluginReport.run(
+ snapshotId,
+ pluginName,
+ data,
+ input.totalScore ?? null,
+ input.scoreCount ?? null,
+ input.scoreWeightSum ?? null,
+ input.scoreCalculatedAt ?? null
+ );
}
public insertPluginRow<T>(input: {
@@ -234,7 +280,7 @@ export class CrawlithDB {
}
public getPluginRows<T>(tableName?: string, snapshotId?: number | string): T[] {
- const targetTable = tableName || (this._pluginName ? `${this._pluginName}_plugin` : undefined);
+ const targetTable = tableName || (this._pluginName ? this._toTableName(this._pluginName) : undefined);
const sid = snapshotId || this._snapshotId;
if (!targetTable || !sid) throw new Error('Table name and snapshotId are required');
@@ -283,10 +329,10 @@ export class CrawlithDB {
const params: any[] = [];
if (options.global) {
- // Join with pages to find the URL globally
- query += ` JOIN pages p ON t.url_id = p.id WHERE p.url = ?`;
- query += ` AND p.snapshot_id = t.snapshot_id`; // Sanity check
+ // Join with pages to find the URL globally across all snapshots
+ query += ` JOIN pages p ON t.url_id = p.id WHERE p.normalized_url = ?`;
params.push(targetUrl);
+
} else {
const urlId = this.getPageIdByUrl(sid!, targetUrl);
if (!urlId) return null;
@@ -343,6 +389,109 @@ export class CrawlithDB {
});
}
+ private async _getOrFetch<T>(url: string, fetchFn: () => Promise<T>): Promise<T | null> {
+ // 1. Check cache (global across snapshots)
+ const cached = this.getPluginRow<T>(url, undefined, undefined, { global: true });
+ if (cached !== null) {
+ return cached; // Always use cache when it exists
+ }
+
+ // 2. No cache. Can we fetch live?
+ const canFetchFresh = this._fetchMode === 'local' || (this._fetchMode === 'network' && this._live);
+
+ if (!canFetchFresh) {
+ return null; // Silent skip, no data to return
+ }
+
+ // 3. Compute/Fetch fresh data
+ const freshData = await fetchFn();
+
+ // 4. Save to DB
+ this.insertPluginRow({ url, data: freshData });
+
+ return freshData;
+ }
+
+ public aggregateScoreProviders(snapshotId: number | string, plugins: CrawlithPlugin[]): void {
+ this._assertSnapshotExists(snapshotId);
+
+ for (const plugin of plugins) {
+ if (!plugin.scoreProvider || !plugin.storage?.perPage?.columns) continue;
+
+ const tableName = this._toTableName(plugin.name);
+ if (!this.registry.isTableRegistered(tableName)) continue;
+
+ try {
+ // Ensure the table schema strictly conforms by ignoring plugins that might have bypassed registry constraints
+ const aggregate = this.db.prepare(`
+ SELECT
+ SUM(score * weight) as total_score,
+ SUM(weight) as score_weight_sum,
+ COUNT(score) as score_count
+ FROM ${tableName}
+ WHERE snapshot_id = ? AND score IS NOT NULL
+ `).get(snapshotId) as any;
+
+ if (!aggregate || aggregate.score_count === 0) continue;
+
+ const { total_score, score_weight_sum, score_count } = aggregate;
+
+ const recentReport = this.db.prepare(`
+ SELECT id FROM plugin_reports
+ WHERE snapshot_id = ? AND plugin_name = ?
+ ORDER BY created_at DESC LIMIT 1
+ `).get(snapshotId, plugin.name) as { id: number } | undefined;
+
+ if (recentReport) {
+ this.db.prepare(`
+ UPDATE plugin_reports
+ SET total_score = ?, score_weight_sum = ?, score_count = ?, score_calculated_at = datetime('now')
+ WHERE id = ?
+ `).run(total_score, score_weight_sum, score_count, recentReport.id);
+ } else {
+ this.db.prepare(`
+ INSERT INTO plugin_reports
+ (snapshot_id, plugin_name, data, total_score, score_weight_sum, score_count, score_calculated_at)
+ VALUES (?, ?, '{}', ?, ?, ?, datetime('now'))
+ `).run(snapshotId, plugin.name, total_score, score_weight_sum, score_count);
+ }
+ } catch (err) {
+ console.error(`[CrawlithDB.Aggregation] Failed to aggregate scores for plugin: ${plugin.name} - ${(err as Error).message}`);
+ }
+ }
+
+ try {
+ // After all plugins are aggregated, tally up the snapshot-level totals
+ const snapshotAggregate = this.db.prepare(`
+ SELECT
+ SUM(total_score) as overall_total_score,
+ SUM(score_weight_sum) as overall_weight_sum,
+ SUM(score_count) as overall_score_count
+ FROM plugin_reports
+ WHERE snapshot_id = ? AND total_score IS NOT NULL
+ `).get(snapshotId) as any;
+
+ if (snapshotAggregate && snapshotAggregate.overall_score_count > 0) {
+ this.db.prepare(`
+ UPDATE snapshots
+ SET
+ total_score = ?,
+ score_weight_sum = ?,
+ score_count = ?,
+ score_calculated_at = datetime('now')
+ WHERE id = ?
+ `).run(
+ snapshotAggregate.overall_total_score,
+ snapshotAggregate.overall_weight_sum,
+ snapshotAggregate.overall_score_count,
+ snapshotId
+ );
+ }
+ } catch (err) {
+ console.error(`[CrawlithDB.Aggregation] Failed to aggregate snapshot-level scores: ${(err as Error).message}`);
+ }
+ }
+
public runInTransaction(fn: () => void): void {
const tx = this.db.transaction(fn);
tx();
@@ -350,11 +499,16 @@ export class CrawlithDB {
private _resolveTableName(name: string): string {
if (this.registry.isTableRegistered(name)) return name;
- const pluginTable = `${name}_plugin`;
+ const pluginTable = this._toTableName(name);
if (this.registry.isTableRegistered(pluginTable)) return pluginTable;
return name; // Will fail assertion later
}
+ /** Converts a plugin name to its canonical SQLite table name, sanitizing invalid characters. */
+ private _toTableName(pluginName: string): string {
+ return `${pluginName.replace(/-/g, '_')}_plugin`;
+ }
+
public close(): void {
this.db.close();
}
diff --git a/packages/core/src/db/migrations.ts b/packages/core/src/db/migrations.ts
index ff3811a..fd7202a 100644
--- a/packages/core/src/db/migrations.ts
+++ b/packages/core/src/db/migrations.ts
@@ -26,6 +26,10 @@ export function runBaseMigrations(db: Database) {
health_score REAL,
orphan_count INTEGER,
thin_content_count INTEGER,
+ total_score REAL,
+ score_count INTEGER,
+ score_weight_sum REAL,
+ score_calculated_at TEXT,
FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
);
`);
@@ -157,6 +161,10 @@ export function runBaseMigrations(db: Database) {
snapshot_id INTEGER NOT NULL,
plugin_name TEXT NOT NULL,
data TEXT NOT NULL,
+ total_score REAL,
+ score_count INTEGER,
+ score_weight_sum REAL,
+ score_calculated_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
);
diff --git a/packages/core/src/db/statements.ts b/packages/core/src/db/statements.ts
index 779c736..6306a96 100644
--- a/packages/core/src/db/statements.ts
+++ b/packages/core/src/db/statements.ts
@@ -1,46 +1,47 @@
import { Database, Statement } from 'better-sqlite3';
export class Statements {
- public getPageIdByUrl: Statement;
- public insertPluginReport: Statement;
- public getPluginReport: Statement;
- public deleteSnapshotPlugins: Statement;
- public getSnapshot: Statement;
- public getMigration: Statement;
- public insertMigration: Statement;
-
- constructor(private db: Database) {
- this.getPageIdByUrl = this.db.prepare(`
+ public getPageIdByUrl: Statement;
+ public insertPluginReport: Statement;
+ public getPluginReport: Statement;
+ public deleteSnapshotPlugins: Statement;
+ public getSnapshot: Statement;
+ public getMigration: Statement;
+ public insertMigration: Statement;
+
+ constructor(private db: Database) {
+ this.getPageIdByUrl = this.db.prepare(`
SELECT id FROM pages
WHERE site_id = (SELECT site_id FROM snapshots WHERE id = ?)
AND normalized_url = ?
`);
- this.insertPluginReport = this.db.prepare(`
- INSERT OR REPLACE INTO plugin_reports (snapshot_id, plugin_name, data)
- VALUES (?, ?, ?)
+ this.insertPluginReport = this.db.prepare(`
+ INSERT OR REPLACE INTO plugin_reports
+ (snapshot_id, plugin_name, data, total_score, score_count, score_weight_sum, score_calculated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
`);
- this.getPluginReport = this.db.prepare(`
+ this.getPluginReport = this.db.prepare(`
SELECT data FROM plugin_reports
WHERE snapshot_id = ? AND plugin_name = ?
ORDER BY created_at DESC LIMIT 1
`);
- this.deleteSnapshotPlugins = this.db.prepare(`
+ this.deleteSnapshotPlugins = this.db.prepare(`
DELETE FROM plugin_reports WHERE snapshot_id = ?
`);
- this.getSnapshot = this.db.prepare(`
+ this.getSnapshot = this.db.prepare(`
SELECT id FROM snapshots WHERE id = ?
`);
- this.getMigration = this.db.prepare(`
+ this.getMigration = this.db.prepare(`
SELECT plugin_name FROM plugin_migrations WHERE plugin_name = ?
`);
- this.insertMigration = this.db.prepare(`
+ this.insertMigration = this.db.prepare(`
INSERT INTO plugin_migrations (plugin_name) VALUES (?)
`);
- }
+ }
}
diff --git a/packages/core/src/plugin-system/plugin-registry.ts b/packages/core/src/plugin-system/plugin-registry.ts
index 5a05e77..3382c40 100644
--- a/packages/core/src/plugin-system/plugin-registry.ts
+++ b/packages/core/src/plugin-system/plugin-registry.ts
@@ -1,5 +1,5 @@
import { Command } from 'commander';
-import { CrawlithPlugin, PluginContext } from './plugin-types.js';
+import { CrawlithPlugin, PluginContext, CLIWriter } from './plugin-types.js';
import { PluginConfig } from './plugin-config.js';
export class PluginRegistry {
@@ -9,8 +9,16 @@ export class PluginRegistry {
this.plugins = plugins;
}
+ public get pluginsList(): CrawlithPlugin[] {
+ return this.plugins;
+ }
+
private registeredCommands = new WeakSet<Command>();
+ /**
+ * Registers all plugin CLI flags on the given command.
+ * Handles both declarative `cli` config and legacy `register(cmd)` callbacks.
+ */
registerPlugins(program: any) {
if (!(program instanceof Command)) return;
@@ -18,33 +26,109 @@ export class PluginRegistry {
if (this.registeredCommands.has(cmd)) return;
this.registeredCommands.add(cmd);
+ const cmdName = cmd.name() as 'page' | 'crawl';
+
for (const plugin of this.plugins) {
+ // Declarative cli registration (preferred)
+ if (plugin.cli) {
+ const targets = plugin.cli.for ?? ['page', 'crawl'];
+ if (targets.includes(cmdName)) {
+ cmd.option(plugin.cli.flag, plugin.cli.description);
+ for (const opt of plugin.cli.options ?? []) {
+ const dv = opt.defaultValue;
+ if (dv !== undefined && dv !== null) {
+ cmd.option(opt.flag, opt.description, dv as string | boolean | string[]);
+ } else {
+ cmd.option(opt.flag, opt.description);
+ }
+ }
+ }
+ }
+
+ // Legacy imperative registration (backwards compat)
if (typeof plugin.register === 'function') {
plugin.register(cmd);
}
}
+
for (const sub of cmd.commands) {
traverse(sub);
}
};
+
traverse(program);
}
+ /**
+ * Applies declarative `storage` schemas for all plugins.
+ * Called by the core after `getCrawlithDB()` is available, before any hooks run.
+ */
+ applyStorage(context: PluginContext): void {
+ for (const plugin of this.plugins) {
+ if (!plugin.storage?.perPage?.columns) continue;
+ if (!context.db) continue;
+
+ try {
+ const scopedDb = context.db.scope(plugin.name, context.snapshotId, {
+ live: context.live,
+ fetchMode: plugin.storage.fetchMode
+ });
+ scopedDb.schema.define(plugin.storage.perPage.columns as any);
+ } catch (err) {
+ context.logger?.error(
+ `[plugin:${plugin.name}] Storage schema failed: ${(err as Error).message}`
+ );
+ }
+ }
+ }
+
+ // ─── Scope guards ─────────────────────────────────────────────────────────
+
+ /** Hooks that only make sense during a full site crawl. */
+ private static readonly CRAWL_ONLY_HOOKS = new Set([
+ 'onCrawlStart', 'onPageParsed', 'onGraphBuilt', 'onMetrics', 'onReport', 'shouldEnqueueUrl'
+ ]);
+
+ /** Hooks that only make sense during a single-page analysis. */
+ private static readonly PAGE_ONLY_HOOKS = new Set([
+ 'onPage'
+ ]);
+
+ // ─── Hook runners ─────────────────────────────────────────────────────────
+
async runHook(hookName: string, context: PluginContext, payload?: any) {
+ // Enforce scope: silently skip hooks that don't belong to this execution context.
+ // Undefined scope (legacy/test) is treated as permissive.
+ if (context.scope === 'page' && PluginRegistry.CRAWL_ONLY_HOOKS.has(hookName)) return;
+ if (context.scope === 'crawl' && PluginRegistry.PAGE_ONLY_HOOKS.has(hookName)) return;
+
for (const plugin of this.plugins) {
const hooks = plugin.hooks as any;
if (hooks && typeof hooks[hookName] === 'function') {
- const scopedDb = context.db?.scope(plugin.name, context.snapshotId || (payload?.snapshotId));
+ const scopedDb = context.db?.scope(plugin.name, context.snapshotId || payload?.snapshotId, {
+ live: context.live,
+ fetchMode: plugin.storage?.fetchMode
+ });
const scopedConfig = new PluginConfig(plugin.name);
// Resolve targetUrl from payload if available (standard result object)
const targetUrl = context.targetUrl || payload?.pages?.[0]?.url;
- const scopedContext = {
+ // Build a CLIWriter that prefixes plugin name — satisfies both cli and logger
+ const cliWriter: CLIWriter = {
+ info: (m) => context.logger?.info(m),
+ warn: (m) => context.logger?.warn(m),
+ error: (m) => context.logger?.error(m),
+ debug: (m) => context.logger?.debug(m),
+ };
+
+ const scopedContext: PluginContext = {
...context,
db: scopedDb,
config: scopedConfig,
- targetUrl
+ targetUrl,
+ cli: cliWriter,
+ logger: cliWriter, // keep logger alias in sync
};
try {
@@ -54,7 +138,9 @@ export class PluginRegistry {
await hooks[hookName](scopedContext);
}
} catch (err) {
- context.logger?.error(`[plugin:${plugin.name}] Hook ${hookName} failed: ${(err as Error).message}`);
+ context.logger?.error(
+ `[plugin:${plugin.name}] Hook ${hookName} failed: ${(err as Error).message}`
+ );
}
}
}
@@ -64,15 +150,20 @@ export class PluginRegistry {
for (const plugin of this.plugins) {
const hooks = plugin.hooks as any;
if (hooks && typeof hooks[hookName] === 'function') {
- const scopedDb = context.db?.scope(plugin.name, context.snapshotId);
+ const scopedDb = context.db?.scope(plugin.name, context.snapshotId, {
+ live: context.live,
+ fetchMode: plugin.storage?.fetchMode
+ });
const scopedConfig = new PluginConfig(plugin.name);
- const scopedContext = { ...context, db: scopedDb, config: scopedConfig };
+ const scopedContext: PluginContext = { ...context, db: scopedDb, config: scopedConfig };
try {
const result = hooks[hookName](scopedContext, ...args);
if (result !== undefined) return result;
} catch (err) {
- context.logger?.error(`[plugin:${plugin.name}] Sync bail hook ${hookName} failed: ${(err as Error).message}`);
+ context.logger?.error(
+ `[plugin:${plugin.name}] Sync bail hook ${hookName} failed: ${(err as Error).message}`
+ );
}
}
}
diff --git a/packages/core/src/plugin-system/plugin-types.ts b/packages/core/src/plugin-system/plugin-types.ts
index 50ba0cd..1f55ac1 100644
--- a/packages/core/src/plugin-system/plugin-types.ts
+++ b/packages/core/src/plugin-system/plugin-types.ts
@@ -1,8 +1,56 @@
-import { Command } from 'commander';
import type { CrawlithDB } from '../db/CrawlithDB.js';
+// ─── Scope ───────────────────────────────────────────────────────────────────
+
+/**
+ * Execution scope — set by the core before any hooks run.
+ * - `page` → single-URL analysis (crawlith page …)
+ * - `crawl` → full site crawl (crawlith crawl …)
+ */
+export type PluginScope = 'page' | 'crawl';
+
+// ─── Output writers ───────────────────────────────────────────────────────────
+
+/** Injected into hook contexts to let plugins emit structured CLI output. */
+export interface CLIWriter {
+ info(message: string): void;
+ warn(message: string): void;
+ error(message: string): void;
+ debug(message: string): void;
+}
+
+/**
+ * Injected into report-phase hook contexts.
+ * Plugins contribute structured data that the core aggregates into the final output.
+ */
+export interface ReportWriter {
+ /** Attach a named data section to the report output. */
+ addSection(pluginName: string, data: unknown): void;
+ /** Optionally contribute a weighted score component. */
+ contributeScore?(input: { label: string; score: number; weight: number }): void;
+}
+
+// ─── Per-page input ───────────────────────────────────────────────────────────
+
+/** Passed to `onPage` — the single URL being analyzed. */
+export interface PageInput {
+ /** Absolute URL of the page being analyzed. */
+ url: string;
+ /** Raw HTML content of the page. */
+ html: string;
+ /** HTTP status code. */
+ status: number;
+}
+
+// ─── Scoped hook contexts ─────────────────────────────────────────────────────
+
+/** Base context available in every hook. */
export interface PluginContext {
+ /** Execution scope. Undefined only in legacy / test contexts — treated as permissive. */
+ scope?: PluginScope;
command?: string;
+ /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+ live?: boolean;
flags?: Record<string, any>;
snapshotId?: number;
targetUrl?: string;
@@ -12,29 +60,136 @@ export interface PluginContext {
require(key?: string): string;
set(value: string): void;
};
- logger?: {
- info(msg: string): void;
- warn(msg: string): void;
- error(msg: string): void;
- debug(msg: string): void;
- };
+ /** CLI writer — populated by the registry before each hook call. */
+ cli?: CLIWriter;
+ /** Legacy logger alias — kept for backwards compatibility. */
+ logger?: CLIWriter;
+ metadata?: Record<string, any>;
[key: string]: any;
}
+// ─── CLI option declaration ───────────────────────────────────────────────────
+
+export interface PluginCliOption {
+ /** e.g. '--my-flag <value>' */
+ flag: string;
+ description: string;
+ defaultValue?: unknown;
+}
+
+// ─── Plugin storage declaration ───────────────────────────────────────────────
+
+export type PluginColumnType = 'INTEGER' | 'REAL' | 'TEXT';
+
+export interface PluginStorage {
+ /** Whether this plugin fetches data from a network or computes locally. Defaults to 'network'. */
+ fetchMode?: 'local' | 'network';
+ /**
+ * Per-URL columns to add to the plugin's scoped data table.
+ * The core creates the table automatically before `onInit` runs,
+ * so plugins never need to call `ctx.db.schema.define()`.
+ */
+ perPage?: {
+ columns: Record<string, PluginColumnType>;
+ };
+}
+
+// ─── Plugin interface ─────────────────────────────────────────────────────────
+
export interface CrawlithPlugin {
name: string;
version?: string;
description?: string;
- register?: (cli: Command) => void;
+ /**
+ * Declarative CLI registration.
+ * The core registers these options on the appropriate commands —
+ * no need to interact with Commander directly.
+ *
+ * `for` controls which commands expose the options:
+ * - `['page', 'crawl']` (default when omitted) — both commands
+ * - `['page']` — page command only
+ * - `['crawl']` — crawl command only
+ */
+ cli?: {
+ flag: string;
+ description: string;
+ for?: ('page' | 'crawl')[];
+ options?: PluginCliOption[];
+ };
+
+ /**
+ * Declarative storage schema.
+ * The core creates the plugin's scoped table before `onInit` runs.
+ * Plugins that don't persist data can omit this entirely.
+ */
+ storage?: PluginStorage;
+
+ /**
+ * Set to true to declare this plugin as a Score Provider.
+ * The core will automatically aggregate the `score` and `weight` columns
+ * from this plugin's storage table during snapshot aggregation.
+ */
+ scoreProvider?: boolean;
+
+ /**
+ * Legacy imperative CLI registration — kept for backwards compatibility.
+ * Prefer `cli` for new plugins.
+ * @deprecated Use `cli` instead.
+ */
+ register?: (cli: any) => void;
hooks?: {
+ /**
+ * Runs on both `page` and `crawl` scopes.
+ * Use for any setup that doesn't depend on the scope —
+ * e.g. initialising in-memory state, reading config.
+ * DB schema is already created by the time this runs (via `storage`).
+ */
onInit?: (ctx: PluginContext) => void | Promise<void>;
+
+ /**
+ * Single-page hook — `page` scope only.
+ * Receives the target URL, its raw HTML, and HTTP status.
+ * Use this for URL-scoped plugins (PageSpeed, heading-health, etc.).
+ */
+ onPage?: (ctx: PluginContext, page: PageInput) => void | Promise<void>;
+
+ /**
+ * Fired at the very start of a crawl — `crawl` scope only.
+ * Use to initialise crawl-wide state or validate config.
+ */
onCrawlStart?: (ctx: PluginContext) => void | Promise<void>;
+
+ /**
+ * URL enqueue filter — `crawl` scope only.
+ * Return `false` to prevent a URL from being crawled.
+ */
shouldEnqueueUrl?: (ctx: PluginContext, url: string, depth: number) => boolean;
+
+ /**
+ * Fired after each page is fetched and parsed — `crawl` scope only.
+ * Use for real-time per-page processing without waiting for the full graph.
+ */
onPageParsed?: (ctx: PluginContext, page: any) => void | Promise<void>;
+
+ /**
+ * Fired after the full link graph is built — `crawl` scope only.
+ * Graph structure is complete; metrics have not been computed yet.
+ */
onGraphBuilt?: (ctx: PluginContext, graph: any) => void | Promise<void>;
- onMetrics?: (ctx: PluginContext, metrics: any) => void | Promise<void>;
+
+ /**
+ * Graph-level metrics phase — `crawl` scope only.
+ * All pages are available; use for cross-page analysis (duplicate
+ * detection, PageRank, heading structure across the site, etc.).
+ */
+ onMetrics?: (ctx: PluginContext, graph: any) => void | Promise<void>;
+
+ /**
+ * Final report phase — `crawl` scope only.
+ * Attach snapshot-level summary data to the result object.
+ */
onReport?: (ctx: PluginContext, report: any) => void | Promise<void>;
};
}
diff --git a/packages/plugins/heading-health/README.md b/packages/plugins/heading-health/README.md
index 38a4876..58de142 100644
--- a/packages/plugins/heading-health/README.md
+++ b/packages/plugins/heading-health/README.md
@@ -1,9 +1,124 @@
-# Heading Health Plugin
+# `@crawlith/plugin-heading-health`
-Analyzes heading structure, hierarchy health, and content distribution across pages.
+Analyzes the heading structure, hierarchy health, and content distribution of every page in a crawl or single-page analysis.
-## Features
-- **Hierarchy Audit:** Detects missing H1s, multiple H1s, and hierarchy skips (e.g., H2 to H4).
-- **Structural Entropy:** Measures the diversity and balance of heading levels.
-- **Content Mapping:** Visualizes the document outline with parent-child relationships.
-- **Divergence Check:** Flags when H1 text significantly differs from the page title.
+---
+
+## What It Does
+
+The heading-health plugin parses the raw HTML of each crawled page, extracts all `<h1>` through `<h6>` tags, and runs a suite of structural checks. It scores each page from **0–100**, assigns a status (`Healthy`, `Moderate`, `Poor`), and surfaces actionable issues. A snapshot-level summary is also produced, giving you a site-wide view of heading quality.
+
+---
+
+## How It Works
+
+The plugin runs across three lifecycle hooks:
+
+### `onInit`
+Registers the plugin's database schema columns (`score`, `status`, `analysis_json`) so per-URL results can be persisted across runs.
+
+### `onMetrics`
+Triggered after crawling, once the full page graph is available. For each page node in the graph:
+
+1. **Extracts headings** from raw HTML using regex-based parsing, preserving heading level and text.
+2. **Builds a tree** tracking parent–child relationships between headings.
+3. **Segments content** between headings to measure word count and keyword concentration per section.
+4. **Computes structural metrics** (see below).
+5. **Detects site-wide duplicates** by comparing H1 norms, H2 set hashes, and structural pattern hashes across all pages (cross-page duplicate risk for sections).
+6. **Scores and statuses** each page, attaches the payload to the graph node, and persists data to the database.
+7. Emits a **snapshot-level summary** to the context.
+
+### `onReport`
+Attaches the snapshot-level `HeadingHealthSummary` to the final result object under `result.plugins.headingHealth`.
+
+---
+
+## CLI Usage
+
+Add `--heading` to the `crawl` or `page` commands to activate the plugin:
+
+```bash
+# Single page
+crawlith page https://example.com --heading
+
+# Full crawl
+crawlith crawl https://example.com --heading
+
+# Force a fresh recompute (bypass 24h cache)
+crawlith crawl https://example.com --heading --heading-force-refresh
+```
+
+---
+
+## Metrics Explained
+
+Each page receives a `HeadingHealthPayload` with the following fields:
+
+| Field | Description |
+| :--- | :--- |
+| `score` | Overall heading health score, 0–100 |
+| `status` | `Healthy` (≥70), `Moderate` (≥40), or `Poor` (<40) |
+| `issues` | List of human-readable issue strings detected |
+| `map` | Ordered array of all heading nodes with level, text, index, and parent index |
+| `missing_h1` | `1` if no H1 exists on the page, else `0` |
+| `multiple_h1` | `1` if more than one H1 exists, else `0` |
+| `entropy` | Shannon entropy of heading level distribution — higher = more chaotic structure |
+| `max_depth` | Deepest heading level used (e.g., `4` means H4 is the deepest) |
+| `avg_depth` | Average heading level across all headings |
+| `heading_density` | Ratio of heading count to total word count |
+| `fragmentation` | Proportion of headings that are H1 or H2 — high fragmentation = too many top-level headings |
+| `volatility` | Average level-jump between consecutive headings — high = erratic structure |
+| `hierarchy_skips` | Number of times heading level jumps by more than 1 (e.g., H2 → H4) |
+| `reverse_jumps` | Number of times heading level drops by more than 1 (e.g., H4 → H2) |
+| `thin_sections` | Number of content sections containing fewer than 80 words |
+| `duplicate_h1_group` | Number of other pages sharing the same normalized H1 text |
+| `similar_h1_group` | Number of pages with a Jaccard similarity > threshold to this page's H1 |
+| `identical_h2_set_group` | Number of pages sharing the exact same ordered set of H2 text |
+| `duplicate_pattern_group` | Number of pages with the same heading level pattern (e.g., `1>2>2>3`) |
+| `template_risk` | Combined risk score indicating the page may use a templated heading structure |
+
+---
+
+## Site-Wide Summary (`HeadingHealthSummary`)
+
+Attached to `result.plugins.headingHealth` after a crawl:
+
+| Field | Description |
+| :--- | :--- |
+| `avgScore` | Mean heading health score across all evaluated pages |
+| `evaluatedPages` | Number of pages that had headings and were scored |
+| `totalMissing` | Total pages missing an H1 |
+| `totalMultiple` | Total pages with multiple H1 tags |
+| `totalSkips` | Sum of hierarchy skips across all pages |
+| `totalReverseJumps` | Sum of reverse hierarchy jumps across all pages |
+| `totalThinSections` | Total count of thin content sections across the site |
+| `avgEntropy` | Mean structural entropy — a site-wide indicator of heading consistency |
+| `poorPages` | Number of pages rated `Poor` |
+
+---
+
+## Detected Issues
+
+The plugin surfaces plain-language issues on each page:
+
+- `Missing H1` — No H1 tag found
+- `Multiple H1 found` — More than one H1 on the page
+- `Empty or near-empty H1` — H1 text is fewer than 6 characters
+- `H1 diverges from <title>` — H1 and page title have low Jaccard similarity (< 0.3)
+- `N hierarchy skips detected` — Heading level jumps more than 1 step forward
+- `N reverse hierarchy jumps detected` — Heading level drops more than 1 step
+- `Thin section under "..."` — A content section has fewer than 80 words
+- `High structural entropy` — Entropy score exceeds 2.1
+- `Section fragmentation is high` — More than 65% of headings are H1/H2
+
+---
+
+## Data Storage
+
+Per-URL results are persisted in the plugin's scoped database table. The stored columns are:
+
+- `score` — Integer health score
+- `status` — `Healthy`, `Moderate`, or `Poor`
+- `analysis_json` — Full `HeadingHealthPayload` serialized as JSON
+
+This data is available for future queries via `ctx.db.data.find()` in other plugins or custom tooling.
diff --git a/packages/plugins/heading-health/index.test.ts b/packages/plugins/heading-health/index.test.ts
index 6bccdd0..80f571b 100644
--- a/packages/plugins/heading-health/index.test.ts
+++ b/packages/plugins/heading-health/index.test.ts
@@ -36,14 +36,12 @@ describe('heading-health plugin', () => {
const graph = { getNodes: () => rawNodes };
- const save = vi.fn();
- const find = vi.fn(() => null);
+ const getOrFetch = vi.fn(async (url, fetchFn) => fetchFn());
const ctx: PluginContext = {
flags: { heading: true },
db: {
- schema: { define: vi.fn() },
- data: { save, find }
+ data: { getOrFetch }
} as any
};
@@ -64,8 +62,7 @@ describe('heading-health plugin', () => {
expect(summary.totalMissing).toBe(0);
expect(summary.totalSkips).toBeGreaterThan(0);
- expect(find).toHaveBeenCalledTimes(2);
- expect(save).toHaveBeenCalledTimes(2);
+ expect(getOrFetch).toHaveBeenCalledTimes(2);
});
it('attaches summary payload on report output', async () => {
diff --git a/packages/plugins/heading-health/index.ts b/packages/plugins/heading-health/index.ts
index f99f355..74183be 100644
--- a/packages/plugins/heading-health/index.ts
+++ b/packages/plugins/heading-health/index.ts
@@ -1,5 +1,4 @@
import type { CrawlithPlugin } from '@crawlith/core';
-import { registerHeadingHealthCli } from './src/cli.js';
import { HeadingHealthHooks } from './src/plugin.js';
/**
@@ -11,10 +10,34 @@ import { HeadingHealthHooks } from './src/plugin.js';
export const HeadingHealthPlugin: CrawlithPlugin = {
name: 'heading-health',
description: 'Analyzes heading structure, hierarchy health, and content distribution',
- register: (cli) => {
- registerHeadingHealthCli(cli);
+
+ cli: {
+ flag: '--heading',
+ description: 'Analyze heading structure and hierarchy health',
+ for: ['page', 'crawl'],
+ options: [
+ { flag: '--heading-force-refresh', description: 'Bypass the 24h heading-health cache and recompute' }
+ ]
+ },
+
+ scoreProvider: true,
+
+ storage: {
+ fetchMode: 'local',
+ perPage: {
+
+ columns: {
+ status: 'TEXT',
+ analysis_json: 'TEXT'
+ }
+ }
},
- hooks: HeadingHealthHooks
+
+ hooks: {
+ onPage: HeadingHealthHooks.onPage,
+ onMetrics: HeadingHealthHooks.onMetrics,
+ onReport: HeadingHealthHooks.onReport
+ }
};
export default HeadingHealthPlugin;
diff --git a/packages/plugins/heading-health/src/HeadingHealthService.ts b/packages/plugins/heading-health/src/HeadingHealthService.ts
index dd54f80..4c11812 100644
--- a/packages/plugins/heading-health/src/HeadingHealthService.ts
+++ b/packages/plugins/heading-health/src/HeadingHealthService.ts
@@ -102,6 +102,51 @@ export class HeadingHealthService {
return { payloadsByUrl, summary };
}
+ /**
+ * Evaluates a single page's heading health directly from raw HTML.
+ * Used by the `onPage` hook (page command). Cross-page duplicate signals are omitted
+ * since there is only one page to compare against.
+ */
+ public evaluateSinglePage(url: string, html: string): HeadingHealthPayload {
+ const analysis = analyzeHeadingHealth(html);
+ analysis.url = url;
+
+ const thinSectionCount = analysis.sections.filter((section) => section.thin).length;
+ const health = scoreHealth({
+ metrics: analysis.metrics,
+ thinSectionCount,
+ duplicateH1GroupSize: 1,
+ similarH1GroupSize: 0,
+ identicalH2SetGroupSize: 1,
+ duplicatePatternGroupSize: 1,
+ templateRisk: 0,
+ issues: analysis.issues
+ });
+
+ return {
+ score: health.score,
+ status: health.status,
+ issues: health.issues,
+ map: analysis.headingNodes,
+ missing_h1: analysis.metrics.missingH1,
+ multiple_h1: analysis.metrics.multipleH1,
+ entropy: analysis.metrics.entropy,
+ max_depth: analysis.metrics.maxDepth,
+ avg_depth: analysis.metrics.avgDepth,
+ heading_density: analysis.metrics.headingDensity,
+ fragmentation: analysis.metrics.fragmentation,
+ volatility: analysis.metrics.levelVolatility,
+ hierarchy_skips: analysis.metrics.hierarchySkips,
+ reverse_jumps: analysis.metrics.reverseJumps,
+ thin_sections: thinSectionCount,
+ duplicate_h1_group: 1,
+ similar_h1_group: 0,
+ identical_h2_set_group: 1,
+ duplicate_pattern_group: 1,
+ template_risk: 0
+ };
+ }
+
private collectAnalyses(nodes: Array<Record<string, any>>): LocalPageAnalysis[] {
const analyses: LocalPageAnalysis[] = [];
diff --git a/packages/plugins/heading-health/src/Output.ts b/packages/plugins/heading-health/src/Output.ts
index c8cdfbe..b60707e 100644
--- a/packages/plugins/heading-health/src/Output.ts
+++ b/packages/plugins/heading-health/src/Output.ts
@@ -1,12 +1,22 @@
import type { PluginContext } from '@crawlith/core';
-import type { HeadingHealthSummary } from './types.js';
+import type { HeadingHealthPayload, HeadingHealthSummary } from './types.js';
/**
* Responsible for plugin logging output.
*/
export class HeadingHealthOutput {
/**
- * Emits a single-line summary for heading-health metrics.
+ * Emits a single-line result for a single-page heading-health analysis (page command).
+ */
+ public static emitSingle(ctx: PluginContext, payload: HeadingHealthPayload): void {
+ ctx.logger?.info(
+ `[plugin:heading-health] score=${payload.score} (${payload.status})` +
+ (payload.issues.length ? ` — ${payload.issues.join(', ')}` : '')
+ );
+ }
+
+ /**
+ * Emits a single-line summary for heading-health metrics (crawl command).
*/
public static emitSummary(ctx: PluginContext, summary: HeadingHealthSummary): void {
ctx.logger?.info(
diff --git a/packages/plugins/heading-health/src/plugin.ts b/packages/plugins/heading-health/src/plugin.ts
index cfc491b..26ec749 100644
--- a/packages/plugins/heading-health/src/plugin.ts
+++ b/packages/plugins/heading-health/src/plugin.ts
@@ -1,57 +1,80 @@
-import type { PluginContext } from '@crawlith/core';
+import type { PluginContext, PageInput } from '@crawlith/core';
import { HeadingHealthOutput } from './Output.js';
import { HeadingHealthService } from './HeadingHealthService.js';
import type { HeadingHealthPayload, HeadingHealthRow, HeadingHealthSummary } from './types.js';
/**
* Lifecycle hooks for the heading-health plugin.
+ * Schema registration is handled declaratively via plugin.storage — no onInit needed.
*/
export const HeadingHealthHooks = {
/**
- * Registers schema columns used for URL-scoped heading-health persistence.
+ * Single-page hook (page command). Analyzes one URL's HTML directly.
+ * No cross-page duplicate signals since only one page is available.
*/
- onInit: async (ctx: PluginContext): Promise<void> => {
- if (!ctx.db) {
- return;
- }
+ onPage: async (ctx: PluginContext, page: PageInput): Promise<void> => {
+ if (!ctx.flags?.heading || !ctx.db) return;
+
+ const service = new HeadingHealthService();
+
+ const row = await ctx.db.data.getOrFetch<HeadingHealthRow>(
+ page.url,
+ async () => {
+ const payload = service.evaluateSinglePage(page.url, page.html);
+ return {
+ score: payload.score,
+ status: payload.status,
+ // Store as string, but getOrFetch/find will auto-parse to object later
+ analysis_json: JSON.stringify(payload)
+ } as unknown as HeadingHealthRow;
+ }
+ );
+
+ if (!row) return;
- ctx.db.schema.define({
- score: 'INTEGER NOT NULL',
- status: "TEXT CHECK(status IN ('Healthy', 'Moderate', 'Poor')) NOT NULL",
- analysis_json: 'TEXT NOT NULL'
- });
+ // analysis_json gets auto-parsed by CrawlithDB._parseRow
+ const payload = (typeof row.analysis_json === 'string'
+ ? JSON.parse(row.analysis_json)
+ : row.analysis_json) as HeadingHealthPayload;
+
+ HeadingHealthOutput.emitSingle(ctx, payload);
},
/**
- * Evaluates heading health for each eligible graph node, persisting URL-scoped data with 24h cache reads.
+ * Graph-level hook (crawl command). Evaluates all nodes with cross-page duplicate detection.
*/
onMetrics: async (ctx: PluginContext, graph: any): Promise<void> => {
- if (!ctx.flags?.heading) {
- return;
- }
+ if (!ctx.flags?.heading || !ctx.db) return;
const nodes = graph?.getNodes?.();
- if (!Array.isArray(nodes)) {
- return;
- }
+ if (!Array.isArray(nodes)) return;
const service = new HeadingHealthService();
const { payloadsByUrl, summary } = service.evaluateNodes(nodes);
for (const node of nodes) {
const url = node?.url;
- if (!url) {
- continue;
- }
+ if (!url) continue;
const livePayload = payloadsByUrl.get(url);
- if (!livePayload) {
- continue;
+ if (!livePayload) continue;
+
+ const row = await ctx.db.data.getOrFetch<HeadingHealthRow>(
+ url,
+ async () => ({
+ score: livePayload.score,
+ status: livePayload.status,
+ analysis_json: JSON.stringify(livePayload)
+ } as unknown as HeadingHealthRow)
+ );
+
+ if (row) {
+ const payload = (typeof row.analysis_json === 'string'
+ ? JSON.parse(row.analysis_json)
+ : row.analysis_json) as HeadingHealthPayload;
+
+ (node as any).headingHealth = payload;
}
-
- const payload = readCachedPayload(ctx, url, livePayload);
- (node as any).headingHealth = payload;
- persistPayload(ctx, url, payload);
}
ctx.metadata = ctx.metadata || {};
@@ -63,49 +86,12 @@ export const HeadingHealthHooks = {
* Attaches snapshot-level heading-health summary to the final report object.
*/
onReport: async (ctx: PluginContext, result: any): Promise<void> => {
- if (!ctx.flags?.heading) {
- return;
- }
+ if (!ctx.flags?.heading) return;
const summary = ctx.metadata?.headingHealthSummary as HeadingHealthSummary | undefined;
- if (!summary) {
- return;
- }
+ if (!summary) return;
result.plugins = result.plugins || {};
result.plugins.headingHealth = summary;
}
};
-
-function readCachedPayload(ctx: PluginContext, url: string, fallback: HeadingHealthPayload): HeadingHealthPayload {
- if (!ctx.db || ctx.flags?.headingForceRefresh) {
- return fallback;
- }
-
- const cached = ctx.db.data.find<HeadingHealthRow>(url, { maxAge: '24h', global: true });
- if (!cached?.analysis_json) {
- return fallback;
- }
-
- try {
- return JSON.parse(cached.analysis_json) as HeadingHealthPayload;
- } catch {
- ctx.logger?.warn(`[plugin:heading-health] invalid cached JSON for ${url}, recomputing`);
- return fallback;
- }
-}
-
-function persistPayload(ctx: PluginContext, url: string, payload: HeadingHealthPayload): void {
- if (!ctx.db) {
- return;
- }
-
- ctx.db.data.save({
- url,
- data: {
- score: payload.score,
- status: payload.status,
- analysis_json: JSON.stringify(payload)
- }
- });
-}
diff --git a/packages/plugins/pagespeed/index.ts b/packages/plugins/pagespeed/index.ts
index 1615cc1..298e9ce 100644
--- a/packages/plugins/pagespeed/index.ts
+++ b/packages/plugins/pagespeed/index.ts
@@ -4,7 +4,7 @@ import { PageSpeedHooks } from './src/plugin.js';
/**
* PageSpeed Insights Plugin for Crawlith.
- *
+ *
* Provides automated performance auditing for scanned URLs using the Google PageSpeed Insights API.
* Features:
* - 24h Global Persistent Caching (via ctx.db)
@@ -13,22 +13,37 @@ import { PageSpeedHooks } from './src/plugin.js';
export const PageSpeedPlugin: CrawlithPlugin = {
name: 'pagespeed',
- /**
- * Registers CLI commands and options for the PageSpeed plugin.
- * This acts as the visual manifest of what the plugin adds to the CLI.
- */
- register: (cli) => {
- registerConfigCommand(cli);
+ cli: {
+ flag: '--pagespeed',
+ description: 'Attach Google PageSpeed Insights report (mobile strategy)',
+ for: ['page'],
+ options: [
+ { flag: '--force', description: 'Force a fresh PageSpeed API request and bypass 24h cache' }
+ ]
+ },
- if (cli.name() === 'page') {
- cli.option('--pagespeed', 'Attach Google PageSpeed Insights report (mobile strategy)');
- cli.option('--force', 'Force a fresh PageSpeed API request and bypass 24h cache');
+ scoreProvider: true,
+
+ storage: {
+ perPage: {
+ columns: {
+ strategy: 'TEXT',
+ performance_score: 'INTEGER',
+ lcp: 'REAL',
+ cls: 'REAL',
+ tbt: 'REAL',
+ raw_json: 'TEXT'
+ }
}
},
/**
- * Internal logic implementation is delegated to src/plugin.js
+ * Keeps the `pagespeed config` subcommand — not replaceable by declarative cli.
*/
+ register: (cli) => {
+ registerConfigCommand(cli);
+ },
+
hooks: PageSpeedHooks
};
diff --git a/packages/plugins/pagespeed/src/plugin.ts b/packages/plugins/pagespeed/src/plugin.ts
index 4485fa5..45164ba 100644
--- a/packages/plugins/pagespeed/src/plugin.ts
+++ b/packages/plugins/pagespeed/src/plugin.ts
@@ -1,87 +1,62 @@
import { PageSpeedService } from './PageSpeedService.js';
import { PageSpeedOutput } from './PageSpeedOutput.js';
import { PageSpeedRow } from './types.js';
-import type { PluginContext } from '@crawlith/core';
+import type { PluginContext, PageInput } from '@crawlith/core';
const STRATEGY = 'mobile';
/**
* PageSpeed Plugin Hooks implementation.
+ * Schema registration is handled declaratively via plugin.storage — no onInit needed.
*/
export const PageSpeedHooks = {
/**
- * Initialization hook: Registers the persistent database schema for PageSpeed metrics.
- */
- onInit: async (ctx: PluginContext) => {
- if (!ctx.db) return;
-
- ctx.db.schema.define({
- strategy: "TEXT CHECK(strategy IN ('mobile', 'desktop')) NOT NULL",
- performance_score: "INTEGER",
- lcp: "REAL",
- cls: "REAL",
- tbt: "REAL",
- raw_json: "TEXT NOT NULL"
- });
- },
-
- /**
- * Report hook: Executes after a page analysis to perform performance auditing.
+ * Page hook: Executes for a single URL to perform performance auditing.
* Implements smart caching and persists results to the database.
*/
- onReport: async (ctx: PluginContext, result: any) => {
+ onPage: async (ctx: PluginContext, page: PageInput) => {
const flags = ctx.flags || {};
- if (ctx.command !== 'page' || !flags.pagespeed) return;
+ if (!flags.pagespeed) return;
if (!ctx.db || !ctx.config) return;
- result.plugins = result.plugins || {};
const service = new PageSpeedService();
try {
- const pageUrl = ctx.targetUrl;
+ const pageUrl = page.url;
if (!pageUrl || !URL.canParse(pageUrl)) {
throw new Error('PageSpeed requires an absolute target URL.');
}
const apiKey = ctx.config.require();
- // Smart Caching: Look globally across snapshots for this URL within last 24h
- let raw: any = null;
- if (!flags.force) {
- const cached = ctx.db.data.find<PageSpeedRow>(pageUrl, {
- maxAge: '24h',
- global: true
- });
- if (cached) raw = cached.raw_json;
- }
-
- let summary;
-
- if (raw) {
- summary = service.summarize(raw, 'cache', STRATEGY);
- } else {
- raw = await service.fetch(pageUrl, apiKey, STRATEGY);
- summary = service.summarize(raw, 'api', STRATEGY);
-
- ctx.db.data.save({
- url: pageUrl,
- data: {
+ const row = await ctx.db.data.getOrFetch<PageSpeedRow>(
+ pageUrl,
+ async () => {
+ const freshRaw = await service.fetch(pageUrl, apiKey, STRATEGY);
+ const freshSummary = service.summarize(freshRaw, 'api', STRATEGY);
+ return {
strategy: STRATEGY,
- performance_score: summary.score,
- lcp: summary.lcp,
- cls: summary.cls,
- tbt: summary.tbt,
- raw_json: raw
- }
- });
+ performance_score: freshSummary.score,
+ lcp: freshSummary.lcp,
+ cls: freshSummary.cls,
+ tbt: freshSummary.tbt,
+ raw_json: freshRaw
+ } as PageSpeedRow;
+ }
+ );
+
+ if (!row) {
+ ctx.cli?.info(`[plugin:pagespeed] No local cache for ${pageUrl}. Run with --live to fetch from API.`);
+ return;
}
- // Only attach summary to the result object (prevents raw JSON leakage)
- result.plugins.pagespeed = { summary };
+ // Summarize the data that was retrieved (either cached or freshly fetched above)
+ const summary = service.summarize(row.raw_json, row.created_at ? 'cache' : 'api', STRATEGY);
+
PageSpeedOutput.emit(ctx, { summary });
+
} catch (error) {
const message = (error as Error).message;
- result.plugins.pagespeed = { error: message };
PageSpeedOutput.emit(ctx, { error: message });
ctx.logger?.error(`[plugin:pagespeed] ${message}`);
}
diff --git a/packages/plugins/signals/index.ts b/packages/plugins/signals/index.ts
index dc7b962..b086d39 100644
--- a/packages/plugins/signals/index.ts
+++ b/packages/plugins/signals/index.ts
@@ -208,6 +208,7 @@ const SignalsPlugin: CrawlithPlugin = {
});
insertTx(buffer);
+ ctx.metadata = ctx.metadata ?? {};
ctx.metadata.signalsReport = computeSignalsReport(snapshotId);
ctx.metadata.signalsBuffer = [];
},
@@ -216,8 +217,8 @@ const SignalsPlugin: CrawlithPlugin = {
const snapshotId = report?.snapshotId || ctx.snapshotId;
let signalsReport = ctx.metadata?.signalsReport;
- if (!signalsReport && ctx.metadata?.signalsBuffer?.length > 0) {
- const buffer = ctx.metadata.signalsBuffer as BufferedSignal[];
+ if (!signalsReport && (ctx.metadata?.signalsBuffer?.length ?? 0) > 0) {
+ const buffer = (ctx.metadata?.signalsBuffer ?? []) as BufferedSignal[];
const coverage = {
og: buffer.every(s => s.hasOg) ? 100 : (buffer.some(s => s.hasOg) ? 50 : 0),
jsonld: buffer.every(s => s.hasJsonld) ? 100 : (buffer.some(s => s.hasJsonld) ? 50 : 0),
From 69a1d411ae9dd066acf7899bc79b2200ffb9238f Mon Sep 17 00:00:00 2001
From: Saurabh <saurabhsharma2u@gmail.com>
Date: Tue, 3 Mar 2026 11:07:28 -0800
Subject: [PATCH 3/3] docs: update Soft404 README and AI Agent Guide with new
ScoreProvider architecture
---
docs/plugins/ai-agent-guide.md | 35 +++++-
packages/core/src/crawler/crawler.ts | 1 -
packages/core/src/crawler/parser.ts | 10 +-
packages/core/src/db/graphLoader.ts | 1 -
packages/core/src/db/index.ts | 1 -
packages/core/src/db/migrations.ts | 2 -
.../src/db/repositories/PageRepository.ts | 13 +-
packages/core/src/db/schema.ts | 1 -
packages/plugins/soft404-detector/README.md | 64 +++++++++-
packages/plugins/soft404-detector/index.ts | 113 +++---------------
.../plugins/soft404-detector/src/Output.ts | 23 ++++
.../plugins/soft404-detector/src/Service.ts | 73 +++++++++++
.../plugins/soft404-detector/src/plugin.ts | 92 ++++++++++++++
.../plugins/soft404-detector/src/types.ts | 9 ++
.../soft404-detector/tests/soft404.test.ts | 13 +-
15 files changed, 322 insertions(+), 129 deletions(-)
create mode 100644 packages/plugins/soft404-detector/src/Output.ts
create mode 100644 packages/plugins/soft404-detector/src/Service.ts
create mode 100644 packages/plugins/soft404-detector/src/plugin.ts
create mode 100644 packages/plugins/soft404-detector/src/types.ts
diff --git a/docs/plugins/ai-agent-guide.md b/docs/plugins/ai-agent-guide.md
index 217fd7b..5fb0a39 100644
--- a/docs/plugins/ai-agent-guide.md
+++ b/docs/plugins/ai-agent-guide.md
@@ -78,12 +78,41 @@ For data tied to specific pages/URLs.
* `global`: Boolean. If `true`, looks across **all snapshots** instead of just the current one.
* `ctx.db.data.all<T>(): T[]` - Get all rows for your plugin in current snapshot.
-**Example (24h Global Cache):**
+* `ctx.db.data.all<T>(): T[]` - Get all rows for your plugin in current snapshot.
+
+### 3. Declarative Storage (`scoreProvider` & `fetchMode`)
+Plugins should use declarative storage to define their caching and scoring strategies natively:
+
```typescript
-const cached = ctx.db.data.find<MyRow>(url, { maxAge: '24h', global: true });
-if (cached) return cached.data;
+export const MyPlugin: CrawlithPlugin = {
+ name: 'my-plugin',
+ scoreProvider: true, // Enables automatic weighting & aggregation in the core
+ storage: {
+ fetchMode: 'local', // 'network' | 'local'
+ perPage: {
+ columns: {
+ score: 'REAL',
+ weight: 'REAL',
+ reason: 'TEXT'
+ }
+ }
+ }
+};
+```
+Note: `score` and `weight` are reserved columns managed automatically if omitted, but you can explicitly use them to define defaults.
+
+### 4. Smart Caching (`getOrFetch`)
+Replaces `db.data.find()` and `db.data.save()`. The core handles the `--live` flag logic natively:
+
+**Example:**
+```typescript
+const row = await ctx.db.data.getOrFetch<MyRow>(
+ page.url,
+ async () => myService.compute(page.html) // Only executes if cache is stale or --live flag is passed
+);
```
+
---
## 🪝 Plugin Hooks Lifecycle
diff --git a/packages/core/src/crawler/crawler.ts b/packages/core/src/crawler/crawler.ts
index 65fc14f..60f2d9e 100644
--- a/packages/core/src/crawler/crawler.ts
+++ b/packages/core/src/crawler/crawler.ts
@@ -487,7 +487,6 @@ export class Crawler {
nofollow: parseResult.nofollow ? 1 : 0,
content_hash: parseResult.contentHash,
simhash: parseResult.simhash,
- soft404_score: parseResult.soft404Score,
etag: res.etag,
last_modified: res.lastModified,
retries: res.retries
diff --git a/packages/core/src/crawler/parser.ts b/packages/core/src/crawler/parser.ts
index f7028f9..4ca9d38 100644
--- a/packages/core/src/crawler/parser.ts
+++ b/packages/core/src/crawler/parser.ts
@@ -17,8 +17,6 @@ export interface ParseResult {
contentHash: string;
simhash?: string;
uniqueTokenRatio?: number;
- soft404Score: number;
- soft404Signals: string[];
}
export class Parser {
@@ -121,10 +119,6 @@ export class Parser {
const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
const simhash = SimHash.generate(tokens).toString();
- // 5. Soft 404 Detection (Migrated to Soft404DetectorPlugin)
- const soft404Score = 0;
- const soft404Signals: string[] = [];
-
return {
links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
html: html, // pass raw HTML for analysis
@@ -133,9 +127,7 @@ export class Parser {
nofollow,
contentHash,
simhash,
- uniqueTokenRatio,
- soft404Score,
- soft404Signals
+ uniqueTokenRatio
}
}
}
diff --git a/packages/core/src/db/graphLoader.ts b/packages/core/src/db/graphLoader.ts
index ec9f5ee..d02a685 100644
--- a/packages/core/src/db/graphLoader.ts
+++ b/packages/core/src/db/graphLoader.ts
@@ -64,7 +64,6 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
etag: p.etag || undefined,
lastModified: p.last_modified || undefined,
html: p.html || undefined,
- soft404Score: p.soft404_score || undefined,
noindex: !!p.noindex,
nofollow: !!p.nofollow,
incrementalStatus,
diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts
index cff489c..dead079 100644
--- a/packages/core/src/db/index.ts
+++ b/packages/core/src/db/index.ts
@@ -43,7 +43,6 @@ export function getCrawlithDB(): CrawlithDB {
// Migrations for existing tables
try { dbInstance.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`); } catch (_e) { /* ignore */ }
- try { dbInstance.exec(`ALTER TABLE pages ADD COLUMN soft404_score REAL;`); } catch (_e) { /* ignore */ }
// Security controls: Ensure file permissions are 600 (user read/write only)
if (dbPath !== ':memory:') {
diff --git a/packages/core/src/db/migrations.ts b/packages/core/src/db/migrations.ts
index fd7202a..4dfbc09 100644
--- a/packages/core/src/db/migrations.ts
+++ b/packages/core/src/db/migrations.ts
@@ -49,7 +49,6 @@ export function runBaseMigrations(db: Database) {
etag TEXT,
last_modified TEXT,
html TEXT,
- soft404_score REAL,
noindex INTEGER DEFAULT 0,
nofollow INTEGER DEFAULT 0,
security_error TEXT,
@@ -72,7 +71,6 @@ export function runBaseMigrations(db: Database) {
// Migrations for existing tables
try { db.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`); } catch (_e) { /* ignore */ }
- try { db.exec(`ALTER TABLE pages ADD COLUMN soft404_score REAL;`); } catch (_e) { /* ignore */ }
db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
diff --git a/packages/core/src/db/repositories/PageRepository.ts b/packages/core/src/db/repositories/PageRepository.ts
index 233e3dc..fe82bf6 100644
--- a/packages/core/src/db/repositories/PageRepository.ts
+++ b/packages/core/src/db/repositories/PageRepository.ts
@@ -13,7 +13,6 @@ export interface Page {
etag: string | null;
last_modified: string | null;
html: string | null;
- soft404_score: number | null;
noindex: number;
nofollow: number;
security_error: string | null;
@@ -38,13 +37,13 @@ export class PageRepository {
INSERT INTO pages (
site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
- soft404_score, noindex, nofollow, security_error, retries, depth,
+ noindex, nofollow, security_error, retries, depth,
discovered_via_sitemap, redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
updated_at
) VALUES (
@site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
@http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
- @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+ @noindex, @nofollow, @security_error, @retries, @depth,
@discovered_via_sitemap, @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
datetime('now')
)
@@ -57,7 +56,6 @@ export class PageRepository {
etag = COALESCE(excluded.etag, pages.etag),
last_modified = COALESCE(excluded.last_modified, pages.last_modified),
html = COALESCE(excluded.html, pages.html),
- soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
security_error = COALESCE(excluded.security_error, pages.security_error),
@@ -88,7 +86,6 @@ export class PageRepository {
etag: page.etag ?? null,
last_modified: page.last_modified ?? null,
html: page.html ?? null,
- soft404_score: page.soft404_score ?? null,
noindex: page.noindex ?? 0,
nofollow: page.nofollow ?? 0,
security_error: page.security_error ?? null,
@@ -142,13 +139,13 @@ export class PageRepository {
INSERT INTO pages (
site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
- soft404_score, noindex, nofollow, security_error, retries, depth,
+ noindex, nofollow, security_error, retries, depth,
discovered_via_sitemap, redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
updated_at
) VALUES (
@site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
@http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
- @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+ @noindex, @nofollow, @security_error, @retries, @depth,
@discovered_via_sitemap, @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
datetime('now')
)
@@ -161,7 +158,6 @@ export class PageRepository {
etag = COALESCE(excluded.etag, pages.etag),
last_modified = COALESCE(excluded.last_modified, pages.last_modified),
html = COALESCE(excluded.html, pages.html),
- soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
security_error = COALESCE(excluded.security_error, pages.security_error),
@@ -192,7 +188,6 @@ export class PageRepository {
etag: page.etag ?? null,
last_modified: page.last_modified ?? null,
html: page.html ?? null,
- soft404_score: page.soft404_score ?? null,
noindex: page.noindex ?? 0,
nofollow: page.nofollow ?? 0,
security_error: page.security_error ?? null,
diff --git a/packages/core/src/db/schema.ts b/packages/core/src/db/schema.ts
index 8a2b8aa..36a7b63 100644
--- a/packages/core/src/db/schema.ts
+++ b/packages/core/src/db/schema.ts
@@ -45,7 +45,6 @@ export function initSchema(db: Database) {
etag TEXT,
last_modified TEXT,
html TEXT,
- soft404_score REAL,
noindex INTEGER DEFAULT 0,
nofollow INTEGER DEFAULT 0,
security_error TEXT,
diff --git a/packages/plugins/soft404-detector/README.md b/packages/plugins/soft404-detector/README.md
index 60b6e73..fa8c169 100644
--- a/packages/plugins/soft404-detector/README.md
+++ b/packages/plugins/soft404-detector/README.md
@@ -1,9 +1,63 @@
-# Soft404 Detector Plugin
+# 🕵️ Soft 404 Detector Plugin
-Crawlith plugin for soft404 detector
+The **Soft 404 Detector** plugin automatically analyzes site pages to detect "Soft 404" responses—pages that return a HTTP `200 OK` status code but conceptually behave like an error page (e.g., stating "Page Not Found", lacking content, or providing no outbound navigation).
-## Installation
-This plugin is built-in.
+It scores pages between `0.0` and `1.0` and outputs detailed reasoning regarding *why* the page triggered an alert. As a registered **Score Provider** natively mapped into Crawlith's core, its analyses are aggregated continuously across the crawl footprint.
+
+## Features
+
+- **Text Pattern Matching:** Flags `<title>` tags and `<h1>` headers containing error phrases (e.g., `not found`, `error`, `unavailable`).
+- **Body Context Extraction:** Detects explicit in-text error declarations (e.g., "page not found").
+- **Content Density Checking:** Severely low word counts (< 50 words) heavily increase the soft 404 probability.
+- **Link Isolation:** Flags pages acting as dead-ends, lacking any outbound links entirely.
+- **Smart Caching (`fetchMode: 'local'`):** Computes entirely on the local device, safely executing against previously downloaded `html` payloads, supporting instant execution during `page` and `crawl` snapshot pipelines without re-triggering remote requests unless bypassing cache thresholds (`--live`).
## Usage
-Include it in your Crawlith configuration or CLI usage.
+
+Enable the plugin via the CLI using the `--detect-soft404` flag.
+
+### Command Line Interface
+
+```bash
+# Analyze a single page locally for Soft 404 violations
+crawlith page --detect-soft404 https://example.com/missing-product
+
+# Deep crawl a fully site, scanning all discovered URLs for Soft 404 structures
+crawlith crawl --detect-soft404 https://example.com
+```
+
+## Output & Interpretation
+
+### Single Page (`page` command)
+For specific URL lookups, the plugin outputs a simple terminal read-out on execution:
+```plaintext
+[Soft404] https://example.com/broken-page | Score: 1.0 | Reason: title_contains_not found, h1_contains_404, very_low_word_count
+```
+
+### Full Site Crawl JSON Output (`crawl` command)
+During extensive crawls, nodes heavily penalized map into the JSON reporter object (`--format json` or `--export json`):
+
+```json
+{
+ "plugins": {
+ "soft404": {
+ "totalDetected": 3,
+ "topSample": [
+ {
+ "url": "https://example.com/missing",
+ "score": 0.8
+ }
+ ]
+ }
+ }
+}
+```
+
+The plugin also explicitly injects `soft404Score` directly onto graph `Node` objects, enabling graph algorithms to dynamically down-weight links originating from dead-end error patterns.
+
+## Architecture Guidelines
+
+This plugin strictly implements the [AI Agent Guide's](../../../docs/plugins/ai-agent-guide.md) architectural standards:
+- All business logic lives inside a standalone `Soft404Service` object (`src/Service.ts`).
+- Storage leverages the `ctx.db.data.getOrFetch` fluent API.
+- Implements `scoreProvider: true` natively inside its manifest for top-level aggregation logic.
diff --git a/packages/plugins/soft404-detector/index.ts b/packages/plugins/soft404-detector/index.ts
index f03fadc..9776790 100644
--- a/packages/plugins/soft404-detector/index.ts
+++ b/packages/plugins/soft404-detector/index.ts
@@ -1,109 +1,30 @@
-
-import { CrawlithPlugin, PluginContext } from '@crawlith/core';
-import * as cheerio from 'cheerio';
-import { Command } from '@crawlith/core';
+import { CrawlithPlugin } from '@crawlith/core';
+import { Soft404Hooks } from './src/plugin.js';
/**
- * Soft404 Detector Plugin
- * Crawlith plugin for soft404 detector
+ * Description: Soft404 Detector Plugin calculates the likelihood of a Soft 404 response.
+ * @requirements None.
*/
export const Soft404DetectorPlugin: CrawlithPlugin = {
name: 'soft404-detector',
- register: (cli: Command) => {
- if (cli.name() === 'crawl' || cli.name() === 'page') {
- cli.option('--detect-soft404', 'Detect soft 404 pages');
- }
- },
-
- hooks: {
- onMetrics: async (ctx: PluginContext, graph: any) => {
- const flags = ctx.flags || {};
-
- if (!flags.detectSoft404) {
- return;
- }
-
- ctx.logger?.info?.('🕵️ Detecting soft 404 pages...');
-
- const nodes = graph.getNodes();
-
- for (const node of nodes) {
- if (node.status === 200 && node.html) {
- let score = 0;
- const signals: string[] = [];
-
- const $ = cheerio.load(node.html);
- $('script, style, noscript, iframe').remove();
-
- const cleanText = $('body').text().replace(/\s+/g, ' ').trim();
- const title = $('title').text().toLowerCase();
- const h1Text = $('h1').first().text().toLowerCase();
- const bodyText = cleanText.toLowerCase();
- const errorPatterns = ['404', 'not found', 'error', "doesn't exist", 'unavailable', 'invalid'];
-
- for (const pattern of errorPatterns) {
- if (title.includes(pattern)) {
- score += 0.4;
- signals.push(`title_contains_${pattern}`);
- break;
- }
- }
-
- for (const pattern of errorPatterns) {
- if (h1Text.includes(pattern)) {
- score += 0.3;
- signals.push(`h1_contains_${pattern}`);
- break;
- }
- }
-
- if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
- score += 0.2;
- signals.push('body_error_phrase');
- }
-
- const words = cleanText.split(/\s+/).filter(w => w.length > 0);
- if (words.length < 50) {
- score += 0.3;
- signals.push('very_low_word_count');
- } else if (words.length < 150) {
- score += 0.1;
- signals.push('low_word_count');
- }
-
- if (node.outLinks === 0) {
- score += 0.2;
- signals.push('no_outbound_links');
- }
-
- score = Math.min(1.0, score);
-
- if (score > 0) {
- (node as any).soft404 = {
- score: Number(score.toFixed(2)),
- reason: signals.join(', ')
- };
- }
- }
- }
+ cli: {
+ flag: '--detect-soft404',
+ description: 'Detect soft 404 pages'
+ },
- ctx.logger?.info?.(`🕵️ Soft 404 detection complete.`);
- },
- onReport: async (ctx: PluginContext, result: any) => {
- const flags = ctx.flags || {};
- if (!flags.detectSoft404) return;
+ scoreProvider: true,
- const soft404Pages = result.pages.filter((p: any) => p.plugins?.soft404?.score > 0.5);
- if (soft404Pages.length > 0) {
- if (!result.plugins) result.plugins = {};
- result.plugins.soft404 = {
- totalDetected: soft404Pages.length,
- topSample: soft404Pages.slice(0, 5).map((p: any) => ({ url: p.url, score: p.plugins.soft404.score }))
- };
+ storage: {
+ fetchMode: 'local',
+ perPage: {
+ columns: {
+ reason: 'TEXT'
}
}
- }
+ },
+
+ hooks: Soft404Hooks
};
export default Soft404DetectorPlugin;
diff --git a/packages/plugins/soft404-detector/src/Output.ts b/packages/plugins/soft404-detector/src/Output.ts
new file mode 100644
index 0000000..9fab8b3
--- /dev/null
+++ b/packages/plugins/soft404-detector/src/Output.ts
@@ -0,0 +1,23 @@
+import { PluginContext } from '@crawlith/core';
+import { Soft404Result } from './types.js';
+
+/**
+ * Handles presentation and standardized logging for the Soft404 plugin.
+ */
+export class Soft404Output {
+ /**
+ * Emits logging output for a single evaluated page.
+ * @param {PluginContext} ctx - Plugin context for logging access.
+ * @param {string} url - Target URL.
+ * @param {Soft404Result} result - The evaluation payload.
+ */
+ public static emitSingle(ctx: PluginContext, url: string, result: Soft404Result): void {
+ if (!ctx.logger) return;
+
+ if (result.score === 0) {
+ ctx.logger.info(`[Soft404] ${url} | Score: 0 (No issues detected)`);
+ } else {
+ ctx.logger.warn(`[Soft404] ${url} | Score: ${result.score} | Reason: ${result.reason}`);
+ }
+ }
+}
diff --git a/packages/plugins/soft404-detector/src/Service.ts b/packages/plugins/soft404-detector/src/Service.ts
new file mode 100644
index 0000000..4c2e8ee
--- /dev/null
+++ b/packages/plugins/soft404-detector/src/Service.ts
@@ -0,0 +1,73 @@
+import * as cheerio from 'cheerio';
+import { Soft404Result } from './types.js';
+
+/**
+ * Service to analyze HTML content for soft 404 signals.
+ * Extracts signals from title, H1, body content length, and outlinks.
+ */
+export class Soft404Service {
+ /**
+ * Analyzes HTML string to determine probability of being a soft 404 page.
+ * @param {string | undefined} html - Raw HTML source code.
+ * @param {number} outLinks - Total number of outbound links extracted during parsing.
+ * @returns {Soft404Result} A calculated score between 0.0 and 1.0, and the matched reasons.
+ */
+ public analyze(html: string | undefined, outLinks: number): Soft404Result {
+ if (!html) return { score: 0, reason: '' };
+
+ let score = 0;
+ const signals: string[] = [];
+
+ const $ = cheerio.load(html);
+ $('script, style, noscript, iframe').remove();
+
+ const cleanText = $('body').text().replace(/\s+/g, ' ').trim();
+ const title = $('title').text().toLowerCase();
+ const h1Text = $('h1').first().text().toLowerCase();
+ const bodyText = cleanText.toLowerCase();
+
+ const errorPatterns = ['404', 'not found', 'error', "doesn't exist", 'unavailable', 'invalid'];
+
+ for (const pattern of errorPatterns) {
+ if (title.includes(pattern)) {
+ score += 0.4;
+ signals.push(`title_contains_${pattern}`);
+ break;
+ }
+ }
+
+ for (const pattern of errorPatterns) {
+ if (h1Text.includes(pattern)) {
+ score += 0.3;
+ signals.push(`h1_contains_${pattern}`);
+ break;
+ }
+ }
+
+ if (bodyText.includes('page not found') || bodyText.includes('404 error')) {
+ score += 0.2;
+ signals.push('body_error_phrase');
+ }
+
+ const words = cleanText.split(/\s+/).filter(w => w.length > 0);
+ if (words.length < 50) {
+ score += 0.3;
+ signals.push('very_low_word_count');
+ } else if (words.length < 150) {
+ score += 0.1;
+ signals.push('low_word_count');
+ }
+
+ if (outLinks === 0) {
+ score += 0.2;
+ signals.push('no_outbound_links');
+ }
+
+ score = Math.min(1.0, score);
+
+ return {
+ score: Number(score.toFixed(2)),
+ reason: signals.join(', ')
+ };
+ }
+}
diff --git a/packages/plugins/soft404-detector/src/plugin.ts b/packages/plugins/soft404-detector/src/plugin.ts
new file mode 100644
index 0000000..d0e3014
--- /dev/null
+++ b/packages/plugins/soft404-detector/src/plugin.ts
@@ -0,0 +1,92 @@
+import { PluginContext, PageInput } from '@crawlith/core';
+import { Soft404Service } from './Service.js';
+import { Soft404Row } from './types.js';
+
+export const Soft404Hooks = {
+ /**
+ * Called during `crawlith page` command to analyze the single page immediately.
+ */
+ onPage: async (ctx: PluginContext, page: PageInput) => {
+ if (!ctx.flags?.detectSoft404 || !ctx.db) return;
+
+ const service = new Soft404Service();
+ // Cache the soft404 result natively inside SQLite via getOrFetch.
+ const row = await ctx.db.data.getOrFetch<Soft404Row>(
+ page.url,
+ async () => service.analyze(page.html, 0 /* onPage command doesn't provide fully built links graph yet */)
+ );
+
+ if (row && ctx.logger) {
+ if (row.score > 0) {
+ ctx.logger.warn(`[soft404-detector] Score for ${page.url}: ${row.score} (${row.reason})`);
+ } else {
+ ctx.logger.info(`[soft404-detector] Score for ${page.url}: 0 (No issues detected)`);
+ }
+ }
+ },
+
+ /**
+ * Called on the fully built graph when a `crawl` evaluates metrics across the snapshot.
+ */
+ onMetrics: async (ctx: PluginContext, graph: any) => {
+ const flags = ctx.flags || {};
+ if (!flags.detectSoft404 || !ctx.db) return;
+
+ ctx.logger?.info?.('🕵️ Detecting soft 404 pages...');
+
+ const service = new Soft404Service();
+ const nodes = graph.getNodes();
+
+ let issueCount = 0;
+
+ for (const node of nodes) {
+ if (node.status === 200) {
+ // Caches internally inside 'soft404_detector_plugin' SQLite table.
+ const row = await ctx.db.data.getOrFetch<Soft404Row>(
+ node.url,
+ async () => service.analyze(node.html, node.outLinks || 0)
+ );
+
+ if (row && row.score > 0) {
+ (node as any).soft404 = {
+ score: row.score,
+ reason: row.reason
+ };
+ // Map back to node so pagerank/crawlers can read it downstream.
+ node.soft404Score = row.score;
+ issueCount++;
+ }
+ }
+ }
+
+ if (issueCount > 0) {
+ ctx.logger?.warn(`🕵️ Soft 404 detection complete. Found ${issueCount} issues!`);
+ } else {
+ ctx.logger?.info(`🕵️ Soft 404 detection complete.`);
+ }
+ },
+
+ /**
+ * Evaluates the active snapshot results and attaches them statically mapped onto the final JSON reporter.
+ */
+ onReport: async (ctx: PluginContext, result: any) => {
+ const flags = ctx.flags || {};
+ if (!flags.detectSoft404 || !ctx.db) return;
+
+ // Pull directly from our scoped SQLite plugin table
+ const allRows = ctx.db.data.all<{ url_id: number; score: number; reason: string }>();
+ const soft404Pages = allRows.filter(r => r.score > 0.5);
+
+ if (soft404Pages.length > 0) {
+ if (!result.plugins) result.plugins = {};
+
+ const mappedPages = (result.pages || []).filter((p: any) => p.soft404Score && p.soft404Score > 0.5)
+ .map((p: any) => ({ url: p.url, score: p.soft404Score }));
+
+ result.plugins.soft404 = {
+ totalDetected: mappedPages.length > 0 ? mappedPages.length : soft404Pages.length,
+ topSample: mappedPages.slice(0, 5)
+ };
+ }
+ }
+};
diff --git a/packages/plugins/soft404-detector/src/types.ts b/packages/plugins/soft404-detector/src/types.ts
new file mode 100644
index 0000000..0ece4fe
--- /dev/null
+++ b/packages/plugins/soft404-detector/src/types.ts
@@ -0,0 +1,9 @@
+export interface Soft404Result {
+ score: number;
+ reason: string;
+}
+
+export interface Soft404Row {
+ score: number;
+ reason: string;
+}
diff --git a/packages/plugins/soft404-detector/tests/soft404.test.ts b/packages/plugins/soft404-detector/tests/soft404.test.ts
index 3fb9833..fb2adc9 100644
--- a/packages/plugins/soft404-detector/tests/soft404.test.ts
+++ b/packages/plugins/soft404-detector/tests/soft404.test.ts
@@ -12,7 +12,18 @@ describe('Soft 404 Detection Plugin', () => {
node.html = html;
node.outLinks = outLinks;
- await Soft404DetectorPlugin.hooks!.onMetrics!({ flags: { detectSoft404: true } } as any, graph);
+ const ctx: any = {
+ flags: { detectSoft404: true },
+ db: {
+ data: {
+ getOrFetch: async (url: string, fetchFn: Function) => {
+ return await fetchFn();
+ }
+ }
+ }
+ };
+
+ await Soft404DetectorPlugin.hooks!.onMetrics!(ctx, graph);
return node as any;
};