Redesign heading-health plugin to match Crawlith plugin standards

docs/plugins/ai-agent-guide.md

@@ -78,12 +78,41 @@ For data tied to specific pages/URLs.
     *   `global`: Boolean. If `true`, looks across **all snapshots** instead of just the current one.
 *   `ctx.db.data.all<T>(): T[]` - Get all rows for your plugin in current snapshot.
 
-**Example (24h Global Cache):**
+*   `ctx.db.data.all<T>(): T[]` - Get all rows for your plugin in current snapshot.
+
+### 3. Declarative Storage (`scoreProvider` & `fetchMode`)
+Plugins should use declarative storage to define their caching and scoring strategies natively:
+
 ```typescript
-const cached = ctx.db.data.find<MyRow>(url, { maxAge: '24h', global: true });
-if (cached) return cached.data;
+export const MyPlugin: CrawlithPlugin = {
+  name: 'my-plugin',
+  scoreProvider: true, // Enables automatic weighting & aggregation in the core
+  storage: {
+    fetchMode: 'local', // 'network' | 'local'
+    perPage: {
+      columns: {
+        score: 'REAL',
+        weight: 'REAL',
+        reason: 'TEXT'
+      }
+    }
+  }
+};
+```
+Note: `score` and `weight` are reserved columns managed automatically if omitted, but you can explicitly use them to define defaults.
+
+### 4. Smart Caching (`getOrFetch`)
+Replaces `db.data.find()` and `db.data.save()`. The core handles the `--live` flag logic natively:
+
+**Example:**
+```typescript
+const row = await ctx.db.data.getOrFetch<MyRow>(
+  page.url,
+  async () => myService.compute(page.html) // Only executes if cache is stale or --live flag is passed
+);
 ```
 
+
 ---
 
 ## 🪝 Plugin Hooks Lifecycle

packages/core/src/analysis/analyze.ts

@@ -215,48 +215,10 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context?
     crawledAt
   };
 
-  if (options.plugins && options.plugins.length > 0) {
-    const { PluginRegistry } = await import('../plugin-system/plugin-registry.js');
-    const { getCrawlithDB } = await import('../db/index.js');
-    const registry = new PluginRegistry(options.plugins);
-    const pluginCtx = options.pluginContext || { command: 'page' };
-    if (!pluginCtx.db) pluginCtx.db = getCrawlithDB();
-
-    await registry.runHook('onInit', pluginCtx);
-    await registry.runHook('onMetrics', pluginCtx, crawlData.graph);
-
-    // Map graph node plugin data back to the results
-    for (const page of result.pages) {
-      const node = crawlData.graph.nodes.get(page.url);
-      if (node) {
-        const extra: Record<string, any> = {};
-        // Common node properties to exclude
-        const internalKeys = [
-          'url', 'status', 'html', 'depth', 'crawlStatus', 'metadata',
-          'inLinks', 'outLinks', 'metrics', 'rank', 'hub', 'authority',
-          'canonical', 'noindex', 'nofollow', 'brokenLinks', 'redirectChain',
-          'incrementalStatus', 'etag', 'lastModified', 'contentHash',
-          'isCollapsed', 'collapseInto', 'simhash', 'uniqueTokenRatio',
-          'securityError', 'retries', 'bytesReceived', 'wordCount',
-          'thinContentScore', 'externalLinkRatio', 'h1Count', 'h2Count', 'title'
-        ];
-        for (const [key, value] of Object.entries(node)) {
-          if (value !== undefined && !internalKeys.includes(key)) {
-            extra[key] = value;
-          }
-        }
-        if (Object.keys(extra).length > 0) {
-          page.plugins = { ...(page.plugins || {}), ...extra };
-        }
-      }
-    }
-
-    await registry.runHook('onReport', pluginCtx, result);
-  }
-
   return result;
 }
 
+
 export function analyzePages(rootUrl: string, pages: Iterable<CrawlPage> | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] {
   const titleCounts = new Map<string, number>();
   const metaCounts = new Map<string, number>();

packages/core/src/application/usecases.ts

@@ -40,13 +40,16 @@ export interface SiteCrawlInput {
 
 export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphResult> {
   async execute(input: SiteCrawlInput): Promise<CrawlSitegraphResult> {
-    const ctx = input.context ?? { command: 'crawl' };
+    const ctx = input.context ?? { command: 'crawl', scope: 'crawl' as const };
+    ctx.scope = 'crawl';
     ctx.db = getCrawlithDB();
     const registry = new PluginRegistry(input.plugins ?? []);
 
+    await registry.applyStorage(ctx);
     await registry.runHook('onInit', ctx);
     if (ctx.terminate) return { snapshotId: 0, graph: undefined as any };
 
+
     await registry.runHook('onCrawlStart', ctx);
     if (ctx.terminate) return { snapshotId: 0, graph: undefined as any };
 
@@ -85,6 +88,10 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes
       computeHITS: false
     });
 
+    if (ctx.db) {
+      ctx.db.aggregateScoreProviders(snapshotId, registry.pluginsList);
+    }
+
     await registry.runHook('onReport', ctx, { snapshotId, graph });
     return { snapshotId, graph };
   }
@@ -96,8 +103,12 @@ export class AnalyzeSnapshot implements UseCase<{ url: string; options: AnalyzeO
 
     if (input.plugins && input.plugins.length > 0) {
       const registry = new PluginRegistry(input.plugins);
-      const ctx = input.context ?? { command: 'analyze' };
-      ctx.db = getCrawlithDB();
+      const ctx: PluginContext = {
+        command: 'analyze',
+        scope: 'crawl',
+        ...(input.context || {}),
+        db: getCrawlithDB()
+      };
       await registry.runHook('onInit', ctx);
       await registry.runHook('onReport', ctx, result);
     }
@@ -143,13 +154,36 @@ export class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisR
       minClusterSize: input.minClusterSize,
       debug: input.debug,
       allPages: input.allPages,
-      plugins: input.plugins,
-      pluginContext: {
-        ...(input.context || { command: 'page' }),
-        db: getCrawlithDB()
-      }
     }, this.context);
 
+    // Run plugins with page scope — only onInit + onPage are called
+    if (input.plugins && input.plugins.length > 0) {
+      const { PluginRegistry } = await import('../plugin-system/plugin-registry.js');
+      const registry = new PluginRegistry(input.plugins);
+
+      const pluginCtx: PluginContext = {
+        command: 'page',
+        scope: 'page',
+        targetUrl: input.url,
+        snapshotId: result.snapshotId,
+        live: input.live || !!(input.context?.flags?.live),
+        ...(input.context || {}),
+        db: getCrawlithDB(),
+      };
+
+      await registry.applyStorage(pluginCtx);
+      await registry.runHook('onInit', pluginCtx);
+
+      // Fire onPage once per analyzed page (normally just 1 for the page command)
+      for (const page of result.pages) {
+        await registry.runHook('onPage', pluginCtx, {
+          url: page.url,
+          html: (page as any).html ?? '',
+          status: page.status,
+        });
+      }
+    }
+
     return result;
   }
 }

packages/core/src/crawler/crawler.ts

@@ -487,7 +487,6 @@ export class Crawler {
       nofollow: parseResult.nofollow ? 1 : 0,
       content_hash: parseResult.contentHash,
       simhash: parseResult.simhash,
-      soft404_score: parseResult.soft404Score,
       etag: res.etag,
       last_modified: res.lastModified,
       retries: res.retries

packages/core/src/crawler/parser.ts

@@ -17,8 +17,6 @@ export interface ParseResult {
   contentHash: string;
   simhash?: string;
   uniqueTokenRatio?: number;
-  soft404Score: number;
-  soft404Signals: string[];
 }
 
 export class Parser {
@@ -121,10 +119,6 @@ export class Parser {
     const uniqueTokenRatio = tokens.length > 0 ? (uniqueTokens.size / tokens.length) : 0;
     const simhash = SimHash.generate(tokens).toString();
 
-    // 5. Soft 404 Detection (Migrated to Soft404DetectorPlugin)
-    const soft404Score = 0;
-    const soft404Signals: string[] = [];
-
     return {
       links: Array.from(links.entries()).map(([url, weight]) => ({ url, weight })),
       html: html, // pass raw HTML for analysis
@@ -133,9 +127,7 @@ export class Parser {
       nofollow,
       contentHash,
       simhash,
-      uniqueTokenRatio,
-      soft404Score,
-      soft404Signals
+      uniqueTokenRatio
     }
   }
 }