Redesign snapshot-diff plugin to compare persisted snapshots

packages/plugins/snapshot-diff/README.md

@@ -1,9 +1,59 @@
 # Snapshot Diff Plugin
 
-Crawlith plugin for snapshot diff
+The `snapshot-diff` plugin adds snapshot-aware graph comparison and incremental baseline loading to the `crawl` command.
 
-## Installation
-This plugin is built-in.
+## What this plugin does
 
-## Usage
-Include it in your Crawlith configuration or CLI usage.
+- Adds `--compare <oldSnapshotId> <newSnapshotId>` to compare **two saved Crawlith snapshots**.
+- Adds `--incremental` to preload the latest completed snapshot graph for the same domain before a new crawl.
+- Produces either:
+  - a human-readable terminal summary (`pretty` default), or
+  - a raw JSON diff payload when `--format json` is used.
+
+## CLI options
+
+- `--compare <oldSnapshotId> <newSnapshotId>`  
+  Internal flag that compares two persisted snapshot IDs and exits before crawling.
+- `--incremental`  
+  Resolves the latest completed snapshot for the target domain and stores it in plugin metadata for incremental graph diffing.
+
+## Usage examples
+
+### Compare two snapshots
+
+```bash
+crawlith crawl https://example.com --compare 101 102
+```
+
+### Compare two snapshots in JSON format
+
+```bash
+crawlith crawl https://example.com --compare 101 102 --format json
+```
+
+### Run a normal incremental crawl
+
+```bash
+crawlith crawl https://example.com --incremental
+```
+
+## Output details
+
+### Pretty mode output
+
+The plugin prints:
+
+- Compared snapshot IDs.
+- Added URL count.
+- Removed URL count.
+- Status change count.
+- Metric delta table (positive/negative/zero colorized values).
+
+### JSON mode output
+
+The plugin emits the full diff object from Crawlith core (the same payload returned by `compareGraphs`).
+
+## Notes
+
+- `--compare` requires exactly two **numeric** snapshot IDs.
+- The plugin does not read graph JSON files from disk.

packages/plugins/snapshot-diff/index.ts

@@ -1,101 +1,20 @@
-
-import { CrawlithPlugin, PluginContext, compareGraphs, Graph, SiteRepository, SnapshotRepository, getDb, loadGraphFromSnapshot } from '@crawlith/core';
-import fs from 'node:fs/promises';
-import chalk from 'chalk';
-import { Command } from '@crawlith/core';
+import type { CrawlithPlugin, Command } from '@crawlith/core';
+import { registerSnapshotDiffCli } from './src/cli.js';
+import { snapshotDiffHooks } from './src/plugin.js';
 
 /**
- * Snapshot Diff Plugin
- * Crawlith plugin for snapshot diff
+ * Snapshot Diff Plugin.
+ *
+ * Provides two crawl-time features:
+ * - snapshot-to-snapshot graph comparison via `--compare <oldSnapshotId> <newSnapshotId>`
+ * - incremental baseline loading via `--incremental`
  */
 export const SnapshotDiffPlugin: CrawlithPlugin = {
   name: 'snapshot-diff',
   register: (cli: Command) => {
-    if (cli.name() === 'crawl') {
-      cli
-        .option("--incremental", "incremental crawl using previous snapshot")
-        .option("--compare <files...>", "internal: compare two graph JSON files");
-    }
+    registerSnapshotDiffCli(cli);
   },
-
-  hooks: {
-    onInit: async (ctx: PluginContext) => {
-      const flags = ctx.flags || {};
-      if (flags.compare) {
-        const files = flags.compare as unknown as string[];
-        if (files.length !== 2) {
-          console.error(chalk.red('Error: --compare requires exactly two file paths (old.json new.json)'));
-          process.exit(1);
-        }
-        const [oldFile, newFile] = files;
-        const fmt = typeof flags.format === 'string' ? flags.format : undefined;
-
-        if (fmt !== 'json') {
-          console.log(chalk.cyan(`\n🔍 Comparing Graphs`));
-          console.log(`${chalk.gray('Old:')} ${oldFile}`);
-          console.log(`${chalk.gray('New:')} ${newFile}\n`);
-        }
-
-        const oldJson = JSON.parse(await fs.readFile(oldFile, 'utf-8'));
-        const newJson = JSON.parse(await fs.readFile(newFile, 'utf-8'));
-
-        const oldGraph = Graph.fromJSON(oldJson);
-        const newGraph = Graph.fromJSON(newJson);
-
-        const diffResult = compareGraphs(oldGraph, newGraph);
-
-        if (fmt !== 'json') {
-          console.log(chalk.bold('📈 Comparison Results:'));
-          console.log(`- Added URLs:   ${chalk.green(diffResult.addedUrls.length)}`);
-          console.log(`- Removed URLs: ${chalk.red(diffResult.removedUrls.length)}`);
-          console.log(`- Status Changes: ${chalk.yellow(diffResult.changedStatus.length)}`);
-
-          console.log(chalk.bold('\n📉 Metric Deltas:'));
-          Object.entries(diffResult.metricDeltas).forEach(([metric, delta]) => {
-            const deltaStr = (delta as number) > 0 ? chalk.green(`+${(delta as number).toFixed(3)}`) : ((delta as number) < 0 ? chalk.red((delta as number).toFixed(3)) : chalk.gray('0'));
-            console.log(`  ${metric.padEnd(20)}: ${deltaStr}`);
-          });
-
-          console.log('\n' + chalk.gray('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') + '\n');
-        } else {
-          console.log(JSON.stringify(diffResult, null, 2));
-        }
-        ctx.terminate = true;
-      }
-    },
-    onCrawlStart: async (ctx: PluginContext) => {
-      const flags = ctx.flags || {};
-
-      if (flags.incremental) {
-        ctx.logger?.info('🔍 Resolving previous snapshot for incremental crawl...');
-        const targetUrl = flags.url || (ctx as any).url;
-
-        if (!targetUrl || targetUrl.startsWith('-')) {
-          return;
-        }
-
-        const db = getDb();
-        const siteRepo = new SiteRepository(db);
-        const snapRepo = new SnapshotRepository(db);
-
-        try {
-          const urlObj = new URL(targetUrl);
-          const domain = urlObj.hostname.replace('www.', '');
-          const site = siteRepo.getSite(domain);
-          if (site) {
-            const latestSnap = snapRepo.getLatestSnapshot(site.id, 'completed');
-            if (latestSnap) {
-              ctx.logger?.info(`Found previous snapshot #${latestSnap.id}, loading graph for delta comparison...`);
-              if (!ctx.metadata) ctx.metadata = {};
-              ctx.metadata.previousGraph = loadGraphFromSnapshot(latestSnap.id);
-            }
-          }
-        } catch (_e) {
-          // ignore url parsing failures
-        }
-      }
-    }
-  }
+  hooks: snapshotDiffHooks
 };
 
 export default SnapshotDiffPlugin;

packages/plugins/snapshot-diff/package.json

@@ -13,5 +13,5 @@
     "test": "vitest run"
   },
   "exports": "./index.ts",
-  "description": "Crawlith plugin for snapshot diff"
+  "description": "Crawlith plugin for snapshot-to-snapshot graph diffing and incremental baselines"
 }

packages/plugins/snapshot-diff/src/Output.ts

@@ -0,0 +1,40 @@
+import chalk from 'chalk';
+
+/**
+ * Renders human-readable output for snapshot comparison.
+ * @param oldSnapshotId Baseline snapshot ID.
+ * @param newSnapshotId Target snapshot ID.
+ * @param diffResult Graph diff payload from core.
+ */
+export function renderPrettyDiffOutput(oldSnapshotId: number, newSnapshotId: number, diffResult: any): void {
+  console.log(chalk.cyan('\n🔍 Comparing Snapshots'));
+  console.log(`${chalk.gray('Old snapshot:')} #${oldSnapshotId}`);
+  console.log(`${chalk.gray('New snapshot:')} #${newSnapshotId}\n`);
+
+  console.log(chalk.bold('📈 Comparison Results:'));
+  console.log(`- Added URLs:      ${chalk.green(diffResult.addedUrls.length)}`);
+  console.log(`- Removed URLs:    ${chalk.red(diffResult.removedUrls.length)}`);
+  console.log(`- Status Changes:  ${chalk.yellow(diffResult.changedStatus.length)}`);
+
+  console.log(chalk.bold('\n📉 Metric Deltas:'));
+  Object.entries(diffResult.metricDeltas).forEach(([metric, delta]) => {
+    const numericDelta = Number(delta);
+    const deltaLabel = numericDelta > 0
+      ? chalk.green(`+${numericDelta.toFixed(3)}`)
+      : numericDelta < 0
+        ? chalk.red(numericDelta.toFixed(3))
+        : chalk.gray('0');
+
+    console.log(`  ${metric.padEnd(20)}: ${deltaLabel}`);
+  });
+
+  console.log(`\n${chalk.gray('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')}\n`);
+}
+
+/**
+ * Emits machine-readable JSON output.
+ * @param diffResult Graph diff payload from core.
+ */
+export function renderJsonDiffOutput(diffResult: unknown): void {
+  console.log(JSON.stringify(diffResult, null, 2));
+}

packages/plugins/snapshot-diff/src/Service.ts

@@ -0,0 +1,90 @@
+import { compareGraphs, getDb, loadGraphFromSnapshot, SiteRepository, SnapshotRepository } from '@crawlith/core';
+import type { Graph, PluginContext } from '@crawlith/core';
+import type { SnapshotCompareRequest, SnapshotDiffFlags, SnapshotDiffOutputFormat } from './types.js';
+
+/**
+ * Encapsulates business logic for snapshot comparison and incremental baseline loading.
+ */
+export class SnapshotDiffService {
+  /**
+   * Parses and validates `--compare` payload as a pair of numeric snapshot IDs.
+   * @param flags CLI flags available on plugin context.
+   * @returns Parsed compare request or `null` when compare mode is disabled.
+   */
+  parseCompareRequest(flags: SnapshotDiffFlags): SnapshotCompareRequest | null {
+    if (!flags.compare) return null;
+
+    const rawValues = Array.isArray(flags.compare)
+      ? flags.compare
+      : [flags.compare];
+
+    if (rawValues.length !== 2) {
+      throw new Error('Error: --compare requires exactly two snapshot IDs (<oldSnapshotId> <newSnapshotId>).');
+    }
+
+    const [oldRaw, newRaw] = rawValues.map(String);
+    const oldSnapshotId = Number.parseInt(oldRaw, 10);
+    const newSnapshotId = Number.parseInt(newRaw, 10);
+
+    if (!Number.isFinite(oldSnapshotId) || !Number.isFinite(newSnapshotId)) {
+      throw new Error('Error: --compare only accepts numeric snapshot IDs.');
+    }
+
+    return { oldSnapshotId, newSnapshotId };
+  }
+
+  /**
+   * Resolves output format from CLI flags.
+   * @param flags CLI flags from plugin context.
+   * @returns Selected output format.
+   */
+  parseOutputFormat(flags: SnapshotDiffFlags): SnapshotDiffOutputFormat {
+    return flags.format === 'json' ? 'json' : 'pretty';
+  }
+
+  /**
+   * Compares two persisted snapshots.
+   * @param request Pair of snapshot IDs.
+   * @returns Diff result generated by Crawlith core.
+   */
+  compareSnapshots(request: SnapshotCompareRequest) {
+    const oldGraph = loadGraphFromSnapshot(request.oldSnapshotId);
+    const newGraph = loadGraphFromSnapshot(request.newSnapshotId);
+    return compareGraphs(oldGraph, newGraph);
+  }
+
+  /**
+   * Resolves previous completed snapshot graph for incremental crawl mode.
+   * @param targetUrl URL passed to crawl command.
+   * @returns Previous graph when found, otherwise `null`.
+   */
+  resolvePreviousGraph(targetUrl: string): Graph | null {
+    const db = getDb();
+    const siteRepo = new SiteRepository(db);
+    const snapshotRepo = new SnapshotRepository(db);
+
+    const domain = new URL(targetUrl).hostname.replace('www.', '');
+    const site = siteRepo.getSite(domain);
+    if (!site) return null;
+
+    const latestSnapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed');
+    if (!latestSnapshot) return null;
+
+    return loadGraphFromSnapshot(latestSnapshot.id);
+  }
+
+  /**
+   * Reads a valid target URL from context flags for incremental mode.
+   * @param ctx Plugin context.
+   * @returns Candidate URL or `null` when missing.
+   */
+  getTargetUrl(ctx: PluginContext): string | null {
+    const flags = (ctx.flags || {}) as SnapshotDiffFlags;
+    const fromFlags = typeof flags.url === 'string' ? flags.url : null;
+    if (fromFlags && !fromFlags.startsWith('-')) return fromFlags;
+
+    return typeof (ctx as PluginContext & { url?: unknown }).url === 'string'
+      ? String((ctx as PluginContext & { url?: unknown }).url)
+      : null;
+  }
+}

packages/plugins/snapshot-diff/src/cli.ts

@@ -0,0 +1,13 @@
+import { Command } from '@crawlith/core';
+
+/**
+ * Registers snapshot-diff CLI options on the crawl command.
+ * @param cli Crawlith command instance.
+ */
+export function registerSnapshotDiffCli(cli: Command): void {
+  if (cli.name() !== 'crawl') return;
+
+  cli
+    .option('--incremental', 'incremental crawl using previous completed snapshot')
+    .option('--compare <snapshots...>', 'internal: compare two snapshot IDs');
+}

packages/plugins/snapshot-diff/src/plugin.ts

@@ -0,0 +1,57 @@
+import type { PluginContext } from '@crawlith/core';
+import { renderJsonDiffOutput, renderPrettyDiffOutput } from './Output.js';
+import { SnapshotDiffService } from './Service.js';
+import type { SnapshotDiffFlags } from './types.js';
+
+const service = new SnapshotDiffService();
+
+/**
+ * Hook implementation for snapshot diff and incremental baseline resolution.
+ */
+export const snapshotDiffHooks = {
+  /**
+   * Handles `--compare` execution before crawl starts.
+   * @param ctx Plugin context for current command invocation.
+   */
+  onInit: async (ctx: PluginContext) => {
+    const flags = (ctx.flags || {}) as SnapshotDiffFlags;
+    const compareRequest = service.parseCompareRequest(flags);
+    if (!compareRequest) return;
+
+    const diffResult = service.compareSnapshots(compareRequest);
+    const outputFormat = service.parseOutputFormat(flags);
+
+    if (outputFormat === 'json') {
+      renderJsonDiffOutput(diffResult);
+    } else {
+      renderPrettyDiffOutput(compareRequest.oldSnapshotId, compareRequest.newSnapshotId, diffResult);
+    }
+
+    ctx.terminate = true;
+  },
+
+  /**
+   * Loads previous completed snapshot graph when incremental mode is enabled.
+   * @param ctx Plugin context for current crawl execution.
+   */
+  onCrawlStart: async (ctx: PluginContext) => {
+    const flags = (ctx.flags || {}) as SnapshotDiffFlags;
+    if (!flags.incremental) return;
+
+    const targetUrl = service.getTargetUrl(ctx);
+    if (!targetUrl) return;
+
+    ctx.logger?.info('🔍 Resolving previous snapshot for incremental crawl...');
+
+    try {
+      const previousGraph = service.resolvePreviousGraph(targetUrl);
+      if (!previousGraph) return;
+
+      if (!ctx.metadata) ctx.metadata = {};
+      ctx.metadata.previousGraph = previousGraph;
+      ctx.logger?.info('Loaded previous snapshot graph for incremental diffing.');
+    } catch {
+      ctx.logger?.warn('Unable to resolve previous snapshot graph for incremental crawl.');
+    }
+  }
+};