From 60fb18d222f4a0128d166b4489398553cf8f2c33 Mon Sep 17 00:00:00 2001 From: Saurabh <41580629+saurabhsharma2u@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:33:14 -0800 Subject: [PATCH] redesign snapshot-diff plugin for snapshot ID comparison --- packages/plugins/snapshot-diff/README.md | 60 +++++++++- packages/plugins/snapshot-diff/index.ts | 101 ++--------------- packages/plugins/snapshot-diff/package.json | 2 +- packages/plugins/snapshot-diff/src/Output.ts | 40 +++++++ packages/plugins/snapshot-diff/src/Service.ts | 90 +++++++++++++++ packages/plugins/snapshot-diff/src/cli.ts | 13 +++ packages/plugins/snapshot-diff/src/plugin.ts | 57 ++++++++++ packages/plugins/snapshot-diff/src/types.ts | 22 ++++ .../snapshot-diff/tests/snapshotDiff.test.ts | 105 ++++++++++++------ 9 files changed, 361 insertions(+), 129 deletions(-) create mode 100644 packages/plugins/snapshot-diff/src/Output.ts create mode 100644 packages/plugins/snapshot-diff/src/Service.ts create mode 100644 packages/plugins/snapshot-diff/src/cli.ts create mode 100644 packages/plugins/snapshot-diff/src/plugin.ts create mode 100644 packages/plugins/snapshot-diff/src/types.ts diff --git a/packages/plugins/snapshot-diff/README.md b/packages/plugins/snapshot-diff/README.md index 11674a5..359541f 100644 --- a/packages/plugins/snapshot-diff/README.md +++ b/packages/plugins/snapshot-diff/README.md @@ -1,9 +1,59 @@ # Snapshot Diff Plugin -Crawlith plugin for snapshot diff +The `snapshot-diff` plugin adds snapshot-aware graph comparison and incremental baseline loading to the `crawl` command. -## Installation -This plugin is built-in. +## What this plugin does -## Usage -Include it in your Crawlith configuration or CLI usage. +- Adds `--compare ` to compare **two saved Crawlith snapshots**. +- Adds `--incremental` to preload the latest completed snapshot graph for the same domain before a new crawl. +- Produces either: + - a human-readable terminal summary (`pretty` default), or + - a raw JSON diff payload when `--format json` is used. + +## CLI options + +- `--compare ` + Internal flag that compares two persisted snapshot IDs and exits before crawling. +- `--incremental` + Resolves the latest completed snapshot for the target domain and stores it in plugin metadata for incremental graph diffing. + +## Usage examples + +### Compare two snapshots + +```bash +crawlith crawl https://example.com --compare 101 102 +``` + +### Compare two snapshots in JSON format + +```bash +crawlith crawl https://example.com --compare 101 102 --format json +``` + +### Run a normal incremental crawl + +```bash +crawlith crawl https://example.com --incremental +``` + +## Output details + +### Pretty mode output + +The plugin prints: + +- Compared snapshot IDs. +- Added URL count. +- Removed URL count. +- Status change count. +- Metric delta table (positive/negative/zero colorized values). + +### JSON mode output + +The plugin emits the full diff object from Crawlith core (the same payload returned by `compareGraphs`). + +## Notes + +- `--compare` requires exactly two **numeric** snapshot IDs. +- The plugin does not read graph JSON files from disk. diff --git a/packages/plugins/snapshot-diff/index.ts b/packages/plugins/snapshot-diff/index.ts index 09aa045..522750c 100644 --- a/packages/plugins/snapshot-diff/index.ts +++ b/packages/plugins/snapshot-diff/index.ts @@ -1,101 +1,20 @@ - -import { CrawlithPlugin, PluginContext, compareGraphs, Graph, SiteRepository, SnapshotRepository, getDb, loadGraphFromSnapshot } from '@crawlith/core'; -import fs from 'node:fs/promises'; -import chalk from 'chalk'; -import { Command } from '@crawlith/core'; +import type { CrawlithPlugin, Command } from '@crawlith/core'; +import { registerSnapshotDiffCli } from './src/cli.js'; +import { snapshotDiffHooks } from './src/plugin.js'; /** - * Snapshot Diff Plugin - * Crawlith plugin for snapshot diff + * Snapshot Diff Plugin. + * + * Provides two crawl-time features: + * - snapshot-to-snapshot graph comparison via `--compare ` + * - incremental baseline loading via `--incremental` */ export const SnapshotDiffPlugin: CrawlithPlugin = { name: 'snapshot-diff', register: (cli: Command) => { - if (cli.name() === 'crawl') { - cli - .option("--incremental", "incremental crawl using previous snapshot") - .option("--compare ", "internal: compare two graph JSON files"); - } + registerSnapshotDiffCli(cli); }, - - hooks: { - onInit: async (ctx: PluginContext) => { - const flags = ctx.flags || {}; - if (flags.compare) { - const files = flags.compare as unknown as string[]; - if (files.length !== 2) { - console.error(chalk.red('Error: --compare requires exactly two file paths (old.json new.json)')); - process.exit(1); - } - const [oldFile, newFile] = files; - const fmt = typeof flags.format === 'string' ? flags.format : undefined; - - if (fmt !== 'json') { - console.log(chalk.cyan(`\nšŸ” Comparing Graphs`)); - console.log(`${chalk.gray('Old:')} ${oldFile}`); - console.log(`${chalk.gray('New:')} ${newFile}\n`); - } - - const oldJson = JSON.parse(await fs.readFile(oldFile, 'utf-8')); - const newJson = JSON.parse(await fs.readFile(newFile, 'utf-8')); - - const oldGraph = Graph.fromJSON(oldJson); - const newGraph = Graph.fromJSON(newJson); - - const diffResult = compareGraphs(oldGraph, newGraph); - - if (fmt !== 'json') { - console.log(chalk.bold('šŸ“ˆ Comparison Results:')); - console.log(`- Added URLs: ${chalk.green(diffResult.addedUrls.length)}`); - console.log(`- Removed URLs: ${chalk.red(diffResult.removedUrls.length)}`); - console.log(`- Status Changes: ${chalk.yellow(diffResult.changedStatus.length)}`); - - console.log(chalk.bold('\nšŸ“‰ Metric Deltas:')); - Object.entries(diffResult.metricDeltas).forEach(([metric, delta]) => { - const deltaStr = (delta as number) > 0 ? chalk.green(`+${(delta as number).toFixed(3)}`) : ((delta as number) < 0 ? chalk.red((delta as number).toFixed(3)) : chalk.gray('0')); - console.log(` ${metric.padEnd(20)}: ${deltaStr}`); - }); - - console.log('\n' + chalk.gray('ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”') + '\n'); - } else { - console.log(JSON.stringify(diffResult, null, 2)); - } - ctx.terminate = true; - } - }, - onCrawlStart: async (ctx: PluginContext) => { - const flags = ctx.flags || {}; - - if (flags.incremental) { - ctx.logger?.info('šŸ” Resolving previous snapshot for incremental crawl...'); - const targetUrl = flags.url || (ctx as any).url; - - if (!targetUrl || targetUrl.startsWith('-')) { - return; - } - - const db = getDb(); - const siteRepo = new SiteRepository(db); - const snapRepo = new SnapshotRepository(db); - - try { - const urlObj = new URL(targetUrl); - const domain = urlObj.hostname.replace('www.', ''); - const site = siteRepo.getSite(domain); - if (site) { - const latestSnap = snapRepo.getLatestSnapshot(site.id, 'completed'); - if (latestSnap) { - ctx.logger?.info(`Found previous snapshot #${latestSnap.id}, loading graph for delta comparison...`); - if (!ctx.metadata) ctx.metadata = {}; - ctx.metadata.previousGraph = loadGraphFromSnapshot(latestSnap.id); - } - } - } catch (_e) { - // ignore url parsing failures - } - } - } - } + hooks: snapshotDiffHooks }; export default SnapshotDiffPlugin; diff --git a/packages/plugins/snapshot-diff/package.json b/packages/plugins/snapshot-diff/package.json index e06f101..c125dd6 100644 --- a/packages/plugins/snapshot-diff/package.json +++ b/packages/plugins/snapshot-diff/package.json @@ -13,5 +13,5 @@ "test": "vitest run" }, "exports": "./index.ts", - "description": "Crawlith plugin for snapshot diff" + "description": "Crawlith plugin for snapshot-to-snapshot graph diffing and incremental baselines" } diff --git a/packages/plugins/snapshot-diff/src/Output.ts b/packages/plugins/snapshot-diff/src/Output.ts new file mode 100644 index 0000000..a8de20f --- /dev/null +++ b/packages/plugins/snapshot-diff/src/Output.ts @@ -0,0 +1,40 @@ +import chalk from 'chalk'; + +/** + * Renders human-readable output for snapshot comparison. + * @param oldSnapshotId Baseline snapshot ID. + * @param newSnapshotId Target snapshot ID. + * @param diffResult Graph diff payload from core. + */ +export function renderPrettyDiffOutput(oldSnapshotId: number, newSnapshotId: number, diffResult: any): void { + console.log(chalk.cyan('\nšŸ” Comparing Snapshots')); + console.log(`${chalk.gray('Old snapshot:')} #${oldSnapshotId}`); + console.log(`${chalk.gray('New snapshot:')} #${newSnapshotId}\n`); + + console.log(chalk.bold('šŸ“ˆ Comparison Results:')); + console.log(`- Added URLs: ${chalk.green(diffResult.addedUrls.length)}`); + console.log(`- Removed URLs: ${chalk.red(diffResult.removedUrls.length)}`); + console.log(`- Status Changes: ${chalk.yellow(diffResult.changedStatus.length)}`); + + console.log(chalk.bold('\nšŸ“‰ Metric Deltas:')); + Object.entries(diffResult.metricDeltas).forEach(([metric, delta]) => { + const numericDelta = Number(delta); + const deltaLabel = numericDelta > 0 + ? chalk.green(`+${numericDelta.toFixed(3)}`) + : numericDelta < 0 + ? chalk.red(numericDelta.toFixed(3)) + : chalk.gray('0'); + + console.log(` ${metric.padEnd(20)}: ${deltaLabel}`); + }); + + console.log(`\n${chalk.gray('ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”')}\n`); +} + +/** + * Emits machine-readable JSON output. + * @param diffResult Graph diff payload from core. + */ +export function renderJsonDiffOutput(diffResult: unknown): void { + console.log(JSON.stringify(diffResult, null, 2)); +} diff --git a/packages/plugins/snapshot-diff/src/Service.ts b/packages/plugins/snapshot-diff/src/Service.ts new file mode 100644 index 0000000..0c13efb --- /dev/null +++ b/packages/plugins/snapshot-diff/src/Service.ts @@ -0,0 +1,90 @@ +import { compareGraphs, getDb, loadGraphFromSnapshot, SiteRepository, SnapshotRepository } from '@crawlith/core'; +import type { Graph, PluginContext } from '@crawlith/core'; +import type { SnapshotCompareRequest, SnapshotDiffFlags, SnapshotDiffOutputFormat } from './types.js'; + +/** + * Encapsulates business logic for snapshot comparison and incremental baseline loading. + */ +export class SnapshotDiffService { + /** + * Parses and validates `--compare` payload as a pair of numeric snapshot IDs. + * @param flags CLI flags available on plugin context. + * @returns Parsed compare request or `null` when compare mode is disabled. + */ + parseCompareRequest(flags: SnapshotDiffFlags): SnapshotCompareRequest | null { + if (!flags.compare) return null; + + const rawValues = Array.isArray(flags.compare) + ? flags.compare + : [flags.compare]; + + if (rawValues.length !== 2) { + throw new Error('Error: --compare requires exactly two snapshot IDs ( ).'); + } + + const [oldRaw, newRaw] = rawValues.map(String); + const oldSnapshotId = Number.parseInt(oldRaw, 10); + const newSnapshotId = Number.parseInt(newRaw, 10); + + if (!Number.isFinite(oldSnapshotId) || !Number.isFinite(newSnapshotId)) { + throw new Error('Error: --compare only accepts numeric snapshot IDs.'); + } + + return { oldSnapshotId, newSnapshotId }; + } + + /** + * Resolves output format from CLI flags. + * @param flags CLI flags from plugin context. + * @returns Selected output format. + */ + parseOutputFormat(flags: SnapshotDiffFlags): SnapshotDiffOutputFormat { + return flags.format === 'json' ? 'json' : 'pretty'; + } + + /** + * Compares two persisted snapshots. + * @param request Pair of snapshot IDs. + * @returns Diff result generated by Crawlith core. + */ + compareSnapshots(request: SnapshotCompareRequest) { + const oldGraph = loadGraphFromSnapshot(request.oldSnapshotId); + const newGraph = loadGraphFromSnapshot(request.newSnapshotId); + return compareGraphs(oldGraph, newGraph); + } + + /** + * Resolves previous completed snapshot graph for incremental crawl mode. + * @param targetUrl URL passed to crawl command. + * @returns Previous graph when found, otherwise `null`. + */ + resolvePreviousGraph(targetUrl: string): Graph | null { + const db = getDb(); + const siteRepo = new SiteRepository(db); + const snapshotRepo = new SnapshotRepository(db); + + const domain = new URL(targetUrl).hostname.replace('www.', ''); + const site = siteRepo.getSite(domain); + if (!site) return null; + + const latestSnapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed'); + if (!latestSnapshot) return null; + + return loadGraphFromSnapshot(latestSnapshot.id); + } + + /** + * Reads a valid target URL from context flags for incremental mode. + * @param ctx Plugin context. + * @returns Candidate URL or `null` when missing. + */ + getTargetUrl(ctx: PluginContext): string | null { + const flags = (ctx.flags || {}) as SnapshotDiffFlags; + const fromFlags = typeof flags.url === 'string' ? flags.url : null; + if (fromFlags && !fromFlags.startsWith('-')) return fromFlags; + + return typeof (ctx as PluginContext & { url?: unknown }).url === 'string' + ? String((ctx as PluginContext & { url?: unknown }).url) + : null; + } +} diff --git a/packages/plugins/snapshot-diff/src/cli.ts b/packages/plugins/snapshot-diff/src/cli.ts new file mode 100644 index 0000000..0402ed7 --- /dev/null +++ b/packages/plugins/snapshot-diff/src/cli.ts @@ -0,0 +1,13 @@ +import { Command } from '@crawlith/core'; + +/** + * Registers snapshot-diff CLI options on the crawl command. + * @param cli Crawlith command instance. + */ +export function registerSnapshotDiffCli(cli: Command): void { + if (cli.name() !== 'crawl') return; + + cli + .option('--incremental', 'incremental crawl using previous completed snapshot') + .option('--compare ', 'internal: compare two snapshot IDs'); +} diff --git a/packages/plugins/snapshot-diff/src/plugin.ts b/packages/plugins/snapshot-diff/src/plugin.ts new file mode 100644 index 0000000..2a95f3e --- /dev/null +++ b/packages/plugins/snapshot-diff/src/plugin.ts @@ -0,0 +1,57 @@ +import type { PluginContext } from '@crawlith/core'; +import { renderJsonDiffOutput, renderPrettyDiffOutput } from './Output.js'; +import { SnapshotDiffService } from './Service.js'; +import type { SnapshotDiffFlags } from './types.js'; + +const service = new SnapshotDiffService(); + +/** + * Hook implementation for snapshot diff and incremental baseline resolution. + */ +export const snapshotDiffHooks = { + /** + * Handles `--compare` execution before crawl starts. + * @param ctx Plugin context for current command invocation. + */ + onInit: async (ctx: PluginContext) => { + const flags = (ctx.flags || {}) as SnapshotDiffFlags; + const compareRequest = service.parseCompareRequest(flags); + if (!compareRequest) return; + + const diffResult = service.compareSnapshots(compareRequest); + const outputFormat = service.parseOutputFormat(flags); + + if (outputFormat === 'json') { + renderJsonDiffOutput(diffResult); + } else { + renderPrettyDiffOutput(compareRequest.oldSnapshotId, compareRequest.newSnapshotId, diffResult); + } + + ctx.terminate = true; + }, + + /** + * Loads previous completed snapshot graph when incremental mode is enabled. + * @param ctx Plugin context for current crawl execution. + */ + onCrawlStart: async (ctx: PluginContext) => { + const flags = (ctx.flags || {}) as SnapshotDiffFlags; + if (!flags.incremental) return; + + const targetUrl = service.getTargetUrl(ctx); + if (!targetUrl) return; + + ctx.logger?.info('šŸ” Resolving previous snapshot for incremental crawl...'); + + try { + const previousGraph = service.resolvePreviousGraph(targetUrl); + if (!previousGraph) return; + + if (!ctx.metadata) ctx.metadata = {}; + ctx.metadata.previousGraph = previousGraph; + ctx.logger?.info('Loaded previous snapshot graph for incremental diffing.'); + } catch { + ctx.logger?.warn('Unable to resolve previous snapshot graph for incremental crawl.'); + } + } +}; diff --git a/packages/plugins/snapshot-diff/src/types.ts b/packages/plugins/snapshot-diff/src/types.ts new file mode 100644 index 0000000..9eca389 --- /dev/null +++ b/packages/plugins/snapshot-diff/src/types.ts @@ -0,0 +1,22 @@ +/** + * Parsed flag subset used by the snapshot-diff plugin. + */ +export interface SnapshotDiffFlags { + incremental?: boolean; + compare?: unknown; + format?: string; + url?: string; +} + +/** + * Output modes supported by the plugin formatter. + */ +export type SnapshotDiffOutputFormat = 'json' | 'pretty'; + +/** + * Normalized compare request containing exactly two snapshot IDs. + */ +export interface SnapshotCompareRequest { + oldSnapshotId: number; + newSnapshotId: number; +} diff --git a/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts b/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts index e31da12..a2acd4f 100644 --- a/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts +++ b/packages/plugins/snapshot-diff/tests/snapshotDiff.test.ts @@ -1,43 +1,84 @@ -import { describe, it, expect, vi } from 'vitest'; -import { SnapshotDiffPlugin } from '../index.js'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const compareGraphs = vi.fn().mockReturnValue({ + addedUrls: ['https://example.com/new'], + removedUrls: [], + changedStatus: [], + metricDeltas: { totalPages: 1 } +}); + +const loadGraphFromSnapshot = vi.fn((snapshotId: number) => ({ snapshotId })); vi.mock('@crawlith/core', () => { - return { - compareGraphs: vi.fn(), - Graph: { - fromJSON: vi.fn() - }, - getDb: vi.fn(), - SiteRepository: class { - getSite = vi.fn().mockReturnValue({ id: 1 }); - }, - SnapshotRepository: class { - getLatestSnapshot = vi.fn().mockReturnValue({ id: 5 }); - }, - loadGraphFromSnapshot: vi.fn().mockReturnValue('mocked-graph') - }; + return { + compareGraphs, + loadGraphFromSnapshot, + getDb: vi.fn(), + SiteRepository: class { + getSite = vi.fn().mockReturnValue({ id: 1 }); + }, + SnapshotRepository: class { + getLatestSnapshot = vi.fn().mockReturnValue({ id: 5 }); + } + }; }); +const { SnapshotDiffPlugin } = await import('../index.js'); + describe('SnapshotDiffPlugin', () => { - it('should resolve previous graph during onCrawlStart when --incremental is passed', async () => { - const ctx = { - flags: { incremental: true, url: 'https://example.com' }, - metadata: {} - }; + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('resolves previous graph during onCrawlStart when --incremental is passed', async () => { + const ctx = { + flags: { incremental: true, url: 'https://example.com' }, + metadata: {}, + logger: { info: vi.fn(), warn: vi.fn() } + }; - await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); + await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); - expect(ctx.metadata).toHaveProperty('previousGraph', 'mocked-graph'); - }); + expect(ctx.metadata).toHaveProperty('previousGraph', { snapshotId: 5 }); + expect(loadGraphFromSnapshot).toHaveBeenCalledWith(5); + }); - it('should ignore resolving previous graph if --incremental is not passed', async () => { - const ctx = { - flags: { incremental: false, url: 'https://example.com' }, - metadata: {} - }; + it('does not resolve previous graph when --incremental is not passed', async () => { + const ctx = { + flags: { incremental: false, url: 'https://example.com' }, + metadata: {}, + logger: { info: vi.fn(), warn: vi.fn() } + }; - await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); + await SnapshotDiffPlugin.hooks!.onCrawlStart!(ctx as any); + + expect(ctx.metadata).not.toHaveProperty('previousGraph'); + }); + + it('compares two snapshot IDs during onInit and terminates execution', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const ctx = { + flags: { compare: ['10', '11'] }, + logger: { info: vi.fn(), warn: vi.fn() } + }; + + await SnapshotDiffPlugin.hooks!.onInit!(ctx as any); + + expect(loadGraphFromSnapshot).toHaveBeenCalledWith(10); + expect(loadGraphFromSnapshot).toHaveBeenCalledWith(11); + expect(compareGraphs).toHaveBeenCalled(); + expect(ctx).toHaveProperty('terminate', true); + logSpy.mockRestore(); + }); + + it('throws when --compare is not a pair of snapshot IDs', async () => { + const ctx = { + flags: { compare: ['10'] }, + logger: { info: vi.fn(), warn: vi.fn() } + }; - expect(ctx.metadata).not.toHaveProperty('previousGraph'); - }); + await expect(SnapshotDiffPlugin.hooks!.onInit!(ctx as any)).rejects.toThrow( + '--compare requires exactly two snapshot IDs' + ); + }); });