feat(graph): add scalable snapshot structure graph explorer (web + server)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlith",
-  "version": "1.0.0",
+  "version": "0.1.1",
   "license": "Apache-2.0",
   "type": "module",
   "private": true,

packages/cli/src/commands/ui.ts

@@ -74,7 +74,8 @@ export const getUiCommand = (registry: PluginRegistry) => {
         host,
         staticPath: distPath,
         siteId: site.id,
-        snapshotId: snapshot.id
+        snapshotId: snapshot.id,
+        plugins: registry.getPlugins()
       });
 
       const displayHost = host === '0.0.0.0' ? 'localhost' : host;

packages/core/src/analysis/analyze.ts

@@ -90,6 +90,7 @@ export interface PageAnalysis {
     noindex?: boolean;
     nofollow?: boolean;
     crawlStatus?: string;
+    canonicalConflict?: boolean;
   };
   soft404?: { score: number; reason: string };
   headingScore?: number;
@@ -182,18 +183,21 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context?
         const robotsParserModule = await import('robots-parser');
         const robotsParser = (robotsParserModule as any).default || robotsParserModule;
         robots = (robotsParser as any)(robotsUrl, robotsRes.body);
-        if (context) context.emit({ type: 'debug', message: `[analyze] Robots fetch took ${Date.now() - start}ms` });
+        if (context) context.emit({ type: 'info', message: `[analyze] Robots fetch took ${Date.now() - start}ms` });
       }
     } catch {
       // Fallback
     }
   }
 
-  // 2. Data Acquisition
+  // Data Acquisition
   if (options.live) {
+    const fullUrl = parsedUrl ? parsedUrl.toString() : (url.startsWith('http') ? url : `https://${url}`);
+    const normalizedFull = normalizeUrl(fullUrl, rootOrigin, { stripQuery: false }) || fullUrl;
+
     const crawlStart = Date.now();
-    crawlData = await runLiveCrawl(normalizedAbs, rootOrigin, options, context, robots);
-    if (context) context.emit({ type: 'debug', message: `[analyze] runLiveCrawl took ${Date.now() - crawlStart}ms` });
+    crawlData = await runLiveCrawl(normalizedFull, rootOrigin, options, context, robots);
+    if (context) context.emit({ type: 'info', message: `[analyze] runLiveCrawl took ${Date.now() - crawlStart}ms` });
   } else {
     try {
       const loadStart = Date.now();
@@ -460,6 +464,9 @@ export function analyzePages(targetPath: string, rootOrigin: string, pages: Iter
     const soft404Service = new Soft404Service();
     const soft404 = soft404Service.analyze(html, links.externalLinks + links.internalLinks);
 
+    const isCanonicalConflict = !!(page.canonical && page.canonical !== page.url && page.canonical !== pageAbsUrl &&
+      page.canonical.replace(/\/$/, '') !== pageAbsUrl.replace(/\/$/, ''));
+
     const resultPage: PageAnalysis = {
       url: page.url,
       status: page.status || 0,
@@ -476,7 +483,8 @@ export function analyzePages(targetPath: string, rootOrigin: string, pages: Iter
         canonical: page.canonical,
         noindex: page.noindex,
         nofollow: page.nofollow,
-        crawlStatus
+        crawlStatus,
+        canonicalConflict: isCanonicalConflict
       },
       soft404
     };
@@ -574,7 +582,7 @@ async function runLiveCrawl(url: string, origin: string, options: AnalyzeOptions
     userAgent: options.userAgent,
     maxRedirects: options.maxRedirects,
     debug: options.debug,
-    snapshotType: 'partial',
+    snapshotRunType: 'single',
     robots,
     sitemap: options.sitemap,
     plugins: options.plugins
@@ -607,16 +615,16 @@ function renderSinglePageHtml(page: PageAnalysis): string {
 }
 
 export function renderAnalysisMarkdown(result: AnalysisResult): string {
-  const summary = ['# Crawlith SEO Analysis Report', '', '## 📊 Summary', `- Pages Analyzed: ${result.site_summary.pages_analyzed}`, `- Overall Site Score: ${result.site_summary.site_score.toFixed(1)}`, `- Avg SEO Score: ${result.site_summary.avg_seo_score.toFixed(1)}`, `- Thin Pages Found: ${result.site_summary.thin_pages}`, `- Duplicate Titles: ${result.site_summary.duplicate_titles}`, '', '## 📄 Page Details', '', '| URL | SEO Score | Thin Score | Title Status | Meta Status |', '| :--- | :--- | :--- | :--- | :--- |'];
-  result.pages.forEach((page) => summary.push(`| ${page.url} | ${page.seoScore} | ${page.thinScore} | ${page.title.status} | ${page.metaDescription.status} |`));
+  const summary = ['# Crawlith SEO Analysis Report', '', '## 📊 Summary', `- Pages Analyzed: ${result.site_summary.pages_analyzed}`, `- Overall Site Score: ${result.site_summary.site_score.toFixed(1)}`, `- Avg SEO Score: ${result.site_summary.site_score.toFixed(1)}`, `- Thin Pages Found: ${result.site_summary.thin_pages}`, `- Duplicate Titles: ${result.site_summary.duplicate_titles}`, '', '## 📄 Page Details', '', '| URL | SEO Score | Thin Score | Title Status | Meta Status | Canonical |', '| :--- | :--- | :--- | :--- | :--- | :--- |'];
+  result.pages.forEach((page) => summary.push(`| ${page.url} | ${page.seoScore} | ${page.thinScore} | ${page.title.status} | ${page.metaDescription.status} | ${page.meta.canonical || '-'} |`));
   return summary.join('\n');
 }
 
 export function renderAnalysisCsv(result: AnalysisResult): string {
-  const headers = ['URL', 'SEO Score', 'Thin Score', 'HTTP Status', 'Title', 'Title Length', 'Meta Description', 'Desc Length', 'Word Count', 'Internal Links', 'External Links'];
+  const headers = ['URL', 'SEO Score', 'Thin Score', 'HTTP Status', 'Title', 'Title Length', 'Meta Description', 'Desc Length', 'Word Count', 'Internal Links', 'External Links', 'Canonical'];
   const rows = result.pages.map((p) => {
     const statusStr = p.status === 0 ? 'Pending/Limit' : p.status;
-    return [p.url, p.seoScore, p.thinScore, statusStr, `"${(p.title.value || '').replace(/"/g, '""')}"`, p.title.length, `"${(p.metaDescription.value || '').replace(/"/g, '""')}"`, p.metaDescription.length, p.content.wordCount, p.links.internalLinks, p.links.externalLinks].join(',');
+    return [p.url, p.seoScore, p.thinScore, statusStr, `"${(p.title.value || '').replace(/"/g, '""')}"`, p.title.length, `"${(p.metaDescription.value || '').replace(/"/g, '""')}"`, p.metaDescription.length, p.content.wordCount, p.links.internalLinks, p.links.externalLinks, p.meta.canonical || ''].join(',');
   });
   return [headers.join(','), ...rows].join('\n');
 }

packages/core/src/analysis/duplicate.ts

@@ -9,6 +9,7 @@ export interface DuplicateCluster {
     id: string;
     type: 'exact' | 'near' | 'template_heavy';
     nodes: GraphNode[];
+    size: number;
     representative?: string;
     severity?: 'low' | 'medium' | 'high';
 }
@@ -71,7 +72,7 @@ export class DuplicateService {
         for (const group of exactMap.values()) {
             if (group.length > 1) {
                 const id = `cluster_exact_${clusterCounter++}`;
-                exactClusters.push({ id, type: 'exact', nodes: group });
+                exactClusters.push({ id, type: 'exact', nodes: group, size: group.length });
                 for (const n of group) {
                     (n as any).duplicateClusterId = id;
                     (n as any).duplicateType = 'exact';
@@ -221,7 +222,7 @@ export class DuplicateService {
             if (groupIndices.length > 1) {
                 const id = `cluster_near_${clusterCounter++}`;
                 const groupNodes = groupIndices.map(idx => candidates[idx]);
-                nearClusters.push({ id, type: 'near', nodes: groupNodes });
+                nearClusters.push({ id, type: 'near', nodes: groupNodes, size: groupNodes.length });
                 for (const n of groupNodes) {
                     (n as any).duplicateClusterId = id;
                     (n as any).duplicateType = 'near';

packages/core/src/analysis/scoring.ts

@@ -68,64 +68,3 @@ export function aggregateSiteScore(metrics: Metrics, pages: PageAnalysis[]): Sit
   };
 }
 
-export function healthStatusLabel(score: number, hasCritical: boolean = false): string {
-  if (hasCritical && score >= 75) return 'Needs Attention';
-  if (score >= 90) return 'Excellent';
-  if (score >= 75) return 'Good';
-  if (score >= 50) return 'Needs Attention';
-  return 'Critical';
-}
-
-export function calculateHealthScore(
-  totalPages: number,
-  issues: any,
-  weights: any = {
-    orphans: 50,
-    brokenLinks: 100,
-    redirectChains: 20,
-    duplicateClusters: 25,
-    thinContent: 15,
-    missingH1: 10,
-    noindexMisuse: 20,
-    canonicalConflicts: 10,
-    lowInternalLinks: 10,
-    excessiveLinks: 5,
-    blockedByRobots: 100
-  }
-): { score: number; status: string; weightedPenalties: any } {
-  const safePages = Math.max(totalPages, 1);
-  const clamp = (v: number, min: number, max: number) => Math.min(max, Math.max(min, v));
-
-  const weightedPenalties = {
-    orphans: clamp((issues.orphanPages / safePages) * weights.orphans, 0, weights.orphans),
-    brokenLinks: clamp((issues.brokenInternalLinks / safePages) * weights.brokenLinks, 0, weights.brokenLinks),
-    redirectChains: clamp((issues.redirectChains / safePages) * weights.redirectChains, 0, weights.redirectChains),
-    duplicateClusters: clamp((issues.duplicateClusters / safePages) * weights.duplicateClusters, 0, weights.duplicateClusters),
-    thinContent: clamp((issues.thinContent / safePages) * weights.thinContent, 0, weights.thinContent),
-    missingH1: clamp((issues.missingH1 / safePages) * weights.missingH1, 0, weights.missingH1),
-    noindexMisuse: clamp((issues.accidentalNoindex / safePages) * weights.noindexMisuse, 0, weights.noindexMisuse),
-    canonicalConflicts: clamp((issues.canonicalConflicts / safePages) * weights.canonicalConflicts, 0, weights.canonicalConflicts),
-    lowInternalLinks: clamp((issues.lowInternalLinkCount / safePages) * weights.lowInternalLinks, 0, weights.lowInternalLinks),
-    excessiveLinks: clamp((issues.excessiveInternalLinkCount / safePages) * weights.excessiveLinks, 0, weights.excessiveLinks),
-    blockedByRobots: clamp((issues.blockedByRobots / safePages) * weights.blockedByRobots, 0, weights.blockedByRobots)
-  };
-
-  const totalPenalty = Object.values(weightedPenalties).reduce((sum, value) => sum + value, 0);
-  const score = Number(clamp(100 - totalPenalty, 0, 100).toFixed(1));
-
-  const hasCritical = (
-    issues.orphanPages > 0 ||
-    issues.brokenInternalLinks > 0 ||
-    issues.redirectChains > 0 ||
-    issues.duplicateClusters > 0 ||
-    issues.canonicalConflicts > 0 ||
-    issues.accidentalNoindex > 0 ||
-    issues.blockedByRobots > 0
-  );
-
-  return {
-    score,
-    status: healthStatusLabel(score, hasCritical),
-    weightedPenalties
-  };
-}

packages/core/src/application/usecases.ts

@@ -4,6 +4,8 @@ import { runPostCrawlMetrics } from '../crawler/metricsRunner.js';
 import { analyzeSite, type AnalyzeOptions, type AnalysisResult } from '../analysis/analyze.js';
 import { loadGraphFromSnapshot } from '../db/graphLoader.js';
 import { compareGraphs } from '../diff/compare.js';
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
 import { PluginRegistry } from '../plugin-system/plugin-registry.js';
 import type { CrawlithPlugin, PluginContext } from '../plugin-system/plugin-types.js';
 import type { UseCase } from './usecase.js';
@@ -102,6 +104,20 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes
     await registry.runHook('onGraphBuilt', ctx, graph);
     await registry.runHook('onMetrics', ctx, graph);
 
+    const db = getCrawlithDB().unsafeGetRawDb();
+    const siteRepo = new SiteRepository(db);
+    const snapshotRepo = new SnapshotRepository(db);
+    const snapshot = snapshotRepo.getSnapshot(snapshotId);
+    let resolvedOrigin = '';
+    if (snapshot) {
+      const site = siteRepo.getSiteById(snapshot.site_id);
+      if (site?.preferred_url) {
+        try {
+          resolvedOrigin = new URL(site.preferred_url).origin;
+        } catch { /* ignore */ }
+      }
+    }
+
     runPostCrawlMetrics(snapshotId, crawlOpts.depth, {
       context: undefined,
       limitReached: false,
@@ -117,7 +133,7 @@ export class CrawlSitegraph implements UseCase<SiteCrawlInput, CrawlSitegraphRes
       orphanSeverity: input.orphanSeverity ?? true,
       includeSoftOrphans: input.includeSoftOrphans ?? true,
       minInbound: input.minInbound,
-      rootOrigin: input.url.startsWith('http') ? new URL(input.url).origin : `https://${input.url}`
+      rootOrigin: resolvedOrigin || (input.url.startsWith('http') ? new URL(input.url).origin : `https://${input.url}`)
     });
 
     if (ctx.db) {
@@ -252,6 +268,10 @@ export class PageAnalysisUseCase implements UseCase<PageAnalysisInput, AnalysisR
           status: page.status,
         });
       }
+
+      if (pluginCtx.db && result.snapshotId) {
+        pluginCtx.db.aggregateScoreProviders(result.snapshotId, registry.pluginsList);
+      }
     }
 
     return result;

packages/core/src/constants.ts

@@ -18,4 +18,6 @@ export const DEFAULTS = {
     // Network timeouts
     HEADERS_TIMEOUT: 10000,
     BODY_TIMEOUT: 10000,
+    // Keep only last 5 snapshots
+    MAX_SNAPSHOTS: 5
 } as const;

packages/core/src/crawler/crawler.ts

@@ -39,7 +39,7 @@ export interface CrawlOptions {
   proxyUrl?: string;
   maxRedirects?: number;
   userAgent?: string;
-  snapshotType?: 'full' | 'partial' | 'incremental';
+  snapshotRunType?: 'completed' | 'incremental' | 'single';
   registry?: PluginRegistry;
   plugins?: any[];
   robots?: any;
@@ -89,6 +89,7 @@ export class Crawler {
   private siteId: number | null = null;
   private snapshotId: number | null = null;
   private reusingSnapshot: boolean = false;
+  private runType: 'completed' | 'incremental' | 'single' = 'completed';
   private rootOrigin: string = '';
 
   // Discovery tracking
@@ -169,22 +170,11 @@ export class Crawler {
       (this.fetcher as any).scopeManager = this.scopeManager;
     }
 
-    // For partial snapshots (page --live), reuse the latest partial snapshot
-    // instead of creating a new one each time
-    if (this.options.snapshotType === 'partial') {
-      const existing = this.snapshotRepo.getLatestPartialSnapshot(this.siteId);
-      if (existing) {
-        this.snapshotId = existing.id;
-        this.reusingSnapshot = true;
-        this.context.emit({ type: 'debug', message: `Reusing partial snapshot #${existing.id}` });
-      } else {
-        this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, 'partial');
-      }
-    } else {
-      const type = this.options.snapshotType || (this.options.previousGraph ? 'incremental' : 'full');
-      this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, type);
-    }
+    // Every scan now creates a new snapshot (no reuse)
+    const runType = this.options.snapshotRunType || (this.options.previousGraph ? 'incremental' : 'completed');
+    this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, runType);
 
+    this.runType = runType;
     this.rootOrigin = urlObj.origin;
     this.startUrl = rootUrl;
 
@@ -257,7 +247,7 @@ export class Crawler {
     const sitemapsToFetch = new Set<string>();
 
     // 1. Explicitly configured sitemap
-    if (this.options.sitemap) {
+    if (this.options.sitemap && this.runType !== 'single') {
       const explicitUrl = this.options.sitemap === 'true' || (this.options.sitemap as any) === true
         ? new URL('/sitemap.xml', this.rootOrigin).toString()
         : this.options.sitemap;
@@ -269,9 +259,9 @@ export class Crawler {
 
     // 2. Discover sitemaps from robots.txt (unless explicitly disabled)
     // Only auto-fetch on the FIRST real crawl (full/incremental).
-    // page --live reuses partial snapshots and should NOT trigger sitemap fetch.
-    const isFirstCrawl = !this.snapshotRepo?.hasFullCrawl(this.siteId!);
-    if (this.options.sitemap !== false && (this.options.sitemap || isFirstCrawl) && this.robots) {
+    // page --live reuses snapshots and should NOT trigger sitemap fetch.
+    const isFirstFullCrawl = this.runType !== 'single' && !this.snapshotRepo?.hasFullCrawl(this.siteId!);
+    if (this.options.sitemap !== false && (this.options.sitemap || isFirstFullCrawl) && this.robots && this.runType !== 'single') {
       const robotsSitemaps = this.robots.getSitemaps();
       for (const s of robotsSitemaps) {
         if (s) sitemapsToFetch.add(s);
@@ -544,9 +534,11 @@ export class Crawler {
       if (sourceAbs && targetAbs) {
         const sourcePath = UrlUtil.toPath(sourceAbs, this.rootOrigin);
         const targetPath = UrlUtil.toPath(targetAbs, this.rootOrigin);
-        this.bufferPage(sourcePath, depth, step.status);
-        this.bufferPage(targetPath, depth, 0);
-        this.bufferEdge(sourcePath, targetPath);
+        const sourceInternal = UrlUtil.isInternal(sourceAbs, this.rootOrigin);
+        const targetInternal = UrlUtil.isInternal(targetAbs, this.rootOrigin);
+        this.bufferPage(sourcePath, depth, step.status, { is_internal: sourceInternal ? 1 : 0 });
+        this.bufferPage(targetPath, depth, 0, { is_internal: targetInternal ? 1 : 0 });
+        this.bufferEdge(sourcePath, targetPath, 1.0, targetInternal ? 'internal' : 'external');
       }
     }
   }
@@ -580,7 +572,8 @@ export class Crawler {
       simhash: parseResult.simhash,
       etag: res.etag,
       last_modified: res.lastModified,
-      retries: res.retries
+      retries: res.retries,
+      bytes_received: res.bytesReceived
     });
 
     try {
@@ -605,7 +598,7 @@ export class Crawler {
 
         if (targetPath !== path) {
           const isInternal = UrlUtil.isInternal(normalizedLink, this.rootOrigin);
-          this.bufferPage(targetPath, depth + 1, 0);
+          this.bufferPage(targetPath, depth + 1, 0, { is_internal: isInternal ? 1 : 0 });
           this.bufferEdge(path, targetPath, 1.0, isInternal ? 'internal' : 'external');
 
           if (isInternal && this.shouldEnqueue(targetPath, depth + 1)) {

packages/core/src/crawler/metricsRunner.ts

@@ -262,10 +262,15 @@ export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, option
         }
     }
 
+    const thinContentCount = graph.getNodes().filter(n => n.wordCount !== undefined && n.wordCount < 200 && n.status === 200).length;
+    const orphanCount = metrics.orphanPages.length;
+
     snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
         node_count: metrics.totalPages,
         edge_count: metrics.totalEdges,
         limit_reached: limitReached ? 1 : 0,
+        thin_content_count: thinContentCount,
+        orphan_count: orphanCount,
         ...(healthScore !== null ? { health_score: healthScore } : {})
     });
 

packages/core/src/crawler/trap.ts

@@ -9,7 +9,7 @@ export interface TrapResult {
 export class TrapDetector {
     private pathCounters = new Map<string, Set<string>>();
     private paginationCounters = new Map<string, number>();
-    private sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token']);
+    private sessionParams = new Set(['sid', 'session', 'phpsessid', 'sessid', 'token', 'intended']);
 
     // Configurable thresholds
     private PARAM_EXPLOSION_THRESHOLD = 30;
@@ -23,7 +23,13 @@ export class TrapDetector {
     /**
      * Checks if a URL represents a potential crawl trap.
      */
-    checkTrap(rawUrl: string, _depth: number): TrapResult {
+    checkTrap(rawUrl: string, _depth: number, isInternal: boolean = true): TrapResult {
+        // If it's not internal (e.g., social sharing links), we don't flag it as a trap
+        // that affects our crawl health, even though technically it might have many params.
+        if (!isInternal) {
+            return { risk: 0, type: null };
+        }
+
         let risk = 0;
         let type: TrapType | null = null;
 
@@ -93,7 +99,7 @@ export class TrapDetector {
         const nodes = graph.getNodes();
         for (const node of nodes) {
             if (node.status === 200 || node.status === 0) {
-                const res = this.checkTrap(node.url, node.depth || 0);
+                const res = this.checkTrap(node.url, node.depth || 0, !!node.isInternal);
                 if (res.risk > 0.4) {
                     node.crawlTrapFlag = true;
                     node.crawlTrapRisk = res.risk;