Replace dummy dashboard data with real SQLite data

plugins/cli/src/commands/crawl.ts

@@ -197,18 +197,6 @@ export const crawlCommand = new Command('crawl')
       //   process.exit(1);
       // }
 
-      if (options.format !== 'json') process.stdout.write(chalk.gray('📊 Calculating metrics and saving to database... '));
-      runPostCrawlMetrics(snapshotId, depth, context);
-      if (options.format !== 'json') process.stdout.write(chalk.green('Done\n'));
-
-
-      // if (nodes.length === 0) {
-      //   console.log(chalk.red('\n❌ No pages were crawled.'));
-      //   console.log(chalk.gray(`The target URL ${chalk.white(url)} could not be reached or is blocked by robots.txt.`));
-      //   console.log(chalk.gray('Try running with ') + chalk.white('--ignore-robots') + chalk.gray(' or ') + chalk.white('--debug') + chalk.gray(' for more details.\n'));
-      //   process.exit(1);
-      // }
-
       if (options.format !== 'json') {
         console.log(chalk.green(`\n✅ Crawl complete.`));
         process.stdout.write(chalk.gray('🔍 Detecting duplicates... '));
@@ -225,6 +213,7 @@ export const crawlCommand = new Command('crawl')
       if (options.format !== 'json') process.stdout.write(chalk.green('Done\n'));
 
       if (options.format !== 'json') process.stdout.write(chalk.gray('📊 Calculating final report metrics... '));
+      runPostCrawlMetrics(snapshotId, depth, context, !!graph.limitReached, graph);
       const metrics = calculateMetrics(graph, depth);
       if (options.format !== 'json') process.stdout.write(chalk.green('Done\n'));
 

plugins/cli/src/commands/crawlFormatter.ts

@@ -1,75 +1,16 @@
 import {
-  analyzeContent,
-  analyzeH1,
-  analyzeImageAlts,
-  analyzeLinks,
   Graph,
   Metrics,
-
+  HealthScoreWeights,
+  DEFAULT_HEALTH_WEIGHTS,
+  CrawlIssueCounts,
+  HealthScoreBreakdown,
+  calculateHealthScore,
+  collectCrawlIssues,
+  THIN_CONTENT_THRESHOLD,
+  EXCESSIVE_INTERNAL_LINK_THRESHOLD
 } from '@crawlith/core';
 
-const THIN_CONTENT_THRESHOLD = 300;
-const LOW_INTERNAL_LINK_THRESHOLD = 2;
-const EXCESSIVE_INTERNAL_LINK_THRESHOLD = 150;
-const HIGH_EXTERNAL_LINK_RATIO_THRESHOLD = 0.6;
-const OPPORTUNITY_AUTHORITY_THRESHOLD = 0.8;
-
-export interface HealthScoreWeights {
-  orphans: number;
-  brokenLinks: number;
-  redirectChains: number;
-  duplicateClusters: number;
-  thinContent: number;
-  missingH1: number;
-  noindexMisuse: number;
-  canonicalConflicts: number;
-  lowInternalLinks: number;
-  excessiveLinks: number;
-  blockedByRobots: number;
-}
-
-export const DEFAULT_HEALTH_WEIGHTS: HealthScoreWeights = {
-  orphans: 20,
-  brokenLinks: 20,
-  redirectChains: 10,
-  duplicateClusters: 15,
-  thinContent: 10,
-  missingH1: 10,
-  noindexMisuse: 10,
-  canonicalConflicts: 5,
-  lowInternalLinks: 10,
-  excessiveLinks: 5,
-  blockedByRobots: 100
-};
-
-export interface CrawlIssueCounts {
-  orphanPages: number;
-  brokenInternalLinks: number;
-  redirectChains: number;
-  duplicateClusters: number;
-  canonicalConflicts: number;
-  accidentalNoindex: number;
-  missingH1: number;
-  thinContent: number;
-  lowInternalLinkCount: number;
-  excessiveInternalLinkCount: number;
-  highExternalLinkRatio: number;
-  imageAltMissing: number;
-  strongPagesUnderLinking: number;
-  cannibalizationClusters: number;
-  nearAuthorityThreshold: number;
-  underlinkedHighAuthorityPages: number;
-  externalLinks: number;
-  blockedByRobots: number;
-}
-
-export interface HealthScoreBreakdown {
-  score: number;
-  status: string;
-  weightedPenalties: Record<keyof HealthScoreWeights, number>;
-  weights: HealthScoreWeights;
-}
-
 export interface CrawlInsightReport {
   pages: number;
   fetchedPages?: number;
@@ -90,167 +31,8 @@ export interface CrawlInsightReport {
   };
 }
 
-function clamp(value: number, min: number, max: number): number {
-  return Math.min(max, Math.max(min, value));
-}
-
-export function healthStatusLabel(score: number, hasCritical: boolean = false): string {
-  if (hasCritical && score >= 75) return 'Needs Attention';
-  if (score >= 90) return 'Excellent';
-  if (score >= 75) return 'Good';
-  if (score >= 50) return 'Needs Attention';
-  return 'Critical';
-}
-
-
-export function calculateHealthScore(
-  totalPages: number,
-  issues: Pick<CrawlIssueCounts, 'orphanPages' | 'brokenInternalLinks' | 'redirectChains' | 'duplicateClusters' | 'thinContent' | 'missingH1' | 'accidentalNoindex' | 'canonicalConflicts' | 'lowInternalLinkCount' | 'excessiveInternalLinkCount' | 'blockedByRobots'>,
-  weights: HealthScoreWeights = DEFAULT_HEALTH_WEIGHTS
-): HealthScoreBreakdown {
-  const safePages = Math.max(totalPages, 1);
-
-  const weightedPenalties = {
-    orphans: clamp((issues.orphanPages / safePages) * weights.orphans, 0, weights.orphans),
-    brokenLinks: clamp((issues.brokenInternalLinks / safePages) * weights.brokenLinks, 0, weights.brokenLinks),
-    redirectChains: clamp((issues.redirectChains / safePages) * weights.redirectChains, 0, weights.redirectChains),
-    duplicateClusters: clamp((issues.duplicateClusters / safePages) * weights.duplicateClusters, 0, weights.duplicateClusters),
-    thinContent: clamp((issues.thinContent / safePages) * weights.thinContent, 0, weights.thinContent),
-    missingH1: clamp((issues.missingH1 / safePages) * weights.missingH1, 0, weights.missingH1),
-    noindexMisuse: clamp((issues.accidentalNoindex / safePages) * weights.noindexMisuse, 0, weights.noindexMisuse),
-    canonicalConflicts: clamp((issues.canonicalConflicts / safePages) * weights.canonicalConflicts, 0, weights.canonicalConflicts),
-    lowInternalLinks: clamp((issues.lowInternalLinkCount / safePages) * weights.lowInternalLinks, 0, weights.lowInternalLinks),
-    excessiveLinks: clamp((issues.excessiveInternalLinkCount / safePages) * weights.excessiveLinks, 0, weights.excessiveLinks),
-    blockedByRobots: clamp((issues.blockedByRobots / safePages) * weights.blockedByRobots, 0, weights.blockedByRobots)
-  };
-
-  const totalPenalty = Object.values(weightedPenalties).reduce((sum, value) => sum + value, 0);
-  const score = Number(clamp(100 - totalPenalty, 0, 100).toFixed(1));
-  // Determine if there are critical issues for the label
-  const hasCritical = (
-    issues.orphanPages > 0 ||
-    issues.brokenInternalLinks > 0 ||
-    issues.redirectChains > 0 ||
-    issues.duplicateClusters > 0 ||
-    issues.canonicalConflicts > 0 ||
-    issues.accidentalNoindex > 0 ||
-    issues.blockedByRobots > 0
-  );
-  return {
-    score,
-    status: healthStatusLabel(score, hasCritical),
-    weightedPenalties,
-    weights
-  };
-}
-
-export function collectCrawlIssues(graph: Graph, metrics: Metrics): CrawlIssueCounts {
-  const nodes = graph.getNodes();
-
-  let brokenInternalLinks = 0;
-  let redirectChains = 0;
-  let canonicalConflicts = 0;
-  let accidentalNoindex = 0;
-  let missingH1 = 0;
-  let thinContent = 0;
-  let highExternalLinkRatio = 0;
-  let imageAltMissing = 0;
-  let lowInternalLinkCount = 0;
-  let excessiveInternalLinkCount = 0;
-  let strongPagesUnderLinking = 0;
-  let nearAuthorityThreshold = 0;
-  let underlinkedHighAuthorityPages = 0;
-  let externalLinks = 0;
-  let blockedByRobots = 0;
 
 
-
-  for (const node of nodes) {
-    if (node.crawlStatus === 'blocked' || node.crawlStatus === 'blocked_by_robots') {
-      blockedByRobots += 1;
-    }
-    brokenInternalLinks += node.brokenLinks?.length || 0;
-    if ((node.redirectChain?.length || 0) > 1) {
-      redirectChains += 1;
-    }
-    if (node.canonical && node.canonical !== node.url) {
-      canonicalConflicts += 1;
-    }
-    if (node.noindex && node.status >= 200 && node.status < 300) {
-      accidentalNoindex += 1;
-    }
-
-    if (node.inLinks < LOW_INTERNAL_LINK_THRESHOLD && node.depth > 0) {
-      lowInternalLinkCount += 1;
-    }
-    if (node.outLinks > EXCESSIVE_INTERNAL_LINK_THRESHOLD) {
-      excessiveInternalLinkCount += 1;
-    }
-
-    if (!node.html) {
-      continue;
-    }
-
-    const h1 = analyzeH1(node.html, null);
-    if (h1.count === 0) {
-      missingH1 += 1;
-    }
-
-    const content = analyzeContent(node.html);
-    if (content.wordCount < THIN_CONTENT_THRESHOLD) {
-      thinContent += 1;
-    }
-
-    const links = analyzeLinks(node.html, node.url, node.url);
-    externalLinks += links.externalLinks;
-    if (links.externalRatio > HIGH_EXTERNAL_LINK_RATIO_THRESHOLD) {
-      highExternalLinkRatio += 1;
-    }
-
-    const imageAlt = analyzeImageAlts(node.html);
-    if (imageAlt.missingAlt > 0) {
-      imageAltMissing += 1;
-    }
-  }
-
-  const duplicateClusters = graph.duplicateClusters?.length || 0;
-  const cannibalizationClusters = graph.duplicateClusters?.filter((cluster) => cluster.type === 'near').length || 0;
-
-  for (const node of nodes) {
-    const authority = node.pageRank || 0;
-    if (authority >= OPPORTUNITY_AUTHORITY_THRESHOLD && node.outLinks < 3) {
-      strongPagesUnderLinking += 1;
-    }
-    if (authority >= 0.65 && authority < OPPORTUNITY_AUTHORITY_THRESHOLD) {
-      nearAuthorityThreshold += 1;
-    }
-    if (authority >= OPPORTUNITY_AUTHORITY_THRESHOLD && node.inLinks < LOW_INTERNAL_LINK_THRESHOLD) {
-      underlinkedHighAuthorityPages += 1;
-    }
-  }
-
-  return {
-    orphanPages: metrics.orphanPages.length,
-    brokenInternalLinks,
-    redirectChains,
-    duplicateClusters,
-    canonicalConflicts,
-    accidentalNoindex,
-    missingH1,
-    thinContent,
-    lowInternalLinkCount,
-    excessiveInternalLinkCount,
-    highExternalLinkRatio,
-    imageAltMissing,
-    strongPagesUnderLinking,
-    cannibalizationClusters,
-    nearAuthorityThreshold,
-    underlinkedHighAuthorityPages,
-    externalLinks,
-    blockedByRobots
-  };
-}
-
 export function buildCrawlInsightReport(
   graph: Graph,
   metrics: Metrics,

plugins/cli/src/commands/ui.ts

@@ -5,21 +5,26 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { startServer } from '@crawlith/server';
+import { getDb, SiteRepository, SnapshotRepository } from '@crawlith/core';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const distPath = path.join(__dirname, 'ui');
 
 export const ui = new Command('ui')
   .description('Start the Crawlith UI Dashboard')
-  .option('--site <url>', 'Site URL to display in dashboard', 'https://example.com')
+  .argument('<url>', 'Site URL or domain to visualize')
   .option('--port <number>', 'Port to run server on', '23484')
   .option('--host <address>', 'Host to bind server to', '127.0.0.1')
-  .action(async (options) => {
+  .action(async (siteUrl, options) => {
+    if (!siteUrl) {
+      console.error(chalk.red('❌ ssquired argument: url'));
+      ui.outputHelp();
+      process.exit(0);
+    }
     try {
       const port = parseInt(options.port, 10);
       const host = options.host;
-      const siteUrl = options.site;
 
       console.log(chalk.bold.cyan(`\n🚀 Starting Crawlith UI`));
 
@@ -29,11 +34,42 @@ export const ui = new Command('ui')
         process.exit(1);
       }
 
+      // 1. Normalize domain
+      let domain = siteUrl;
+      try {
+        const url = new URL(siteUrl.startsWith('http') ? siteUrl : `https://${siteUrl}`);
+        domain = url.hostname;
+      } catch (_e) {
+        // use raw string if URL parsing fails
+      }
+
+      console.log(chalk.gray(`   Resolving site: ${domain}`));
+
+      // 2. Connect to DB and resolve site/snapshot
+      const db = getDb();
+      const siteRepo = new SiteRepository(db);
+      const snapshotRepo = new SnapshotRepository(db);
+
+      const site = siteRepo.getSite(domain);
+      if (!site) {
+        console.error(chalk.red(`❌ Site not found: ${domain}`));
+        console.error(chalk.yellow(`   Run "crawlith crawl ${domain}" first to generate data.`));
+        process.exit(1);
+      }
+
+      const snapshot = snapshotRepo.getLatestSnapshot(site.id);
+      if (!snapshot) {
+        console.error(chalk.red(`❌ No snapshots found for site: ${domain}`));
+        process.exit(1);
+      }
+
+      // 3. Start Server with context
       await startServer({
         port,
         host,
         staticPath: distPath,
-        siteName: siteUrl
+        siteId: site.id,
+        snapshotId: snapshot.id
       });
 
       const displayHost = host === '0.0.0.0' ? 'localhost' : host;

plugins/cli/tests/crawlFormatter.test.ts

@@ -1,12 +1,10 @@
 import { describe, expect, test } from 'vitest';
 import {
-  calculateHealthScore,
-  healthStatusLabel,
   renderInsightOutput,
   hasCriticalIssues,
   buildCrawlInsightReport
 } from '../src/commands/crawlFormatter.js';
-import { Graph, Metrics } from '@crawlith/core';
+import { Graph, Metrics, calculateHealthScore, healthStatusLabel } from '@crawlith/core';
 
 function createMetrics(overrides: Partial<Metrics> = {}): Metrics {
   return {
@@ -46,8 +44,7 @@ describe('health score calculation', () => {
       blockedByRobots: 0
     });
 
-    expect(score.score).toBe(94);
-    expect(score.score).toBe(94);
+    expect(score.score).toBe(80);
     expect(score.status).toBe('Needs Attention');
   });
 });

plugins/core/src/crawler/crawler.ts

@@ -500,10 +500,14 @@ export class Crawler {
 
       const isStringStatus = typeof res.status === 'string';
       if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
-        this.bufferPage(finalUrl, depth, typeof res.status === 'number' ? res.status : 0, {
+        const statusNum = typeof res.status === 'number' ? res.status : 0;
+        this.bufferPage(finalUrl, depth, statusNum, {
           security_error: isStringStatus ? res.status : undefined,
           retries: res.retries
         });
+        this.bufferMetrics(finalUrl, {
+          crawl_status: isStringStatus ? res.status : 'fetched_error'
+        });
         return;
       }