Crawlith · saurabhsharma2u · Feb 27, 2026 · Feb 27, 2026 · Feb 27, 2026 · Feb 27, 2026
diff --git a/plugins/cli/src/commands/ui.ts b/plugins/cli/src/commands/ui.ts
@@ -57,7 +57,7 @@ export const ui = new Command('ui')
         process.exit(1);
       }
 
-      const snapshot = snapshotRepo.getLatestSnapshot(site.id);
+      const snapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed');
       if (!snapshot) {
         console.error(chalk.red(`❌ No snapshots found for site: ${domain}`));
         process.exit(1);

diff --git a/plugins/core/src/analysis/analyze.ts b/plugins/core/src/analysis/analyze.ts
diff --git a/plugins/core/src/analysis/content.ts b/plugins/core/src/analysis/content.ts
@@ -1,4 +1,4 @@
-import { load } from 'cheerio';
+import { CheerioAPI, load } from 'cheerio';
 
 export interface ContentAnalysis {
   wordCount: number;
@@ -18,17 +18,24 @@ const DEFAULT_WEIGHTS: ThinScoreWeights = {
   dupWeight: 0.25
 };
 
-export function analyzeContent(html: string): ContentAnalysis {
-  const $ = load(html || '<html></html>');
-  $('script,style,nav,footer').remove();
+export function analyzeContent($: CheerioAPI | string): ContentAnalysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
 
-  const text = $('body').length ? $('body').text() : $.text();
+  // We don't want to modify the shared $ object if we remove elements
+  // So we create a localized copy of the body text or use selection
+  const body = cheerioObj('body').length ? cheerioObj('body') : cheerioObj('html');
+
+  // To avoid removing from shared $, we extract text from a clone if possible, 
+  // but cloning in cheerio is expensive. 
+  // Better: just get the text and clean it or use a filter.
+  const text = body.clone().find('script,style,nav,footer').remove().end().text();
   const cleanText = text.replace(/\s+/g, ' ').trim();
 
   const words = cleanText ? cleanText.split(/\s+/).filter(Boolean) : [];
   const wordCount = words.length;
 
-  const htmlLength = Math.max(html.length, 1);
+  const htmlLength = isString ? ($.length || 1) : 1000; // Fallback if we don't have original HTML length
   const textHtmlRatio = cleanText.length / htmlLength;
 
   const sentenceSet = new Set(

diff --git a/plugins/core/src/analysis/images.ts b/plugins/core/src/analysis/images.ts
@@ -1,18 +1,20 @@
-import { load } from 'cheerio';
+import { CheerioAPI, load } from 'cheerio';
 
 export interface ImageAltAnalysis {
   totalImages: number;
   missingAlt: number;
   emptyAlt: number;
 }
 
-export function analyzeImageAlts(html: string): ImageAltAnalysis {
-  const $ = load(html);
+export function analyzeImageAlts($: CheerioAPI | string): ImageAltAnalysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
+
   let missingAlt = 0;
   let emptyAlt = 0;
 
-  $('img').each((_idx, el) => {
-    const alt = $(el).attr('alt');
+  cheerioObj('img').each((_idx, el) => {
+    const alt = cheerioObj(el).attr('alt');
     if (alt === undefined) {
       missingAlt += 1;
       return;
@@ -23,6 +25,6 @@ export function analyzeImageAlts(html: string): ImageAltAnalysis {
     }
   });
 
-  const totalImages = $('img').length;
+  const totalImages = cheerioObj('img').length;
   return { totalImages, missingAlt, emptyAlt };
 }
diff --git a/plugins/core/src/analysis/links.ts b/plugins/core/src/analysis/links.ts
@@ -1,4 +1,4 @@
-import { load } from 'cheerio';
+import { load, CheerioAPI } from 'cheerio';
 import { normalizeUrl } from '../crawler/normalize.js';
 
 export interface LinkRatioAnalysis {
@@ -8,28 +8,33 @@ export interface LinkRatioAnalysis {
   externalRatio: number;
 }
 
-export function analyzeLinks(html: string, pageUrl: string, rootUrl: string): LinkRatioAnalysis {
-  const $ = load(html);
+export function analyzeLinks($: CheerioAPI | string, pageUrl: string, rootUrl: string): LinkRatioAnalysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
   const rootOrigin = new URL(rootUrl).origin;
 
   let internalLinks = 0;
   let externalLinks = 0;
   let nofollowCount = 0;
 
-  $('a[href]').each((_idx, el) => {
-    const href = $(el).attr('href');
+  cheerioObj('a[href]').each((_idx, el) => {
+    const href = cheerioObj(el).attr('href');
     if (!href) return;
     const normalized = normalizeUrl(href, pageUrl, { stripQuery: false });
     if (!normalized) return;
 
-    const rel = ($(el).attr('rel') || '').toLowerCase();
+    const rel = (cheerioObj(el).attr('rel') || '').toLowerCase();
     if (rel.includes('nofollow')) {
       nofollowCount += 1;
     }
 
-    if (new URL(normalized).origin === rootOrigin) {
-      internalLinks += 1;
-    } else {
+    try {
+      if (new URL(normalized).origin === rootOrigin) {
+        internalLinks += 1;
+      } else {
+        externalLinks += 1;
+      }
+    } catch {
       externalLinks += 1;
     }
   });

diff --git a/plugins/core/src/analysis/seo.ts b/plugins/core/src/analysis/seo.ts
@@ -1,4 +1,4 @@
-import { load } from 'cheerio';
+import { CheerioAPI, load } from 'cheerio';
 
 export type SeoStatus = 'ok' | 'missing' | 'too_short' | 'too_long' | 'duplicate';
 
@@ -18,9 +18,11 @@ function normalizedText(value: string | null): string {
   return (value ?? '').trim().toLowerCase();
 }
 
-export function analyzeTitle(html: string): TextFieldAnalysis {
-  const $ = load(html);
-  const title = $('title').first().text().trim();
+export function analyzeTitle($: CheerioAPI | string): TextFieldAnalysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
+
+  const title = cheerioObj('title').first().text().trim();
   if (!title) {
     return { value: null, length: 0, status: 'missing' };
   }
@@ -30,9 +32,11 @@ export function analyzeTitle(html: string): TextFieldAnalysis {
   return { value: title, length: title.length, status: 'ok' };
 }
 
-export function analyzeMetaDescription(html: string): TextFieldAnalysis {
-  const $ = load(html);
-  const raw = $('meta[name="description"]').attr('content');
+export function analyzeMetaDescription($: CheerioAPI | string): TextFieldAnalysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
+
+  const raw = cheerioObj('meta[name="description"]').attr('content');
   if (raw === undefined) {
     return { value: null, length: 0, status: 'missing' };
   }
@@ -47,27 +51,11 @@ export function analyzeMetaDescription(html: string): TextFieldAnalysis {
   return { value: description, length: description.length, status: 'ok' };
 }
 
-export function applyDuplicateStatuses<T extends TextFieldAnalysis>(fields: T[]): T[] {
-  const counts = new Map<string, number>();
-  for (const field of fields) {
-    const key = normalizedText(field.value);
-    if (!key) continue;
-    counts.set(key, (counts.get(key) || 0) + 1);
-  }
-
-  return fields.map((field) => {
-    const key = normalizedText(field.value);
-    if (!key) return field;
-    if ((counts.get(key) || 0) > 1) {
-      return { ...field, status: 'duplicate' };
-    }
-    return field;
-  });
-}
+export function analyzeH1($: CheerioAPI | string, titleValue: string | null): H1Analysis {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
 
-export function analyzeH1(html: string, titleValue: string | null): H1Analysis {
-  const $ = load(html);
-  const h1Values = $('h1').toArray().map((el) => $(el).text().trim()).filter(Boolean);
+  const h1Values = cheerioObj('h1').toArray().map((el) => cheerioObj(el).text().trim()).filter(Boolean);
   const count = h1Values.length;
   const first = h1Values[0] || null;
   const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue));

diff --git a/plugins/core/src/analysis/structuredData.ts b/plugins/core/src/analysis/structuredData.ts
@@ -1,14 +1,16 @@
-import { load } from 'cheerio';
+import { CheerioAPI, load } from 'cheerio';
 
 export interface StructuredDataResult {
   present: boolean;
   types: string[];
   valid: boolean;
 }
 
-export function analyzeStructuredData(html: string): StructuredDataResult {
-  const $ = load(html);
-  const scripts = $('script[type="application/ld+json"]').toArray();
+export function analyzeStructuredData($: CheerioAPI | string): StructuredDataResult {
+  const isString = typeof $ === 'string';
+  const cheerioObj = isString ? load($ || '<html></html>') : $;
+
+  const scripts = cheerioObj('script[type="application/ld+json"]').toArray();
   if (scripts.length === 0) {
     return { present: false, types: [], valid: false };
   }
@@ -17,7 +19,7 @@ export function analyzeStructuredData(html: string): StructuredDataResult {
   let valid = true;
 
   for (const script of scripts) {
-    const raw = $(script).text().trim();
+    const raw = cheerioObj(script).text().trim();
     if (!raw) {
       valid = false;
       continue;

diff --git a/plugins/core/src/crawler/crawler.ts b/plugins/core/src/crawler/crawler.ts
@@ -38,6 +38,7 @@ export interface CrawlOptions {
   maxRedirects?: number;
   userAgent?: string;
   snapshotType?: 'full' | 'partial' | 'incremental';
+  robots?: any;
 }
 
 interface QueueItem {
@@ -522,7 +523,11 @@ export class Crawler {
   async run(): Promise<number> {
     await this.initialize();
     this.setupModules();
-    await this.fetchRobots();
+    if (this.options.robots) {
+      this.robots = this.options.robots;
+    } else {
+      await this.fetchRobots();
+    }
     await this.seedQueue();
 
     return new Promise((resolve) => {

diff --git a/plugins/core/src/db/repositories/SnapshotRepository.ts b/plugins/core/src/db/repositories/SnapshotRepository.ts
@@ -28,8 +28,11 @@ export class SnapshotRepository {
     return info.lastInsertRowid as number;
   }
 
-  getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined {
-    let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\'';
+  getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed', includePartial: boolean = false): Snapshot | undefined {
+    let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+    if (!includePartial) {
+      sql += ' AND type != \'partial\'';
+    }
     const params: any[] = [siteId];
     if (status) {
       sql += ' AND status = ?';