⚡ Optimize analysis data loading with iterators

plugins/core/src/analysis/analyze.ts

@@ -5,7 +5,7 @@ import { normalizeUrl } from '../crawler/normalize.js';
 import { calculateMetrics, Metrics } from '../graph/metrics.js';
 import { Graph, ClusterInfo } from '../graph/graph.js';
 import { analyzeContent, calculateThinContentScore } from './content.js';
-import { analyzeH1, analyzeMetaDescription, analyzeTitle, applyDuplicateStatuses, H1Analysis, TextFieldAnalysis } from './seo.js';
+import { analyzeH1, analyzeMetaDescription, analyzeTitle, H1Analysis, TextFieldAnalysis } from './seo.js';
 import { analyzeImageAlts, ImageAltAnalysis } from './images.js';
 import { analyzeLinks, LinkRatioAnalysis } from './links.js';
 import { analyzeStructuredData, StructuredDataResult } from './structuredData.js';
@@ -80,7 +80,7 @@ export interface AnalysisResult {
 }
 
 interface CrawlData {
-  pages: CrawlPage[];
+  pages: Iterable<CrawlPage> | CrawlPage[];
   metrics: Metrics;
   graph: Graph;
 }
@@ -347,52 +347,84 @@ function escapeHtml(value: string): string {
   return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
 }
 
-function analyzePages(rootUrl: string, pages: CrawlPage[]): PageAnalysis[] {
-  const titleCandidates = pages.map((page) => analyzeTitle(page.html || ''));
-  const metaCandidates = pages.map((page) => analyzeMetaDescription(page.html || ''));
-  const titles = applyDuplicateStatuses(titleCandidates);
-  const metas = applyDuplicateStatuses(metaCandidates);
-
+export function analyzePages(rootUrl: string, pages: Iterable<CrawlPage> | CrawlPage[]): PageAnalysis[] {
+  const titleCounts = new Map<string, number>();
+  const metaCounts = new Map<string, number>();
   const sentenceCountFrequency = new Map<number, number>();
-  const baseContent = pages.map((page) => analyzeContent(page.html || ''));
-  for (const item of baseContent) {
-    sentenceCountFrequency.set(item.uniqueSentenceCount, (sentenceCountFrequency.get(item.uniqueSentenceCount) || 0) + 1);
-  }
 
-  return pages.map((page, index) => {
+  const results: PageAnalysis[] = [];
+
+  for (const page of pages) {
     const html = page.html || '';
-    const title = titles[index];
-    const metaDescription = metas[index];
+
+    // 1. Analyze Individual Components
+    const title = analyzeTitle(html);
+    const metaDescription = analyzeMetaDescription(html);
     const h1 = analyzeH1(html, title.value);
-    const content = baseContent[index];
-    const duplicationScore = (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) > 1 ? 100 : 0;
-    const thinScore = calculateThinContentScore(content, duplicationScore);
+    const content = analyzeContent(html);
     const images = analyzeImageAlts(html);
     const links = analyzeLinks(html, page.url, rootUrl);
     const structuredData = analyzeStructuredData(html);
 
-    const analysis: PageAnalysis = {
+    // 2. Accumulate Frequencies for Duplicates
+    if (title.value) {
+      const key = (title.value || '').trim().toLowerCase();
+      titleCounts.set(key, (titleCounts.get(key) || 0) + 1);
+    }
+    if (metaDescription.value) {
+      const key = (metaDescription.value || '').trim().toLowerCase();
+      metaCounts.set(key, (metaCounts.get(key) || 0) + 1);
+    }
+    sentenceCountFrequency.set(content.uniqueSentenceCount, (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) + 1);
+
+    // 3. Store Preliminary Result
+    results.push({
       url: page.url,
       status: page.status || 0,
       title,
       metaDescription,
       h1,
       content,
-      thinScore,
+      thinScore: 0, // Calculated in pass 2
       images,
       links,
       structuredData,
-      seoScore: 0,
+      seoScore: 0, // Calculated in pass 2
       meta: {
         canonical: page.canonical,
         noindex: page.noindex,
         nofollow: page.nofollow
       }
-    };
+    });
+  }
+
+  // 4. Finalize Statuses and Scores (Pass 2)
+  for (const analysis of results) {
+    // Check Title Duplicates
+    if (analysis.title.value) {
+      const key = (analysis.title.value || '').trim().toLowerCase();
+      if ((titleCounts.get(key) || 0) > 1) {
+        analysis.title.status = 'duplicate';
+      }
+    }
+
+    // Check Meta Duplicates
+    if (analysis.metaDescription.value) {
+      const key = (analysis.metaDescription.value || '').trim().toLowerCase();
+      if ((metaCounts.get(key) || 0) > 1) {
+        analysis.metaDescription.status = 'duplicate';
+      }
+    }
+
+    // Check Content Duplication
+    const duplicationScore = (sentenceCountFrequency.get(analysis.content.uniqueSentenceCount) || 0) > 1 ? 100 : 0;
+    analysis.thinScore = calculateThinContentScore(analysis.content, duplicationScore);
 
+    // Calculate Final SEO Score
     analysis.seoScore = scorePageSeo(analysis);
-    return analysis;
-  });
+  }
+
+  return results;
 }
 
 function filterPageModules(
@@ -450,21 +482,25 @@ async function loadCrawlData(rootUrl: string, fromCrawl?: string): Promise<Crawl
   const graph = loadGraphFromSnapshot(snapshot.id);
   const metrics = calculateMetrics(graph, 5);
 
-  // We also need the `pages` array for analysis. 
-  // It needs `html` which might not be fully available unless we look up from the DB or Graph.
-  // Wait, the Graph stores Node which doesn't contain HTML since we removed it from memory? 
-  // Actually, `loadGraphFromSnapshot` does NOT load actual raw HTML from nodes to save memory.
-  // We need HTML for `analyzeSite` module! So we must fetch it from `pageRepo`.
-
-  const dbPages = pageRepo.getPagesBySnapshot(snapshot.id);
-  const pages: CrawlPage[] = dbPages.map((p: any) => ({
-    url: p.normalized_url,
-    status: p.http_status || 0,
-    html: p.html || '',
-    depth: p.depth || 0
-  }));
+  // Use iterator to save memory
+  const dbPagesIterator = pageRepo.getPagesIteratorBySnapshot(snapshot.id);
+
+  // We need to map the DB pages to CrawlPage format lazily
+  const pagesGenerator = function* () {
+    for (const p of dbPagesIterator) {
+      yield {
+        url: p.normalized_url,
+        status: p.http_status || 0,
+        html: p.html || '',
+        depth: p.depth || 0,
+        canonical: p.canonical_url || undefined,
+        noindex: !!p.noindex,
+        nofollow: !!p.nofollow
+      } as CrawlPage;
+    }
+  };
 
-  return { pages, metrics, graph };
+  return { pages: pagesGenerator(), metrics, graph };
 }
 
 function parsePages(raw: Record<string, unknown>): CrawlPage[] {

plugins/core/src/db/repositories/PageRepository.ts

@@ -121,6 +121,10 @@ export class PageRepository {
     return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId) as Page[];
   }
 
+  getPagesIteratorBySnapshot(snapshotId: number): IterableIterator<Page> {
+    return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').iterate(snapshotId) as IterableIterator<Page>;
+  }
+
   getIdByUrl(siteId: number, url: string): number | undefined {
     const row = this.getIdStmt.get(siteId, url) as { id: number } | undefined;
     return row?.id;