diff --git a/plugins/cli/src/commands/ui.ts b/plugins/cli/src/commands/ui.ts index 8f80879..3c7920d 100644 --- a/plugins/cli/src/commands/ui.ts +++ b/plugins/cli/src/commands/ui.ts @@ -57,7 +57,7 @@ export const ui = new Command('ui') process.exit(1); } - const snapshot = snapshotRepo.getLatestSnapshot(site.id); + const snapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed'); if (!snapshot) { console.error(chalk.red(`❌ No snapshots found for site: ${domain}`)); process.exit(1); diff --git a/plugins/core/src/analysis/analyze.ts b/plugins/core/src/analysis/analyze.ts index f3a0bb6..23c89e2 100644 --- a/plugins/core/src/analysis/analyze.ts +++ b/plugins/core/src/analysis/analyze.ts @@ -1,3 +1,4 @@ +import { load } from 'cheerio'; import { crawl } from '../crawler/crawl.js'; import { loadGraphFromSnapshot } from '../db/graphLoader.js'; import { normalizeUrl } from '../crawler/normalize.js'; @@ -94,11 +95,6 @@ interface CrawlData { /** * Analyzes a site for SEO, content, and accessibility. * Supports live crawling or loading from a database snapshot. - * Note: File-based data loading is not supported. - * - * @param url The root URL to analyze - * @param options Analysis options - * @param context Engine context for event emission */ export async function analyzeSite(url: string, options: AnalyzeOptions, context?: EngineContext): Promise { const normalizedRoot = normalizeUrl(url, '', { stripQuery: false }); @@ -106,66 +102,70 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? throw new Error('Invalid URL for analysis'); } + const start = Date.now(); let crawlData: CrawlData; let robots: any = null; - // Always try to fetch robots.txt for the analysis session - // to ensure we have the latest rules for visibility reporting. + // 1. Robots fetch try { const robotsUrl = new URL('/robots.txt', normalizedRoot).toString(); - const robotsRes = await (new (await import('../crawler/fetcher.js')).Fetcher()).fetch(robotsUrl, { maxBytes: 500000 }); - const status = robotsRes.status; - if (typeof status === 'number' && status >= 200 && status < 300) { + const { Fetcher } = await import('../crawler/fetcher.js'); + const fetcher = new Fetcher({ + rate: options.live ? 10 : options.rate, + proxyUrl: options.proxyUrl, + userAgent: options.userAgent + }); + const robotsRes = await fetcher.fetch(robotsUrl, { maxBytes: 500000 }); + if (typeof robotsRes.status === 'number' && robotsRes.status >= 200 && robotsRes.status < 300) { const robotsParserModule = await import('robots-parser'); const robotsParser = (robotsParserModule as any).default || robotsParserModule; robots = (robotsParser as any)(robotsUrl, robotsRes.body); + if (context) context.emit({ type: 'info', message: `[analyze] Robots fetch took ${Date.now() - start}ms` }); } } catch { - // Silence robots fetch errors, fallback to existing or none + // Fallback } + + // 2. Data Acquisition if (options.live) { - crawlData = await runLiveCrawl(normalizedRoot, options, context); + const crawlStart = Date.now(); + crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); + if (context) context.emit({ type: 'info', message: `[analyze] runLiveCrawl took ${Date.now() - crawlStart}ms` }); } else { try { + const loadStart = Date.now(); crawlData = await loadCrawlData(normalizedRoot); + if (context) context.emit({ type: 'info', message: `[analyze] loadCrawlData took ${Date.now() - loadStart}ms` }); - // Convert generator to array so it can be reused multiple times const allPages = Array.from(crawlData.pages); crawlData.pages = allPages; - // Check if the requested URL actually exists in this snapshot const exists = allPages.some(p => p.url === normalizedRoot); if (!exists) { - options.live = true; // Mark as live so the analysis knows to pick the first page if exact match fails - if (context) { - context.emit({ type: 'info', message: `URL ${normalizedRoot} not found in latest snapshot. Fetching live...` }); - } - crawlData = await runLiveCrawl(normalizedRoot, options, context); + if (context) context.emit({ type: 'info', message: `URL ${normalizedRoot} not found. Fetching live...` }); + crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); } } catch (error: any) { - const isNotFound = error.code === 'ENOENT' || - error.message.includes('Crawl data not found') || - error.message.includes('No completed snapshot found') || - error.message.includes('not found in database'); - if (isNotFound) { - options.live = true; // Force live mode - if (context) { - context.emit({ type: 'info', message: 'No local crawl data found. Switching to live analysis mode...' }); - } - crawlData = await runLiveCrawl(normalizedRoot, options, context); - } else { - throw error; - } + if (context) context.emit({ type: 'info', message: 'No local crawl data found. Switching to live...' }); + crawlData = await runLiveCrawl(normalizedRoot, options, context, robots); } } const snapshotId = crawlData.snapshotId; const crawledAt = crawlData.crawledAt; - // Run clustering if requested or as default - detectContentClusters(crawlData.graph, options.clusterThreshold, options.minClusterSize); + // 3. Post-Processing + const clusterStart = Date.now(); + if (options.allPages) { + detectContentClusters(crawlData.graph, options.clusterThreshold, options.minClusterSize); + if (context) context.emit({ type: 'info', message: `[analyze] detectContentClusters took ${Date.now() - clusterStart}ms` }); + } else { + if (context) context.emit({ type: 'info', message: `[analyze] Skipping clustering for single-page view` }); + } - const pages = analyzePages(normalizedRoot, crawlData.pages, robots); + const pagesStart = Date.now(); + const pages = analyzePages(normalizedRoot, crawlData.pages, robots, options); + if (context) context.emit({ type: 'info', message: `[analyze] analyzePages took ${Date.now() - pagesStart}ms` }); const activeModules = { seo: !!options.seo, @@ -174,12 +174,8 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? }; const hasFilters = activeModules.seo || activeModules.content || activeModules.accessibility; + const filteredPages = hasFilters ? pages.map((page) => filterPageModules(page, activeModules)) : pages; - const filteredPages = hasFilters - ? pages.map((page) => filterPageModules(page, activeModules)) - : pages; - - // Filter to only the requested URL const targetPage = filteredPages.find(p => p.url === normalizedRoot); let resultPages: PageAnalysis[]; @@ -193,6 +189,8 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? const thinPages = pages.filter((page) => page.thinScore >= 70).length; const siteScores = aggregateSiteScore(crawlData.metrics, resultPages.length === 1 ? resultPages : pages); + if (context) context.emit({ type: 'info', message: `[analyze] Total analysis completed in ${Date.now() - start}ms` }); + return { site_summary: { pages_analyzed: resultPages.length, @@ -210,152 +208,52 @@ export async function analyzeSite(url: string, options: AnalyzeOptions, context? }; } -export function renderAnalysisHtml(result: AnalysisResult): string { - if (result.pages.length === 1) { - return renderSinglePageHtml(result.pages[0]); - } - const rows = result.pages - .map((page) => `${escapeHtml(page.url)}${page.seoScore}${page.thinScore}${page.title.status}${page.metaDescription.status}`) - .join(''); - - return ANALYSIS_LIST_TEMPLATE - .replace('{{PAGES_ANALYZED}}', result.site_summary.pages_analyzed.toString()) - .replace('{{AVG_SEO_SCORE}}', result.site_summary.avg_seo_score.toString()) - .replace('{{ROWS}}', rows); -} - -function renderSinglePageHtml(page: PageAnalysis): string { - const structuredDataStatus = page.structuredData.present - ? (page.structuredData.valid ? 'Valid' : 'Invalid JSON') - : 'Not detected'; - - const structuredDataTypesRow = page.structuredData.present ? ` - - Types Found - ${page.structuredData.types.map(t => `${t}`).join(', ')} - - ` : ''; - - return ANALYSIS_PAGE_TEMPLATE - .replaceAll('{{URL}}', escapeHtml(page.url)) - .replace('{{SEO_SCORE}}', page.seoScore.toString()) - .replace('{{THIN_SCORE}}', page.thinScore.toString()) - .replace('{{HTTP_STATUS}}', page.status === 0 ? 'Pending/Limit' : page.status.toString()) - .replace('{{TITLE_VALUE}}', escapeHtml(page.title.value || '(missing)')) - .replace('{{TITLE_LENGTH}}', page.title.length.toString()) - .replaceAll('{{TITLE_STATUS}}', page.title.status) - .replace('{{META_DESCRIPTION_VALUE}}', escapeHtml(page.metaDescription.value || '(missing)')) - .replace('{{META_DESCRIPTION_LENGTH}}', page.metaDescription.length.toString()) - .replaceAll('{{META_DESCRIPTION_STATUS}}', page.metaDescription.status) - .replace('{{CANONICAL}}', page.meta.canonical ? escapeHtml(page.meta.canonical) : '(none)') - .replace('{{ROBOTS_INDEX}}', (!page.meta.noindex).toString()) - .replace('{{ROBOTS_FOLLOW}}', (!page.meta.nofollow).toString()) - .replaceAll('{{H1_STATUS}}', page.h1.status) - .replace('{{H1_COUNT}}', page.h1.count.toString()) - .replace('{{H1_MATCHES_TITLE}}', page.h1.matchesTitle ? ' | Matches Title' : '') - .replace('{{WORD_COUNT}}', page.content.wordCount.toString()) - .replace('{{UNIQUE_SENTENCES}}', page.content.uniqueSentenceCount.toString()) - .replace('{{TEXT_HTML_RATIO}}', (page.content.textHtmlRatio * 100).toFixed(2)) - .replace('{{INTERNAL_LINKS}}', page.links.internalLinks.toString()) - .replace('{{EXTERNAL_LINKS}}', page.links.externalLinks.toString()) - .replace('{{EXTERNAL_RATIO}}', (page.links.externalRatio * 100).toFixed(1)) - .replace('{{TOTAL_IMAGES}}', page.images.totalImages.toString()) - .replace('{{MISSING_ALT}}', page.images.missingAlt.toString()) - .replace('{{STRUCTURED_DATA_STATUS}}', structuredDataStatus) - .replace('{{STRUCTURED_DATA_TYPES_ROW}}', structuredDataTypesRow); -} - -export function renderAnalysisMarkdown(result: AnalysisResult): string { - const summary = [ - '# Crawlith SEO Analysis Report', - '', - '## 📊 Summary', - `- Pages Analyzed: ${result.site_summary.pages_analyzed}`, - `- Overall Site Score: ${result.site_summary.site_score.toFixed(1)}`, - `- Avg SEO Score: ${result.site_summary.avg_seo_score.toFixed(1)}`, - `- Thin Pages Found: ${result.site_summary.thin_pages}`, - `- Duplicate Titles: ${result.site_summary.duplicate_titles}`, - '', - '## 📄 Page Details', - '', - '| URL | SEO Score | Thin Score | Title Status | Meta Status |', - '| :--- | :--- | :--- | :--- | :--- |', - ]; - - result.pages.forEach((page) => { - summary.push(`| ${page.url} | ${page.seoScore} | ${page.thinScore} | ${page.title.status} | ${page.metaDescription.status} |`); - }); - - return summary.join('\n'); -} - -export function renderAnalysisCsv(result: AnalysisResult): string { - const headers = ['URL', 'SEO Score', 'Thin Score', 'HTTP Status', 'Title', 'Title Length', 'Meta Description', 'Desc Length', 'Word Count', 'Internal Links', 'External Links']; - const rows = result.pages.map((p) => { - const statusStr = p.status === 0 ? 'Pending/Limit' : p.status; - return [ - p.url, - p.seoScore, - p.thinScore, - statusStr, - `"${(p.title.value || '').replace(/"/g, '""')}"`, - p.title.length, - `"${(p.metaDescription.value || '').replace(/"/g, '""')}"`, - p.metaDescription.length, - p.content.wordCount, - p.links.internalLinks, - p.links.externalLinks - ].join(','); - }); - - return [headers.join(','), ...rows].join('\n'); -} - -function escapeHtml(value: string): string { - return value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); -} - -export function analyzePages(rootUrl: string, pages: Iterable | CrawlPage[], robots?: any): PageAnalysis[] { +export function analyzePages(rootUrl: string, pages: Iterable | CrawlPage[], robots?: any, options: AnalyzeOptions = {}): PageAnalysis[] { const titleCounts = new Map(); const metaCounts = new Map(); const sentenceCountFrequency = new Map(); const results: PageAnalysis[] = []; + const normalizedRoot = rootUrl; for (const page of pages) { + const isTarget = page.url === normalizedRoot; + + // In single-page mode, if it's not the target, we skip it entirely for speed. + // Duplicate title detection is sacrificed for single-page live analysis speed. + // Full site audits will correctly handle site-wide duplication. + if (!options.allPages && !isTarget) continue; + const html = page.html || ''; + const $ = load(html || ''); - // 0. Update crawl status based on current robots rules let crawlStatus = page.crawlStatus; if (robots) { const isBlocked = !robots.isAllowed(page.url, 'crawlith') || (!page.url.endsWith('/') && !robots.isAllowed(page.url + '/', 'crawlith')); - if (isBlocked) { - crawlStatus = 'blocked_by_robots'; - } + if (isBlocked) crawlStatus = 'blocked_by_robots'; } - // 1. Analyze Individual Components - const title = analyzeTitle(html); - const metaDescription = analyzeMetaDescription(html); - const h1 = analyzeH1(html, title.value); - const content = analyzeContent(html); - const images = analyzeImageAlts(html); - const links = analyzeLinks(html, page.url, rootUrl); - const structuredData = analyzeStructuredData(html); + // Shared DOM Analysis + const title = analyzeTitle($); + const metaDescription = analyzeMetaDescription($); + + const h1 = analyzeH1($, title.value); + const content = analyzeContent($); + const images = analyzeImageAlts($); + const links = analyzeLinks($, page.url, rootUrl); + const structuredData = analyzeStructuredData($); - // 2. Accumulate Frequencies for Duplicates if (title.value) { - const key = (title.value || '').trim().toLowerCase(); + const key = title.value.trim().toLowerCase(); titleCounts.set(key, (titleCounts.get(key) || 0) + 1); } if (metaDescription.value) { - const key = (metaDescription.value || '').trim().toLowerCase(); + const key = metaDescription.value.trim().toLowerCase(); metaCounts.set(key, (metaCounts.get(key) || 0) + 1); } sentenceCountFrequency.set(content.uniqueSentenceCount, (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) + 1); - // 3. Store Preliminary Result results.push({ url: page.url, status: page.status || 0, @@ -363,11 +261,11 @@ export function analyzePages(rootUrl: string, pages: Iterable | Crawl metaDescription, h1, content, - thinScore: 0, // Calculated in pass 2 + thinScore: 0, images, links, structuredData, - seoScore: 0, // Calculated in pass 2 + seoScore: 0, meta: { canonical: page.canonical, noindex: page.noindex, @@ -377,53 +275,34 @@ export function analyzePages(rootUrl: string, pages: Iterable | Crawl }); } - // 4. Finalize Statuses and Scores (Pass 2) for (const analysis of results) { - // Check Title Duplicates if (analysis.title.value) { - const key = (analysis.title.value || '').trim().toLowerCase(); - if ((titleCounts.get(key) || 0) > 1) { - analysis.title.status = 'duplicate'; - } + const key = analysis.title.value.trim().toLowerCase(); + if ((titleCounts.get(key) || 0) > 1) analysis.title.status = 'duplicate'; } - - // Check Meta Duplicates if (analysis.metaDescription.value) { - const key = (analysis.metaDescription.value || '').trim().toLowerCase(); - if ((metaCounts.get(key) || 0) > 1) { - analysis.metaDescription.status = 'duplicate'; - } + const key = analysis.metaDescription.value.trim().toLowerCase(); + if ((metaCounts.get(key) || 0) > 1) analysis.metaDescription.status = 'duplicate'; } - - // Check Content Duplication const duplicationScore = (sentenceCountFrequency.get(analysis.content.uniqueSentenceCount) || 0) > 1 ? 100 : 0; analysis.thinScore = calculateThinContentScore(analysis.content, duplicationScore); - - // Calculate Final SEO Score analysis.seoScore = scorePageSeo(analysis); } return results; } -function filterPageModules( - page: PageAnalysis, - modules: { seo: boolean; content: boolean; accessibility: boolean } -): PageAnalysis { - const keepSeo = modules.seo; - const keepContent = modules.content; - const keepAccessibility = modules.accessibility; - +function filterPageModules(page: PageAnalysis, modules: { seo: boolean; content: boolean; accessibility: boolean }): PageAnalysis { return { ...page, - title: keepSeo ? page.title : { value: null, length: 0, status: 'missing' }, - metaDescription: keepSeo ? page.metaDescription : { value: null, length: 0, status: 'missing' }, - h1: (keepSeo || keepContent) ? page.h1 : { count: 0, status: 'critical', matchesTitle: false }, - links: keepSeo ? page.links : { internalLinks: 0, externalLinks: 0, nofollowCount: 0, externalRatio: 0 }, - structuredData: keepSeo ? page.structuredData : { present: false, valid: false, types: [] }, - content: keepContent ? page.content : { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 }, - thinScore: keepContent ? page.thinScore : 0, - images: keepAccessibility ? page.images : { totalImages: 0, missingAlt: 0, emptyAlt: 0 } + title: modules.seo ? page.title : { value: null, length: 0, status: 'missing' }, + metaDescription: modules.seo ? page.metaDescription : { value: null, length: 0, status: 'missing' }, + h1: (modules.seo || modules.content) ? page.h1 : { count: 0, status: 'critical', matchesTitle: false }, + links: modules.seo ? page.links : { internalLinks: 0, externalLinks: 0, nofollowCount: 0, externalRatio: 0 }, + structuredData: modules.seo ? page.structuredData : { present: false, valid: false, types: [] }, + content: modules.content ? page.content : { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 }, + thinScore: modules.content ? page.thinScore : 0, + images: modules.accessibility ? page.images : { totalImages: 0, missingAlt: 0, emptyAlt: 0 } }; } @@ -437,27 +316,18 @@ async function loadCrawlData(rootUrl: string): Promise { const domain = urlObj.hostname.replace('www.', ''); const site = siteRepo.firstOrCreateSite(domain); - let snapshot; + let snapshot = null; const page = pageRepo.getPage(site.id, rootUrl); - if (page && page.last_seen_snapshot_id) { + if (page?.last_seen_snapshot_id) { snapshot = snapshotRepo.getSnapshot(page.last_seen_snapshot_id); } - - if (!snapshot) { - snapshot = snapshotRepo.getLatestSnapshot(site.id); - } - - if (!snapshot) { - throw new Error(`No crawl data found for ${rootUrl} in database.`); - } + if (!snapshot) snapshot = snapshotRepo.getLatestSnapshot(site.id); + if (!snapshot) throw new Error(`No crawl data found for ${rootUrl}`); const graph = loadGraphFromSnapshot(snapshot.id); const metrics = calculateMetrics(graph, 5); - - // Use iterator to save memory const dbPagesIterator = pageRepo.getPagesIteratorBySnapshot(snapshot.id); - // We need to map the DB pages to CrawlPage format lazily const pagesGenerator = function* () { for (const p of dbPagesIterator) { yield { @@ -476,30 +346,56 @@ async function loadCrawlData(rootUrl: string): Promise { return { pages: pagesGenerator(), metrics, graph, snapshotId: snapshot.id, crawledAt: snapshot.created_at }; } - -async function runLiveCrawl(url: string, options: AnalyzeOptions, context?: EngineContext): Promise { +async function runLiveCrawl(url: string, options: AnalyzeOptions, context?: EngineContext, robots?: any): Promise { const snapshotId = await crawl(url, { - limit: 1, // Always limit to 1 for single page live analysis + limit: 1, depth: 0, rate: options.rate, proxyUrl: options.proxyUrl, userAgent: options.userAgent, maxRedirects: options.maxRedirects, debug: options.debug, - snapshotType: 'partial' + snapshotType: 'partial', + robots }, context) as number; const graph = loadGraphFromSnapshot(snapshotId); const pages = graph.getNodes().map((node) => ({ url: node.url, status: node.status, - html: node.html || '', // Include HTML + html: node.html || '', depth: node.depth, crawlStatus: node.crawlStatus })); - return { - pages, - metrics: calculateMetrics(graph, 1), - graph, - snapshotId - }; + return { pages, metrics: calculateMetrics(graph, 1), graph, snapshotId }; +} + +export function escapeHtml(value: string): string { + return value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); +} + +export function renderAnalysisHtml(result: AnalysisResult): string { + if (result.pages.length === 1) return renderSinglePageHtml(result.pages[0]); + const rows = result.pages.map((page) => `${escapeHtml(page.url)}${page.seoScore}${page.thinScore}${page.title.status}${page.metaDescription.status}`).join(''); + return ANALYSIS_LIST_TEMPLATE.replace('{{PAGES_ANALYZED}}', result.site_summary.pages_analyzed.toString()).replace('{{AVG_SEO_SCORE}}', result.site_summary.avg_seo_score.toString()).replace('{{ROWS}}', rows); +} + +function renderSinglePageHtml(page: PageAnalysis): string { + const structuredDataStatus = page.structuredData.present ? (page.structuredData.valid ? 'Valid' : 'Invalid JSON') : 'Not detected'; + const structuredDataTypesRow = page.structuredData.present ? `Types Found${page.structuredData.types.map(t => `${t}`).join(', ')}` : ''; + return ANALYSIS_PAGE_TEMPLATE.replaceAll('{{URL}}', escapeHtml(page.url)).replace('{{SEO_SCORE}}', page.seoScore.toString()).replace('{{THIN_SCORE}}', page.thinScore.toString()).replace('{{HTTP_STATUS}}', page.status === 0 ? 'Pending/Limit' : page.status.toString()).replace('{{TITLE_VALUE}}', escapeHtml(page.title.value || '(missing)')).replace('{{TITLE_LENGTH}}', page.title.length.toString()).replaceAll('{{TITLE_STATUS}}', page.title.status).replace('{{META_DESCRIPTION_VALUE}}', escapeHtml(page.metaDescription.value || '(missing)')).replace('{{META_DESCRIPTION_LENGTH}}', page.metaDescription.length.toString()).replaceAll('{{META_DESCRIPTION_STATUS}}', page.metaDescription.status).replace('{{CANONICAL}}', page.meta.canonical ? escapeHtml(page.meta.canonical) : '(none)').replace('{{ROBOTS_INDEX}}', (!page.meta.noindex).toString()).replace('{{ROBOTS_FOLLOW}}', (!page.meta.nofollow).toString()).replaceAll('{{H1_STATUS}}', page.h1.status).replace('{{H1_COUNT}}', page.h1.count.toString()).replace('{{H1_MATCHES_TITLE}}', page.h1.matchesTitle ? ' | Matches Title' : '').replace('{{WORD_COUNT}}', page.content.wordCount.toString()).replace('{{UNIQUE_SENTENCES}}', page.content.uniqueSentenceCount.toString()).replace('{{TEXT_HTML_RATIO}}', (page.content.textHtmlRatio * 100).toFixed(2)).replace('{{INTERNAL_LINKS}}', page.links.internalLinks.toString()).replace('{{EXTERNAL_LINKS}}', page.links.externalLinks.toString()).replace('{{EXTERNAL_RATIO}}', (page.links.externalRatio * 100).toFixed(1)).replace('{{TOTAL_IMAGES}}', page.images.totalImages.toString()).replace('{{MISSING_ALT}}', page.images.missingAlt.toString()).replace('{{STRUCTURED_DATA_STATUS}}', structuredDataStatus).replace('{{STRUCTURED_DATA_TYPES_ROW}}', structuredDataTypesRow); +} + +export function renderAnalysisMarkdown(result: AnalysisResult): string { + const summary = ['# Crawlith SEO Analysis Report', '', '## 📊 Summary', `- Pages Analyzed: ${result.site_summary.pages_analyzed}`, `- Overall Site Score: ${result.site_summary.site_score.toFixed(1)}`, `- Avg SEO Score: ${result.site_summary.avg_seo_score.toFixed(1)}`, `- Thin Pages Found: ${result.site_summary.thin_pages}`, `- Duplicate Titles: ${result.site_summary.duplicate_titles}`, '', '## 📄 Page Details', '', '| URL | SEO Score | Thin Score | Title Status | Meta Status |', '| :--- | :--- | :--- | :--- | :--- |']; + result.pages.forEach((page) => summary.push(`| ${page.url} | ${page.seoScore} | ${page.thinScore} | ${page.title.status} | ${page.metaDescription.status} |`)); + return summary.join('\n'); +} + +export function renderAnalysisCsv(result: AnalysisResult): string { + const headers = ['URL', 'SEO Score', 'Thin Score', 'HTTP Status', 'Title', 'Title Length', 'Meta Description', 'Desc Length', 'Word Count', 'Internal Links', 'External Links']; + const rows = result.pages.map((p) => { + const statusStr = p.status === 0 ? 'Pending/Limit' : p.status; + return [p.url, p.seoScore, p.thinScore, statusStr, `"${(p.title.value || '').replace(/"/g, '""')}"`, p.title.length, `"${(p.metaDescription.value || '').replace(/"/g, '""')}"`, p.metaDescription.length, p.content.wordCount, p.links.internalLinks, p.links.externalLinks].join(','); + }); + return [headers.join(','), ...rows].join('\n'); } diff --git a/plugins/core/src/analysis/content.ts b/plugins/core/src/analysis/content.ts index 815d4e3..694feac 100644 --- a/plugins/core/src/analysis/content.ts +++ b/plugins/core/src/analysis/content.ts @@ -1,4 +1,4 @@ -import { load } from 'cheerio'; +import { CheerioAPI, load } from 'cheerio'; export interface ContentAnalysis { wordCount: number; @@ -18,17 +18,24 @@ const DEFAULT_WEIGHTS: ThinScoreWeights = { dupWeight: 0.25 }; -export function analyzeContent(html: string): ContentAnalysis { - const $ = load(html || ''); - $('script,style,nav,footer').remove(); +export function analyzeContent($: CheerioAPI | string): ContentAnalysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; - const text = $('body').length ? $('body').text() : $.text(); + // We don't want to modify the shared $ object if we remove elements + // So we create a localized copy of the body text or use selection + const body = cheerioObj('body').length ? cheerioObj('body') : cheerioObj('html'); + + // To avoid removing from shared $, we extract text from a clone if possible, + // but cloning in cheerio is expensive. + // Better: just get the text and clean it or use a filter. + const text = body.clone().find('script,style,nav,footer').remove().end().text(); const cleanText = text.replace(/\s+/g, ' ').trim(); const words = cleanText ? cleanText.split(/\s+/).filter(Boolean) : []; const wordCount = words.length; - const htmlLength = Math.max(html.length, 1); + const htmlLength = isString ? ($.length || 1) : 1000; // Fallback if we don't have original HTML length const textHtmlRatio = cleanText.length / htmlLength; const sentenceSet = new Set( diff --git a/plugins/core/src/analysis/images.ts b/plugins/core/src/analysis/images.ts index 4504f79..b03f6f2 100644 --- a/plugins/core/src/analysis/images.ts +++ b/plugins/core/src/analysis/images.ts @@ -1,4 +1,4 @@ -import { load } from 'cheerio'; +import { CheerioAPI, load } from 'cheerio'; export interface ImageAltAnalysis { totalImages: number; @@ -6,13 +6,15 @@ export interface ImageAltAnalysis { emptyAlt: number; } -export function analyzeImageAlts(html: string): ImageAltAnalysis { - const $ = load(html); +export function analyzeImageAlts($: CheerioAPI | string): ImageAltAnalysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; + let missingAlt = 0; let emptyAlt = 0; - $('img').each((_idx, el) => { - const alt = $(el).attr('alt'); + cheerioObj('img').each((_idx, el) => { + const alt = cheerioObj(el).attr('alt'); if (alt === undefined) { missingAlt += 1; return; @@ -23,6 +25,6 @@ export function analyzeImageAlts(html: string): ImageAltAnalysis { } }); - const totalImages = $('img').length; + const totalImages = cheerioObj('img').length; return { totalImages, missingAlt, emptyAlt }; } diff --git a/plugins/core/src/analysis/links.ts b/plugins/core/src/analysis/links.ts index 846d8f9..c9c7469 100644 --- a/plugins/core/src/analysis/links.ts +++ b/plugins/core/src/analysis/links.ts @@ -1,4 +1,4 @@ -import { load } from 'cheerio'; +import { load, CheerioAPI } from 'cheerio'; import { normalizeUrl } from '../crawler/normalize.js'; export interface LinkRatioAnalysis { @@ -8,28 +8,33 @@ export interface LinkRatioAnalysis { externalRatio: number; } -export function analyzeLinks(html: string, pageUrl: string, rootUrl: string): LinkRatioAnalysis { - const $ = load(html); +export function analyzeLinks($: CheerioAPI | string, pageUrl: string, rootUrl: string): LinkRatioAnalysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; const rootOrigin = new URL(rootUrl).origin; let internalLinks = 0; let externalLinks = 0; let nofollowCount = 0; - $('a[href]').each((_idx, el) => { - const href = $(el).attr('href'); + cheerioObj('a[href]').each((_idx, el) => { + const href = cheerioObj(el).attr('href'); if (!href) return; const normalized = normalizeUrl(href, pageUrl, { stripQuery: false }); if (!normalized) return; - const rel = ($(el).attr('rel') || '').toLowerCase(); + const rel = (cheerioObj(el).attr('rel') || '').toLowerCase(); if (rel.includes('nofollow')) { nofollowCount += 1; } - if (new URL(normalized).origin === rootOrigin) { - internalLinks += 1; - } else { + try { + if (new URL(normalized).origin === rootOrigin) { + internalLinks += 1; + } else { + externalLinks += 1; + } + } catch { externalLinks += 1; } }); diff --git a/plugins/core/src/analysis/seo.ts b/plugins/core/src/analysis/seo.ts index 1eec87a..d7ee2f3 100644 --- a/plugins/core/src/analysis/seo.ts +++ b/plugins/core/src/analysis/seo.ts @@ -1,4 +1,4 @@ -import { load } from 'cheerio'; +import { CheerioAPI, load } from 'cheerio'; export type SeoStatus = 'ok' | 'missing' | 'too_short' | 'too_long' | 'duplicate'; @@ -18,9 +18,11 @@ function normalizedText(value: string | null): string { return (value ?? '').trim().toLowerCase(); } -export function analyzeTitle(html: string): TextFieldAnalysis { - const $ = load(html); - const title = $('title').first().text().trim(); +export function analyzeTitle($: CheerioAPI | string): TextFieldAnalysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; + + const title = cheerioObj('title').first().text().trim(); if (!title) { return { value: null, length: 0, status: 'missing' }; } @@ -30,9 +32,11 @@ export function analyzeTitle(html: string): TextFieldAnalysis { return { value: title, length: title.length, status: 'ok' }; } -export function analyzeMetaDescription(html: string): TextFieldAnalysis { - const $ = load(html); - const raw = $('meta[name="description"]').attr('content'); +export function analyzeMetaDescription($: CheerioAPI | string): TextFieldAnalysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; + + const raw = cheerioObj('meta[name="description"]').attr('content'); if (raw === undefined) { return { value: null, length: 0, status: 'missing' }; } @@ -47,27 +51,11 @@ export function analyzeMetaDescription(html: string): TextFieldAnalysis { return { value: description, length: description.length, status: 'ok' }; } -export function applyDuplicateStatuses(fields: T[]): T[] { - const counts = new Map(); - for (const field of fields) { - const key = normalizedText(field.value); - if (!key) continue; - counts.set(key, (counts.get(key) || 0) + 1); - } - - return fields.map((field) => { - const key = normalizedText(field.value); - if (!key) return field; - if ((counts.get(key) || 0) > 1) { - return { ...field, status: 'duplicate' }; - } - return field; - }); -} +export function analyzeH1($: CheerioAPI | string, titleValue: string | null): H1Analysis { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; -export function analyzeH1(html: string, titleValue: string | null): H1Analysis { - const $ = load(html); - const h1Values = $('h1').toArray().map((el) => $(el).text().trim()).filter(Boolean); + const h1Values = cheerioObj('h1').toArray().map((el) => cheerioObj(el).text().trim()).filter(Boolean); const count = h1Values.length; const first = h1Values[0] || null; const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue)); diff --git a/plugins/core/src/analysis/structuredData.ts b/plugins/core/src/analysis/structuredData.ts index f5462b4..da54fe0 100644 --- a/plugins/core/src/analysis/structuredData.ts +++ b/plugins/core/src/analysis/structuredData.ts @@ -1,4 +1,4 @@ -import { load } from 'cheerio'; +import { CheerioAPI, load } from 'cheerio'; export interface StructuredDataResult { present: boolean; @@ -6,9 +6,11 @@ export interface StructuredDataResult { valid: boolean; } -export function analyzeStructuredData(html: string): StructuredDataResult { - const $ = load(html); - const scripts = $('script[type="application/ld+json"]').toArray(); +export function analyzeStructuredData($: CheerioAPI | string): StructuredDataResult { + const isString = typeof $ === 'string'; + const cheerioObj = isString ? load($ || '') : $; + + const scripts = cheerioObj('script[type="application/ld+json"]').toArray(); if (scripts.length === 0) { return { present: false, types: [], valid: false }; } @@ -17,7 +19,7 @@ export function analyzeStructuredData(html: string): StructuredDataResult { let valid = true; for (const script of scripts) { - const raw = $(script).text().trim(); + const raw = cheerioObj(script).text().trim(); if (!raw) { valid = false; continue; diff --git a/plugins/core/src/crawler/crawler.ts b/plugins/core/src/crawler/crawler.ts index 3018371..071f61f 100644 --- a/plugins/core/src/crawler/crawler.ts +++ b/plugins/core/src/crawler/crawler.ts @@ -38,6 +38,7 @@ export interface CrawlOptions { maxRedirects?: number; userAgent?: string; snapshotType?: 'full' | 'partial' | 'incremental'; + robots?: any; } interface QueueItem { @@ -522,7 +523,11 @@ export class Crawler { async run(): Promise { await this.initialize(); this.setupModules(); - await this.fetchRobots(); + if (this.options.robots) { + this.robots = this.options.robots; + } else { + await this.fetchRobots(); + } await this.seedQueue(); return new Promise((resolve) => { diff --git a/plugins/core/src/db/repositories/SnapshotRepository.ts b/plugins/core/src/db/repositories/SnapshotRepository.ts index 29eaf97..83a9199 100644 --- a/plugins/core/src/db/repositories/SnapshotRepository.ts +++ b/plugins/core/src/db/repositories/SnapshotRepository.ts @@ -28,8 +28,11 @@ export class SnapshotRepository { return info.lastInsertRowid as number; } - getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined { - let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\''; + getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed', includePartial: boolean = false): Snapshot | undefined { + let sql = 'SELECT * FROM snapshots WHERE site_id = ?'; + if (!includePartial) { + sql += ' AND type != \'partial\''; + } const params: any[] = [siteId]; if (status) { sql += ' AND status = ?'; diff --git a/plugins/server/src/index.ts b/plugins/server/src/index.ts index c48213b..ea46041 100644 --- a/plugins/server/src/index.ts +++ b/plugins/server/src/index.ts @@ -7,7 +7,10 @@ import { SiteRepository, SnapshotRepository, - Snapshot + Snapshot, + crawl, + analyzeSite, + analyzePages } from '@crawlith/core'; export interface ServerOptions { @@ -70,9 +73,13 @@ export function startServer(options: ServerOptions): Promise { // 4.1 GET /api/context api.get('/context', (req, res) => { + const latestSnapshot = snapshotRepo.getLatestSnapshot(siteId as number, 'completed', true); + const latestFullSnapshot = snapshotRepo.getLatestSnapshot(siteId as number, 'completed', false); + res.json({ siteId, - snapshotId, // Default boot snapshot + snapshotId: latestFullSnapshot?.id || snapshotId, + latestSnapshotId: latestSnapshot?.id || snapshotId, domain: site.domain, createdAt: site.created_at }); @@ -150,11 +157,11 @@ export function startServer(options: ServerOptions): Promise { durationMs: 0, // Not stored currently avgDepth: 0, // Need to compute efficiency: 100 - } + }, + snapshotId: currentSnapshotId }); }); - // 4.3 GET /api/issues api.get('/issues', validateSnapshot, (req, res) => { const currentSnapshotId = (req as any).snapshotId as number; const severity = req.query.severity as string; @@ -328,10 +335,338 @@ export function startServer(options: ServerOptions): Promise { // 4.7 GET /api/snapshots api.get('/snapshots', (req, res) => { - const rows = db.prepare('SELECT id, created_at as createdAt FROM snapshots WHERE site_id = ? ORDER BY created_at DESC').all(siteId); + const rows = db.prepare('SELECT id, type, created_at as createdAt FROM snapshots WHERE site_id = ? ORDER BY created_at DESC').all(siteId); res.json({ results: rows }); }); + // 5.1 GET /api/page + api.get('/page', (req, res) => { + const url = req.query.url as string; + + if (!url) { + return res.status(400).json({ error: 'URL parameter is required' }); + } + + // ALWAYS select the latest snapshot where this URL was analyzed, regardless of dashboard snapshot + const latestSnapshotForUrl = db.prepare(` + SELECT m.snapshot_id + FROM metrics m + JOIN pages p ON m.page_id = p.id + WHERE p.site_id = ? AND p.normalized_url = ? AND (m.pagerank_score IS NOT NULL OR (p.html IS NOT NULL AND p.html != '')) + ORDER BY m.snapshot_id DESC LIMIT 1 + `).get(siteId, url) as { snapshot_id: number } | undefined; + + if (!latestSnapshotForUrl) { + return res.status(404).json({ error: 'Page not found' }); + } + + const targetSnapshotId = latestSnapshotForUrl.snapshot_id; + + let page = db.prepare(` + SELECT + p.*, + m.authority_score, m.hub_score, m.pagerank, m.pagerank_score, + m.link_role, m.word_count, m.thin_content_score, + m.external_link_ratio, m.orphan_score, + m.duplicate_cluster_id, m.duplicate_type, m.is_cluster_primary + FROM pages p + LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE p.site_id = ? AND p.normalized_url = ? + `).get(targetSnapshotId, siteId, url) as any; + + if (!page) { + return res.status(404).json({ error: 'Page not found' }); + } + + // Perform full semantic analysis from stored HTML + const analysis = analyzePages(url, [{ + url: page.normalized_url, + status: page.http_status, + html: page.html || '', + canonical: page.canonical_url, + noindex: !!page.noindex, + nofollow: !!page.nofollow, + crawlStatus: page.security_error ? 'failed' : (page.http_status ? 'fetched' : 'discovered') + }])[0]; + + // Calculate health issues + const criticalCount = (page.http_status >= 400 || page.http_status === 0 || page.security_error || analysis.title.status === 'missing' || analysis.h1.status === 'critical') ? 1 : 0; + const warningCount = (analysis.title.status === 'too_long' || analysis.title.status === 'too_short' || analysis.content.wordCount < 300 || analysis.h1.status === 'warning') ? 1 : 0; + + // Inlinks count + const inlinksCount = db.prepare(` + SELECT COUNT(*) as count FROM edges + WHERE snapshot_id = ? AND target_page_id = ? AND rel = 'internal' + `).get(targetSnapshotId, page.id) as { count: number }; + + // Outlinks count + const outlinksCount = db.prepare(` + SELECT COUNT(*) as count FROM edges + WHERE snapshot_id = ? AND source_page_id = ? AND rel = 'internal' + `).get(targetSnapshotId, page.id) as { count: number }; + + res.json({ + identity: { + url: page.normalized_url, + status: page.http_status, + canonical: page.canonical_url, + title: analysis.title, + metaDescription: analysis.metaDescription, + h1: analysis.h1, + crawlError: page.security_error, + crawlDate: page.updated_at + }, + metrics: { + pageRank: page.pagerank_score || 0, + rawPageRank: page.pagerank || 0, + authority: page.authority_score || 0, + hub: page.hub_score || 0, + depth: page.depth || 0, + inlinks: inlinksCount.count, + outlinks: outlinksCount.count + }, + health: { + status: analysis.seoScore > 80 ? 'Good' : analysis.seoScore > 50 ? 'Warning' : 'Critical', + criticalCount, + warningCount, + isThinContent: analysis.thinScore > 70, + isDuplicate: analysis.title.status === 'duplicate', + indexabilityRisk: page.noindex === 1 + }, + content: analysis.content, + images: analysis.images, + links: analysis.links, + structuredData: analysis.structuredData, + snapshotId: targetSnapshotId + }); + }); + + // 5.2 GET /api/page/inlinks + api.get('/page/inlinks', validateSnapshot, (req, res) => { + const currentSnapshotId = (req as any).snapshotId as number; + const url = req.query.url as string; + const pageNum = parseInt(req.query.page as string || '1', 10); + const pageSize = parseInt(req.query.pageSize as string || '50', 10); + const offset = (pageNum - 1) * pageSize; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const page = db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as { id: number }; + if (!page) return res.status(404).json({ error: 'Page not found' }); + + const total = db.prepare(` + SELECT COUNT(*) as count + FROM edges + WHERE snapshot_id = ? AND target_page_id = ? AND rel = 'internal' + `).get(currentSnapshotId, page.id) as { count: number }; + + const rows = db.prepare(` + SELECT + p.normalized_url as sourceUrl, + m.pagerank_score as sourcePageRank, + e.rel as linkType, + 'Follow' as followState -- Needs column in edges if we distinguish nofollow per link + FROM edges e + JOIN pages p ON e.source_page_id = p.id + LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE e.snapshot_id = ? AND e.target_page_id = ? AND e.rel = 'internal' + ORDER BY m.pagerank_score DESC + LIMIT ? OFFSET ? + `).all(currentSnapshotId, currentSnapshotId, page.id, pageSize, offset); + + res.json({ + total: total.count, + page: pageNum, + pageSize, + results: rows + }); + }); + + // 5.3 GET /api/page/outlinks + api.get('/page/outlinks', validateSnapshot, (req, res) => { + const currentSnapshotId = (req as any).snapshotId as number; + const url = req.query.url as string; + const pageNum = parseInt(req.query.page as string || '1', 10); + const pageSize = parseInt(req.query.pageSize as string || '50', 10); + const offset = (pageNum - 1) * pageSize; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const page = db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as { id: number }; + if (!page) return res.status(404).json({ error: 'Page not found' }); + + const total = db.prepare(` + SELECT COUNT(*) as count + FROM edges + WHERE snapshot_id = ? AND source_page_id = ? + `).get(currentSnapshotId, page.id) as { count: number }; + + const rows = db.prepare(` + SELECT + p.normalized_url as targetUrl, + p.http_status as status, + e.rel as type, + CASE WHEN e.rel = 'nofollow' THEN 0 ELSE 1 END as follow + FROM edges e + JOIN pages p ON e.target_page_id = p.id + WHERE e.snapshot_id = ? AND e.source_page_id = ? + ORDER BY p.http_status DESC + LIMIT ? OFFSET ? + `).all(currentSnapshotId, page.id, pageSize, offset); + + res.json({ + total: total.count, + page: pageNum, + pageSize, + results: rows + }); + }); + + // 5.4 GET /api/page/cluster + api.get('/page/cluster', validateSnapshot, (req, res) => { + const currentSnapshotId = (req as any).snapshotId as number; + const url = req.query.url as string; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const page = db.prepare(` + SELECT p.id, m.duplicate_cluster_id + FROM pages p + JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE p.site_id = ? AND p.normalized_url = ? + `).get(currentSnapshotId, siteId, url) as any; + + if (!page || !page.duplicate_cluster_id) { + return res.json({ hasCluster: false }); + } + + const cluster = db.prepare(` + SELECT * FROM duplicate_clusters WHERE snapshot_id = ? AND id = ? + `).get(currentSnapshotId, page.duplicate_cluster_id) as any; + + const similarPages = db.prepare(` + SELECT p.normalized_url + FROM metrics m + JOIN pages p ON m.page_id = p.id + WHERE m.snapshot_id = ? AND m.duplicate_cluster_id = ? AND p.id != ? + LIMIT 10 + `).all(currentSnapshotId, page.duplicate_cluster_id, page.id); + + res.json({ + hasCluster: true, + clusterSize: cluster.size, + representative: cluster.representative, + similarity: 'High', // Simplified + similarUrls: similarPages.map((r: any) => r.normalized_url) + }); + }); + + // 5.5 GET /api/page/technical + api.get('/page/technical', validateSnapshot, (req, res) => { + const url = req.query.url as string; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const page = db.prepare(` + SELECT * FROM pages WHERE site_id = ? AND normalized_url = ? + `).get(siteId, url) as any; + + if (!page) return res.status(404).json({ error: 'Page not found' }); + + res.json({ + redirectChain: page.redirect_chain ? JSON.parse(page.redirect_chain) : null, + headers: [], // Not stored in DB currently + responseTime: null, // Not stored + contentType: 'text/html', + contentSize: page.bytes_received, + serverError: page.http_status >= 500, + status: page.http_status + }); + }); + + // 5.6 GET /api/page/graph-context + api.get('/page/graph-context', validateSnapshot, (req, res) => { + const currentSnapshotId = (req as any).snapshotId as number; + const url = req.query.url as string; + + if (!url) return res.status(400).json({ error: 'URL is required' }); + + const page = db.prepare(` + SELECT p.id, m.pagerank_score + FROM pages p + LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE p.site_id = ? AND p.normalized_url = ? + `).get(currentSnapshotId, siteId, url) as any; + + if (!page) return res.status(404).json({ error: 'Page not found' }); + + // Get neighbors (depth 1) + const incoming = db.prepare(` + SELECT p.normalized_url, m.pagerank_score + FROM edges e + JOIN pages p ON e.source_page_id = p.id + LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE e.snapshot_id = ? AND e.target_page_id = ? AND e.rel = 'internal' + LIMIT 10 + `).all(currentSnapshotId, currentSnapshotId, page.id); + + const outgoing = db.prepare(` + SELECT p.normalized_url, m.pagerank_score + FROM edges e + JOIN pages p ON e.target_page_id = p.id + LEFT JOIN metrics m ON p.id = m.page_id AND m.snapshot_id = ? + WHERE e.snapshot_id = ? AND e.source_page_id = ? AND e.rel = 'internal' + LIMIT 10 + `).all(currentSnapshotId, currentSnapshotId, page.id); + + res.json({ + centrality: page.pagerank_score, + incoming: incoming, + outgoing: outgoing, + // Calculate a simple "equity ratio" + equityRatio: incoming.length > 0 ? (outgoing.length / incoming.length) : 0 + }); + }); + + // 5.7 POST /api/page/crawl (Live crawl of single page) + api.post('/page/crawl', express.json(), async (req, res) => { + const { url } = req.body; + if (!url) return res.status(400).json({ error: 'URL is required' }); + + try { + console.log(chalk.cyan(` Live crawl requested: ${url}`)); + const start = Date.now(); + + // Context to pipe events to terminal + const context = { + emit: (event: any) => { + if (event.type === 'info' && event.message.includes('[analyze]')) { + console.log(chalk.gray(` ${event.message}`)); + } + } + }; + + // We use analyzeSite with live: true and limit to that specific URL + const result = await analyzeSite(url, { + live: true, + seo: true, + content: true, + accessibility: true + }, context as any); + + console.log(chalk.green(` ✅ Live crawl completed in ${Date.now() - start}ms`)); + + res.json({ + success: true, + snapshotId: result.snapshotId, + message: 'Live crawl completed successfully' + }); + } catch (error: any) { + console.error(chalk.red(`❌ Live crawl failed: ${error.message}`)); + res.status(500).json({ error: error.message }); + } + }); + // 4.8 GET /api/history (List of snapshots with key stats) api.get('/history', (req, res) => { // Fetch snapshots with summary data from the snapshots table. @@ -339,6 +674,7 @@ export function startServer(options: ServerOptions): Promise { const sql = ` SELECT id, + type, created_at as createdAt, node_count as pages, health_score as health, @@ -361,7 +697,7 @@ export function startServer(options: ServerOptions): Promise { // Actually, 'broken links' is not in snapshots table directly. We need to count. const snapshots = db.prepare(` - SELECT id, created_at, node_count, health_score, orphan_count + SELECT id, type, created_at, node_count, health_score, orphan_count FROM snapshots WHERE site_id = ? ORDER BY created_at ASC @@ -507,7 +843,7 @@ export function startServer(options: ServerOptions): Promise { // Check if it's the ONLY snapshot if (snapshotRepo.getSnapshotCount(siteId) <= 1) { - return res.status(400).json({ error: 'Cannot delete the only snapshot.' }); + return res.status(400).json({ error: 'Cannot delete the only snapshot.' }); } try { diff --git a/plugins/server/tests/server.test.ts b/plugins/server/tests/server.test.ts index 80d93fa..d42845a 100644 --- a/plugins/server/tests/server.test.ts +++ b/plugins/server/tests/server.test.ts @@ -2,6 +2,16 @@ import { test, expect, vi, beforeEach } from 'vitest'; import express from 'express'; import { startServer } from '../src/index.js'; +vi.mock('chalk', () => ({ + default: { + red: vi.fn((msg) => msg), + green: vi.fn((msg) => msg), + yellow: vi.fn((msg) => msg), + gray: vi.fn((msg) => msg), + blue: vi.fn((msg) => msg) + } +})); + vi.mock('express', () => { const mockApp = { use: vi.fn(), @@ -49,18 +59,16 @@ vi.mock('@crawlith/core', () => { })), closeDb: vi.fn(), // Important: Use function expression to support 'new' keyword - SiteRepository: vi.fn(function () { - return { - getSiteById: vi.fn(() => ({ domain: 'test.com', created_at: '2024-01-01' })) - }; - }), - SnapshotRepository: vi.fn(function () { - return { - getSnapshot: vi.fn(() => ({ id: 1, site_id: 1, health_score: 90, node_count: 10 })) - }; - }), - PageRepository: vi.fn(function () { return {}; }), - MetricsRepository: vi.fn(function () { return {}; }) + SiteRepository: class { + constructor() {} + getSiteById() { return { domain: 'test.com', created_at: '2024-01-01' }; } + }, + SnapshotRepository: class { + constructor() {} + getSnapshot() { return { id: 1, site_id: 1, health_score: 90, node_count: 10 }; } + }, + PageRepository: class {}, + MetricsRepository: class {} }; }); diff --git a/plugins/web/package.json b/plugins/web/package.json index 13d14a9..6b24f2b 100644 --- a/plugins/web/package.json +++ b/plugins/web/package.json @@ -21,6 +21,7 @@ "lucide-react": "^0.378.0", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-router-dom": "^7.13.1", "tailwindcss": "^3.4.3" }, "devDependencies": { diff --git a/plugins/web/src/App.tsx b/plugins/web/src/App.tsx index 117fcb7..a33a49f 100644 --- a/plugins/web/src/App.tsx +++ b/plugins/web/src/App.tsx @@ -1,37 +1,36 @@ import React, { useState, useEffect } from 'react'; +import { BrowserRouter, Routes, Route } from 'react-router-dom'; import { Sidebar } from './components/Sidebar'; import { Header } from './components/Header'; -import { HealthScoreCard } from './components/Metrics/HealthScoreCard'; -import { CriticalIssuesCard } from './components/Metrics/CriticalIssuesCard'; -import { IndexabilityRiskCard } from './components/Metrics/IndexabilityRiskCard'; -import { SecondaryMetricCard } from './components/Metrics/SecondaryMetricCard'; -import { IssuesTable } from './components/IssuesTable'; -import { CriticalPanel } from './components/CriticalPanel'; -import { GraphIntelligenceSection } from './components/GraphIntelligenceSection'; -import { HistoryView } from './components/History/HistoryView'; import * as API from './api'; +import { Dashboard } from './pages/Dashboard'; +import { SinglePage } from './pages/SinglePage'; +import { HistoryView } from './components/History/HistoryView'; export const DashboardContext = React.createContext<{ overview: API.OverviewData | null; currentSnapshot: number | null; snapshots: API.Snapshot[]; setSnapshot: (id: number) => void; + setSnapshots: (snaps: API.Snapshot[]) => void; + setOverview: (ov: API.OverviewData | null) => void; domain: string; }>({ overview: null, currentSnapshot: null, snapshots: [], setSnapshot: () => { }, + setSnapshots: () => { }, + setOverview: () => { }, domain: '' }); function App() { const [sidebarOpen, setSidebarOpen] = useState(false); const [showCompare, setShowCompare] = useState(false); - const [currentView, setCurrentView] = useState('dashboard'); // Data State - const [loading, setLoading] = useState(true); + const [isBooting, setIsBooting] = useState(true); const [error, setError] = useState(null); const [context, setContext] = useState<{ siteId: number, snapshotId: number, domain: string } | null>(null); @@ -39,45 +38,25 @@ function App() { const [currentSnapshotId, setCurrentSnapshotId] = useState(null); const [overview, setOverview] = useState(null); - // Initial Boot + // Initial Boot: Only get domain and site ID useEffect(() => { const init = async () => { try { const ctx = await API.fetchContext(); setContext(ctx); - setCurrentSnapshotId(ctx.snapshotId); - const snaps = await API.fetchSnapshots(); - setSnapshots(snaps.results); + // Default Boot + setCurrentSnapshotId(ctx.snapshotId); } catch (e) { setError('Failed to initialize dashboard. Is the server running?'); console.error(e); + } finally { + setIsBooting(false); } }; init(); }, []); - // Fetch Data on Snapshot Change - useEffect(() => { - if (!currentSnapshotId) return; - - const loadData = async () => { - setLoading(true); - try { - const ov = await API.fetchOverview(currentSnapshotId); - setOverview(ov); - setError(null); - } catch (e) { - setError('Failed to load snapshot data.'); - console.error(e); - } finally { - setLoading(false); - } - }; - - loadData(); - }, [currentSnapshotId]); - if (error) { return (
@@ -89,7 +68,7 @@ function App() { ); } - if (loading && !overview) { + if (isBooting) { return (
Loading Crawlith Context...
@@ -97,86 +76,39 @@ function App() { ); } - const secondaryMetrics = overview ? [ - { label: 'Pages Discovered', value: overview.totals.discovered, delta: 0, tooltip: 'Total raw URLs found during the crawl (includes blocked or broken links).' }, - { label: 'Successfully Crawled', value: overview.totals.crawled, delta: 0, tooltip: 'Pages that returned a successful 200 OK status.' }, - { label: 'Duplicate Clusters', value: overview.totals.duplicateClusters, delta: 0, tooltip: 'Groups of pages with highly identical content.' }, - { label: 'Thin Content', value: overview.totals.thinContent, delta: 0, tooltip: 'Pages with very little text (often under 300 words).' }, - { label: 'Crawl Efficiency', value: overview.crawl.efficiency, unit: '%', delta: 0, tooltip: 'Percentage of discovered URLs that were successfully fetched.' }, - { label: 'Internal Links', value: overview.totals.internalLinks, delta: 0, tooltip: 'Total number of valid internal hyperlinks found.' }, - ] : []; - return (
- - -
setSidebarOpen(!sidebarOpen)} - showCompare={showCompare} - setShowCompare={setShowCompare} - /> - -
-
- - {currentView === 'history' ? ( - - ) : currentView === 'dashboard' ? ( - <> - {/* Primary Metrics Row */} -
- - - -
- - {/* Secondary Metrics Row */} -
- {secondaryMetrics.map((metric, index) => ( - - ))} + + +
setSidebarOpen(!sidebarOpen)} + showCompare={showCompare} + setShowCompare={setShowCompare} + /> +
+ + } /> + +
- - {/* Main Section */} -
-
- -
-
- -
-
- - {/* Lower Section: Graph Intelligence */} - - - ) : ( -
- View "{currentView}" not implemented yet. -
- )} -
-
+ } /> + } /> + + +
); diff --git a/plugins/web/src/api.ts b/plugins/web/src/api.ts index 3916bc9..f643820 100644 --- a/plugins/web/src/api.ts +++ b/plugins/web/src/api.ts @@ -27,6 +27,7 @@ export interface OverviewData { avgDepth: number; efficiency: number; }; + snapshotId?: number; } export interface Issue { @@ -61,6 +62,7 @@ export interface TopPage { export interface Snapshot { id: number; + type?: 'full' | 'partial' | 'incremental'; createdAt: string; pages?: number; health?: number; @@ -94,6 +96,126 @@ export interface SnapshotComparison { }; } +// --- New Interfaces for Single Page View --- + +export interface TextFieldAnalysis { + value: string | null; + length: number; + status: 'ok' | 'too_long' | 'too_short' | 'missing' | 'duplicate'; +} + +export interface H1Analysis { + count: number; + status: 'ok' | 'critical' | 'warning'; + matchesTitle: boolean; +} + +export interface PageDetails { + identity: { + url: string; + status: number; + canonical: string | null; + title: TextFieldAnalysis; + metaDescription: TextFieldAnalysis; + h1: H1Analysis; + crawlError?: string | null; + crawlDate?: string; + }; + metrics: { + pageRank: number; + rawPageRank: number; + authority: number; + hub: number; + depth: number; + inlinks: number; + outlinks: number; + }; + health: { + status: string; + criticalCount: number; + warningCount: number; + isThinContent: boolean; + isDuplicate: boolean; + indexabilityRisk: boolean; + }; + content: { + wordCount: number; + textHtmlRatio: number; + uniqueSentenceCount: number; + }; + images: { + totalImages: number; + missingAlt: number; + emptyAlt: number; + }; + links: { + internalLinks: number; + externalLinks: number; + externalRatio: number; + }; + structuredData: { + present: boolean; + valid: boolean; + types: string[]; + }; + snapshotId: number; +} + +export interface Inlink { + sourceUrl: string; + sourcePageRank: number; + linkType: string; + followState: string; +} + +export interface InlinksResponse { + total: number; + page: number; + pageSize: number; + results: Inlink[]; +} + +export interface Outlink { + targetUrl: string; + status: number; + type: string; + follow: number; +} + +export interface OutlinksResponse { + total: number; + page: number; + pageSize: number; + results: Outlink[]; +} + +export interface ClusterInfo { + hasCluster: boolean; + clusterSize?: number; + representative?: string; + similarity?: string; + similarUrls?: string[]; +} + +export interface TechnicalSignals { + redirectChain: string[] | null; + headers: any[]; + responseTime: number | null; + contentType: string; + contentSize: number; + serverError: boolean; + status: number; +} + +export interface GraphContext { + centrality: number; + incoming: { normalized_url: string; pagerank_score: number }[]; + outgoing: { normalized_url: string; pagerank_score: number }[]; + equityRatio: number; +} + +// --- Existing Functions --- + export async function fetchOverview(snapshotId?: number): Promise { const query = snapshotId ? `?snapshot=${snapshotId}` : ''; const res = await fetch(`${API_PREFIX}/overview${query}`); @@ -140,12 +262,80 @@ export async function fetchSnapshots(): Promise<{ results: Snapshot[] }> { return res.json(); } -export async function fetchContext(): Promise<{ siteId: number, snapshotId: number, domain: string, createdAt: string }> { +export async function fetchContext(): Promise<{ siteId: number, snapshotId: number, latestSnapshotId: number, domain: string, createdAt: string }> { const res = await fetch(`${API_PREFIX}/context`); if (!res.ok) throw new Error('Failed to fetch context'); return res.json(); } + +// --- New Functions for Single Page View --- + +export async function fetchPageDetails(url: string, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page?${params.toString()}`); + if (!res.ok) { + if (res.status === 404) throw new Error('Page not found'); + throw new Error('Failed to fetch page details'); + } + return res.json(); +} + +export async function fetchPageInlinks(url: string, page: number = 1, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + params.append('page', page.toString()); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/inlinks?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch inlinks'); + return res.json(); +} + +export async function fetchPageOutlinks(url: string, page: number = 1, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + params.append('page', page.toString()); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/outlinks?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch outlinks'); + return res.json(); +} + +export async function fetchPageCluster(url: string, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/cluster?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch cluster info'); + return res.json(); +} + +export async function fetchPageTechnical(url: string, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/technical?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch technical signals'); + return res.json(); +} + +export async function fetchPageGraphContext(url: string, snapshotId?: number): Promise { + const params = new URLSearchParams(); + params.append('url', url); + if (snapshotId) params.append('snapshot', snapshotId.toString()); + + const res = await fetch(`${API_PREFIX}/page/graph-context?${params.toString()}`); + if (!res.ok) throw new Error('Failed to fetch graph context'); + return res.json(); +} + export async function fetchHistory(): Promise<{ results: Snapshot[] }> { const res = await fetch(`${API_PREFIX}/history`); if (!res.ok) throw new Error('Failed to fetch history'); @@ -171,3 +361,16 @@ export async function deleteSnapshot(id: number): Promise { throw new Error(err.error || 'Failed to delete snapshot'); } } + +export async function crawlPage(url: string): Promise<{ success: boolean, snapshotId: number, message: string }> { + const res = await fetch(`${API_PREFIX}/page/crawl`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }) + }); + if (!res.ok) { + const err = await res.json(); + throw new Error(err.error || 'Failed to trigger crawl'); + } + return res.json(); +} diff --git a/plugins/web/src/components/Header.tsx b/plugins/web/src/components/Header.tsx index 6da68e5..179ce18 100644 --- a/plugins/web/src/components/Header.tsx +++ b/plugins/web/src/components/Header.tsx @@ -1,5 +1,6 @@ -import { Moon, Sun, Clock, ChevronDown, Menu } from 'lucide-react'; +import { Moon, Sun, Clock, ChevronDown, Menu, ArrowLeft } from 'lucide-react'; import { useEffect, useState, useContext } from 'react'; +import { useLocation, useNavigate } from 'react-router-dom'; import { DashboardContext } from '../App'; interface HeaderProps { @@ -10,6 +11,10 @@ interface HeaderProps { export const Header = ({ toggleSidebar, showCompare: _showCompare, setShowCompare: _setShowCompare }: HeaderProps) => { const { overview, currentSnapshot, snapshots, setSnapshot, domain } = useContext(DashboardContext); + const location = useLocation(); + const navigate = useNavigate(); + const isDashboard = location.pathname === '/'; + const [dropdownOpen, setDropdownOpen] = useState(false); const [theme, setTheme] = useState(() => { if (typeof window !== 'undefined') { @@ -35,7 +40,7 @@ export const Header = ({ toggleSidebar, showCompare: _showCompare, setShowCompar const currentSnapshotData = snapshots.find(s => s.id === currentSnapshot); return ( -
+
{/* Left Section: Domain & Context */}
@@ -46,6 +51,16 @@ export const Header = ({ toggleSidebar, showCompare: _showCompare, setShowCompar + {!isDashboard && ( + + )} +

@@ -57,47 +72,77 @@ export const Header = ({ toggleSidebar, showCompare: _showCompare, setShowCompar

-
-
- - {currentSnapshotData?.createdAt ? new Date(currentSnapshotData.createdAt).toLocaleString() : 'N/A'} + {isDashboard && ( +
+
+ + {currentSnapshotData?.createdAt ? new Date(currentSnapshotData.createdAt).toLocaleString() : 'N/A'} +
-
+ )}
{/* Right Section: Actions & Settings */}
- {/* Snapshot Selector */} -
- - - {/* Dropdown */} -
- {snapshots.map(snap => ( - - ))} -
-
+ {/* Snapshot Selector (Dashboard Only) */} + {isDashboard && ( +
+ + + {/* Dropdown */} + {dropdownOpen && ( +
+
+ Audit History +
+
+ {snapshots.filter(s => s.type !== 'partial').map(snap => ( + + ))} +
+
+ )} + + {/* Backdrop to close dropdown */} + {dropdownOpen && ( +
setDropdownOpen(false)}>
+ )} +
+ )} - {/* Health Delta Badge (Mobile/Tablet Compact) */} - {overview && ( -
+ {/* Health Delta Badge (Dashboard Only) */} + {isDashboard && overview && ( +
= 0 ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'}`}> - {overview.health.delta > 0 ? '+' : ''}{overview.health.delta} + {overview.health.delta > 0 ? '+' : ''}{overview.health.delta}
Health Delta -
+
)}
diff --git a/plugins/web/src/components/IssuesTable.tsx b/plugins/web/src/components/IssuesTable.tsx index 3a76201..580f5c6 100644 --- a/plugins/web/src/components/IssuesTable.tsx +++ b/plugins/web/src/components/IssuesTable.tsx @@ -1,9 +1,11 @@ import React, { useState, useEffect, useContext } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; import { AlertTriangle, AlertCircle, Info, ChevronRight, Search, ArrowUp, ArrowDown } from 'lucide-react'; import { DashboardContext } from '../App'; import * as API from '../api'; export const IssuesTable = () => { + const navigate = useNavigate(); const { currentSnapshot } = useContext(DashboardContext); const [issues, setIssues] = useState([]); @@ -126,10 +128,13 @@ export const IssuesTable = () => { {issues.map((issue, idx) => ( navigate(`/page?url=${encodeURIComponent(issue.url)}`)} className="hover:bg-slate-50 dark:hover:bg-slate-800/30 transition-colors group cursor-pointer" > - {issue.url} + e.stopPropagation()}> + {issue.url} + {issue.issueType} diff --git a/plugins/web/src/components/Sidebar.tsx b/plugins/web/src/components/Sidebar.tsx index 80baa82..b8a0c96 100644 --- a/plugins/web/src/components/Sidebar.tsx +++ b/plugins/web/src/components/Sidebar.tsx @@ -1,14 +1,23 @@ import React from 'react'; +import { useNavigate, useLocation } from 'react-router-dom'; import { LayoutDashboard, Network, FileText, Settings, Layers, ChevronRight, X } from 'lucide-react'; interface SidebarProps { isOpen: boolean; setIsOpen: (isOpen: boolean) => void; - currentView: string; - setCurrentView: (view: string) => void; } -export const Sidebar = ({ isOpen, setIsOpen, currentView, setCurrentView }: SidebarProps) => { +export const Sidebar = ({ isOpen, setIsOpen }: SidebarProps) => { + const navigate = useNavigate(); + const location = useLocation(); + + const currentPath = location.pathname; + + const handleNavigation = (path: string) => { + navigate(path); + setIsOpen(false); + }; + return ( <> {/* Mobile Overlay */} @@ -47,20 +56,20 @@ export const Sidebar = ({ isOpen, setIsOpen, currentView, setCurrentView }: Side { setCurrentView('dashboard'); setIsOpen(false); }} + active={currentPath === '/'} + onClick={() => handleNavigation('/')} /> { setCurrentView('graph'); setIsOpen(false); }} + active={currentPath === '/graph'} + onClick={() => handleNavigation('/graph')} /> { setCurrentView('audit'); setIsOpen(false); }} + active={currentPath === '/audit'} + onClick={() => handleNavigation('/audit')} /> @@ -68,14 +77,14 @@ export const Sidebar = ({ isOpen, setIsOpen, currentView, setCurrentView }: Side { setCurrentView('history'); setIsOpen(false); }} + active={currentPath === '/history'} + onClick={() => handleNavigation('/history')} /> { setCurrentView('settings'); setIsOpen(false); }} + active={currentPath === '/settings'} + onClick={() => handleNavigation('/settings')} /> @@ -102,7 +111,7 @@ const SidebarItem = ({ icon: Icon, label, active, onClick }: any) => ( className={`w-full flex items-center justify-between px-3 py-2 rounded-lg text-sm font-medium transition-all group ${active ? 'bg-blue-600 text-white shadow-lg shadow-blue-900/20' : 'text-slate-400 hover:bg-slate-800 hover:text-slate-100' - }`}> + }`}>
{label} diff --git a/plugins/web/src/components/Tabs/ClusterTab.tsx b/plugins/web/src/components/Tabs/ClusterTab.tsx new file mode 100644 index 0000000..551ff38 --- /dev/null +++ b/plugins/web/src/components/Tabs/ClusterTab.tsx @@ -0,0 +1,99 @@ +import { useState, useEffect } from 'react'; +import * as API from '../../api'; +import { Layers } from 'lucide-react'; + +export const ClusterTab = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!snapshotId) return; + const fetch = async () => { + setLoading(true); + try { + const res = await API.fetchPageCluster(url, snapshotId); + setData(res); + } catch (e) { + console.error(e); + } finally { + setLoading(false); + } + }; + fetch(); + }, [url, snapshotId]); + + if (!data && loading) return
Loading Cluster Info...
; + if (!data) return null; + + if (!data.hasCluster) { + return ( +
+
+ +
+

No significant similarity detected.

+

This page appears unique within the current crawl snapshot and does not belong to any duplicate content clusters.

+
+ ); + } + + return ( +
+
+

+ + Duplicate Cluster Found +

+
+ Risk: High +
+
+ +
+
+
+
Cluster Details
+
+
+
Cluster Size
+
{data.clusterSize} Pages
+
+
+
Similarity
+
{data.similarity}
+
+
+
+ +
+
Representative URL
+
+ {data.representative} +
+
+ The canonical or highest-authority version of this content. +
+
+
+ +
+
Similar URLs in Cluster
+
+
    + {data.similarUrls?.map((simUrl, idx) => ( +
  • + {simUrl} +
    • + ))} + {data.similarUrls && data.similarUrls.length >= 10 && ( +
  • + + {data.clusterSize ? data.clusterSize - 10 : 'more'} others... +
    • + )} +
+
+
+
+
+ ); +}; diff --git a/plugins/web/src/components/Tabs/ContentTab.tsx b/plugins/web/src/components/Tabs/ContentTab.tsx new file mode 100644 index 0000000..a73db0b --- /dev/null +++ b/plugins/web/src/components/Tabs/ContentTab.tsx @@ -0,0 +1,87 @@ +import { HelpCircle } from 'lucide-react'; +import { PageDetails } from '../../api'; + +export const ContentTab = ({ details }: { details: PageDetails }) => { + const { content, identity } = details; + + return ( +
+
+

Content Analysis

+
+ +
+
+ + + 0 ? (identity.h1.status === 'ok' ? 'Correct' : identity.h1.status) : 'Missing'} + meta={identity.h1?.count ? `${identity.h1.count} tag(s)` : undefined} + status={identity.h1?.status === 'ok' ? 'good' : 'warning'} + tooltip="The primary heading of the page structure." + /> +
+ +
+ + + + +
+
+
+ ); +}; + +const ContentRow = ({ label, value, meta, status, tooltip }: { label: string, value: string | number, meta?: string, status?: 'good' | 'warning', tooltip: string }) => ( +
+
+ {label} +
+ + {/* Tooltip */} +
+
{label}
+
{tooltip}
+
+
+
+
+
+
+ {value} +
+ {meta &&
{meta}
} +
+
+); diff --git a/plugins/web/src/components/Tabs/GraphTab.tsx b/plugins/web/src/components/Tabs/GraphTab.tsx new file mode 100644 index 0000000..cabfac2 --- /dev/null +++ b/plugins/web/src/components/Tabs/GraphTab.tsx @@ -0,0 +1,137 @@ +import { useState, useEffect } from 'react'; +import * as API from '../../api'; +import { Share2, HelpCircle } from 'lucide-react'; + +export const GraphTab = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!snapshotId) return; + const fetch = async () => { + setLoading(true); + try { + const res = await API.fetchPageGraphContext(url, snapshotId); + setData(res); + } catch (e) { + console.error(e); + } finally { + setLoading(false); + } + }; + fetch(); + }, [url, snapshotId]); + + if (!data && loading) return
Loading Graph Context...
; + if (!data) return null; + + return ( +
+ {/* Metrics Column */} +
+
+

+ + Node Centrality +

+ +
+
+
+ Centrality Score +
+ +
+ Eigenvector centrality of the node within the internal sitewide link graph. +
+
+
+
+ {(data.centrality || 0).toFixed(2)} +
+
+ +
+
+ Equity Ratio (In/Out) +
+ +
+ Ratio of inbound links to outbound links. Helps identify "sink" nodes vs "hub" nodes. +
+
+
+
+ {(data.equityRatio || 0).toFixed(2)} +
+
+
+
+
+ + {/* Visualization Column */} +
+
+ Immediate Neighbors (Depth 1) +
+ +
+ This visualization shows internal pages linking directly to this page (Green) and pages this page links to (Amber). Node size indicates logical importance. +
+
+
+ + {/* SVG Visualization */} + + {/* Central Node */} + + + Current Node + + + {/* Incoming Nodes (Left) */} + {(data.incoming || []).map((node, i) => { + const x = -150; + const arrayLength = (data.incoming || []).length; + const y = (i - (arrayLength - 1) / 2) * 45; + const shortUrl = (node.normalized_url || '').split('/').pop() || '/'; + return ( + + + + + {shortUrl} + + {node.normalized_url} (PR: {(node.pagerank_score || 0).toFixed(1)}) + + ); + })} + + {/* Outgoing Nodes (Right) */} + {(data.outgoing || []).map((node, i) => { + const x = 150; + const arrayLength = (data.outgoing || []).length; + const y = (i - (arrayLength - 1) / 2) * 45; + const shortUrl = (node.normalized_url || '').split('/').pop() || '/'; + return ( + + + + + {shortUrl} + + {node.normalized_url} (PR: {(node.pagerank_score || 0).toFixed(1)}) + + ); + })} + + +
+
Incoming Sources
+
Current Page
+
Outgoing Targets
+
+
+
+ ); +}; diff --git a/plugins/web/src/components/Tabs/LinkingTab.tsx b/plugins/web/src/components/Tabs/LinkingTab.tsx new file mode 100644 index 0000000..291e8cc --- /dev/null +++ b/plugins/web/src/components/Tabs/LinkingTab.tsx @@ -0,0 +1,185 @@ +import { useState, useEffect } from 'react'; +import { Link } from 'react-router-dom'; +import { ArrowUpRight, ArrowDownLeft } from 'lucide-react'; +import * as API from '../../api'; + +export const LinkingTab = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [subTab, setSubTab] = useState<'inlinks' | 'outlinks'>('inlinks'); + + return ( +
+
+ + +
+ +
+ {subTab === 'inlinks' ? : } +
+
+ ); +}; + +const InlinksTable = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [page, setPage] = useState(1); + + useEffect(() => { + if (!snapshotId) return; + const fetch = async () => { + setLoading(true); + try { + const res = await API.fetchPageInlinks(url, page, snapshotId); + setData(res); + } catch (e) { + console.error(e); + } finally { + setLoading(false); + } + }; + fetch(); + }, [url, page, snapshotId]); + + if (!data && loading) return
Loading Inlinks...
; + if (!data) return
No data available
; + + return ( +
+
+ + + + + + + + + + {data.results.map((row, idx) => ( + + + + + + ))} + +
Source URLSource PageRankLink Type
+ + + {row.sourceUrl} + + + {Math.round(row.sourcePageRank)} + + + {row.linkType} + +
+
+ +
+ ); +}; + +const OutlinksTable = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [page, setPage] = useState(1); + + useEffect(() => { + if (!snapshotId) return; + const fetch = async () => { + setLoading(true); + try { + const res = await API.fetchPageOutlinks(url, page, snapshotId); + setData(res); + } catch (e) { + console.error(e); + } finally { + setLoading(false); + } + }; + fetch(); + }, [url, page, snapshotId]); + + if (!data && loading) return
Loading Outlinks...
; + if (!data) return
No data available
; + + return ( +
+
+ + + + + + + + + + {data.results.map((row, idx) => ( + + + + + + ))} + +
Target URLStatusType
+ + + {row.targetUrl} + + + = 200 && row.status < 300 ? 'bg-green-100 text-green-700 dark:bg-green-900/30 dark:text-green-400' : + row.status >= 300 && row.status < 400 ? 'bg-blue-100 text-blue-700 dark:bg-blue-900/30 dark:text-blue-400' : + 'bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400' + }`}> + {row.status} + + + + {row.type} + +
+
+ +
+ ); +}; + +const Pagination = ({ total, pageSize, page, setPage, loading }: any) => { + const totalPages = Math.ceil(total / pageSize); + return ( +
+ + + Page {page} of {totalPages} ({total} links) + + +
+ ); +}; diff --git a/plugins/web/src/components/Tabs/TechnicalTab.tsx b/plugins/web/src/components/Tabs/TechnicalTab.tsx new file mode 100644 index 0000000..8067db6 --- /dev/null +++ b/plugins/web/src/components/Tabs/TechnicalTab.tsx @@ -0,0 +1,89 @@ +import { useState, useEffect } from 'react'; +import * as API from '../../api'; +import { ArrowRight, Server, Activity } from 'lucide-react'; + +export const TechnicalTab = ({ url, snapshotId }: { url: string; snapshotId: number }) => { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!snapshotId) return; + const fetch = async () => { + setLoading(true); + try { + const res = await API.fetchPageTechnical(url, snapshotId); + setData(res); + } catch (e) { + console.error(e); + } finally { + setLoading(false); + } + }; + fetch(); + }, [url, snapshotId]); + + if (!data && loading) return
Loading Technical Signals...
; + if (!data) return null; + + return ( +
+ {/* Redirect Chain */} +
+

+ + Redirect Chain +

+ + {data.redirectChain && data.redirectChain.length > 0 ? ( +
+ {data.redirectChain.map((hop, index) => ( +
+
+
Hop {index + 1}
+
+ {hop} +
+ {index < (data.redirectChain?.length || 0) - 1 && ( +
+ 301 Moved Permanently +
+ )} +
+ ))} +
+ ) : ( +
No redirects detected. This URL resolves directly (200 OK).
+ )} +
+ + {/* Server & Response */} +
+

+ + Response & Headers +

+ +
+
+ Content Type + {data.contentType} +
+
+ Page Size + {(data.contentSize / 1024).toFixed(2)} KB +
+
+ Response Time + {data.responseTime ? `${data.responseTime}ms` : 'N/A'} +
+
+ Server Status + = 400 || data.status === 0 ? 'bg-red-100 text-red-700' : data.status >= 300 ? 'bg-blue-100 text-blue-700' : 'bg-green-100 text-green-700'}`}> + {data.status === 0 ? 'Network Error' : `${data.status} ${data.status < 300 ? 'OK' : data.status < 400 ? 'Redirect' : 'Error'}`} + +
+
+
+
+ ); +}; diff --git a/plugins/web/src/pages/Dashboard.tsx b/plugins/web/src/pages/Dashboard.tsx new file mode 100644 index 0000000..cb9babd --- /dev/null +++ b/plugins/web/src/pages/Dashboard.tsx @@ -0,0 +1,110 @@ +import { useContext, useEffect, useState } from 'react'; +import { DashboardContext } from '../App'; +import * as API from '../api'; +import { HealthScoreCard } from '../components/Metrics/HealthScoreCard'; +import { CriticalIssuesCard } from '../components/Metrics/CriticalIssuesCard'; +import { IndexabilityRiskCard } from '../components/Metrics/IndexabilityRiskCard'; +import { SecondaryMetricCard } from '../components/Metrics/SecondaryMetricCard'; +import { IssuesTable } from '../components/IssuesTable'; +import { CriticalPanel } from '../components/CriticalPanel'; +import { GraphIntelligenceSection } from '../components/GraphIntelligenceSection'; + +interface DashboardProps { + showCompare: boolean; +} + +export function Dashboard({ showCompare }: DashboardProps) { + const { currentSnapshot, overview, setOverview, snapshots, setSnapshots } = useContext(DashboardContext); + const [localLoading, setLocalLoading] = useState(false); + + useEffect(() => { + if (snapshots.length === 0) { + API.fetchSnapshots().then(res => setSnapshots(res.results)); + } + }, [snapshots.length, setSnapshots]); + + useEffect(() => { + if (!currentSnapshot) return; + + // Skip if we already have the correct snapshot overview + if (overview && (overview as any).snapshotId === currentSnapshot) return; + + const loadData = async () => { + setLocalLoading(true); + try { + const ov = await API.fetchOverview(currentSnapshot); + setOverview(ov); + } catch (e) { + console.error('Failed to load dashboard overview', e); + } finally { + setLocalLoading(false); + } + }; + + loadData(); + }, [currentSnapshot, setOverview, overview]); + + if (!overview || localLoading) { + return ( +
+
+ {[1, 2, 3].map(i => ( +
+ ))} +
+
+
+

Calculating site-wide intelligence...

+
+
+ ); + } + + const secondaryMetrics = [ + { label: 'Pages Discovered', value: overview.totals.discovered, delta: 0, tooltip: 'Total raw URLs found during the crawl (includes blocked or broken links).' }, + { label: 'Successfully Crawled', value: overview.totals.crawled, delta: 0, tooltip: 'Pages that returned a successful 200 OK status.' }, + { label: 'Duplicate Clusters', value: overview.totals.duplicateClusters, delta: 0, tooltip: 'Groups of pages with highly identical content.' }, + { label: 'Thin Content', value: overview.totals.thinContent, delta: 0, tooltip: 'Pages with very little text (often under 300 words).' }, + { label: 'Crawl Efficiency', value: overview.crawl.efficiency, unit: '%', delta: 0, tooltip: 'Percentage of discovered URLs that were successfully fetched.' }, + { label: 'Internal Links', value: overview.totals.internalLinks, delta: 0, tooltip: 'Total number of valid internal hyperlinks found.' }, + ]; + + return ( +
+ {/* Primary Metrics Row */} +
+ + + +
+ + {/* Secondary Metrics Row */} +
+ {secondaryMetrics.map((metric, index) => ( + + ))} +
+ + {/* Main Section */} +
+
+ +
+
+ +
+
+ + {/* Lower Section: Graph Intelligence */} + +
+ ); +} diff --git a/plugins/web/src/pages/SinglePage.tsx b/plugins/web/src/pages/SinglePage.tsx new file mode 100644 index 0000000..8e5be58 --- /dev/null +++ b/plugins/web/src/pages/SinglePage.tsx @@ -0,0 +1,328 @@ +import { useEffect, useState, useContext } from 'react'; +import { useSearchParams } from 'react-router-dom'; +import { DashboardContext } from '../App'; +import * as API from '../api'; +import { HelpCircle, AlertTriangle, CheckCircle, AlertOctagon, Copy, CornerDownRight, Activity, Layers, Clock } from 'lucide-react'; +import { ContentTab } from '../components/Tabs/ContentTab'; +import { LinkingTab } from '../components/Tabs/LinkingTab'; +import { ClusterTab } from '../components/Tabs/ClusterTab'; +import { TechnicalTab } from '../components/Tabs/TechnicalTab'; +import { GraphTab } from '../components/Tabs/GraphTab'; + +export const SinglePage = () => { + const [searchParams] = useSearchParams(); + const url = searchParams.get('url'); + const { currentSnapshot, setSnapshot } = useContext(DashboardContext); + + const [details, setDetails] = useState(null); + const [loading, setLoading] = useState(true); + const [isCrawling, setIsCrawling] = useState(false); + const [error, setError] = useState(null); + const [activeTab, setActiveTab] = useState('content'); + + useEffect(() => { + if (!url || !currentSnapshot) { + setLoading(false); + return; + } + + const fetchData = async () => { + setLoading(true); + try { + const data = await API.fetchPageDetails(url, currentSnapshot); + setDetails(data); + setError(null); + } catch (e: any) { + setError(e.message || 'Failed to load page details.'); + } finally { + setLoading(false); + } + }; + + fetchData(); + setActiveTab('content'); + }, [url, currentSnapshot]); + + const handleLiveCrawl = async () => { + if (!url) return; + setIsCrawling(true); + try { + const result = await API.crawlPage(url); + if (result.success) { + setSnapshot(result.snapshotId); + } + } catch (e: any) { + alert(`Live crawl failed: ${e.message}`); + } finally { + setIsCrawling(false); + } + }; + + if (!url) { + return ( +
+
+ +
+

No URL Selected

+

Please select a page from the dashboard to view its detailed intelligence reports.

+
+ ); + } + + if (loading) { + return ( +
+
+
+
+
+
+
+
+ ); + } + + if (error || !details) { + return ( +
+
+
+ +
+

+ {error ? 'Data Fetch Error' : 'Page Not Found'} +

+

+ {error || `The URL "${url}" was not found in snapshot #${currentSnapshot}.`} +

+
+ Suggestion: Try selecting a different snapshot or verifying the URL in the discovery list. +
+
+
+ ); + } + + return ( +
+ {/* Header Section */} +
+
+
+
+
+ + + {details.identity.canonical ? `Canonical: ${details.identity.canonical}` : 'Self-canonical'} + +
+
+

+ {details.identity.url} + +

+ +
+ +
+
+ + Snapshot #{details.snapshotId || currentSnapshot} +
+
+ + Crawled: {details.identity.crawlDate ? new Date(details.identity.crawlDate).toLocaleString() : 'N/A'} +
+
+ + {details.identity.crawlError && ( +
+ +
+ Crawl Intelligence Error + {details.identity.crawlError} +
+
+ )} + +
+ + + + + + +
+
+ + {/* Page Health Summary Panel */} +
+ +
+
+ + {/* Tabs Navigation */} +
+ {['content', 'linking', 'cluster', 'technical', 'graph'].map((tab) => ( + + ))} +
+
+
+ + {/* Tab Content */} +
+ {activeTab === 'content' && } + {activeTab === 'linking' && } + {activeTab === 'cluster' && } + {activeTab === 'technical' && } + {activeTab === 'graph' && } +
+
+ ); +}; + +const MetricDisplay = ({ label, value, tooltip }: { label: string, value: string | number, tooltip: string }) => ( +
+
+ {label} + +
+ {tooltip} +
+
+
+
+ {value} +
+
+); + +const StatusBadge = ({ status }: { status: number }) => { + let color = 'bg-slate-100 text-slate-700 border-slate-200 dark:bg-slate-800 dark:text-slate-300 dark:border-slate-700'; + if (status >= 200 && status < 300) color = 'bg-green-50 text-green-700 border-green-200 dark:bg-green-900/20 dark:text-green-400 dark:border-green-900/30'; + if (status >= 300 && status < 400) color = 'bg-blue-50 text-blue-700 border-blue-200 dark:bg-blue-900/20 dark:text-blue-400 dark:border-blue-900/30'; + if (status >= 400 && status < 500) color = 'bg-red-50 text-red-700 border-red-200 dark:bg-red-900/20 dark:text-red-400 dark:border-red-900/30'; + if (status >= 500) color = 'bg-amber-50 text-amber-700 border-amber-200 dark:bg-amber-900/20 dark:text-amber-400 dark:border-amber-900/30'; + if (status === 0) color = 'bg-red-50 text-red-700 border-red-200 dark:bg-red-900/20 dark:text-red-400 dark:border-red-900/30'; + + return ( + + HTTP {status === 0 ? 'ERR' : status} + + ); +}; + +const HealthPanel = ({ health }: { health: API.PageDetails['health'] }) => { + const isHealthy = health.status === 'Healthy'; + const isCritical = health.status === 'Critical'; + + return ( +
+
+

Page Health

+ + {health.status} + +
+ +
+
+ + 0 ? 'text-red-500' : 'text-slate-300'} /> + Critical Issues + + {health.criticalCount} +
+
+ + 0 ? 'text-amber-500' : 'text-slate-300'} /> + Warnings + + {health.warningCount} +
+ +
+ {health.isThinContent && ( +
+ Thin Content Detected +
+ )} + {health.isDuplicate && ( +
+ Duplicate Content +
+ )} + {health.indexabilityRisk && ( +
+ Indexability Risk +
+ )} + {!health.isThinContent && !health.isDuplicate && !health.indexabilityRisk && health.status === 'Healthy' && ( +
+ No major issues +
+ )} +
+
+
+ ); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3b44fc4..96cf810 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,6 +162,9 @@ importers: react-dom: specifier: ^18.2.0 version: 18.3.1(react@18.3.1) + react-router-dom: + specifier: ^7.13.1 + version: 7.13.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) tailwindcss: specifier: ^3.4.3 version: 3.4.19 @@ -658,79 +661,66 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1239,6 +1229,10 @@ packages: resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -2252,6 +2246,23 @@ packages: resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} engines: {node: '>=0.10.0'} + react-router-dom@7.13.1: + resolution: {integrity: sha512-UJnV3Rxc5TgUPJt2KJpo1Jpy0OKQr0AjgbZzBFjaPJcFOb2Y8jA5H3LT8HUJAiRLlWrEXWHbF1Z4SCZaQjWDHw==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + + react-router@7.13.1: + resolution: {integrity: sha512-td+xP4X2/6BJvZoX6xw++A2DdEi++YypA69bJUV5oVvqf6/9/9nNlD70YO1e9d3MyamJEBQFEzk6mbfDYbqrSA==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + peerDependenciesMeta: + react-dom: + optional: true + react@18.3.1: resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} engines: {node: '>=0.10.0'} @@ -2342,6 +2353,9 @@ packages: resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} engines: {node: '>= 0.8.0'} + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -3896,6 +3910,8 @@ snapshots: cookie@0.7.2: {} + cookie@1.1.1: {} + create-require@1.1.1: {} cross-spawn@7.0.6: @@ -4865,6 +4881,20 @@ snapshots: react-refresh@0.17.0: {} + react-router-dom@7.13.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-router: 7.13.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + + react-router@7.13.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + cookie: 1.1.1 + react: 18.3.1 + set-cookie-parser: 2.7.2 + optionalDependencies: + react-dom: 18.3.1(react@18.3.1) + react@18.3.1: dependencies: loose-envify: 1.4.0 @@ -4992,6 +5022,8 @@ snapshots: transitivePeerDependencies: - supports-color + set-cookie-parser@2.7.2: {} + setprototypeof@1.2.0: {} shebang-command@2.0.0: