Skip to content

Refactor Crawler.processPage for better maintainability #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
saurabhsharma2u merged 2 commits into from
Feb 25, 2026
Merged

Refactor Crawler.processPage for better maintainability #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7808dd3
Refactor `processPage` in `Crawler` class to improve maintainability
saurabhsharma2u Feb 25, 2026
8f69c47
Fix CI linting errors and unused variables
saurabhsharma2u Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 121 additions & 99 deletions plugins/core/src/crawler/crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { request } from 'undici';
import pLimit from 'p-limit';
import chalk from 'chalk';
import robotsParser from 'robots-parser';
import { Graph } from '../graph/graph.js';
import { Fetcher, FetchOptions } from './fetcher.js';
import { Graph, GraphNode } from '../graph/graph.js';
import { Fetcher, FetchResult } from './fetcher.js';
import { Parser } from './parser.js';
import { Sitemap } from './sitemap.js';
import { normalizeUrl } from './normalize.js';
Expand Down Expand Up @@ -343,15 +343,7 @@ export class Crawler {
this.flushMetrics();
}

private async processPage(item: QueueItem): Promise<void> {
const { url, depth } = item;
if (this.scopeManager!.isUrlEligible(url) !== 'allowed') {
this.bufferPage(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
return;
}

const prevNode = this.options.previousGraph?.nodes.get(url);

private async fetchPage(url: string, depth: number, prevNode?: GraphNode): Promise<FetchResult | null> {
try {
const res = await this.fetcher!.fetch(url, {
maxBytes: this.options.maxBytes,
Expand All @@ -363,55 +355,131 @@ export class Crawler {
if (this.options.debug) {
console.log(`${chalk.gray(`[D:${depth}]`)} ${res.status} ${chalk.blue(url)}`);
}
return res;
} catch (e) {
console.error(`Error processing ${url}:`, e);
return null;
}
}

const finalUrl = normalizeUrl(res.finalUrl, '', this.options);
if (!finalUrl) return;

if (res.status === 304 && prevNode) {
this.bufferPage(finalUrl, depth, 200, {
html: prevNode.html,
canonical_url: prevNode.canonical,
content_hash: prevNode.contentHash,
simhash: prevNode.simhash,
etag: prevNode.etag,
last_modified: prevNode.lastModified,
noindex: prevNode.noindex ? 1 : 0,
nofollow: prevNode.nofollow ? 1 : 0
});
this.bufferMetrics(finalUrl, {
crawl_status: 'cached'
});
private handleCachedResponse(url: string, finalUrl: string, depth: number, prevNode: GraphNode): void {
this.bufferPage(finalUrl, depth, 200, {
html: prevNode.html,
canonical_url: prevNode.canonical,
content_hash: prevNode.contentHash,
simhash: prevNode.simhash,
etag: prevNode.etag,
last_modified: prevNode.lastModified,
noindex: prevNode.noindex ? 1 : 0,
nofollow: prevNode.nofollow ? 1 : 0
});
this.bufferMetrics(finalUrl, {
crawl_status: 'cached'
});

// Re-discovery links from previous graph to continue crawling if needed
const prevLinks = this.options.previousGraph?.getEdges()
.filter(e => e.source === url)
.map(e => e.target);

if (prevLinks) {
for (const link of prevLinks) {
const normalizedLink = normalizeUrl(link, '', this.options);
if (normalizedLink && normalizedLink !== finalUrl) {
this.bufferPage(normalizedLink, depth + 1, 0);
this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
if (this.shouldEnqueue(normalizedLink, depth + 1)) {
this.addToQueue(normalizedLink, depth + 1);
}
}
// Re-discovery links from previous graph to continue crawling if needed
const prevLinks = this.options.previousGraph?.getEdges()
.filter(e => e.source === url)
.map(e => e.target);

if (prevLinks) {
for (const link of prevLinks) {
const normalizedLink = normalizeUrl(link, '', this.options);
if (normalizedLink && normalizedLink !== finalUrl) {
this.bufferPage(normalizedLink, depth + 1, 0);
this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
if (this.shouldEnqueue(normalizedLink, depth + 1)) {
this.addToQueue(normalizedLink, depth + 1);
}
}
return;
}
}
}

private handleRedirects(chain: FetchResult['redirectChain'], depth: number): void {
for (const step of chain) {
const source = normalizeUrl(step.url, '', this.options);
const target = normalizeUrl(step.target, '', this.options);
if (source && target) {
this.bufferPage(source, depth, step.status);
this.bufferPage(target, depth, 0);
this.bufferEdge(source, target);
}
}
}

const chain = res.redirectChain;
for (const step of chain) {
const source = normalizeUrl(step.url, '', this.options);
const target = normalizeUrl(step.target, '', this.options);
if (source && target) {
this.bufferPage(source, depth, step.status);
this.bufferPage(target, depth, 0);
this.bufferEdge(source, target);
private handleSuccessResponse(res: FetchResult, finalUrl: string, depth: number): void {
const contentTypeHeader = res.headers['content-type'];
const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
if (!contentType || !contentType.toLowerCase().includes('text/html')) {
this.bufferPage(finalUrl, depth, typeof res.status === 'number' ? res.status : 0);
return;
}

const parseResult = this.parser!.parse(res.body, finalUrl, res.status as number);

this.bufferPage(finalUrl, depth, res.status as number, {
html: parseResult.html,
canonical_url: parseResult.canonical || undefined,
noindex: parseResult.noindex ? 1 : 0,
nofollow: parseResult.nofollow ? 1 : 0,
content_hash: parseResult.contentHash,
simhash: parseResult.simhash,
soft404_score: parseResult.soft404Score,
etag: res.etag,
last_modified: res.lastModified,
retries: res.retries
});

try {
const contentAnalysis = analyzeContent(parseResult.html);
const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, this.rootOrigin);
const thinScore = calculateThinContentScore(contentAnalysis, 0);

this.bufferMetrics(finalUrl, {
crawl_status: 'fetched',
word_count: contentAnalysis.wordCount,
thin_content_score: thinScore,
external_link_ratio: linkAnalysis.externalRatio
});
} catch (e) {
console.error(`Error calculating per-page metrics for ${finalUrl}:`, e);
}

for (const linkItem of parseResult.links) {
const normalizedLink = normalizeUrl(linkItem.url, '', this.options);
if (normalizedLink && normalizedLink !== finalUrl) {
this.bufferPage(normalizedLink, depth + 1, 0);
this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
if (this.shouldEnqueue(normalizedLink, depth + 1)) {
this.addToQueue(normalizedLink, depth + 1);
}
}
}
}

private async processPage(item: QueueItem): Promise<void> {
const { url, depth } = item;
if (this.scopeManager!.isUrlEligible(url) !== 'allowed') {
this.bufferPage(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
return;
}

try {
const prevNode = this.options.previousGraph?.nodes.get(url);
const res = await this.fetchPage(url, depth, prevNode);

if (!res) return;

const finalUrl = normalizeUrl(res.finalUrl, '', this.options);
if (!finalUrl) return;

if (res.status === 304 && prevNode) {
this.handleCachedResponse(url, finalUrl, depth, prevNode);
return;
}

this.handleRedirects(res.redirectChain, depth);

const isStringStatus = typeof res.status === 'string';
if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
Expand All @@ -423,53 +491,7 @@ export class Crawler {
}

if (res.status === 200) {
const contentTypeHeader = res.headers['content-type'];
const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
if (!contentType || !contentType.toLowerCase().includes('text/html')) {
this.bufferPage(finalUrl, depth, res.status);
return;
}

const parseResult = this.parser!.parse(res.body, finalUrl, res.status);

this.bufferPage(finalUrl, depth, res.status, {
html: parseResult.html,
canonical_url: parseResult.canonical || undefined,
noindex: parseResult.noindex ? 1 : 0,
nofollow: parseResult.nofollow ? 1 : 0,
content_hash: parseResult.contentHash,
simhash: parseResult.simhash,
soft404_score: parseResult.soft404Score,
etag: res.etag,
last_modified: res.lastModified,
retries: res.retries
});

try {
const contentAnalysis = analyzeContent(parseResult.html);
const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, this.rootOrigin);
const thinScore = calculateThinContentScore(contentAnalysis, 0);

this.bufferMetrics(finalUrl, {
crawl_status: 'fetched',
word_count: contentAnalysis.wordCount,
thin_content_score: thinScore,
external_link_ratio: linkAnalysis.externalRatio
});
} catch (e) {
console.error(`Error calculating per-page metrics for ${finalUrl}:`, e);
}

for (const linkItem of parseResult.links) {
const normalizedLink = normalizeUrl(linkItem.url, '', this.options);
if (normalizedLink && normalizedLink !== finalUrl) {
this.bufferPage(normalizedLink, depth + 1, 0);
this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
if (this.shouldEnqueue(normalizedLink, depth + 1)) {
this.addToQueue(normalizedLink, depth + 1);
}
}
}
this.handleSuccessResponse(res, finalUrl, depth);
}
} catch (e) {
console.error(`Error processing ${url}:`, e);
Expand Down
1 change: 0 additions & 1 deletion plugins/core/tests/audit/transport.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { describe, it, expect, vi, afterEach } from 'vitest';
import { analyzeTransport } from '../../src/audit/transport.js';
import https from 'node:https';
import http from 'node:http';
import tls from 'node:tls';
import { EventEmitter } from 'events';

Expand Down