diff --git a/package.json b/package.json index a74eb68..519b1ff 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5", "typescript-eslint": "^8.56.0", + "undici": "^6.23.0", "vitest": "^4.0.18" }, "engines": { diff --git a/plugins/core/src/crawler/crawler.ts b/plugins/core/src/crawler/crawler.ts index 124c7fa..05ddae2 100644 --- a/plugins/core/src/crawler/crawler.ts +++ b/plugins/core/src/crawler/crawler.ts @@ -3,7 +3,7 @@ import pLimit from 'p-limit'; import chalk from 'chalk'; import robotsParser from 'robots-parser'; import { Graph } from '../graph/graph.js'; -import { Fetcher, FetchOptions } from './fetcher.js'; +import { Fetcher } from './fetcher.js'; import { Parser } from './parser.js'; import { Sitemap } from './sitemap.js'; import { normalizeUrl } from './normalize.js'; diff --git a/plugins/core/src/crawler/fetcher.ts b/plugins/core/src/crawler/fetcher.ts index 1c5c852..63259ee 100644 --- a/plugins/core/src/crawler/fetcher.ts +++ b/plugins/core/src/crawler/fetcher.ts @@ -1,4 +1,5 @@ import { request, Dispatcher } from 'undici'; +import * as net from 'net'; import { IPGuard } from '../core/security/ipGuard.js'; import { RateLimiter } from '../core/network/rateLimiter.js'; import { RetryPolicy } from '../core/network/retryPolicy.js'; @@ -85,8 +86,9 @@ export class Fetcher { const urlObj = new URL(currentUrl); // 1. SSRF Guard - const isSafe = await IPGuard.validateHost(urlObj.hostname); - if (!isSafe) { + // We only validate explicit IP literals here. Domain names are validated + // by the secureDispatcher during connection to prevent DNS Rebinding attacks. + if (net.isIP(urlObj.hostname) && IPGuard.isInternal(urlObj.hostname)) { return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries); } @@ -212,6 +214,10 @@ export class Fetcher { } } catch (error: any) { + if (error.code === 'EBLOCKED' || error.message?.includes('Blocked internal IP')) { + return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries); + } + // Map common network errors to specific statuses if needed const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED'; const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error'; diff --git a/plugins/core/tests/audit/transport.test.ts b/plugins/core/tests/audit/transport.test.ts index 6fbb770..44ff7aa 100644 --- a/plugins/core/tests/audit/transport.test.ts +++ b/plugins/core/tests/audit/transport.test.ts @@ -1,7 +1,6 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; import { analyzeTransport } from '../../src/audit/transport.js'; import https from 'node:https'; -import http from 'node:http'; import tls from 'node:tls'; import { EventEmitter } from 'events'; diff --git a/plugins/core/tests/fetcher_dispatcher.test.ts b/plugins/core/tests/fetcher_dispatcher.test.ts new file mode 100644 index 0000000..81554ff --- /dev/null +++ b/plugins/core/tests/fetcher_dispatcher.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { Fetcher } from '../src/crawler/fetcher.js'; +import { request } from 'undici'; + +// Hoist the mock dispatcher so it's available in the mock factory +const { mockDispatcher } = vi.hoisted(() => { + return { mockDispatcher: { dispatch: vi.fn() } }; +}); + +// Mock undici +vi.mock('undici', () => { + return { + request: vi.fn(), + Agent: class { + dispatch = vi.fn(); + }, + Dispatcher: class {} + }; +}); + +// Mock IPGuard to bypass the initial check for domains +vi.mock('../src/core/security/ipGuard.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + IPGuard: { + ...actual.IPGuard, + validateHost: vi.fn().mockResolvedValue(true), // Bypass initial check + getSecureDispatcher: vi.fn().mockReturnValue(mockDispatcher) + } + }; +}); + +describe('SSRF Fix Reproduction', () => { + let fetcher: Fetcher; + + beforeEach(() => { + vi.clearAllMocks(); + fetcher = new Fetcher(); + }); + + it('should return blocked_internal_ip when secureDispatcher blocks an IP', async () => { + const mockRequest = vi.mocked(request); + + // Simulate secureDispatcher throwing EBLOCKED + const blockedError = new Error('Blocked internal IP: 127.0.0.1'); + (blockedError as any).code = 'EBLOCKED'; + mockRequest.mockRejectedValue(blockedError); + + const res = await fetcher.fetch('http://example.com'); + + // Should return blocked_internal_ip now + expect(res.status).toBe('blocked_internal_ip'); + + // Verify that the secure dispatcher was actually passed to the request + expect(mockRequest).toHaveBeenCalledWith( + expect.stringContaining('http://example.com'), + expect.objectContaining({ + dispatcher: mockDispatcher + }) + ); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1f609c..1bf74e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,6 +48,9 @@ importers: typescript-eslint: specifier: ^8.56.0 version: 8.56.1(eslint@10.0.2(jiti@1.21.7))(typescript@5.9.3) + undici: + specifier: ^6.23.0 + version: 6.23.0 vitest: specifier: ^4.0.18 version: 4.0.18(@types/node@20.19.33)(jiti@1.21.7) @@ -637,79 +640,66 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==}