diff --git a/plugins/core/src/core/security/ipGuard.ts b/plugins/core/src/core/security/ipGuard.ts index feaadc9..3592309 100644 --- a/plugins/core/src/core/security/ipGuard.ts +++ b/plugins/core/src/core/security/ipGuard.ts @@ -1,6 +1,7 @@ import * as dns from 'dns'; import * as net from 'net'; import { promisify } from 'util'; +import { Agent } from 'undici'; const resolve4 = promisify(dns.resolve4); const resolve6 = promisify(dns.resolve6); @@ -76,6 +77,62 @@ export class IPGuard { } } + /** + * Custom lookup function for undici that validates the resolved IP. + * Prevents DNS Rebinding attacks by checking the IP immediately before connection. + */ + static secureLookup( + hostname: string, + options: dns.LookupOneOptions | dns.LookupAllOptions, + callback: (err: NodeJS.ErrnoException | null, address: string | dns.LookupAddress[], family: number) => void + ): void { + dns.lookup(hostname, options as any, (err: NodeJS.ErrnoException | null, address: string | dns.LookupAddress[], family: number) => { + if (err) { + return callback(err, address as any, family); + } + + const checkIP = (ip: string) => { + if (IPGuard.isInternal(ip)) { + return new Error(`Blocked internal IP: ${ip}`); + } + return null; + }; + + if (typeof address === 'string') { + const error = checkIP(address); + if (error) { + // Return a custom error that undici will propagate + const blockedError = new Error(`Blocked internal IP: ${address}`); + (blockedError as any).code = 'EBLOCKED'; + return callback(blockedError, address, family); + } + } else if (Array.isArray(address)) { + // Handle array of addresses (if options.all is true) + for (const addr of address) { + const error = checkIP(addr.address); + if (error) { + const blockedError = new Error(`Blocked internal IP: ${addr.address}`); + (blockedError as any).code = 'EBLOCKED'; + return callback(blockedError, address, family); + } + } + } + + callback(null, address, family); + }); + } + + /** + * Returns an undici Agent configured with secure DNS lookup. + */ + static getSecureDispatcher(): Agent { + return new Agent({ + connect: { + lookup: IPGuard.secureLookup as any + } + }); + } + private static expandIPv6(ip: string): string { if (ip === '::') return '0000:0000:0000:0000:0000:0000:0000:0000'; let full = ip; diff --git a/plugins/core/src/crawler/fetcher.ts b/plugins/core/src/crawler/fetcher.ts index 9d430fc..1c5c852 100644 --- a/plugins/core/src/crawler/fetcher.ts +++ b/plugins/core/src/crawler/fetcher.ts @@ -1,4 +1,4 @@ -import { request } from 'undici'; +import { request, Dispatcher } from 'undici'; import { IPGuard } from '../core/security/ipGuard.js'; import { RateLimiter } from '../core/network/rateLimiter.js'; import { RetryPolicy } from '../core/network/retryPolicy.js'; @@ -47,6 +47,7 @@ export class Fetcher { private userAgent = 'crawlith/1.0'; private rateLimiter: RateLimiter; private proxyAdapter: ProxyAdapter; + private secureDispatcher: Dispatcher; private scopeManager?: ScopeManager; private maxRedirects: number; @@ -59,6 +60,13 @@ export class Fetcher { } = {}) { this.rateLimiter = new RateLimiter(options.rate || 2); this.proxyAdapter = new ProxyAdapter(options.proxyUrl); + + if (this.proxyAdapter.dispatcher) { + this.secureDispatcher = this.proxyAdapter.dispatcher; + } else { + this.secureDispatcher = IPGuard.getSecureDispatcher(); + } + this.scopeManager = options.scopeManager; this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11); this.userAgent = options.userAgent || `crawlith/${version}`; @@ -113,7 +121,7 @@ export class Fetcher { method: 'GET', headers, maxRedirections: 0, - dispatcher: this.proxyAdapter.dispatcher, + dispatcher: this.secureDispatcher, headersTimeout: 10000, bodyTimeout: 10000 }); diff --git a/plugins/core/tests/crawler.test.ts b/plugins/core/tests/crawler.test.ts index 3052834..49a0b80 100644 --- a/plugins/core/tests/crawler.test.ts +++ b/plugins/core/tests/crawler.test.ts @@ -1,8 +1,9 @@ -import { test, expect, beforeEach, afterEach } from 'vitest'; +import { test, expect, beforeEach, afterEach, vi } from 'vitest'; import { crawl } from '../src/crawler/crawl.js'; import { loadGraphFromSnapshot } from '../src/db/graphLoader.js'; import { closeDb } from '../src/db/index.js'; import { MockAgent, setGlobalDispatcher } from 'undici'; +import { IPGuard } from '../src/core/security/ipGuard.js'; let mockAgent: MockAgent; @@ -11,6 +12,9 @@ beforeEach(() => { mockAgent = new MockAgent(); mockAgent.disableNetConnect(); setGlobalDispatcher(mockAgent); + + // IPGuard.getSecureDispatcher must return the mockAgent so Fetcher uses it + vi.spyOn(IPGuard, 'getSecureDispatcher').mockReturnValue(mockAgent as any); }); afterEach(() => { diff --git a/plugins/core/tests/fetcher.test.ts b/plugins/core/tests/fetcher.test.ts index 7a3889b..7397ecd 100644 --- a/plugins/core/tests/fetcher.test.ts +++ b/plugins/core/tests/fetcher.test.ts @@ -1,6 +1,7 @@ -import { test, expect, beforeEach } from 'vitest'; +import { test, expect, beforeEach, vi } from 'vitest'; import { Fetcher } from '../src/crawler/fetcher.js'; import { MockAgent, setGlobalDispatcher } from 'undici'; +import { IPGuard } from '../src/core/security/ipGuard.js'; let mockAgent: MockAgent; @@ -8,6 +9,9 @@ beforeEach(() => { mockAgent = new MockAgent(); mockAgent.disableNetConnect(); setGlobalDispatcher(mockAgent); + + // IPGuard.getSecureDispatcher must return the mockAgent so Fetcher uses it + vi.spyOn(IPGuard, 'getSecureDispatcher').mockReturnValue(mockAgent as any); }); test('fetches simple page', async () => { diff --git a/plugins/core/tests/fetcher_safety.test.ts b/plugins/core/tests/fetcher_safety.test.ts index 9a09c62..278700b 100644 --- a/plugins/core/tests/fetcher_safety.test.ts +++ b/plugins/core/tests/fetcher_safety.test.ts @@ -2,9 +2,15 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { Fetcher } from '../src/crawler/fetcher.js'; import { request } from 'undici'; -vi.mock('undici', () => ({ - request: vi.fn(), -})); +vi.mock('undici', () => { + return { + request: vi.fn(), + Agent: class { + dispatch = vi.fn(); + }, + Dispatcher: class {} + }; +}); describe('Fetcher Safety Integration', () => { let fetcher: Fetcher; diff --git a/plugins/core/tests/ipGuard.test.ts b/plugins/core/tests/ipGuard.test.ts new file mode 100644 index 0000000..27129d2 --- /dev/null +++ b/plugins/core/tests/ipGuard.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect, vi } from 'vitest'; +import { IPGuard } from '../src/core/security/ipGuard.js'; +import * as dns from 'dns'; + +vi.mock('dns', () => ({ + lookup: vi.fn(), + resolve4: vi.fn(), + resolve6: vi.fn(), +})); + +describe('IPGuard Secure Lookup', () => { + it('should resolve safe IPs', () => { + const lookupMock = vi.mocked(dns.lookup); + // Mock successful resolution + lookupMock.mockImplementation((hostname, options, callback) => { + callback(null, '8.8.8.8', 4); + }); + + const callback = vi.fn(); + IPGuard.secureLookup('google.com', {}, callback); + + expect(callback).toHaveBeenCalledWith(null, '8.8.8.8', 4); + }); + + it('should block internal IPs', () => { + const lookupMock = vi.mocked(dns.lookup); + // Mock internal IP resolution + lookupMock.mockImplementation((hostname, options, callback) => { + callback(null, '127.0.0.1', 4); + }); + + const callback = vi.fn(); + IPGuard.secureLookup('localhost', {}, callback); + + expect(callback).toHaveBeenCalledWith(expect.any(Error), '127.0.0.1', 4); + const error = callback.mock.calls[0][0]; + expect(error.message).toContain('Blocked internal IP'); + expect(error.code).toBe('EBLOCKED'); + }); + + it('should handle array of IPs (IPv4)', () => { + const lookupMock = vi.mocked(dns.lookup); + // Mock array resolution + lookupMock.mockImplementation((hostname, options, callback) => { + // Mocking address array structure + const addresses = [ + { address: '1.1.1.1', family: 4 }, + { address: '127.0.0.1', family: 4 } + ]; + callback(null, addresses as any, 4); + }); + + const callback = vi.fn(); + IPGuard.secureLookup('mixed.com', { all: true } as any, callback); + + expect(callback).toHaveBeenCalledWith(expect.any(Error), expect.anything(), 4); + const error = callback.mock.calls[0][0]; + expect(error.message).toContain('Blocked internal IP'); + }); + + it('should pass through DNS errors', () => { + const lookupMock = vi.mocked(dns.lookup); + const dnsError = new Error('ENOTFOUND'); + lookupMock.mockImplementation((hostname, options, callback) => { + callback(dnsError as any, undefined as any, 0); + }); + + const callback = vi.fn(); + IPGuard.secureLookup('invalid.domain', {}, callback); + + expect(callback).toHaveBeenCalledWith(dnsError, undefined, 0); + }); +}); diff --git a/plugins/core/tests/redirect_safety.test.ts b/plugins/core/tests/redirect_safety.test.ts index e1baa04..0cb923f 100644 --- a/plugins/core/tests/redirect_safety.test.ts +++ b/plugins/core/tests/redirect_safety.test.ts @@ -5,7 +5,11 @@ import { request } from 'undici'; vi.mock('undici', () => ({ request: vi.fn(), - ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} })) + ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} })), + Agent: class { + dispatch = vi.fn(); + }, + Dispatcher: class {} })); describe('RedirectController', () => {