🔒 Fix SSRF vulnerability in Fetcher (DNS Rebinding)

package.json

@@ -27,6 +27,7 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.4.5",
     "typescript-eslint": "^8.56.0",
+    "undici": "^6.23.0",
     "vitest": "^4.0.18"
   },
   "engines": {

plugins/core/src/crawler/crawler.ts

@@ -3,7 +3,7 @@ import pLimit from 'p-limit';
 import chalk from 'chalk';
 import robotsParser from 'robots-parser';
 import { Graph } from '../graph/graph.js';
-import { Fetcher, FetchOptions } from './fetcher.js';
+import { Fetcher } from './fetcher.js';
 import { Parser } from './parser.js';
 import { Sitemap } from './sitemap.js';
 import { normalizeUrl } from './normalize.js';

plugins/core/src/crawler/fetcher.ts

@@ -1,4 +1,5 @@
 import { request, Dispatcher } from 'undici';
+import * as net from 'net';
 import { IPGuard } from '../core/security/ipGuard.js';
 import { RateLimiter } from '../core/network/rateLimiter.js';
 import { RetryPolicy } from '../core/network/retryPolicy.js';
@@ -85,8 +86,9 @@ export class Fetcher {
       const urlObj = new URL(currentUrl);
 
       // 1. SSRF Guard
-      const isSafe = await IPGuard.validateHost(urlObj.hostname);
-      if (!isSafe) {
+      // We only validate explicit IP literals here. Domain names are validated
+      // by the secureDispatcher during connection to prevent DNS Rebinding attacks.
+      if (net.isIP(urlObj.hostname) && IPGuard.isInternal(urlObj.hostname)) {
         return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
       }
 
@@ -212,6 +214,10 @@ export class Fetcher {
         }
 
       } catch (error: any) {
+        if (error.code === 'EBLOCKED' || error.message?.includes('Blocked internal IP')) {
+          return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+        }
+
         // Map common network errors to specific statuses if needed
         const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED';
         const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error';

plugins/core/tests/audit/transport.test.ts

@@ -1,7 +1,6 @@
 import { describe, it, expect, vi, afterEach } from 'vitest';
 import { analyzeTransport } from '../../src/audit/transport.js';
 import https from 'node:https';
-import http from 'node:http';
 import tls from 'node:tls';
 import { EventEmitter } from 'events';
 

plugins/core/tests/fetcher_dispatcher.test.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Fetcher } from '../src/crawler/fetcher.js';
+import { request } from 'undici';
+
+// Hoist the mock dispatcher so it's available in the mock factory
+const { mockDispatcher } = vi.hoisted(() => {
+    return { mockDispatcher: { dispatch: vi.fn() } };
+});
+
+// Mock undici
+vi.mock('undici', () => {
+    return {
+        request: vi.fn(),
+        Agent: class {
+            dispatch = vi.fn();
+        },
+        Dispatcher: class {}
+    };
+});
+
+// Mock IPGuard to bypass the initial check for domains
+vi.mock('../src/core/security/ipGuard.js', async (importOriginal) => {
+    const actual = await importOriginal<any>();
+    return {
+        ...actual,
+        IPGuard: {
+            ...actual.IPGuard,
+            validateHost: vi.fn().mockResolvedValue(true), // Bypass initial check
+            getSecureDispatcher: vi.fn().mockReturnValue(mockDispatcher)
+        }
+    };
+});
+
+describe('SSRF Fix Reproduction', () => {
+    let fetcher: Fetcher;
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        fetcher = new Fetcher();
+    });
+
+    it('should return blocked_internal_ip when secureDispatcher blocks an IP', async () => {
+        const mockRequest = vi.mocked(request);
+
+        // Simulate secureDispatcher throwing EBLOCKED
+        const blockedError = new Error('Blocked internal IP: 127.0.0.1');
+        (blockedError as any).code = 'EBLOCKED';
+        mockRequest.mockRejectedValue(blockedError);
+
+        const res = await fetcher.fetch('http://example.com');
+
+        // Should return blocked_internal_ip now
+        expect(res.status).toBe('blocked_internal_ip');
+
+        // Verify that the secure dispatcher was actually passed to the request
+        expect(mockRequest).toHaveBeenCalledWith(
+            expect.stringContaining('http://example.com'),
+            expect.objectContaining({
+                dispatcher: mockDispatcher
+            })
+        );
+    });
+});