From 10b91a53e8d15cf971e8a81e036f916e04a37a91 Mon Sep 17 00:00:00 2001 From: Rafia minhaj Date: Fri, 10 Jul 2026 03:04:30 +0530 Subject: [PATCH 1/2] feat: enforce max-redirects and response-size limit on crawler (#1816) --- backend/secuscan/crawler.py | 26 ++++++- testing/backend/unit/test_crawler_limits.py | 82 +++++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 testing/backend/unit/test_crawler_limits.py diff --git a/backend/secuscan/crawler.py b/backend/secuscan/crawler.py index 183a50d30..2d2256915 100644 --- a/backend/secuscan/crawler.py +++ b/backend/secuscan/crawler.py @@ -78,19 +78,41 @@ async def crawl_target( timeout: int = 10, cookies: Dict[str, str] | None = None, extra_headers: Dict[str, Any] | None = None, + max_redirects: int = 10, + max_size: int = 5 * 1024 * 1024, ) -> Dict[str, Any]: """Fetch a target and normalize discovered links/forms/scripts/API hints.""" headers = _build_headers(extra_headers) async with httpx.AsyncClient( follow_redirects=True, + max_redirects=max_redirects, timeout=timeout, headers=headers, cookies=cookies or {}, verify=False, ) as client: - response = await client.get(url) + async with client.stream("GET", url) as response: + # Check Content-Length header if present + content_length = response.headers.get("content-length") + if content_length: + try: + cl_val = int(content_length) + except ValueError: + cl_val = 0 + if cl_val > max_size: + raise ValueError(f"Response size exceeds limit of {max_size} bytes") - body = response.text + # Read response in chunks to enforce size limit + body_chunks = [] + bytes_read = 0 + async for chunk in response.aiter_bytes(): + bytes_read += len(chunk) + if bytes_read > max_size: + raise ValueError(f"Response size exceeds limit of {max_size} bytes") + body_chunks.append(chunk) + + body_bytes = b"".join(body_chunks) + body = body_bytes.decode("utf-8", errors="replace") parser = _SurfaceParser() parser.feed(body) diff --git a/testing/backend/unit/test_crawler_limits.py b/testing/backend/unit/test_crawler_limits.py new file mode 100644 index 000000000..0b65fb7d2 --- /dev/null +++ b/testing/backend/unit/test_crawler_limits.py @@ -0,0 +1,82 @@ +"""Unit tests for verifying crawl_target max-redirects and max-size constraints.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch +import httpx +import pytest + +from backend.secuscan.crawler import crawl_target + + +@pytest.mark.asyncio +async def test_crawl_target_max_size_via_content_length(): + """Verify that crawl_target raises ValueError if the Content-Length header exceeds max_size.""" + mock_response = MagicMock() + mock_response.headers = {"content-length": "10000000"} # 10MB + mock_response.status_code = 200 + mock_response.url = "http://example.com" + mock_response.history = [] + + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + + mock_stream_ctx = MagicMock() + mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response) + mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) + mock_client.stream.return_value = mock_stream_ctx + + with patch("httpx.AsyncClient", return_value=mock_client): + with pytest.raises(ValueError, match="Response size exceeds limit"): + await crawl_target("http://example.com", max_size=1000) + + +@pytest.mark.asyncio +async def test_crawl_target_max_size_via_streaming(): + """Verify that crawl_target raises ValueError if the streamed chunks exceed max_size.""" + mock_response = MagicMock() + mock_response.headers = {} + mock_response.status_code = 200 + mock_response.url = "http://example.com" + mock_response.history = [] + + async def mock_aiter_bytes(): + yield b"hello " + yield b"world of pentesting" + + mock_response.aiter_bytes = mock_aiter_bytes + + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + + mock_stream_ctx = MagicMock() + mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response) + mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) + mock_client.stream.return_value = mock_stream_ctx + + with patch("httpx.AsyncClient", return_value=mock_client): + # Setting max_size to 10 bytes: + # First chunk (b"hello ") is 6 bytes (ok). + # Second chunk adds 19 bytes, total 25 bytes (exceeds limit). + with pytest.raises(ValueError, match="Response size exceeds limit"): + await crawl_target("http://example.com", max_size=10) + + +@pytest.mark.asyncio +async def test_crawl_target_max_redirects_exceeded(): + """Verify that crawl_target raises httpx.TooManyRedirects when the redirect limit is hit.""" + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_request = httpx.Request("GET", "http://example.com") + + mock_stream_ctx = MagicMock() + mock_stream_ctx.__aenter__ = AsyncMock(side_effect=httpx.TooManyRedirects("Too many redirects", request=mock_request)) + mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) + mock_client.stream.return_value = mock_stream_ctx + + with patch("httpx.AsyncClient", return_value=mock_client): + with pytest.raises(httpx.TooManyRedirects): + await crawl_target("http://example.com", max_redirects=2) From 3c455e8014668f49c606c7cced480122a9f66c02 Mon Sep 17 00:00:00 2001 From: Rafia minhaj Date: Fri, 10 Jul 2026 03:19:48 +0530 Subject: [PATCH 2/2] style: remove trailing whitespaces to pass formatting hygiene check --- testing/backend/unit/test_crawler_limits.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/testing/backend/unit/test_crawler_limits.py b/testing/backend/unit/test_crawler_limits.py index 0b65fb7d2..05132a0b8 100644 --- a/testing/backend/unit/test_crawler_limits.py +++ b/testing/backend/unit/test_crawler_limits.py @@ -17,16 +17,16 @@ async def test_crawl_target_max_size_via_content_length(): mock_response.status_code = 200 mock_response.url = "http://example.com" mock_response.history = [] - + mock_client = MagicMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) - + mock_stream_ctx = MagicMock() mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response) mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) mock_client.stream.return_value = mock_stream_ctx - + with patch("httpx.AsyncClient", return_value=mock_client): with pytest.raises(ValueError, match="Response size exceeds limit"): await crawl_target("http://example.com", max_size=1000) @@ -40,22 +40,22 @@ async def test_crawl_target_max_size_via_streaming(): mock_response.status_code = 200 mock_response.url = "http://example.com" mock_response.history = [] - + async def mock_aiter_bytes(): yield b"hello " yield b"world of pentesting" - + mock_response.aiter_bytes = mock_aiter_bytes - + mock_client = MagicMock() mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) - + mock_stream_ctx = MagicMock() mock_stream_ctx.__aenter__ = AsyncMock(return_value=mock_response) mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) mock_client.stream.return_value = mock_stream_ctx - + with patch("httpx.AsyncClient", return_value=mock_client): # Setting max_size to 10 bytes: # First chunk (b"hello ") is 6 bytes (ok). @@ -71,12 +71,12 @@ async def test_crawl_target_max_redirects_exceeded(): mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=None) mock_request = httpx.Request("GET", "http://example.com") - + mock_stream_ctx = MagicMock() mock_stream_ctx.__aenter__ = AsyncMock(side_effect=httpx.TooManyRedirects("Too many redirects", request=mock_request)) mock_stream_ctx.__aexit__ = AsyncMock(return_value=None) mock_client.stream.return_value = mock_stream_ctx - + with patch("httpx.AsyncClient", return_value=mock_client): with pytest.raises(httpx.TooManyRedirects): await crawl_target("http://example.com", max_redirects=2)