From e89b71b64a92ae3d271e40c6d5ee924dac233adc Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 12 Aug 2026 14:06:03 +0300 Subject: [PATCH] fix(harness): route 202/499 and JS-shell pages through jina reader fetch_webpage returns a raw HTTP GET, so JS-rendered pages (weatherspark) came back as a 202 "empty" shell, and paywalled sites (climate-data.org, HTTP 499 "Pay for usage") hard-failed. Both are cases the jina.ai reader is designed for, but the fallback only fired on 401/403/429. - add 202 (soft-block) and 499 (paywall) to the jina fallback status set - after HTML extraction, if the body was non-empty but yielded no title/text (a JS-rendered shell), fall back to the reader as well --- openjiuwen/harness/tools/web/fetch_webpage.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/openjiuwen/harness/tools/web/fetch_webpage.py b/openjiuwen/harness/tools/web/fetch_webpage.py index 4dd42b28e..0f4acb486 100644 --- a/openjiuwen/harness/tools/web/fetch_webpage.py +++ b/openjiuwen/harness/tools/web/fetch_webpage.py @@ -185,7 +185,13 @@ async def _fetch_webpage( timeout_seconds: int, byte_cap: int, ) -> dict[str, Any]: - """Fetch webpage content, falling back to the jina.ai reader on 401/403/429.""" + """Fetch webpage content, falling back to the jina.ai reader when the + direct GET is blocked or returns a JS-rendered shell. + + Hard blocks (401/403/429) and paywalls (499) go straight to the reader. + A 202 soft-block, or an HTML body with no extractable text (a JS shell), + also falls back — the reader renders the page server-side. + """ status, headers, body, final_url, truncated = await _http.request( session, "GET", @@ -194,7 +200,7 @@ async def _fetch_webpage( timeout_seconds=timeout_seconds, max_bytes=byte_cap, ) - if status in {401, 403, 429}: + if status in {202, 401, 403, 429, 499}: return await WebFetchWebpageTool._fetch_via_jina_reader(session, url, timeout_seconds, byte_cap) _raise_fetch_http_error(url, status, body) @@ -204,6 +210,10 @@ async def _fetch_webpage( if "html" in content_type.lower(): title, text = WebFetchWebpageTool._extract_main_text_from_html(text) + # A JS-rendered page returns an HTML shell with no extractable text; + # only fall back when there actually was a body to render. + if not title.strip() and not text.strip() and body: + return await WebFetchWebpageTool._fetch_via_jina_reader(session, url, timeout_seconds, byte_cap) else: text = re.sub(r"\s+", " ", text).strip()