diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.md b/packages/agent-core/src/tools/builtin/web/fetch-url.md index 30e98d6f9a..897cbde946 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.md +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.md @@ -1,3 +1,3 @@ -Fetch content from a URL. The content is returned either as the main text extracted from the page, or as the full response body verbatim; a note at the top of the result states which of the two you received, so you can judge how complete it is. Use this when you need to read a specific web page. +Fetch content from a URL. The content is returned either as the main text extracted from the page, or as the full response body verbatim; a note at the top of the result states which of the two you received, so you can judge how complete it is. When the URL points to an image, the image is fetched and returned directly so you can view it. Use this when you need to read a specific web page or view an image from a URL. -Only fully-formed public `http`/`https` URLs are supported; other schemes and private or loopback addresses are not fetched. Very large pages may be truncated or refused. The fetch carries no login or session for the target site, so pages behind authentication (private repositories, internal dashboards) return a login page or an error instead of the real content — if the text you get back looks like a generic landing or sign-in page, treat that as the login wall, not the answer, and reach the content through a credentialed route (an authenticated CLI or MCP tool) instead. +Only fully-formed public `http`/`https` URLs are supported; other schemes and private or loopback addresses are not fetched. Very large pages or images may be truncated or refused. The fetch carries no login or session for the target site, so pages behind authentication (private repositories, internal dashboards) return a login page or an error instead of the real content — if the text you get back looks like a generic landing or sign-in page, treat that as the login wall, not the answer, and reach the content through a credentialed route (an authenticated CLI or MCP tool) instead. diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.ts b/packages/agent-core/src/tools/builtin/web/fetch-url.ts index d130e3977a..a1b795b210 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.ts +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.ts @@ -6,6 +6,7 @@ * should not be registered (not exposed to the LLM). */ +import type { ContentPart } from '@moonshot-ai/kosong'; import { z } from 'zod'; import type { BuiltinTool } from '../../../agent/tool'; @@ -26,13 +27,20 @@ import DESCRIPTION from './fetch-url.md?raw'; * - `extracted` — the body was an HTML page; only the main article text * was extracted and returned. */ -export type UrlFetchKind = 'passthrough' | 'extracted'; +export type UrlFetchKind = 'passthrough' | 'extracted' | 'image'; + +export interface ImageFetchData { + mimeType: string; + base64: string; +} export interface UrlFetchResult { /** The text handed to the LLM. */ content: string; /** Whether `content` is a verbatim passthrough or extracted main text. */ kind: UrlFetchKind; + /** Optional image data when the response is an image. */ + imageData?: ImageFetchData; } export interface UrlFetcher { @@ -89,7 +97,18 @@ export class FetchURLTool implements BuiltinTool { }: ExecutableToolContext, ): Promise { try { - const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId }); + const { content, kind, imageData } = await this.fetcher.fetch(args.url, { toolCallId }); + + if (imageData) { + const output: ContentPart[] = [ + { type: 'text', text: `Fetched image from ${args.url}` }, + { + type: 'image_url', + imageUrl: { url: `data:${imageData.mimeType};base64,${imageData.base64}` }, + }, + ]; + return { output, isError: false }; + } if (!content) { return { @@ -99,12 +118,6 @@ export class FetchURLTool implements BuiltinTool { } const builder = new ToolResultBuilder({ maxLineLength: null }); - // Tell the LLM whether it received the whole body or only the extracted - // article text, so it can judge how complete the content is, and remind it - // to cite this page when it uses the content. Both notes must ride in - // `output`: the result's `message` field is dropped from the transcript, so - // `output` is the only place the model can read them. Put them at the front - // so they survive any downstream truncation of the body. const note = kind === 'passthrough' ? 'The returned content is the full response body, returned verbatim.' diff --git a/packages/agent-core/src/tools/providers/local-fetch-url.ts b/packages/agent-core/src/tools/providers/local-fetch-url.ts index af10a8ca31..f9f2ebde1a 100644 --- a/packages/agent-core/src/tools/providers/local-fetch-url.ts +++ b/packages/agent-core/src/tools/providers/local-fetch-url.ts @@ -172,6 +172,22 @@ export class LocalFetchURLProvider implements UrlFetcher { } } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + if (contentType.startsWith('image/')) { + // Image response: read as binary, convert to base64, return as image data. + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const actualBytes = buffer.length; + if (actualBytes > this.maxBytes) { + throw new Error( + `Response body too large: ${String(actualBytes)} bytes exceeds maxBytes (${String(this.maxBytes)}).`, + ); + } + const base64 = buffer.toString('base64'); + return { content: '', kind: 'image', imageData: { mimeType: contentType, base64 } }; + } + const body = await response.text(); // Servers may omit content-length — measure again defensively. @@ -182,7 +198,6 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } - const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) { return { content: body, kind: 'passthrough' }; } diff --git a/packages/agent-core/test/tools/fetch-url.test.ts b/packages/agent-core/test/tools/fetch-url.test.ts index 06f319633e..4918fcd280 100644 --- a/packages/agent-core/test/tools/fetch-url.test.ts +++ b/packages/agent-core/test/tools/fetch-url.test.ts @@ -20,9 +20,10 @@ const signal = new AbortController().signal; function fakeFetcher( content = '', - kind: 'passthrough' | 'extracted' = 'extracted', + kind: 'passthrough' | 'extracted' | 'image' = 'extracted', + imageData?: { mimeType: string; base64: string }, ): UrlFetcher { - return { fetch: vi.fn().mockResolvedValue({ content, kind }) }; + return { fetch: vi.fn().mockResolvedValue({ content, kind, imageData }) }; } describe('FetchURLTool', () => { @@ -290,6 +291,24 @@ describe('FetchURLTool', () => { // (py wording was "full content"; main #238 uses "full response body"). expect(out).toContain('full response body'); }); + + it('returns image content parts when fetcher returns image data', async () => { + const imageData = { mimeType: 'image/png', base64: 'fakebase64' }; + const tool = new FetchURLTool(fakeFetcher('', 'image', imageData)); + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_img', + args: { url: 'https://example.com/image.png' }, + signal, + }); + + expect(result.isError).toBe(false); + expect(Array.isArray(result.output)).toBe(true); + const parts = result.output as Array<{ type: string } & Record>; + expect(parts.length).toBe(2); + expect(parts[0]).toMatchObject({ type: 'text', text: 'Fetched image from https://example.com/image.png' }); + expect(parts[1]).toMatchObject({ type: 'image_url', imageUrl: { url: 'data:image/png;base64,fakebase64' } }); + }); }); describe('MoonshotFetchURLProvider', () => { diff --git a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts index 2c0ce931f1..3c1c4110c8 100644 --- a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts +++ b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts @@ -55,4 +55,51 @@ describe('LocalFetchURLProvider content kind', () => { expect(result.kind).toBe('extracted'); expect(result.content).toContain('quick brown fox'); }); + + it('returns image data for image content types', async () => { + const imageBuffer = Buffer.from('fake-png-data'); + const fetchImpl = vi.fn().mockResolvedValue( + new Response(imageBuffer, { + status: 200, + headers: { 'content-type': 'image/png' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.png'); + + expect(result.kind).toBe('image'); + expect(result.imageData).toBeDefined(); + expect(result.imageData?.mimeType).toBe('image/png'); + expect(result.imageData?.base64).toBe(imageBuffer.toString('base64')); + }); + + it('returns image data for image/jpeg content type', async () => { + const imageBuffer = Buffer.from('fake-jpeg-data'); + const fetchImpl = vi.fn().mockResolvedValue( + new Response(imageBuffer, { + status: 200, + headers: { 'content-type': 'image/jpeg' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/photo.jpg'); + + expect(result.kind).toBe('image'); + expect(result.imageData?.mimeType).toBe('image/jpeg'); + }); + + it('rejects oversized images', async () => { + const largeBuffer = Buffer.alloc(11 * 1024 * 1024); // 11MB, over default 10MB limit + const fetchImpl = vi.fn().mockResolvedValue( + new Response(largeBuffer, { + status: 200, + headers: { 'content-type': 'image/png', 'content-length': String(largeBuffer.length) }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + await expect(provider.fetch('https://example.com/huge.png')).rejects.toThrow('too large'); + }); });