From 659898e52c3d8eadfe132ecd620c54a1e71b9469 Mon Sep 17 00:00:00 2001 From: qer Date: Mon, 22 Jun 2026 18:13:05 +0800 Subject: [PATCH] feat: FetchURL tool supports downloading images - Detect image/* content types in LocalFetchURLProvider - Read binary body for images, convert to base64 data URI - Sniff image dimensions when available - Return image as ContentPart[] with image_url for LLM vision models - Update tool description to mention image support - Add tests for image fetching in both tool and provider --- .../src/tools/builtin/web/fetch-url.md | 2 +- .../src/tools/builtin/web/fetch-url.ts | 38 ++++++++++-- .../src/tools/providers/local-fetch-url.ts | 27 +++++++-- .../agent-core/test/tools/fetch-url.test.ts | 54 +++++++++++++++++ .../tools/providers/local-fetch-url.test.ts | 60 +++++++++++++++++++ 5 files changed, 172 insertions(+), 9 deletions(-) diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.md b/packages/agent-core/src/tools/builtin/web/fetch-url.md index f2356e6904..5fc3e9d0e5 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.md +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.md @@ -1,3 +1,3 @@ -Fetch content from a URL. Returns the main text content extracted from the page. Use this when you need to read a specific web page. +Fetch content from a URL. Returns the main text content extracted from the page, or images when the URL points to an image file. Use this when you need to read a specific web page or view an image from the web. Only public `http`/`https` URLs are supported. Requests to private, loopback, or link-local addresses are refused, and responses larger than 10 MiB are rejected. diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.ts b/packages/agent-core/src/tools/builtin/web/fetch-url.ts index 9ea5b126c6..e8a90b91ab 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.ts +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.ts @@ -6,6 +6,8 @@ * should not be registered (not exposed to the LLM). */ +import type { ContentPart } from '@moonshot-ai/kosong'; + import { z } from 'zod'; import type { BuiltinTool } from '../../../agent/tool'; @@ -25,14 +27,19 @@ import DESCRIPTION from './fetch-url.md?raw'; * returned verbatim, in full. * - `extracted` — the body was an HTML page; only the main article text * was extracted and returned. + * - `image` — the body is an image; returned as a base64 data URI. */ -export type UrlFetchKind = 'passthrough' | 'extracted'; +export type UrlFetchKind = 'passthrough' | 'extracted' | 'image'; export interface UrlFetchResult { - /** The text handed to the LLM. */ + /** The text handed to the LLM, or a base64 data URI for images. */ content: string; - /** Whether `content` is a verbatim passthrough or extracted main text. */ + /** Whether `content` is a verbatim passthrough, extracted main text, or image. */ kind: UrlFetchKind; + /** The MIME type of the response, when known. */ + mimeType?: string | undefined; + /** Image dimensions, when the response is an image and they can be determined. */ + dimensions?: { width: number; height: number } | null | undefined; } export interface UrlFetcher { @@ -89,7 +96,7 @@ export class FetchURLTool implements BuiltinTool { }: ExecutableToolContext, ): Promise { try { - const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId }); + const { content, kind, mimeType, dimensions } = await this.fetcher.fetch(args.url, { toolCallId }); if (!content) { return { @@ -98,6 +105,29 @@ export class FetchURLTool implements BuiltinTool { }; } + if (kind === 'image') { + const systemParts: string[] = ['Fetched image from URL.']; + if (mimeType) { + systemParts.push(`Mime type: ${mimeType}.`); + } + if (dimensions) { + systemParts.push( + `Original dimensions: ${String(dimensions.width)}x${String(dimensions.height)} pixels.`, + ); + systemParts.push( + 'If you need to output coordinates, output relative coordinates first ' + + 'and compute absolute coordinates using the original image size.', + ); + } + const output: ContentPart[] = [ + { type: 'text', text: `${systemParts.join(' ')}` }, + { type: 'text', text: `` }, + { type: 'image_url', imageUrl: { url: content } }, + { type: 'text', text: '' }, + ]; + return { output, isError: false }; + } + const builder = new ToolResultBuilder({ maxLineLength: null }); builder.write(content); // Tell the LLM whether it received the whole body or only the diff --git a/packages/agent-core/src/tools/providers/local-fetch-url.ts b/packages/agent-core/src/tools/providers/local-fetch-url.ts index af10a8ca31..4435ce450a 100644 --- a/packages/agent-core/src/tools/providers/local-fetch-url.ts +++ b/packages/agent-core/src/tools/providers/local-fetch-url.ts @@ -6,8 +6,10 @@ * 2. Reject HTTP >= 400 with the status code in the message. * 3. Reject responses larger than `maxBytes` (content-length first, * then measured body length as a defensive second check). - * 4. `text/plain` / `text/markdown` → passthrough verbatim. - * 5. Otherwise (assumed HTML) → run Readability over a linkedom + * 4. `image/*` → read binary body, encode as base64 data URI, and + * return as `image` kind with optional dimension sniffing. + * 5. `text/plain` / `text/markdown` → passthrough verbatim. + * 6. Otherwise (assumed HTML) → run Readability over a linkedom * document. Return `# ${title}\n\n${text}` (title omitted when * absent). If extraction yields no meaningful text, fall back to * common content containers (`
` / `
` / ``) @@ -18,6 +20,7 @@ import { Readability } from '@mozilla/readability'; import { parseHTML as rawParseHTML } from 'linkedom'; import { HttpFetchError, type UrlFetcher, type UrlFetchResult } from '../builtin'; +import { sniffImageDimensions } from '../support/file-type'; // Readability's .d.ts references the global `Document` type, but this // package compiles with `lib: ES2023` (no DOM). Extracting the @@ -172,6 +175,23 @@ export class LocalFetchURLProvider implements UrlFetcher { } } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + const mimeType = contentType.split(';')[0]!.trim(); + + if (mimeType.startsWith('image/')) { + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + if (buffer.length > this.maxBytes) { + throw new Error( + `Response body too large: ${String(buffer.length)} bytes exceeds maxBytes (${String(this.maxBytes)}).`, + ); + } + const base64 = buffer.toString('base64'); + const dataUri = `data:${mimeType};base64,${base64}`; + const dimensions = sniffImageDimensions(buffer); + return { content: dataUri, kind: 'image', mimeType, dimensions }; + } + const body = await response.text(); // Servers may omit content-length — measure again defensively. @@ -182,8 +202,7 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } - const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); - if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) { + if (mimeType.startsWith('text/plain') || mimeType.startsWith('text/markdown')) { return { content: body, kind: 'passthrough' }; } diff --git a/packages/agent-core/test/tools/fetch-url.test.ts b/packages/agent-core/test/tools/fetch-url.test.ts index 0e5c55ee6d..84f1a50b78 100644 --- a/packages/agent-core/test/tools/fetch-url.test.ts +++ b/packages/agent-core/test/tools/fetch-url.test.ts @@ -259,6 +259,60 @@ describe('FetchURLTool', () => { const message = (result as { message?: string }).message ?? ''; expect(message).toContain('full response body'); }); + + it('returns image content as ContentPart[] when fetcher returns image kind', async () => { + const dataUri = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; + const fetcher: UrlFetcher = { + fetch: vi.fn().mockResolvedValue({ + content: dataUri, + kind: 'image', + mimeType: 'image/png', + dimensions: { width: 1, height: 1 }, + }), + }; + const tool = new FetchURLTool(fetcher); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_img', + args: { url: 'https://example.com/image.png' }, + signal, + }); + + expect(result.isError).toBe(false); + const output = (result as { output?: unknown }).output; + expect(Array.isArray(output)).toBe(true); + const parts = output as Array>; + expect(parts).toHaveLength(4); + expect(parts[0]).toMatchObject({ type: 'text', text: expect.stringContaining('Fetched image') }); + expect(parts[1]).toMatchObject({ type: 'text', text: '' }); + expect(parts[2]).toMatchObject({ type: 'image_url', imageUrl: { url: dataUri } }); + expect(parts[3]).toMatchObject({ type: 'text', text: '' }); + }); + + it('returns image without dimensions when fetcher omits them', async () => { + const dataUri = 'data:image/jpeg;base64,abc123'; + const fetcher: UrlFetcher = { + fetch: vi.fn().mockResolvedValue({ + content: dataUri, + kind: 'image', + mimeType: 'image/jpeg', + }), + }; + const tool = new FetchURLTool(fetcher); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_img2', + args: { url: 'https://example.com/image.jpg' }, + signal, + }); + + expect(result.isError).toBe(false); + const output = (result as { output?: unknown }).output as Array>; + expect(output[0]).toMatchObject({ type: 'text', text: expect.stringContaining('Mime type: image/jpeg') }); + expect(output[0]).toMatchObject({ type: 'text', text: expect.not.stringContaining('Original dimensions') }); + }); }); describe('MoonshotFetchURLProvider', () => { diff --git a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts index 2c0ce931f1..df89c19c55 100644 --- a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts +++ b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts @@ -17,6 +17,24 @@ function htmlResponse(body: string, contentType: string): Response { }); } +function binaryResponse(body: Buffer, contentType: string): Response { + return new Response(body, { + status: 200, + headers: { 'content-type': contentType }, + }); +} + +// Minimal 1x1 PNG (signature + truncated IHDR — enough for sniffing). +const minimalPng = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, // signature + 0x00, 0x00, 0x00, 0x0d, // IHDR length + 0x49, 0x48, 0x44, 0x52, // IHDR + 0x00, 0x00, 0x00, 0x01, // width = 1 + 0x00, 0x00, 0x00, 0x01, // height = 1 + 0x08, 0x02, 0x00, 0x00, 0x00, // bit depth, color type, etc. + 0x00, 0x00, 0x00, 0x00, // dummy CRC +]); + describe('LocalFetchURLProvider content kind', () => { it('reports text/plain bodies as a verbatim passthrough', async () => { const fetchImpl = vi @@ -55,4 +73,46 @@ describe('LocalFetchURLProvider content kind', () => { expect(result.kind).toBe('extracted'); expect(result.content).toContain('quick brown fox'); }); + + it('reports image/png bodies as image kind with base64 data URI and dimensions', async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(binaryResponse(minimalPng, 'image/png')); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.png'); + + expect(result.kind).toBe('image'); + expect(result.mimeType).toBe('image/png'); + expect(result.dimensions).toEqual({ width: 1, height: 1 }); + expect(result.content).toMatch(/^data:image\/png;base64,/); + }); + + it('reports image/jpeg bodies as image kind', async () => { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46]); + const fetchImpl = vi + .fn() + .mockResolvedValue(binaryResponse(jpeg, 'image/jpeg')); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.jpg'); + + expect(result.kind).toBe('image'); + expect(result.mimeType).toBe('image/jpeg'); + expect(result.content).toMatch(/^data:image\/jpeg;base64,/); + }); + + it('rejects image responses larger than maxBytes', async () => { + const largePng = Buffer.alloc(11 * 1024 * 1024, 0x00); + largePng[0] = 0x89; + largePng[1] = 0x50; + largePng[2] = 0x4e; + largePng[3] = 0x47; + const fetchImpl = vi + .fn() + .mockResolvedValue(binaryResponse(largePng, 'image/png')); + const provider = new LocalFetchURLProvider({ fetchImpl, maxBytes: 10 * 1024 * 1024 }); + + await expect(provider.fetch('https://example.com/huge.png')).rejects.toThrow('too large'); + }); });