feat: FetchURL tool supports downloading images

packages/agent-core/src/tools/builtin/web/fetch-url.md

@@ -1,3 +1,3 @@
-Fetch content from a URL. Returns the main text content extracted from the page. Use this when you need to read a specific web page.
+Fetch content from a URL. Returns the main text content extracted from the page, or images when the URL points to an image file. Use this when you need to read a specific web page or view an image from the web.
 
 Only public `http`/`https` URLs are supported. Requests to private, loopback, or link-local addresses are refused, and responses larger than 10 MiB are rejected.

packages/agent-core/src/tools/builtin/web/fetch-url.ts

@@ -6,6 +6,8 @@
  * should not be registered (not exposed to the LLM).
  */
 
+import type { ContentPart } from '@moonshot-ai/kosong';
+
 import { z } from 'zod';
 
 import type { BuiltinTool } from '../../../agent/tool';
@@ -25,14 +27,19 @@ import DESCRIPTION from './fetch-url.md?raw';
  *   returned verbatim, in full.
  * - `extracted` — the body was an HTML page; only the main article text
  *   was extracted and returned.
+ * - `image` — the body is an image; returned as a base64 data URI.
  */
-export type UrlFetchKind = 'passthrough' | 'extracted';
+export type UrlFetchKind = 'passthrough' | 'extracted' | 'image';
 
 export interface UrlFetchResult {
-  /** The text handed to the LLM. */
+  /** The text handed to the LLM, or a base64 data URI for images. */
   content: string;
-  /** Whether `content` is a verbatim passthrough or extracted main text. */
+  /** Whether `content` is a verbatim passthrough, extracted main text, or image. */
   kind: UrlFetchKind;
+  /** The MIME type of the response, when known. */
+  mimeType?: string | undefined;
+  /** Image dimensions, when the response is an image and they can be determined. */
+  dimensions?: { width: number; height: number } | null | undefined;
 }
 
 export interface UrlFetcher {
@@ -89,7 +96,7 @@ export class FetchURLTool implements BuiltinTool<FetchURLInput> {
     }: ExecutableToolContext,
   ): Promise<ExecutableToolResult> {
     try {
-      const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId });
+      const { content, kind, mimeType, dimensions } = await this.fetcher.fetch(args.url, { toolCallId });
 
       if (!content) {
         return {
@@ -98,6 +105,29 @@ export class FetchURLTool implements BuiltinTool<FetchURLInput> {
         };
       }
 
+      if (kind === 'image') {
+        const systemParts: string[] = ['Fetched image from URL.'];
+        if (mimeType) {
+          systemParts.push(`Mime type: ${mimeType}.`);
+        }
+        if (dimensions) {
+          systemParts.push(
+            `Original dimensions: ${String(dimensions.width)}x${String(dimensions.height)} pixels.`,
+          );
+          systemParts.push(
+            'If you need to output coordinates, output relative coordinates first ' +
+              'and compute absolute coordinates using the original image size.',
+          );
+        }
+        const output: ContentPart[] = [
+          { type: 'text', text: `<system>${systemParts.join(' ')}</system>` },
+          { type: 'text', text: `<image url="${args.url}">` },
+          { type: 'image_url', imageUrl: { url: content } },
+          { type: 'text', text: '</image>' },
+        ];
+        return { output, isError: false };
+      }
+
       const builder = new ToolResultBuilder({ maxLineLength: null });
       builder.write(content);
       // Tell the LLM whether it received the whole body or only the

packages/agent-core/src/tools/providers/local-fetch-url.ts

@@ -6,8 +6,10 @@
  *   2. Reject HTTP >= 400 with the status code in the message.
  *   3. Reject responses larger than `maxBytes` (content-length first,
  *      then measured body length as a defensive second check).
- *   4. `text/plain` / `text/markdown` → passthrough verbatim.
- *   5. Otherwise (assumed HTML) → run Readability over a linkedom
+ *   4. `image/*` → read binary body, encode as base64 data URI, and
+ *      return as `image` kind with optional dimension sniffing.
+ *   5. `text/plain` / `text/markdown` → passthrough verbatim.
+ *   6. Otherwise (assumed HTML) → run Readability over a linkedom
  *      document. Return `# ${title}\n\n${text}` (title omitted when
  *      absent). If extraction yields no meaningful text, fall back to
  *      common content containers (`<article>` / `<main>` / `<body>`)
@@ -18,6 +20,7 @@ import { Readability } from '@mozilla/readability';
 import { parseHTML as rawParseHTML } from 'linkedom';
 
 import { HttpFetchError, type UrlFetcher, type UrlFetchResult } from '../builtin';
+import { sniffImageDimensions } from '../support/file-type';
 
 // Readability's .d.ts references the global `Document` type, but this
 // package compiles with `lib: ES2023` (no DOM). Extracting the
@@ -172,6 +175,23 @@ export class LocalFetchURLProvider implements UrlFetcher {
       }
     }
 
+    const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
+    const mimeType = contentType.split(';')[0]!.trim();
+
+    if (mimeType.startsWith('image/')) {
+      const arrayBuffer = await response.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      if (buffer.length > this.maxBytes) {
+        throw new Error(
+          `Response body too large: ${String(buffer.length)} bytes exceeds maxBytes (${String(this.maxBytes)}).`,
+        );
+      }
+      const base64 = buffer.toString('base64');
+      const dataUri = `data:${mimeType};base64,${base64}`;
+      const dimensions = sniffImageDimensions(buffer);
+      return { content: dataUri, kind: 'image', mimeType, dimensions };
+    }
+
     const body = await response.text();
 
     // Servers may omit content-length — measure again defensively.
@@ -182,8 +202,7 @@ export class LocalFetchURLProvider implements UrlFetcher {
       );
     }
 
-    const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
-    if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) {
+    if (mimeType.startsWith('text/plain') || mimeType.startsWith('text/markdown')) {
       return { content: body, kind: 'passthrough' };
     }
 

packages/agent-core/test/tools/fetch-url.test.ts

@@ -259,6 +259,60 @@ describe('FetchURLTool', () => {
     const message = (result as { message?: string }).message ?? '';
     expect(message).toContain('full response body');
   });
+
+  it('returns image content as ContentPart[] when fetcher returns image kind', async () => {
+    const dataUri = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=';
+    const fetcher: UrlFetcher = {
+      fetch: vi.fn().mockResolvedValue({
+        content: dataUri,
+        kind: 'image',
+        mimeType: 'image/png',
+        dimensions: { width: 1, height: 1 },
+      }),
+    };
+    const tool = new FetchURLTool(fetcher);
+
+    const result = await executeTool(tool, {
+      turnId: 't1',
+      toolCallId: 'c_img',
+      args: { url: 'https://example.com/image.png' },
+      signal,
+    });
+
+    expect(result.isError).toBe(false);
+    const output = (result as { output?: unknown }).output;
+    expect(Array.isArray(output)).toBe(true);
+    const parts = output as Array<Record<string, unknown>>;
+    expect(parts).toHaveLength(4);
+    expect(parts[0]).toMatchObject({ type: 'text', text: expect.stringContaining('Fetched image') });
+    expect(parts[1]).toMatchObject({ type: 'text', text: '<image url="https://example.com/image.png">' });
+    expect(parts[2]).toMatchObject({ type: 'image_url', imageUrl: { url: dataUri } });
+    expect(parts[3]).toMatchObject({ type: 'text', text: '</image>' });
+  });
+
+  it('returns image without dimensions when fetcher omits them', async () => {
+    const dataUri = 'data:image/jpeg;base64,abc123';
+    const fetcher: UrlFetcher = {
+      fetch: vi.fn().mockResolvedValue({
+        content: dataUri,
+        kind: 'image',
+        mimeType: 'image/jpeg',
+      }),
+    };
+    const tool = new FetchURLTool(fetcher);
+
+    const result = await executeTool(tool, {
+      turnId: 't1',
+      toolCallId: 'c_img2',
+      args: { url: 'https://example.com/image.jpg' },
+      signal,
+    });
+
+    expect(result.isError).toBe(false);
+    const output = (result as { output?: unknown }).output as Array<Record<string, unknown>>;
+    expect(output[0]).toMatchObject({ type: 'text', text: expect.stringContaining('Mime type: image/jpeg') });
+    expect(output[0]).toMatchObject({ type: 'text', text: expect.not.stringContaining('Original dimensions') });
+  });
 });
 
 describe('MoonshotFetchURLProvider', () => {

packages/agent-core/test/tools/providers/local-fetch-url.test.ts

@@ -17,6 +17,24 @@ function htmlResponse(body: string, contentType: string): Response {
   });
 }
 
+function binaryResponse(body: Buffer, contentType: string): Response {
+  return new Response(body, {
+    status: 200,
+    headers: { 'content-type': contentType },
+  });
+}
+
+// Minimal 1x1 PNG (signature + truncated IHDR — enough for sniffing).
+const minimalPng = Buffer.from([
+  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, // signature
+  0x00, 0x00, 0x00, 0x0d, // IHDR length
+  0x49, 0x48, 0x44, 0x52, // IHDR
+  0x00, 0x00, 0x00, 0x01, // width = 1
+  0x00, 0x00, 0x00, 0x01, // height = 1
+  0x08, 0x02, 0x00, 0x00, 0x00, // bit depth, color type, etc.
+  0x00, 0x00, 0x00, 0x00, // dummy CRC
+]);
+
 describe('LocalFetchURLProvider content kind', () => {
   it('reports text/plain bodies as a verbatim passthrough', async () => {
     const fetchImpl = vi
@@ -55,4 +73,46 @@ describe('LocalFetchURLProvider content kind', () => {
     expect(result.kind).toBe('extracted');
     expect(result.content).toContain('quick brown fox');
   });
+
+  it('reports image/png bodies as image kind with base64 data URI and dimensions', async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValue(binaryResponse(minimalPng, 'image/png'));
+    const provider = new LocalFetchURLProvider({ fetchImpl });
+
+    const result = await provider.fetch('https://example.com/image.png');
+
+    expect(result.kind).toBe('image');
+    expect(result.mimeType).toBe('image/png');
+    expect(result.dimensions).toEqual({ width: 1, height: 1 });
+    expect(result.content).toMatch(/^data:image\/png;base64,/);
+  });
+
+  it('reports image/jpeg bodies as image kind', async () => {
+    const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46]);
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValue(binaryResponse(jpeg, 'image/jpeg'));
+    const provider = new LocalFetchURLProvider({ fetchImpl });
+
+    const result = await provider.fetch('https://example.com/image.jpg');
+
+    expect(result.kind).toBe('image');
+    expect(result.mimeType).toBe('image/jpeg');
+    expect(result.content).toMatch(/^data:image\/jpeg;base64,/);
+  });
+
+  it('rejects image responses larger than maxBytes', async () => {
+    const largePng = Buffer.alloc(11 * 1024 * 1024, 0x00);
+    largePng[0] = 0x89;
+    largePng[1] = 0x50;
+    largePng[2] = 0x4e;
+    largePng[3] = 0x47;
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValue(binaryResponse(largePng, 'image/png'));
+    const provider = new LocalFetchURLProvider({ fetchImpl, maxBytes: 10 * 1024 * 1024 });
+
+    await expect(provider.fetch('https://example.com/huge.png')).rejects.toThrow('too large');
+  });
 });