Skip to content

feat: support image URLs (fileUri) in image input fields #522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gennadijbgl wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: support image URLs (fileUri) in image input fields #522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
84de80c
feat: support image URLs (fileUri) in image input fields
gennadijbgl May 29, 2026
a89f584
refactor: dedupe media validation and Gemini part mapping
gennadijbgl May 29, 2026
577d7e0
refactor: extract toImagePart to dedupe image render branches
gennadijbgl May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/ax/ai/anthropic/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1183,11 +1183,17 @@ function createMessages(
case 'image':
return {
type: 'image' as const,
source: {
type: 'base64' as const,
media_type: v.mimeType,
data: v.image,
},
source:
'fileUri' in v
? {
type: 'url' as const,
url: v.fileUri,
}
: {
type: 'base64' as const,
media_type: v.mimeType,
data: v.image,
},
...(v.cache
? { cache_control: { type: 'ephemeral' as const } }
: {}),
Expand Down
4 changes: 3 additions & 1 deletion src/ax/ai/anthropic/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ export type AxAIAnthropicChatRequest = {
} & AxAIAnthropicChatRequestCacheParam)
| ({
type: 'image';
source: { type: 'base64'; media_type: string; data: string };
source:
| { type: 'base64'; media_type: string; data: string }
| { type: 'url'; url: string };
} & AxAIAnthropicChatRequestCacheParam)
| ({
type: 'tool_result';
Expand Down
6 changes: 5 additions & 1 deletion src/ax/ai/base.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,11 @@ function hashContentPart(
if (part.type === 'text') {
hasher.update(`text:${part.text}`);
} else if (part.type === 'image') {
hasher.update(`image:${part.mimeType}:${part.image.slice(0, 100)}`);
if ('fileUri' in part) {
hasher.update(`image:${part.mimeType}:${part.fileUri}`);
} else {
hasher.update(`image:${part.mimeType}:${part.image.slice(0, 100)}`);
}
} else if (part.type === 'audio') {
hasher.update(`audio:${part.format}:${part.data.slice(0, 100)}`);
} else if (part.type === 'file') {
Expand Down
84 changes: 84 additions & 0 deletions src/ax/ai/google-gemini/api.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2174,3 +2174,87 @@ describe('AxAIGoogleGemini Live audio chat', () => {
}
});
});

describe('AxAIGoogleGemini image content mapping', () => {
it('maps inline base64 images to inlineData', async () => {
const ai = new AxAIGoogleGemini({
apiKey: 'key',
config: { model: AxAIGoogleGeminiModel.Gemini25Flash },
models: [],
});

const capture: { lastBody?: any } = {};
ai.setOptions({
fetch: createMockFetch(
{
candidates: [
{ content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' },
],
},
capture
),
});

await ai.chat(
{
chatPrompt: [
{
role: 'user',
content: [
{ type: 'image', mimeType: 'image/png', image: 'base64data' },
],
},
],
},
{ stream: false }
);

const parts = capture.lastBody?.contents?.[0]?.parts;
expect(parts).toContainEqual({
inlineData: { mimeType: 'image/png', data: 'base64data' },
});
});

it('maps fileUri images to fileData', async () => {
const ai = new AxAIGoogleGemini({
apiKey: 'key',
config: { model: AxAIGoogleGeminiModel.Gemini25Flash },
models: [],
});

const capture: { lastBody?: any } = {};
ai.setOptions({
fetch: createMockFetch(
{
candidates: [
{ content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' },
],
},
capture
),
});

await ai.chat(
{
chatPrompt: [
{
role: 'user',
content: [
{
type: 'image',
mimeType: 'image/png',
fileUri: 'gs://my-bucket/cat.png',
},
],
},
],
},
{ stream: false }
);

const parts = capture.lastBody?.contents?.[0]?.parts;
expect(parts).toContainEqual({
fileData: { mimeType: 'image/png', fileUri: 'gs://my-bucket/cat.png' },
});
});
});
69 changes: 24 additions & 45 deletions src/ax/ai/google-gemini/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,24 @@ export interface AxAIGoogleGeminiArgs<TModelKey> {
modelInfo?: AxModelInfo[];
}

// Maps an image/file content part to a Gemini part: fileUri -> fileData,
// otherwise inline base64 -> inlineData. Image base64 lives on `image`,
// file base64 on `data`.
const toGeminiMediaPart = (
c:
| { mimeType: string; image: string }
| { mimeType: string; data: string }
| { mimeType: string; fileUri: string }
) =>
'fileUri' in c
? { fileData: { mimeType: c.mimeType, fileUri: c.fileUri } }
: {
inlineData: {
mimeType: c.mimeType,
data: 'image' in c ? c.image : c.data,
},
};

class AxAIGoogleGeminiImpl
implements
AxAIServiceImpl<
Expand Down Expand Up @@ -772,9 +790,8 @@ class AxAIGoogleGeminiImpl
case 'text':
return { text: c.text };
case 'image':
return {
inlineData: { mimeType: c.mimeType, data: c.image },
};
case 'file':
return toGeminiMediaPart(c);
case 'audio':
return {
inlineData: {
Expand All @@ -783,20 +800,6 @@ class AxAIGoogleGeminiImpl
data: c.data,
},
};
case 'file':
// Support both inline data and fileUri formats
if ('fileUri' in c) {
return {
fileData: {
mimeType: c.mimeType,
fileUri: c.fileUri,
},
};
} else {
return {
inlineData: { mimeType: c.mimeType, data: c.data },
};
}
default:
throw new Error(
`Chat prompt content type not supported (index: ${idx})`
Expand Down Expand Up @@ -1837,9 +1840,8 @@ class AxAIGoogleGeminiImpl
parts.push({ text: c.text });
break;
case 'image':
parts.push({
inlineData: { mimeType: c.mimeType, data: c.image },
});
case 'file':
parts.push(toGeminiMediaPart(c));
break;
case 'audio':
parts.push({
Expand All @@ -1850,17 +1852,6 @@ class AxAIGoogleGeminiImpl
},
});
break;
case 'file':
if ('fileUri' in c) {
parts.push({
fileData: { mimeType: c.mimeType, fileUri: c.fileUri },
});
} else {
parts.push({
inlineData: { mimeType: c.mimeType, data: c.data },
});
}
break;
}
}
}
Expand Down Expand Up @@ -1996,9 +1987,8 @@ class AxAIGoogleGeminiImpl
parts.push({ text: c.text });
break;
case 'image':
parts.push({
inlineData: { mimeType: c.mimeType, data: c.image },
});
case 'file':
parts.push(toGeminiMediaPart(c));
break;
case 'audio':
parts.push({
Expand All @@ -2009,17 +1999,6 @@ class AxAIGoogleGeminiImpl
},
});
break;
case 'file':
if ('fileUri' in c) {
parts.push({
fileData: { mimeType: c.mimeType, fileUri: c.fileUri },
});
} else {
parts.push({
inlineData: { mimeType: c.mimeType, data: c.data },
});
}
break;
}
}
}
Expand Down
5 changes: 4 additions & 1 deletion src/ax/ai/openai/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,10 @@ function createMessages<TModel>(
case 'text':
return { type: 'text' as const, text: c.text };
case 'image': {
const url = `data:${c.mimeType};base64,${c.image}`;
const url =
'fileUri' in c
? c.fileUri
: `data:${c.mimeType};base64,${c.image}`;
return {
type: 'image_url' as const,
image_url: { url, details: c.details ?? 'auto' },
Expand Down
2 changes: 1 addition & 1 deletion src/ax/ai/processor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ export async function axProcessContentForProvider(
} else if (item.altText) {
// Fallback to alt text
processedContent.push({ type: 'text', text: item.altText });
} else if (options.imageToText) {
} else if (options.imageToText && 'image' in item) {
// Use AI vision service to describe image
try {
const description = await options.imageToText(item.image);
Expand Down
14 changes: 14 additions & 0 deletions src/ax/ai/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ export type AxChatRequest<TModel = string> = {
cache?: boolean;
}
| {
/** Image content type with inline base64 data */
type: 'image';
mimeType: string;
image: string;
Expand All @@ -410,6 +411,19 @@ export type AxChatRequest<TModel = string> = {
/** Fallback text description when images aren't supported */
altText?: string;
}
| {
/** Image content type with a URI (e.g. https:// or gs:// URL) */
type: 'image';
/** Image URI (e.g., https:// or gs:// URL) */
fileUri: string;
mimeType: string;
details?: 'high' | 'low' | 'auto';
cache?: boolean;
/** Optimization preference for image processing */
optimize?: 'quality' | 'size' | 'auto';
/** Fallback text description when images aren't supported */
altText?: string;
}
| {
type: 'audio';
data: string;
Expand Down
12 changes: 10 additions & 2 deletions src/ax/ai/validate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,23 @@ export function axValidateChatRequestMessage(item: AxChatRequestMessage): void {
'image' in contentItem && typeof contentItem.image === 'string'
? contentItem.image
: undefined;
const fileUri =
'fileUri' in contentItem &&
typeof contentItem.fileUri === 'string'
? contentItem.fileUri
: undefined;
const mimeType =
'mimeType' in contentItem &&
typeof contentItem.mimeType === 'string'
? contentItem.mimeType
: undefined;

if (!image || image.trim() === '') {
if (
(!image || image.trim() === '') &&
(!fileUri || fileUri.trim() === '')
) {
throw new Error(
`User message image content at index ${index} cannot be empty, received: ${value(image)}`
`User message image content at index ${index} must have either data or fileUri, received: ${value(image ?? fileUri)}`
);
}
if (!mimeType || mimeType.trim() === '') {
Expand Down
3 changes: 2 additions & 1 deletion src/ax/dsp/loggers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ const formatChatMessage = (
return colorize(item.text, 'green');
}
if (item.type === 'image') {
const content = hideContent ? '[Image]' : `[Image: ${item.image}]`;
const source = 'fileUri' in item ? item.fileUri : item.image;
const content = hideContent ? '[Image]' : `[Image: ${source}]`;
return colorize(content, 'green');
}
if (item.type === 'audio') {
Expand Down
Loading