diff --git a/src/ax/ai/anthropic/api.ts b/src/ax/ai/anthropic/api.ts index 46cf5ba2f..fa47a5aa3 100644 --- a/src/ax/ai/anthropic/api.ts +++ b/src/ax/ai/anthropic/api.ts @@ -1183,11 +1183,17 @@ function createMessages( case 'image': return { type: 'image' as const, - source: { - type: 'base64' as const, - media_type: v.mimeType, - data: v.image, - }, + source: + 'fileUri' in v + ? { + type: 'url' as const, + url: v.fileUri, + } + : { + type: 'base64' as const, + media_type: v.mimeType, + data: v.image, + }, ...(v.cache ? { cache_control: { type: 'ephemeral' as const } } : {}), diff --git a/src/ax/ai/anthropic/types.ts b/src/ax/ai/anthropic/types.ts index 1bcb18eca..25bccb410 100644 --- a/src/ax/ai/anthropic/types.ts +++ b/src/ax/ai/anthropic/types.ts @@ -130,7 +130,9 @@ export type AxAIAnthropicChatRequest = { } & AxAIAnthropicChatRequestCacheParam) | ({ type: 'image'; - source: { type: 'base64'; media_type: string; data: string }; + source: + | { type: 'base64'; media_type: string; data: string } + | { type: 'url'; url: string }; } & AxAIAnthropicChatRequestCacheParam) | ({ type: 'tool_result'; diff --git a/src/ax/ai/base.ts b/src/ax/ai/base.ts index c7d400505..fd11a855d 100644 --- a/src/ax/ai/base.ts +++ b/src/ax/ai/base.ts @@ -338,7 +338,11 @@ function hashContentPart( if (part.type === 'text') { hasher.update(`text:${part.text}`); } else if (part.type === 'image') { - hasher.update(`image:${part.mimeType}:${part.image.slice(0, 100)}`); + if ('fileUri' in part) { + hasher.update(`image:${part.mimeType}:${part.fileUri}`); + } else { + hasher.update(`image:${part.mimeType}:${part.image.slice(0, 100)}`); + } } else if (part.type === 'audio') { hasher.update(`audio:${part.format}:${part.data.slice(0, 100)}`); } else if (part.type === 'file') { diff --git a/src/ax/ai/google-gemini/api.test.ts b/src/ax/ai/google-gemini/api.test.ts index f0f53bf72..60de97327 100644 --- a/src/ax/ai/google-gemini/api.test.ts +++ b/src/ax/ai/google-gemini/api.test.ts @@ -2174,3 +2174,87 @@ describe('AxAIGoogleGemini Live audio chat', () => { } }); }); + +describe('AxAIGoogleGemini image content mapping', () => { + it('maps inline base64 images to inlineData', async () => { + const ai = new AxAIGoogleGemini({ + apiKey: 'key', + config: { model: AxAIGoogleGeminiModel.Gemini25Flash }, + models: [], + }); + + const capture: { lastBody?: any } = {}; + ai.setOptions({ + fetch: createMockFetch( + { + candidates: [ + { content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' }, + ], + }, + capture + ), + }); + + await ai.chat( + { + chatPrompt: [ + { + role: 'user', + content: [ + { type: 'image', mimeType: 'image/png', image: 'base64data' }, + ], + }, + ], + }, + { stream: false } + ); + + const parts = capture.lastBody?.contents?.[0]?.parts; + expect(parts).toContainEqual({ + inlineData: { mimeType: 'image/png', data: 'base64data' }, + }); + }); + + it('maps fileUri images to fileData', async () => { + const ai = new AxAIGoogleGemini({ + apiKey: 'key', + config: { model: AxAIGoogleGeminiModel.Gemini25Flash }, + models: [], + }); + + const capture: { lastBody?: any } = {}; + ai.setOptions({ + fetch: createMockFetch( + { + candidates: [ + { content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' }, + ], + }, + capture + ), + }); + + await ai.chat( + { + chatPrompt: [ + { + role: 'user', + content: [ + { + type: 'image', + mimeType: 'image/png', + fileUri: 'gs://my-bucket/cat.png', + }, + ], + }, + ], + }, + { stream: false } + ); + + const parts = capture.lastBody?.contents?.[0]?.parts; + expect(parts).toContainEqual({ + fileData: { mimeType: 'image/png', fileUri: 'gs://my-bucket/cat.png' }, + }); + }); +}); diff --git a/src/ax/ai/google-gemini/api.ts b/src/ax/ai/google-gemini/api.ts index 9e2e5aa02..7d832aaad 100644 --- a/src/ax/ai/google-gemini/api.ts +++ b/src/ax/ai/google-gemini/api.ts @@ -255,6 +255,24 @@ export interface AxAIGoogleGeminiArgs { modelInfo?: AxModelInfo[]; } +// Maps an image/file content part to a Gemini part: fileUri -> fileData, +// otherwise inline base64 -> inlineData. Image base64 lives on `image`, +// file base64 on `data`. +const toGeminiMediaPart = ( + c: + | { mimeType: string; image: string } + | { mimeType: string; data: string } + | { mimeType: string; fileUri: string } +) => + 'fileUri' in c + ? { fileData: { mimeType: c.mimeType, fileUri: c.fileUri } } + : { + inlineData: { + mimeType: c.mimeType, + data: 'image' in c ? c.image : c.data, + }, + }; + class AxAIGoogleGeminiImpl implements AxAIServiceImpl< @@ -772,9 +790,8 @@ class AxAIGoogleGeminiImpl case 'text': return { text: c.text }; case 'image': - return { - inlineData: { mimeType: c.mimeType, data: c.image }, - }; + case 'file': + return toGeminiMediaPart(c); case 'audio': return { inlineData: { @@ -783,20 +800,6 @@ class AxAIGoogleGeminiImpl data: c.data, }, }; - case 'file': - // Support both inline data and fileUri formats - if ('fileUri' in c) { - return { - fileData: { - mimeType: c.mimeType, - fileUri: c.fileUri, - }, - }; - } else { - return { - inlineData: { mimeType: c.mimeType, data: c.data }, - }; - } default: throw new Error( `Chat prompt content type not supported (index: ${idx})` @@ -1837,9 +1840,8 @@ class AxAIGoogleGeminiImpl parts.push({ text: c.text }); break; case 'image': - parts.push({ - inlineData: { mimeType: c.mimeType, data: c.image }, - }); + case 'file': + parts.push(toGeminiMediaPart(c)); break; case 'audio': parts.push({ @@ -1850,17 +1852,6 @@ class AxAIGoogleGeminiImpl }, }); break; - case 'file': - if ('fileUri' in c) { - parts.push({ - fileData: { mimeType: c.mimeType, fileUri: c.fileUri }, - }); - } else { - parts.push({ - inlineData: { mimeType: c.mimeType, data: c.data }, - }); - } - break; } } } @@ -1996,9 +1987,8 @@ class AxAIGoogleGeminiImpl parts.push({ text: c.text }); break; case 'image': - parts.push({ - inlineData: { mimeType: c.mimeType, data: c.image }, - }); + case 'file': + parts.push(toGeminiMediaPart(c)); break; case 'audio': parts.push({ @@ -2009,17 +1999,6 @@ class AxAIGoogleGeminiImpl }, }); break; - case 'file': - if ('fileUri' in c) { - parts.push({ - fileData: { mimeType: c.mimeType, fileUri: c.fileUri }, - }); - } else { - parts.push({ - inlineData: { mimeType: c.mimeType, data: c.data }, - }); - } - break; } } } diff --git a/src/ax/ai/openai/api.ts b/src/ax/ai/openai/api.ts index ccc632ed0..e0583b396 100644 --- a/src/ax/ai/openai/api.ts +++ b/src/ax/ai/openai/api.ts @@ -683,7 +683,10 @@ function createMessages( case 'text': return { type: 'text' as const, text: c.text }; case 'image': { - const url = `data:${c.mimeType};base64,${c.image}`; + const url = + 'fileUri' in c + ? c.fileUri + : `data:${c.mimeType};base64,${c.image}`; return { type: 'image_url' as const, image_url: { url, details: c.details ?? 'auto' }, diff --git a/src/ax/ai/processor.ts b/src/ax/ai/processor.ts index c6e36ac36..7629f9208 100644 --- a/src/ax/ai/processor.ts +++ b/src/ax/ai/processor.ts @@ -117,7 +117,7 @@ export async function axProcessContentForProvider( } else if (item.altText) { // Fallback to alt text processedContent.push({ type: 'text', text: item.altText }); - } else if (options.imageToText) { + } else if (options.imageToText && 'image' in item) { // Use AI vision service to describe image try { const description = await options.imageToText(item.image); diff --git a/src/ax/ai/types.ts b/src/ax/ai/types.ts index 6cd157830..3ccac5727 100644 --- a/src/ax/ai/types.ts +++ b/src/ax/ai/types.ts @@ -400,6 +400,7 @@ export type AxChatRequest = { cache?: boolean; } | { + /** Image content type with inline base64 data */ type: 'image'; mimeType: string; image: string; @@ -410,6 +411,19 @@ export type AxChatRequest = { /** Fallback text description when images aren't supported */ altText?: string; } + | { + /** Image content type with a URI (e.g. https:// or gs:// URL) */ + type: 'image'; + /** Image URI (e.g., https:// or gs:// URL) */ + fileUri: string; + mimeType: string; + details?: 'high' | 'low' | 'auto'; + cache?: boolean; + /** Optimization preference for image processing */ + optimize?: 'quality' | 'size' | 'auto'; + /** Fallback text description when images aren't supported */ + altText?: string; + } | { type: 'audio'; data: string; diff --git a/src/ax/ai/validate.ts b/src/ax/ai/validate.ts index 37aa6fcfb..88c02627f 100644 --- a/src/ax/ai/validate.ts +++ b/src/ax/ai/validate.ts @@ -136,15 +136,23 @@ export function axValidateChatRequestMessage(item: AxChatRequestMessage): void { 'image' in contentItem && typeof contentItem.image === 'string' ? contentItem.image : undefined; + const fileUri = + 'fileUri' in contentItem && + typeof contentItem.fileUri === 'string' + ? contentItem.fileUri + : undefined; const mimeType = 'mimeType' in contentItem && typeof contentItem.mimeType === 'string' ? contentItem.mimeType : undefined; - if (!image || image.trim() === '') { + if ( + (!image || image.trim() === '') && + (!fileUri || fileUri.trim() === '') + ) { throw new Error( - `User message image content at index ${index} cannot be empty, received: ${value(image)}` + `User message image content at index ${index} must have either data or fileUri, received: ${value(image ?? fileUri)}` ); } if (!mimeType || mimeType.trim() === '') { diff --git a/src/ax/dsp/loggers.ts b/src/ax/dsp/loggers.ts index f74b80ffc..32493503a 100644 --- a/src/ax/dsp/loggers.ts +++ b/src/ax/dsp/loggers.ts @@ -40,7 +40,8 @@ const formatChatMessage = ( return colorize(item.text, 'green'); } if (item.type === 'image') { - const content = hideContent ? '[Image]' : `[Image: ${item.image}]`; + const source = 'fileUri' in item ? item.fileUri : item.image; + const content = hideContent ? '[Image]' : `[Image: ${source}]`; return colorize(content, 'green'); } if (item.type === 'audio') { diff --git a/src/ax/dsp/prompt.test.ts b/src/ax/dsp/prompt.test.ts index dc3453d12..c702052b6 100644 --- a/src/ax/dsp/prompt.test.ts +++ b/src/ax/dsp/prompt.test.ts @@ -686,6 +686,157 @@ describe('AxPromptTemplate.render', () => { }); }); + describe('Image field handling', () => { + it('should render image field with data (base64)', () => { + const sig = new AxSignature('imageInput:image -> responseText:string'); + const pt = new AxPromptTemplate(sig); + + const result = pt.render( + { + imageInput: { + mimeType: 'image/png', + data: 'base64data', + }, + }, + {} + ); + + expect(result).toHaveLength(2); // system + user message + + const userMessage = result.find((m) => m.role === 'user'); + expect(userMessage).toBeDefined(); + expect(userMessage!.content).toHaveLength(2); + expect(userMessage!.content[0]).toEqual({ + type: 'text', + text: 'Image Input: \n', + }); + expect(userMessage!.content[1]).toEqual({ + type: 'image', + mimeType: 'image/png', + image: 'base64data', + }); + }); + + it('should render image field with fileUri (URL)', () => { + const sig = new AxSignature('imageInput:image -> responseText:string'); + const pt = new AxPromptTemplate(sig); + + const result = pt.render( + { + imageInput: { + mimeType: 'image/png', + fileUri: 'https://example.com/cat.png', + }, + }, + {} + ); + + expect(result).toHaveLength(2); // system + user message + + const userMessage = result.find((m) => m.role === 'user'); + expect(userMessage).toBeDefined(); + expect(userMessage!.content).toHaveLength(2); + expect(userMessage!.content[0]).toEqual({ + type: 'text', + text: 'Image Input: \n', + }); + expect(userMessage!.content[1]).toEqual({ + type: 'image', + mimeType: 'image/png', + fileUri: 'https://example.com/cat.png', + }); + }); + + it('should render array of images with mixed formats', () => { + const sig = new AxSignature('imageInputs:image[] -> responseText:string'); + const pt = new AxPromptTemplate(sig); + + const result = pt.render( + { + imageInputs: [ + { + mimeType: 'image/png', + data: 'base64data1', + }, + { + mimeType: 'image/jpeg', + fileUri: 'https://example.com/dog.jpg', + }, + ], + }, + {} + ); + + expect(result).toHaveLength(2); // system + user message + + const userMessage = result.find((m) => m.role === 'user'); + expect(userMessage).toBeDefined(); + expect(userMessage!.content).toHaveLength(3); + + expect(userMessage!.content[0]).toEqual({ + type: 'text', + text: 'Image Inputs: \n', + }); + + // First image with data + expect(userMessage!.content[1]).toEqual({ + type: 'image', + mimeType: 'image/png', + image: 'base64data1', + }); + + // Second image with fileUri + expect(userMessage!.content[2]).toEqual({ + type: 'image', + mimeType: 'image/jpeg', + fileUri: 'https://example.com/dog.jpg', + }); + }); + + it('should validate image field requirements', () => { + const sig = new AxSignature('imageInput:image -> responseText:string'); + const pt = new AxPromptTemplate(sig); + + // Missing mimeType + expect(() => + pt.render( + { + imageInput: { + data: 'base64data', + }, + }, + {} + ) + ).toThrow(/mimeType.*data.*fileUri/); + + // Missing both data and fileUri + expect(() => + pt.render( + { + imageInput: { + mimeType: 'image/png', + }, + }, + {} + ) + ).toThrow(/mimeType.*data.*fileUri/); + + // Both data and fileUri present + expect(() => + pt.render( + { + imageInput: { + mimeType: 'image/png', + data: 'base64data', + fileUri: 'https://example.com/cat.png', + }, + }, + {} + ) + ).toThrow(/mimeType.*data.*fileUri/); + }); + }); + describe('Examples as alternating message pairs (new default behavior)', () => { it('should render multiple examples as alternating user/assistant pairs', () => { const signature = new AxSignature( diff --git a/src/ax/dsp/prompt.ts b/src/ax/dsp/prompt.ts index ef68fe290..8b532f719 100644 --- a/src/ax/dsp/prompt.ts +++ b/src/ax/dsp/prompt.ts @@ -1157,7 +1157,9 @@ export class AxPromptTemplate { if (field.type?.name === 'image') { const validateImage = ( value: Readonly - ): { mimeType: string; data: string } => { + ): + | { mimeType: string; data: string } + | { mimeType: string; fileUri: string } => { if (!value) { throw new Error('Image field value is required.'); } @@ -1168,10 +1170,36 @@ export class AxPromptTemplate { if (!('mimeType' in value)) { throw new Error('Image field must have mimeType'); } - if (!('data' in value)) { - throw new Error('Image field must have data'); + + // Support both data and fileUri formats + const hasData = 'data' in value; + const hasFileUri = 'fileUri' in value; + + if (!hasData && !hasFileUri) { + throw new Error('Image field must have either data or fileUri'); + } + if (hasData && hasFileUri) { + throw new Error('Image field cannot have both data and fileUri'); } - return value as { mimeType: string; data: string }; + + return value as + | { mimeType: string; data: string } + | { mimeType: string; fileUri: string }; + }; + + const toImagePart = (value: Readonly) => { + const validated = validateImage(value); + return 'fileUri' in validated + ? { + type: 'image' as const, + mimeType: validated.mimeType, + fileUri: validated.fileUri, + } + : { + type: 'image' as const, + mimeType: validated.mimeType, + image: validated.data, + }; }; let result: ChatRequestUserMessage = [ @@ -1183,23 +1211,10 @@ export class AxPromptTemplate { throw new Error('Image field value must be an array.'); } result = result.concat( - (value as unknown[]).map((v) => { - // Cast to unknown[] before map - const validated = validateImage(v as AxFieldValue); - return { - type: 'image', - mimeType: validated.mimeType, - image: validated.data, - }; - }) + (value as unknown[]).map((v) => toImagePart(v as AxFieldValue)) ); } else { - const validated = validateImage(value); - result.push({ - type: 'image', - mimeType: validated.mimeType, - image: validated.data, - }); + result.push(toImagePart(value)); } return result; } diff --git a/src/ax/dsp/types.ts b/src/ax/dsp/types.ts index a6ced3326..dfaebb15f 100644 --- a/src/ax/dsp/types.ts +++ b/src/ax/dsp/types.ts @@ -105,7 +105,11 @@ export type AxFieldValue = | null | undefined | { mimeType: string; data: string } - | { mimeType: string; data: string }[] + | { mimeType: string; fileUri: string } + | ( + | { mimeType: string; data: string } + | { mimeType: string; fileUri: string } + )[] | { format?: string; data?: string; diff --git a/src/ax/dsp/util.ts b/src/ax/dsp/util.ts index 185f191e4..b2dbbd9c8 100644 --- a/src/ax/dsp/util.ts +++ b/src/ax/dsp/util.ts @@ -49,34 +49,21 @@ export const validateValue = ( } }; - const validImage = (val: Readonly): boolean => { - if ( - !val || - typeof val !== 'object' || - !('mimeType' in val) || - !('data' in val) - ) { + // image and file share the same shape: mimeType + exactly one of data | fileUri + const validMediaContent = (val: Readonly): boolean => { + if (!val || typeof val !== 'object' || !('mimeType' in val)) { return false; } - return true; + return 'data' in val !== 'fileUri' in val; }; - if (field.type?.name === 'image') { - let msg: string | undefined; - if (Array.isArray(value)) { - for (const item of value) { - if (!validImage(item)) { - msg = 'object ({ mimeType: string; data: string })'; - break; - } - } - } else if (!validImage(value)) { - msg = 'object ({ mimeType: string; data: string })'; - } - - if (msg) { + if (field.type?.name === 'image' || field.type?.name === 'file') { + const mediaMsg = + 'object ({ mimeType: string; data: string } | { mimeType: string; fileUri: string })'; + const items = Array.isArray(value) ? value : [value]; + if (items.some((item) => !validMediaContent(item))) { throw new Error( - `Validation failed: Expected '${field.name}' to be type '${msg}' instead got '${value}'` + `Validation failed: Expected '${field.name}' to be type '${mediaMsg}' instead got '${value}'` ); } return; @@ -117,48 +104,6 @@ export const validateValue = ( return; } - const validFile = (val: Readonly): boolean => { - if (!val || typeof val !== 'object' || !('mimeType' in val)) { - return false; - } - - // Support both data and fileUri formats - const hasData = 'data' in val; - const hasFileUri = 'fileUri' in val; - - if (!hasData && !hasFileUri) { - return false; - } - if (hasData && hasFileUri) { - return false; // Cannot have both - } - - return true; - }; - - if (field.type?.name === 'file') { - let msg: string | undefined; - if (Array.isArray(value)) { - for (const item of value) { - if (!validFile(item)) { - msg = - 'object ({ mimeType: string; data: string } | { mimeType: string; fileUri: string })'; - break; - } - } - } else if (!validFile(value)) { - msg = - 'object ({ mimeType: string; data: string } | { mimeType: string; fileUri: string })'; - } - - if (msg) { - throw new Error( - `Validation failed: Expected '${field.name}' to be type '${msg}' instead got '${value}'` - ); - } - return; - } - const validUrl = (val: Readonly): boolean => { if (typeof val === 'string') { return true; // Simple URL string