11import React , { Suspense } from "react" ;
22
33import {
4- AnsiStreamOutput ,
5- OutputData ,
6- ToolCallData ,
7- ToolResultData ,
8- } from "../outputs/index.js" ;
4+ MediaContainer ,
5+ isInlineContainer ,
6+ isArtifactContainer ,
7+ isAiToolCallData ,
8+ AI_TOOL_CALL_MIME_TYPE ,
9+ AI_TOOL_RESULT_MIME_TYPE ,
10+ TEXT_MIME_TYPES ,
11+ APPLICATION_MIME_TYPES ,
12+ IMAGE_MIME_TYPES ,
13+ JUPYTER_MIME_TYPES ,
14+ } from "@runt/schema" ;
15+ import { AnsiStreamOutput } from "../outputs/index.js" ;
916import { AnsiErrorOutput } from "./AnsiOutput.js" ;
1017import "../outputs/outputs.css" ;
1118
@@ -44,7 +51,7 @@ const PlainTextOutput = React.lazy(() =>
4451) ;
4552
4653interface RichOutputProps {
47- data : Record < string , unknown > ;
54+ data : string | Record < string , MediaContainer > ;
4855 metadata ?: Record < string , unknown > ;
4956 outputType ?:
5057 | "multimedia_display"
@@ -99,37 +106,31 @@ export const RichOutput: React.FC<RichOutputProps> = ({
99106 }
100107
101108 // Handle multimedia outputs (multimedia_display, multimedia_result)
102- let outputData : OutputData ;
109+ let outputData : Record < string , unknown > = { } ;
103110
104- // Check if data contains representations (new format)
111+ // Check if data contains media containers (new format)
105112 if ( data && typeof data === "object" && ! Array . isArray ( data ) ) {
106- const potentialRepresentations = data as Record < string , any > ;
107-
108- // Check if this looks like representations (has MediaRepresentation structure)
109- const hasRepresentations = Object . values ( potentialRepresentations ) . some (
110- ( value : any ) =>
111- value &&
112- typeof value === "object" &&
113- ( value . type === "inline" || value . type === "artifact" )
113+ const potentialContainers = data as Record < string , MediaContainer > ;
114+
115+ // Check if this looks like media containers
116+ const hasContainers = Object . values ( potentialContainers ) . some (
117+ ( value : any ) => isInlineContainer ( value ) || isArtifactContainer ( value )
114118 ) ;
115119
116- if ( hasRepresentations ) {
117- // Convert from representations to rendering format
118- outputData = { } ;
119- for ( const [ mimeType , representation ] of Object . entries (
120- potentialRepresentations
121- ) ) {
122- if (
123- representation &&
124- typeof representation === "object" &&
125- representation . data !== undefined
126- ) {
127- outputData [ mimeType ] = representation . data ;
120+ if ( hasContainers ) {
121+ // Convert from media containers to rendering format
122+ for ( const [ mimeType , container ] of Object . entries ( potentialContainers ) ) {
123+ if ( isInlineContainer ( container ) ) {
124+ outputData [ mimeType ] = container . data ;
125+ } else if ( isArtifactContainer ( container ) ) {
126+ // For artifacts, we'll need to handle them differently
127+ // For now, just mark as artifact reference
128+ outputData [ mimeType ] = `[Artifact: ${ container . artifactId } ]` ;
128129 }
129130 }
130131 } else {
131- // Direct data format
132- outputData = potentialRepresentations as OutputData ;
132+ // Direct data format (legacy support)
133+ outputData = potentialContainers as Record < string , unknown > ;
133134 }
134135 } else {
135136 // Fallback for simple data
@@ -139,16 +140,21 @@ export const RichOutput: React.FC<RichOutputProps> = ({
139140 // Determine the best media type to render, in order of preference
140141 const getPreferredMediaType = ( ) : string | null => {
141142 const preferenceOrder = [
142- "application/vnd.anode.aitool+json" ,
143- "application/vnd.anode.aitool.result+json" ,
144- "text/markdown" ,
145- "text/html" ,
146- "image/png" ,
147- "image/jpeg" ,
148- "image/svg+xml" ,
149- "image/svg" ,
150- "application/json" ,
151- "text/plain" ,
143+ AI_TOOL_CALL_MIME_TYPE ,
144+ AI_TOOL_RESULT_MIME_TYPE ,
145+ // Jupyter rich formats (plots, widgets, etc.)
146+ ...JUPYTER_MIME_TYPES ,
147+ // Text formats
148+ TEXT_MIME_TYPES [ 2 ] , // text/markdown
149+ TEXT_MIME_TYPES [ 1 ] , // text/html
150+ // Images
151+ IMAGE_MIME_TYPES [ 0 ] , // image/png
152+ IMAGE_MIME_TYPES [ 1 ] , // image/jpeg
153+ IMAGE_MIME_TYPES [ 2 ] , // image/svg+xml
154+ "image/svg" , // legacy SVG format
155+ // Application formats
156+ APPLICATION_MIME_TYPES [ 0 ] , // application/json
157+ TEXT_MIME_TYPES [ 0 ] , // text/plain
152158 ] ;
153159
154160 for ( const mediaType of preferenceOrder ) {
@@ -175,25 +181,39 @@ export const RichOutput: React.FC<RichOutputProps> = ({
175181
176182 const renderContent = ( ) => {
177183 switch ( mediaType ) {
178- case "application/vnd.anode.aitool+json" :
179- return (
180- < Suspense fallback = { < LoadingSpinner /> } >
181- < AiToolCallOutput
182- toolData = { outputData [ mediaType ] as ToolCallData }
183- />
184- </ Suspense >
185- ) ;
184+ case AI_TOOL_CALL_MIME_TYPE : {
185+ const toolData = outputData [ mediaType ] ;
186+ if ( isAiToolCallData ( toolData ) ) {
187+ return (
188+ < Suspense fallback = { < LoadingSpinner /> } >
189+ < AiToolCallOutput toolData = { toolData } />
190+ </ Suspense >
191+ ) ;
192+ }
193+ return < div className = "text-red-500" > Invalid tool call data</ div > ;
194+ }
186195
187- case "application/vnd.anode.aitool.result+json" :
188- return (
189- < Suspense fallback = { < LoadingSpinner /> } >
190- < AiToolResultOutput
191- resultData = { outputData [ mediaType ] as ToolResultData }
192- />
193- </ Suspense >
194- ) ;
196+ case AI_TOOL_RESULT_MIME_TYPE : {
197+ const resultData = outputData [ mediaType ] ;
198+ // Handle actual runtime data structure (more flexible than strict schema)
199+ if (
200+ resultData &&
201+ typeof resultData === "object" &&
202+ "tool_call_id" in resultData &&
203+ "status" in resultData &&
204+ typeof ( resultData as any ) . tool_call_id === "string" &&
205+ typeof ( resultData as any ) . status === "string"
206+ ) {
207+ return (
208+ < Suspense fallback = { < LoadingSpinner /> } >
209+ < AiToolResultOutput resultData = { resultData as any } />
210+ </ Suspense >
211+ ) ;
212+ }
213+ return < div className = "text-red-500" > Invalid tool result data</ div > ;
214+ }
195215
196- case " text/markdown" :
216+ case TEXT_MIME_TYPES [ 2 ] : // text/markdown
197217 return (
198218 < Suspense fallback = { < LoadingSpinner /> } >
199219 < MarkdownRenderer
@@ -203,15 +223,15 @@ export const RichOutput: React.FC<RichOutputProps> = ({
203223 </ Suspense >
204224 ) ;
205225
206- case " text/html" :
226+ case TEXT_MIME_TYPES [ 1 ] : // text/html
207227 return (
208228 < Suspense fallback = { < LoadingSpinner /> } >
209229 < HtmlOutput content = { String ( outputData [ mediaType ] || "" ) } />
210230 </ Suspense >
211231 ) ;
212232
213- case " image/png" :
214- case " image/jpeg" :
233+ case IMAGE_MIME_TYPES [ 0 ] : // image/png
234+ case IMAGE_MIME_TYPES [ 1 ] : // image/jpeg
215235 return (
216236 < Suspense fallback = { < LoadingSpinner /> } >
217237 < ImageOutput
@@ -221,22 +241,50 @@ export const RichOutput: React.FC<RichOutputProps> = ({
221241 </ Suspense >
222242 ) ;
223243
224- case " image/svg+xml" :
225- case "image/svg" :
244+ case IMAGE_MIME_TYPES [ 2 ] : // image/svg+xml
245+ case "image/svg" : // legacy SVG format
226246 return (
227247 < Suspense fallback = { < LoadingSpinner /> } >
228248 < SvgOutput content = { String ( outputData [ mediaType ] || "" ) } />
229249 </ Suspense >
230250 ) ;
231251
232- case "application/json" :
252+ case "application/vnd.plotly.v1+json" :
253+ return (
254+ < Suspense fallback = { < LoadingSpinner /> } >
255+ < JsonOutput data = { outputData [ mediaType ] } />
256+ </ Suspense >
257+ ) ;
258+
259+ case "application/vnd.vegalite.v2+json" :
260+ case "application/vnd.vegalite.v3+json" :
261+ case "application/vnd.vegalite.v4+json" :
262+ case "application/vnd.vegalite.v5+json" :
263+ case "application/vnd.vegalite.v6+json" :
264+ case "application/vnd.vega.v3+json" :
265+ case "application/vnd.vega.v4+json" :
266+ case "application/vnd.vega.v5+json" :
267+ return (
268+ < Suspense fallback = { < LoadingSpinner /> } >
269+ < JsonOutput data = { outputData [ mediaType ] } />
270+ </ Suspense >
271+ ) ;
272+
273+ case "application/geo+json" :
274+ return (
275+ < Suspense fallback = { < LoadingSpinner /> } >
276+ < JsonOutput data = { outputData [ mediaType ] } />
277+ </ Suspense >
278+ ) ;
279+
280+ case APPLICATION_MIME_TYPES [ 0 ] : // application/json
233281 return (
234282 < Suspense fallback = { < LoadingSpinner /> } >
235283 < JsonOutput data = { outputData [ mediaType ] } />
236284 </ Suspense >
237285 ) ;
238286
239- case " text/plain" :
287+ case TEXT_MIME_TYPES [ 0 ] : // text/plain
240288 default :
241289 return (
242290 < Suspense fallback = { < LoadingSpinner /> } >
0 commit comments