From 4b0f6ce79b6d4d6a5217368ce58a6ea3aaab7450 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Sun, 24 May 2026 22:22:33 +0200 Subject: [PATCH 1/4] feat: overhaul ai devtools --- .changeset/ai-devtools-hook-dashboard.md | 12 + docs/getting-started/devtools.md | 19 + .../ts-react-chat/src/components/Header.tsx | 14 + examples/ts-react-chat/src/routeTree.gen.ts | 21 + .../src/routes/api.structured-output.ts | 205 +- .../src/routes/generation-hooks.tsx | 804 ++++++++ .../routes/generations.structured-output.tsx | 226 +-- .../typescript/ai-client/src/chat-client.ts | 740 +++++++- packages/typescript/ai-client/src/devtools.ts | 691 +++++++ packages/typescript/ai-client/src/events.ts | 153 +- .../ai-client/src/generation-client.ts | 397 +++- .../ai-client/src/generation-types.ts | 4 + packages/typescript/ai-client/src/index.ts | 8 + packages/typescript/ai-client/src/types.ts | 21 +- .../ai-client/src/video-generation-client.ts | 441 ++++- .../ai-client/tests/devtools.test.ts | 1677 +++++++++++++++++ .../typescript/ai-client/tests/events.test.ts | 193 +- .../ai-client/tests/generation-client.test.ts | 38 +- .../tests/generation-devtools.test.ts | 923 +++++++++ .../tests/video-generation-client.test.ts | 46 +- .../src/components/ConversationDetails.tsx | 216 --- .../src/components/ConversationsList.tsx | 21 - .../ai-devtools/src/components/Shell.tsx | 91 +- .../conversation/ActivityEventsTab.tsx | 57 - .../components/conversation/ChunkBadges.tsx | 51 - .../src/components/conversation/ChunkItem.tsx | 273 --- .../conversation/ChunksCollapsible.tsx | 56 - .../src/components/conversation/ChunksTab.tsx | 87 - .../conversation/ConversationHeader.tsx | 106 -- .../conversation/ConversationTabs.tsx | 205 -- .../components/conversation/IterationCard.tsx | 259 ++- .../conversation/IterationTimeline.tsx | 54 +- .../components/conversation/MessageCard.tsx | 174 -- .../components/conversation/MessageGroup.tsx | 101 - .../components/conversation/MessagesTab.tsx | 30 - .../components/conversation/SummariesTab.tsx | 91 - .../conversation/ToolCallDisplay.tsx | 99 - .../src/components/conversation/index.ts | 9 - .../src/components/hooks/GenerationPanel.tsx | 802 ++++++++ .../src/components/hooks/HookDashboard.tsx | 223 +++ .../src/components/hooks/HookDetails.tsx | 1618 ++++++++++++++++ .../src/components/hooks/ToolFixtureForm.tsx | 324 ++++ .../components/hooks/hook-dashboard-model.ts | 102 + .../ai-devtools/src/components/hooks/index.ts | 4 + .../src/components/hooks/preview-messages.ts | 117 ++ .../src/components/hooks/preview-model.ts | 195 ++ .../src/components/list/ConversationRow.tsx | 85 - .../ai-devtools/src/components/list/index.ts | 1 - .../src/components/utils/format.ts | 58 - .../ai-devtools/src/store/ai-context.tsx | 1267 +++++++++++-- .../ai-devtools/src/store/hook-registry.ts | 668 +++++++ .../src/store/message-event-utils.ts | 60 + .../ai-devtools/src/styles/use-styles.ts | 1655 +++++++++++++++- .../tests/hook-dashboard-model.test.ts | 111 ++ .../ai-devtools/tests/hook-registry.test.ts | 475 +++++ .../tests/message-event-utils.test.ts | 115 ++ .../tests/preview-messages.test.ts | 218 +++ .../ai-devtools/tests/preview-model.test.ts | 146 ++ .../src/devtools-middleware.ts | 4 + .../ai-event-client/src/envelope.ts | 135 ++ .../typescript/ai-event-client/src/index.ts | 242 ++- .../ai-event-client/tests/emit.test.ts | 133 ++ .../ai-event-client/tests/envelope.test.ts | 126 ++ packages/typescript/ai-preact/src/types.ts | 9 +- packages/typescript/ai-preact/src/use-chat.ts | 9 + packages/typescript/ai-react/src/use-chat.ts | 9 + .../ai-react/src/use-generate-audio.ts | 9 +- .../ai-react/src/use-generate-image.ts | 9 +- .../ai-react/src/use-generate-speech.ts | 9 +- .../ai-react/src/use-generate-video.ts | 9 +- .../typescript/ai-react/src/use-generation.ts | 12 +- .../typescript/ai-react/src/use-summarize.ts | 11 +- .../ai-react/src/use-transcription.ts | 11 +- packages/typescript/ai-solid/src/use-chat.ts | 12 + .../ai-solid/src/use-generate-audio.ts | 9 +- .../ai-solid/src/use-generate-image.ts | 9 +- .../ai-solid/src/use-generate-speech.ts | 9 +- .../ai-solid/src/use-generate-video.ts | 19 +- .../typescript/ai-solid/src/use-generation.ts | 22 +- .../typescript/ai-solid/src/use-summarize.ts | 11 +- .../ai-solid/src/use-transcription.ts | 11 +- .../ai-svelte/src/create-chat.svelte.ts | 13 + .../src/create-generate-audio.svelte.ts | 9 +- .../src/create-generate-image.svelte.ts | 9 +- .../src/create-generate-speech.svelte.ts | 11 +- .../src/create-generate-video.svelte.ts | 18 +- .../ai-svelte/src/create-generation.svelte.ts | 21 +- .../ai-svelte/src/create-summarize.svelte.ts | 9 +- .../src/create-transcription.svelte.ts | 9 +- packages/typescript/ai-svelte/src/types.ts | 5 + packages/typescript/ai-vue/src/use-chat.ts | 12 + .../ai-vue/src/use-generate-audio.ts | 9 +- .../ai-vue/src/use-generate-image.ts | 9 +- .../ai-vue/src/use-generate-speech.ts | 9 +- .../ai-vue/src/use-generate-video.ts | 22 +- .../typescript/ai-vue/src/use-generation.ts | 25 +- .../typescript/ai-vue/src/use-summarize.ts | 11 +- .../ai-vue/src/use-transcription.ts | 11 +- .../ai/src/activities/chat/index.ts | 1 + .../src/activities/chat/middleware/types.ts | 2 + .../src/activities/chat/stream/processor.ts | 92 + 101 files changed, 15592 insertions(+), 2411 deletions(-) create mode 100644 .changeset/ai-devtools-hook-dashboard.md create mode 100644 examples/ts-react-chat/src/routes/generation-hooks.tsx create mode 100644 packages/typescript/ai-client/src/devtools.ts create mode 100644 packages/typescript/ai-client/tests/devtools.test.ts create mode 100644 packages/typescript/ai-client/tests/generation-devtools.test.ts delete mode 100644 packages/typescript/ai-devtools/src/components/ConversationDetails.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/ConversationsList.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ActivityEventsTab.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ChunkBadges.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ChunkItem.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ChunksCollapsible.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ChunksTab.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ConversationHeader.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ConversationTabs.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/MessageCard.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/MessageGroup.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/MessagesTab.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/SummariesTab.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/conversation/ToolCallDisplay.tsx create mode 100644 packages/typescript/ai-devtools/src/components/hooks/GenerationPanel.tsx create mode 100644 packages/typescript/ai-devtools/src/components/hooks/HookDashboard.tsx create mode 100644 packages/typescript/ai-devtools/src/components/hooks/HookDetails.tsx create mode 100644 packages/typescript/ai-devtools/src/components/hooks/ToolFixtureForm.tsx create mode 100644 packages/typescript/ai-devtools/src/components/hooks/hook-dashboard-model.ts create mode 100644 packages/typescript/ai-devtools/src/components/hooks/index.ts create mode 100644 packages/typescript/ai-devtools/src/components/hooks/preview-messages.ts create mode 100644 packages/typescript/ai-devtools/src/components/hooks/preview-model.ts delete mode 100644 packages/typescript/ai-devtools/src/components/list/ConversationRow.tsx delete mode 100644 packages/typescript/ai-devtools/src/components/list/index.ts create mode 100644 packages/typescript/ai-devtools/src/store/hook-registry.ts create mode 100644 packages/typescript/ai-devtools/src/store/message-event-utils.ts create mode 100644 packages/typescript/ai-devtools/tests/hook-dashboard-model.test.ts create mode 100644 packages/typescript/ai-devtools/tests/hook-registry.test.ts create mode 100644 packages/typescript/ai-devtools/tests/message-event-utils.test.ts create mode 100644 packages/typescript/ai-devtools/tests/preview-messages.test.ts create mode 100644 packages/typescript/ai-devtools/tests/preview-model.test.ts create mode 100644 packages/typescript/ai-event-client/src/envelope.ts create mode 100644 packages/typescript/ai-event-client/tests/emit.test.ts create mode 100644 packages/typescript/ai-event-client/tests/envelope.test.ts diff --git a/.changeset/ai-devtools-hook-dashboard.md b/.changeset/ai-devtools-hook-dashboard.md new file mode 100644 index 000000000..6c2623365 --- /dev/null +++ b/.changeset/ai-devtools-hook-dashboard.md @@ -0,0 +1,12 @@ +--- +'@tanstack/ai-client': minor +'@tanstack/ai-devtools-core': minor +'@tanstack/ai-event-client': minor +'@tanstack/ai-preact': patch +'@tanstack/ai-react': patch +'@tanstack/ai-solid': patch +'@tanstack/ai-svelte': patch +'@tanstack/ai-vue': patch +--- + +Add hook-aware AI devtools registration, run tracking, state snapshots, and tool fixture replay. diff --git a/docs/getting-started/devtools.md b/docs/getting-started/devtools.md index 5178dbef0..d1f9d857d 100644 --- a/docs/getting-started/devtools.md +++ b/docs/getting-started/devtools.md @@ -16,11 +16,30 @@ keywords: TanStack Devtools is a unified devtools panel for inspecting and debugging TanStack libraries, including TanStack AI. It provides real-time insights into AI interactions, tool calls, and state changes, making it easier to develop and troubleshoot AI-powered applications. ## Features +- **Hook dashboard** - Discover every active TanStack AI hook on the page, including chat, structured output, image, video, audio, speech, transcription, and summarize hooks. +- **Run timeline** - Inspect user turns, linked runs, stream events, client snapshots, and server-only events by `threadId` and `runId`. - **Real-time Monitoring** - View live chat messages, tool invocations, and AI responses. - **Tool Call Inspection** - Inspect input and output of tool calls. +- **Tool Fixture Replay** - Build tool payloads from a tool's standard-schema input, append the result into chat messages, and save fixtures in localStorage for repeated UI iteration. - **State Visualization** - Visualize chat state and message history. - **Error Tracking** - Monitor errors and exceptions in AI interactions. +## Hook Dashboard + +The AI devtools panel listens for active TanStack AI clients and shows them in the left sidebar. Hooks register when they are created, emit a snapshot immediately, and respond again whenever the devtools panel opens or requests state. This keeps hooks discoverable even when the panel is opened after the app has already rendered. + +Each hook entry includes its type, lifecycle, message count, run count, and the latest linked `threadId`. Selecting a hook opens the full timeline for that hook. Chat hooks keep the current turn-based view: a user message wraps every run and event that happened while answering that turn. The details view also includes lightweight client/server state snapshots between runs so you can see exactly what changed. + +## Tool Fixtures + +When a `useChat` hook receives tools, the devtools panel lists those tools and their schemas. For standard-schema-compatible inputs, the panel renders a small form from the input schema so you can create a tool call payload without hand-writing JSON. + +Applying a tool fixture appends the tool call and result into the real chat messages for that hook. Saved fixtures are stored in browser localStorage under the AI devtools namespace so they are available the next time you open the panel. + +## Event Sources + +Client-visible state is emitted by the headless client. Server-only details, such as middleware and provider stream events that never exist on the client, are emitted from the server counterpart. Events include a source descriptor and stable envelope id so the panel can link related events and avoid displaying duplicates. + ## Installation To use TanStack Devtools with TanStack AI, install the `@tanstack/react-ai-devtools` package: diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index d0b02b740..5d2e5e4e0 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -2,6 +2,7 @@ import { Link } from '@tanstack/react-router' import { useState } from 'react' import { + Activity, Braces, FileAudio, FileText, @@ -75,6 +76,19 @@ export default function Header() { Generations

+ setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-1" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-1', + }} + > + + Generation Hooks + + setIsOpen(false)} diff --git a/examples/ts-react-chat/src/routeTree.gen.ts b/examples/ts-react-chat/src/routeTree.gen.ts index d0966d88a..4551a8375 100644 --- a/examples/ts-react-chat/src/routeTree.gen.ts +++ b/examples/ts-react-chat/src/routeTree.gen.ts @@ -12,6 +12,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as RealtimeRouteImport } from './routes/realtime' import { Route as Issue176ToolResultRouteImport } from './routes/issue-176-tool-result' import { Route as ImageGenRouteImport } from './routes/image-gen' +import { Route as GenerationHooksRouteImport } from './routes/generation-hooks' import { Route as IndexRouteImport } from './routes/index' import { Route as GenerationsVideoRouteImport } from './routes/generations.video' import { Route as GenerationsTranscriptionRouteImport } from './routes/generations.transcription' @@ -49,6 +50,11 @@ const ImageGenRoute = ImageGenRouteImport.update({ path: '/image-gen', getParentRoute: () => rootRouteImport, } as any) +const GenerationHooksRoute = GenerationHooksRouteImport.update({ + id: '/generation-hooks', + path: '/generation-hooks', + getParentRoute: () => rootRouteImport, +} as any) const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', @@ -160,6 +166,7 @@ const ApiGenerateAudioRoute = ApiGenerateAudioRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute + '/generation-hooks': typeof GenerationHooksRoute '/image-gen': typeof ImageGenRoute '/issue-176-tool-result': typeof Issue176ToolResultRoute '/realtime': typeof RealtimeRoute @@ -186,6 +193,7 @@ export interface FileRoutesByFullPath { } export interface FileRoutesByTo { '/': typeof IndexRoute + '/generation-hooks': typeof GenerationHooksRoute '/image-gen': typeof ImageGenRoute '/issue-176-tool-result': typeof Issue176ToolResultRoute '/realtime': typeof RealtimeRoute @@ -213,6 +221,7 @@ export interface FileRoutesByTo { export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute + '/generation-hooks': typeof GenerationHooksRoute '/image-gen': typeof ImageGenRoute '/issue-176-tool-result': typeof Issue176ToolResultRoute '/realtime': typeof RealtimeRoute @@ -241,6 +250,7 @@ export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '/' + | '/generation-hooks' | '/image-gen' | '/issue-176-tool-result' | '/realtime' @@ -267,6 +277,7 @@ export interface FileRouteTypes { fileRoutesByTo: FileRoutesByTo to: | '/' + | '/generation-hooks' | '/image-gen' | '/issue-176-tool-result' | '/realtime' @@ -293,6 +304,7 @@ export interface FileRouteTypes { id: | '__root__' | '/' + | '/generation-hooks' | '/image-gen' | '/issue-176-tool-result' | '/realtime' @@ -320,6 +332,7 @@ export interface FileRouteTypes { } export interface RootRouteChildren { IndexRoute: typeof IndexRoute + GenerationHooksRoute: typeof GenerationHooksRoute ImageGenRoute: typeof ImageGenRoute Issue176ToolResultRoute: typeof Issue176ToolResultRoute RealtimeRoute: typeof RealtimeRoute @@ -368,6 +381,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ImageGenRouteImport parentRoute: typeof rootRouteImport } + '/generation-hooks': { + id: '/generation-hooks' + path: '/generation-hooks' + fullPath: '/generation-hooks' + preLoaderRoute: typeof GenerationHooksRouteImport + parentRoute: typeof rootRouteImport + } '/': { id: '/' path: '/' @@ -520,6 +540,7 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, + GenerationHooksRoute: GenerationHooksRoute, ImageGenRoute: ImageGenRoute, Issue176ToolResultRoute: Issue176ToolResultRoute, RealtimeRoute: RealtimeRoute, diff --git a/examples/ts-react-chat/src/routes/api.structured-output.ts b/examples/ts-react-chat/src/routes/api.structured-output.ts index cba72ffdb..7c5928fe6 100644 --- a/examples/ts-react-chat/src/routes/api.structured-output.ts +++ b/examples/ts-react-chat/src/routes/api.structured-output.ts @@ -1,5 +1,9 @@ import { createFileRoute } from '@tanstack/react-router' -import { chat, toServerSentEventsResponse } from '@tanstack/ai' +import { + chat, + chatParamsFromRequestBody, + toServerSentEventsResponse, +} from '@tanstack/ai' import { openaiChatCompletions, openaiText } from '@tanstack/ai-openai' import { grokText } from '@tanstack/ai-grok' import { groqText } from '@tanstack/ai-groq' @@ -45,19 +49,35 @@ async function* withTrailingPhaseCounts( snapshot: () => Record, model: string, ): AsyncIterable { + let yieldedCounts = false for await (const chunk of stream) { + if ( + chunk.type === EventType.RUN_FINISHED || + chunk.type === EventType.RUN_ERROR + ) { + yieldedCounts = true + yield { + type: EventType.CUSTOM, + name: 'phase-counts', + value: snapshot(), + model, + timestamp: Date.now(), + } + } yield chunk } - yield { - type: EventType.CUSTOM, - name: 'phase-counts', - value: snapshot(), - model, - timestamp: Date.now(), + if (!yieldedCounts) { + yield { + type: EventType.CUSTOM, + name: 'phase-counts', + value: snapshot(), + model, + timestamp: Date.now(), + } } } -const GuitarRecommendationSchema = z.object({ +export const GuitarRecommendationSchema = z.object({ title: z.string().describe('Short headline for the recommendation'), summary: z.string().describe('One paragraph summary'), recommendations: z @@ -75,29 +95,22 @@ const GuitarRecommendationSchema = z.object({ nextSteps: z.array(z.string()).describe('Practical follow-up actions'), }) -type Provider = - | 'openai' - | 'openai-chat' - | 'grok' - | 'groq' - | 'openrouter' - | 'openrouter-responses' +export type GuitarRecommendation = z.infer -const StructuredOutputRequestSchema = z.object({ - prompt: z.string().min(1), - provider: z - .enum([ - 'openai', - 'openai-chat', - 'grok', - 'groq', - 'openrouter', - 'openrouter-responses', - ]) - .optional(), - model: z.string().optional(), - stream: z.boolean().optional(), -}) +const PROVIDERS = [ + 'openai', + 'openai-chat', + 'grok', + 'groq', + 'openrouter', + 'openrouter-responses', +] as const + +type Provider = (typeof PROVIDERS)[number] + +function isProvider(value: unknown): value is Provider { + return typeof value === 'string' && PROVIDERS.includes(value as Provider) +} function adapterFor(provider: Provider, model?: string): AnyTextAdapter { switch (provider) { @@ -181,42 +194,104 @@ function reasoningOptionsFor( } } +async function* structuredOutputResultStream(args: { + result: GuitarRecommendation + phaseCounts: Record + threadId: string + runId: string + model: string +}): AsyncIterable { + const messageId = `structured-output-${args.runId}` + const raw = JSON.stringify(args.result) + const timestamp = Date.now() + + yield { + type: EventType.RUN_STARTED, + threadId: args.threadId, + runId: args.runId, + model: args.model, + timestamp, + } + yield { + type: EventType.CUSTOM, + name: 'structured-output.start', + value: { messageId }, + model: args.model, + timestamp: Date.now(), + } + yield { + type: EventType.CUSTOM, + name: 'structured-output.complete', + value: { object: args.result, raw }, + model: args.model, + timestamp: Date.now(), + } + yield { + type: EventType.CUSTOM, + name: 'phase-counts', + value: args.phaseCounts, + model: args.model, + timestamp: Date.now(), + } + yield { + type: EventType.RUN_FINISHED, + threadId: args.threadId, + runId: args.runId, + model: args.model, + timestamp: Date.now(), + finishReason: 'stop', + } +} + export const Route = createFileRoute('/api/structured-output')({ server: { handlers: { POST: async ({ request }) => { + if (request.signal.aborted) { + return new Response(null, { status: 499 }) + } + + const abortController = new AbortController() + const onAbort = () => abortController.abort() + request.signal.addEventListener('abort', onAbort, { once: true }) + if (request.signal.aborted) { + onAbort() + } + + let params: Awaited> try { - const parsed = StructuredOutputRequestSchema.safeParse( - await request.json(), + params = await chatParamsFromRequestBody(await request.json()) + } catch (error) { + return new Response( + error instanceof Error ? error.message : 'Bad request', + { status: 400 }, ) - if (!parsed.success) { - return new Response( - JSON.stringify({ error: 'Invalid request body' }), - { - status: 400, - headers: { 'Content-Type': 'application/json' }, - }, - ) - } - const { prompt, provider, model, stream } = parsed.data - const resolvedProvider: Provider = provider || 'openrouter' - const modelOptions = reasoningOptionsFor(resolvedProvider, model) + } + try { + const providerValue = params.forwardedProps.provider + const resolvedProvider: Provider = isProvider(providerValue) + ? providerValue + : 'openrouter' + const model = + typeof params.forwardedProps.model === 'string' + ? params.forwardedProps.model + : undefined + const stream = params.forwardedProps.stream !== false + const adapter = adapterFor(resolvedProvider, model) + const modelOptions = reasoningOptionsFor(resolvedProvider, model) const counter = phaseCounterMiddleware() if (stream) { - const abortController = new AbortController() - request.signal.addEventListener('abort', () => - abortController.abort(), - ) - const adapter = adapterFor(resolvedProvider, model) const streamIterable = chat({ adapter, modelOptions: modelOptions as never, - messages: [{ role: 'user', content: prompt }], + messages: params.messages, outputSchema: GuitarRecommendationSchema, stream: true, middleware: [counter.middleware], + threadId: params.threadId, + runId: params.runId, abortController, }) as AsyncIterable const withCounts = withTrailingPhaseCounts( @@ -229,28 +304,26 @@ export const Route = createFileRoute('/api/structured-output')({ }) } - const abortController = new AbortController() - request.signal.addEventListener('abort', () => - abortController.abort(), - ) const result = await chat({ - adapter: adapterFor(resolvedProvider, model), + adapter, modelOptions: modelOptions as never, - messages: [{ role: 'user', content: prompt }], + messages: params.messages, outputSchema: GuitarRecommendationSchema, middleware: [counter.middleware], + threadId: params.threadId, + runId: params.runId, abortController, }) - return new Response( - JSON.stringify({ - data: result, - _diagnostics: { phaseCounts: counter.snapshot() }, - }), - { - headers: { 'Content-Type': 'application/json' }, - }, - ) + const responseStream = structuredOutputResultStream({ + result, + phaseCounts: counter.snapshot(), + threadId: params.threadId, + runId: params.runId, + model: adapter.model, + }) + + return toServerSentEventsResponse(responseStream, { abortController }) } catch (error: unknown) { const message = error instanceof Error ? error.message : 'An error occurred' @@ -259,6 +332,8 @@ export const Route = createFileRoute('/api/structured-output')({ status: 500, headers: { 'Content-Type': 'application/json' }, }) + } finally { + request.signal.removeEventListener('abort', onAbort) } }, }, diff --git a/examples/ts-react-chat/src/routes/generation-hooks.tsx b/examples/ts-react-chat/src/routes/generation-hooks.tsx new file mode 100644 index 000000000..a4913116b --- /dev/null +++ b/examples/ts-react-chat/src/routes/generation-hooks.tsx @@ -0,0 +1,804 @@ +import { useState } from 'react' +import type { ReactNode } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { + FileAudio, + FileText, + Image, + Mic, + Music, + Play, + RotateCcw, + Video, +} from 'lucide-react' +import { + useGenerateAudio, + useGenerateImage, + useGenerateSpeech, + useGenerateVideo, + useSummarize, + useTranscription, +} from '@tanstack/ai-react' +import { EventType } from '@tanstack/ai' +import { + GENERATION_EVENTS, + type ConnectConnectionAdapter, + type VideoGenerateResult, +} from '@tanstack/ai-client' +import type { + AudioGenerationResult, + ImageGenerationResult, + StreamChunk, + SummarizationResult, + TranscriptionResult, + TTSResult, +} from '@tanstack/ai' +import type { LucideIcon } from 'lucide-react' + +const SAMPLE_WAV_BASE64 = createToneWavBase64() +const SAMPLE_AUDIO_DATA_URL = `data:audio/wav;base64,${SAMPLE_WAV_BASE64}` +const SAMPLE_VIDEO_URL = + 'https://interactive-examples.mdn.mozilla.net/media/cc0-videos/flower.mp4' +const SAMPLE_TRANSCRIPTION_AUDIO = SAMPLE_AUDIO_DATA_URL + +const imageConnection = createGenerationConnection( + 'image', + (data) => { + const prompt = stringField(data, 'prompt', 'Generated devtools test image') + const count = Math.max( + 1, + Math.min(4, numberField(data, 'numberOfImages', 1)), + ) + return { + id: `local-image-${Date.now()}`, + images: Array.from({ length: count }, (_, index) => ({ + url: svgDataUrl( + `Image ${index + 1}`, + prompt, + index % 2 === 0 ? '#0ea5e9' : '#f97316', + ), + revisedPrompt: `${prompt} (${index + 1})`, + })), + model: 'local-devtools-image-fixture', + } + }, +) + +const audioConnection = createGenerationConnection( + 'audio', + (data) => ({ + id: `local-audio-${Date.now()}`, + audio: { + url: SAMPLE_AUDIO_DATA_URL, + contentType: 'audio/wav', + duration: numberField(data, 'duration', 3), + }, + model: 'local-devtools-audio-fixture', + }), +) + +const speechConnection = createGenerationConnection( + 'speech', + () => ({ + id: `local-speech-${Date.now()}`, + model: 'local-devtools-speech-fixture', + audio: SAMPLE_WAV_BASE64, + contentType: 'audio/wav', + format: 'wav', + duration: 0.8, + }), +) + +const transcriptionConnection = createGenerationConnection( + 'transcription', + (data) => ({ + id: `local-transcription-${Date.now()}`, + model: 'local-devtools-transcription-fixture', + text: `Transcribed local fixture in ${stringField(data, 'language', 'en')}.`, + language: stringField(data, 'language', 'en'), + duration: 3, + segments: [ + { + id: 0, + start: 0, + end: 1.5, + text: 'Transcribed local fixture', + }, + { + id: 1, + start: 1.5, + end: 3, + text: 'ready for devtools inspection', + }, + ], + }), +) + +const summarizeConnection = createGenerationConnection( + 'summarize', + (data) => { + const text = stringField(data, 'text', SAMPLE_SUMMARY_TEXT) + const style = stringField(data, 'style', 'concise') + return { + id: `local-summary-${Date.now()}`, + summary: `${style}: ${text.split(/\s+/).slice(0, 24).join(' ')}.`, + model: 'local-devtools-summary-fixture', + usage: { + promptTokens: text.split(/\s+/).length, + completionTokens: 24, + totalTokens: text.split(/\s+/).length + 24, + }, + } + }, +) + +const videoConnection = createVideoConnection() + +const SAMPLE_SUMMARY_TEXT = + 'Generation hooks emit core devtools snapshots with input, progress, result, and renderable previews for media and text outputs.' + +export const Route = createFileRoute('/generation-hooks')({ + component: GenerationHooksPage, +}) + +function GenerationHooksPage() { + const [prompt, setPrompt] = useState( + 'A compact diagnostics console showing every TanStack AI generation hook', + ) + const [speechText, setSpeechText] = useState( + 'This local fixture exercises the speech generation hook.', + ) + const [summaryText, setSummaryText] = useState(SAMPLE_SUMMARY_TEXT) + const [imageCount, setImageCount] = useState(2) + const [audioDuration, setAudioDuration] = useState(3) + + const image = useGenerateImage({ + id: 'generation-hooks:useGenerateImage', + connection: imageConnection, + }) + + const audio = useGenerateAudio({ + id: 'generation-hooks:useGenerateAudio', + connection: audioConnection, + }) + + const speech = useGenerateSpeech({ + id: 'generation-hooks:useGenerateSpeech', + connection: speechConnection, + }) + + const transcription = useTranscription({ + id: 'generation-hooks:useTranscription', + connection: transcriptionConnection, + }) + + const summarize = useSummarize({ + id: 'generation-hooks:useSummarize', + connection: summarizeConnection, + }) + + const video = useGenerateVideo({ + id: 'generation-hooks:useGenerateVideo', + connection: videoConnection, + }) + + const loadingCount = [ + image.isLoading, + audio.isLoading, + speech.isLoading, + transcription.isLoading, + summarize.isLoading, + video.isLoading, + ].filter(Boolean).length + + const runImage = () => image.generate({ prompt, numberOfImages: imageCount }) + const runAudio = () => audio.generate({ prompt, duration: audioDuration }) + const runSpeech = () => speech.generate({ text: speechText, voice: 'local' }) + const runTranscription = () => + transcription.generate({ + audio: SAMPLE_TRANSCRIPTION_AUDIO, + language: 'en', + }) + const runSummarize = () => + summarize.generate({ + text: summaryText, + style: 'bullet-points', + }) + const runVideo = () => video.generate({ prompt }) + + const runAll = async () => { + await Promise.all([ + runImage(), + runAudio(), + runSpeech(), + runTranscription(), + runSummarize(), + runVideo(), + ]) + } + + const resetAll = () => { + image.reset() + audio.reset() + speech.reset() + transcription.reset() + summarize.reset() + video.reset() + } + + const stopAll = () => { + image.stop() + audio.stop() + speech.stop() + transcription.stop() + summarize.stop() + video.stop() + } + + return ( +
+
+
+
+

+ Devtools fixture route +

+

+ Generation Hooks +

+

+ Six mounted hooks, stable IDs, local streaming fixtures, and + media-shaped results for the devtools panel. +

+
+
+ + + +
+
+ +
+