From fce42d045dffc714dc06fc864a90045cf5171e2c Mon Sep 17 00:00:00 2001 From: Christophe Blefari Date: Thu, 23 Jul 2026 10:14:37 +0200 Subject: [PATCH 1/4] First version of filters --- .env.example | 2 + .github/workflows/docker.yml | 1 + apps/backend/src/agents/tools/execute-sql.ts | 69 +++- apps/backend/src/agents/tools/story.ts | 21 +- apps/backend/src/env.ts | 6 + .../src/queries/execute-sql.queries.ts | 90 +++++ apps/backend/src/services/live-story.ts | 14 +- apps/backend/src/services/ping.ts | 2 + apps/backend/src/services/story-filters.ts | 107 ++++++ apps/backend/src/services/update-sql.ts | 103 ++++++ apps/backend/src/trpc/router.ts | 2 + apps/backend/src/trpc/shared-story.routes.ts | 35 ++ apps/backend/src/trpc/sql.routes.ts | 32 ++ apps/backend/src/trpc/story.routes.ts | 28 ++ apps/backend/src/trpc/system.routes.ts | 1 + apps/backend/src/utils/sql-identifiers.ts | 14 + apps/backend/src/utils/story-html.tsx | 2 + apps/backend/tests/license.test.ts | 2 + apps/backend/tests/story-filters.test.ts | 51 +++ .../src/components/fix-in-chat-button.tsx | 26 ++ .../src/components/mcp-app/story-app-view.tsx | 52 ++- .../src/components/side-panel/sql-editor.tsx | 215 +++++++++++- .../components/side-panel/story-code-view.tsx | 40 ++- .../components/side-panel/story-preview.tsx | 52 ++- .../components/side-panel/story-viewer.tsx | 2 + apps/frontend/src/components/story-embeds.tsx | 112 +++++-- .../src/components/story-filter-bar.tsx | 308 ++++++++++++++++++ .../src/components/story-rendering.tsx | 2 + .../src/components/story-tabbed-content.tsx | 56 +++- .../components/tool-calls/display-chart.tsx | 4 + .../src/components/tool-calls/execute-sql.tsx | 28 +- apps/frontend/src/hooks/use-story-filters.ts | 206 ++++++++++++ apps/frontend/src/lib/charts.utils.test.ts | 19 +- apps/frontend/src/lib/charts.utils.ts | 45 +++ apps/frontend/src/lib/execute-sql-messages.ts | 47 +++ ...out.stories.preview.$chatId.$storySlug.tsx | 59 +++- ...sidebar-layout.stories.shared.$shareId.tsx | 57 +++- ...bar-layout.stories.standalone.$storyId.tsx | 81 ++++- apps/shared/package.json | 3 +- apps/shared/src/chart-block.ts | 20 ++ apps/shared/src/chart-builder.tsx | 19 +- apps/shared/src/index.ts | 1 + apps/shared/src/sql-template.ts | 121 +++++++ apps/shared/src/story-segments.ts | 76 ++++- apps/shared/src/story-validation.ts | 97 +++++- apps/shared/src/tools/execute-sql.ts | 3 + apps/shared/tests/sql-template.test.ts | 88 +++++ apps/shared/tests/story-segments.test.ts | 73 ++++- apps/shared/tests/story-validation.test.ts | 33 ++ 49 files changed, 2400 insertions(+), 127 deletions(-) create mode 100644 apps/backend/src/queries/execute-sql.queries.ts create mode 100644 apps/backend/src/services/story-filters.ts create mode 100644 apps/backend/src/services/update-sql.ts create mode 100644 apps/backend/src/trpc/sql.routes.ts create mode 100644 apps/backend/src/utils/sql-identifiers.ts create mode 100644 apps/backend/tests/story-filters.test.ts create mode 100644 apps/frontend/src/components/fix-in-chat-button.tsx create mode 100644 apps/frontend/src/components/story-filter-bar.tsx create mode 100644 apps/frontend/src/hooks/use-story-filters.ts create mode 100644 apps/frontend/src/lib/execute-sql-messages.ts create mode 100644 apps/shared/src/sql-template.ts create mode 100644 apps/shared/tests/sql-template.test.ts diff --git a/.env.example b/.env.example index 28bb55f02..9e2e35d76 100644 --- a/.env.example +++ b/.env.example @@ -167,3 +167,5 @@ POSTHOG_DISABLED=false # (used to opt-out of tracking) # BETA_AUTOMATIONS_ENABLED=false # Set to true to enable the Context Recommendations feature. # BETA_CONTEXT_RECOMMENDATIONS_ENABLED=false +# Set to true to enable interactive story filters. +# BETA_STORY_FILTERS_ENABLED=false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c92f28920..fb6f0f411 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -218,6 +218,7 @@ jobs: -e "ENABLE_USER_SIGNUP=false" \ -e "BETA_AUTOMATIONS_ENABLED=true" \ -e "BETA_CONTEXT_RECOMMENDATIONS_ENABLED=true" \ + -e "BETA_STORY_FILTERS_ENABLED=true" \ -e "SMTP_PASSWORD=${{ secrets.SMTP_PASSWORD }}" \ -e "SMTP_HOST=${{ secrets.SMTP_HOST }}" \ -e "SMTP_PORT=${{ secrets.SMTP_PORT }}" \ diff --git a/apps/backend/src/agents/tools/execute-sql.ts b/apps/backend/src/agents/tools/execute-sql.ts index 04c658d72..ef97730f7 100644 --- a/apps/backend/src/agents/tools/execute-sql.ts +++ b/apps/backend/src/agents/tools/execute-sql.ts @@ -1,19 +1,22 @@ +import { stripSqlFilterBlocks } from '@nao/shared/sql-template'; import type { executeSql } from '@nao/shared/tools'; import { executeSql as schemas } from '@nao/shared/tools'; import { ExecuteSqlOutput, renderToModelOutput } from '../../components/tool-outputs'; import { env } from '../../env'; +import { getExecuteSqlPartByQueryIdInChat, updateExecuteSqlPart } from '../../queries/execute-sql.queries'; import { ToolContext } from '../../types/tools'; import { detectQueryRowLimit, isReadOnlySqlQuery } from '../../utils/sql-filter'; import { createTool } from '../../utils/tools'; import { queryAppDb } from './query-app-db'; export async function executeQuery( - { sql_query, database_id }: executeSql.Input, + { sql_query, database_id, query_id }: executeSql.Input, context: ToolContext, ): Promise { + const effectiveSql = stripSqlFilterBlocks(sql_query); const writePermEnabled = context.agentSettings?.sql?.dangerouslyWritePermEnabled ?? false; - if (!writePermEnabled && !(await isReadOnlySqlQuery(sql_query))) { + if (!writePermEnabled && !(await isReadOnlySqlQuery(effectiveSql))) { throw new Error( 'Write SQL operations are disabled. Only SELECT queries are allowed. ' + 'Enable "Dangerous write permissions" in the admin panel to allow INSERT, UPDATE, DELETE and DDL queries.', @@ -21,7 +24,7 @@ export async function executeQuery( } if (context.adminMode) { - return executeAppDbQuery(sql_query, context); + return executeAppDbQuery(effectiveSql, context, query_id); } const naoProjectFolder = context.projectFolder; @@ -32,7 +35,7 @@ export async function executeQuery( 'Content-Type': 'application/json', }, body: JSON.stringify({ - sql: sql_query, + sql: effectiveSql, nao_project_folder: naoProjectFolder, ...(database_id && { database_id }), ...(Object.keys(envVars).length > 0 && { env_vars: envVars }), @@ -46,11 +49,11 @@ export async function executeQuery( } const data = await response.json(); - const id = `query_${crypto.randomUUID().slice(0, 8)}` as const; + const id = query_id ?? (`query_${crypto.randomUUID().slice(0, 8)}` as const); context.queryResults.set(id, { columns: data.columns, data: data.data }); - const appliedLimit = detectQueryRowLimit(sql_query); + const appliedLimit = detectQueryRowLimit(effectiveSql); return { _version: '1', @@ -60,9 +63,13 @@ export async function executeQuery( }; } -async function executeAppDbQuery(sqlQuery: string, context: ToolContext): Promise { +async function executeAppDbQuery( + sqlQuery: string, + context: ToolContext, + queryId?: `query_${string}`, +): Promise { const { columns, rows } = await queryAppDb(context.projectId, sqlQuery); - const id = `query_${crypto.randomUUID().slice(0, 8)}` as const; + const id = queryId ?? (`query_${crypto.randomUUID().slice(0, 8)}` as const); context.queryResults.set(id, { columns, data: rows }); const appliedLimit = detectQueryRowLimit(sqlQuery); return { @@ -75,11 +82,51 @@ async function executeAppDbQuery(sqlQuery: string, context: ToolContext): Promis }; } -export default createTool({ - description: +async function updateExistingQuery( + input: executeSql.Input & { query_id: `query_${string}` }, + context: ToolContext, +): Promise { + const existing = await getExecuteSqlPartByQueryIdInChat(context.chatId, input.query_id); + if (!existing) { + throw new Error( + `Query ${input.query_id} not found in this chat. Use execute_sql without query_id to create a new query.`, + ); + } + + const nextInput: executeSql.Input = { + sql_query: input.sql_query, + database_id: input.database_id ?? existing.toolInput.database_id, + name: input.name ?? existing.toolInput.name, + }; + + const output = await executeQuery({ ...nextInput, query_id: input.query_id }, context); + await updateExecuteSqlPart(existing.toolCallId, nextInput, output); + return output; +} + +function buildExecuteSqlToolDescription() { + return [ 'Execute a SQL query against the connected database and return the results. If multiple databases are configured, specify the database_id.', + ...(env.BETA_STORY_FILTERS_ENABLED + ? [ + 'Story filters may be embedded as SQL template blocks that are stripped when this tool runs in chat:', + 'WHERE 1 = 1 {% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %}.', + 'Prefer query_id when adding story filter templates so existing / tags keep working.', + ] + : []), + 'To edit a previous query in-place (keep the same query_id for charts/stories), pass query_id from an earlier execute_sql result.', + ].join(' '); +} + +export default createTool({ + description: buildExecuteSqlToolDescription(), inputSchema: schemas.InputSchema, outputSchema: schemas.OutputSchema, - execute: executeQuery, + execute: async (input, context) => { + if (input.query_id) { + return updateExistingQuery({ ...input, query_id: input.query_id }, context); + } + return executeQuery(input, context); + }, toModelOutput: ({ output }) => renderToModelOutput(ExecuteSqlOutput({ output }), output), }); diff --git a/apps/backend/src/agents/tools/story.ts b/apps/backend/src/agents/tools/story.ts index 04b906ea2..83be66777 100644 --- a/apps/backend/src/agents/tools/story.ts +++ b/apps/backend/src/agents/tools/story.ts @@ -3,26 +3,41 @@ import { story } from '@nao/shared/tools'; import { renderToModelOutput, StoryOutput } from '../../components/tool-outputs'; import { db } from '../../db/db'; +import { env } from '../../env'; import { getDisplayChartTableFormatsForChat } from '../../queries/chart-image'; import * as storyQueries from '../../queries/story.queries'; import * as storyFolderQueries from '../../queries/story-folder.queries'; import type { ToolContext } from '../../types/tools'; import { createTool } from '../../utils/tools'; -export default createTool({ - description: [ +const STORY_FILTER_DESCRIPTION = [ + 'Story-level filters are declared via .', + 'For select/multi_select, provide either table+column (options from SELECT DISTINCT) or options=\'["a","b"]\' (hardcoded values).', + 'When using table+column and multiple databases are configured, set database_id on the filter so option loading targets the correct database.', + 'Matching SQL must use template blocks that reference the same filter id, e.g. WHERE 1 = 1 {% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %}.', + 'Chat and live refresh strip unset filter blocks so the query still runs; active story filter selections re-render and re-execute SQL.', + 'When adding filters to existing charts, prefer execute_sql with query_id set to the existing query so chart/table tags keep the same query_id.', +].join(' '); + +function buildStoryToolDescription() { + return [ 'Create or modify a nao Story — an interactive document combining markdown text and chart visualizations.', 'Use "create" to initialize a new story, "update" to search-and-replace within it (producing a new version),', 'or "replace" to overwrite the entire content (producing a new version).', 'Charts are embedded via .', 'SQL result tables are embedded via
.', + ...(env.BETA_STORY_FILTERS_ENABLED ? [STORY_FILTER_DESCRIPTION] : []), 'Use ... to display charts side by side in a responsive grid.', 'Use consecutive ... blocks to organize a story into top-level tabs.', 'Default to a single flowing story. Use tabs only when the user asks for tabs, or when the content splits into clearly distinct sections that are better separated than stacked (e.g. overview vs. detail, one topic/department/metric per tab). Avoid tabs for a short or single-topic story. Always follow the user\'s explicit request (e.g. "a tab per chart" means one chart per tab). When using tabs, the entire story must consist of ... blocks — no content outside a tab.', 'A story can also be refered as a "canva", an "artifact" or a "report".', 'Users may edit stories directly; the tool result always reflects the latest version, including user edits.', 'Unless explicitly stated, dont use the stories to display a chart, but the display_chart tool.', - ].join(' '), + ].join(' '); +} + +export default createTool({ + description: buildStoryToolDescription(), inputSchema: story.InputSchema, outputSchema: story.OutputSchema, diff --git a/apps/backend/src/env.ts b/apps/backend/src/env.ts index 0610f1ae9..4b024c0b6 100644 --- a/apps/backend/src/env.ts +++ b/apps/backend/src/env.ts @@ -132,6 +132,12 @@ const envSchema = z.object({ .optional() .default('false') .transform((val) => val === 'true'), + + BETA_STORY_FILTERS_ENABLED: z + .enum(['true', 'false']) + .optional() + .default('false') + .transform((val) => val === 'true'), }); const result = envSchema.safeParse(process.env); diff --git a/apps/backend/src/queries/execute-sql.queries.ts b/apps/backend/src/queries/execute-sql.queries.ts new file mode 100644 index 000000000..b5a942d72 --- /dev/null +++ b/apps/backend/src/queries/execute-sql.queries.ts @@ -0,0 +1,90 @@ +import { executeSql } from '@nao/shared/tools'; +import { and, eq, sql } from 'drizzle-orm'; + +import s from '../db/abstractSchema'; +import { db } from '../db/db'; +import dbConfig, { Dialect } from '../db/dbConfig'; +import { takeFirstOrThrow } from '../utils/queries'; + +const EXECUTE_SQL_TOOL_NAME = 'execute_sql'; + +function queryIdFilter(queryId: string) { + return dbConfig.dialect === Dialect.Postgres + ? sql`${s.messagePart.toolOutput}->>'id' = ${queryId}` + : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; +} + +export async function getExecuteSqlOwnerByQueryId( + queryId: string, +): Promise<{ projectId: string; userId: string; chatId: string; toolCallId: string } | null> { + const [row] = await db + .select({ + projectId: s.chat.projectId, + userId: s.chat.userId, + chatId: s.chat.id, + toolCallId: s.messagePart.toolCallId, + }) + .from(s.messagePart) + .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) + .innerJoin(s.chat, eq(s.chatMessage.chatId, s.chat.id)) + .where(and(eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), queryIdFilter(queryId))) + .execute(); + + if (!row?.toolCallId) { + return null; + } + + return { + projectId: row.projectId, + userId: row.userId, + chatId: row.chatId, + toolCallId: row.toolCallId, + }; +} + +export async function getExecuteSqlPartByQueryIdInChat( + chatId: string, + queryId: string, +): Promise<{ toolCallId: string; toolInput: executeSql.Input; toolOutput: executeSql.Output } | null> { + const [row] = await db + .select({ + toolCallId: s.messagePart.toolCallId, + toolInput: s.messagePart.toolInput, + toolOutput: s.messagePart.toolOutput, + }) + .from(s.messagePart) + .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) + .where( + and( + eq(s.chatMessage.chatId, chatId), + eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), + queryIdFilter(queryId), + ), + ) + .execute(); + + if (!row?.toolCallId || !row.toolInput || !row.toolOutput) { + return null; + } + + return { + toolCallId: row.toolCallId, + toolInput: executeSql.InputSchema.parse(row.toolInput), + toolOutput: executeSql.OutputSchema.parse(row.toolOutput), + }; +} + +export async function updateExecuteSqlPart( + toolCallId: string, + toolInput: executeSql.Input, + toolOutput: executeSql.Output, +): Promise { + await takeFirstOrThrow( + db + .update(s.messagePart) + .set({ toolInput, toolOutput }) + .where(and(eq(s.messagePart.toolCallId, toolCallId), eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME))) + .returning({ toolCallId: s.messagePart.toolCallId }) + .execute(), + ); +} diff --git a/apps/backend/src/services/live-story.ts b/apps/backend/src/services/live-story.ts index 0351391be..ab1a4b03e 100644 --- a/apps/backend/src/services/live-story.ts +++ b/apps/backend/src/services/live-story.ts @@ -1,3 +1,4 @@ +import { stripSqlFilterBlocks } from '@nao/shared/sql-template'; import { TAG_ATTRS } from '@nao/shared/story-segments'; import { generateText, Output } from 'ai'; import { CronExpressionParser } from 'cron-parser'; @@ -36,7 +37,7 @@ export async function executeLiveQuery( } const envVars = await projectQueries.getEnvVars(projectId); - return executeRawSql(query.sqlQuery, project.path, query.databaseId, envVars); + return executeRawSql(stripSqlFilterBlocks(query.sqlQuery), project.path, query.databaseId, envVars); } export interface RefreshResult { @@ -69,7 +70,12 @@ export async function refreshStoryData(chatId: string, slug: string): Promise { const projectEnvVars = await projectQueries.getEnvVars(projectId); - const result = await executeRawSql(sqlQuery, project.path!, databaseId, projectEnvVars); + const result = await executeRawSql( + stripSqlFilterBlocks(sqlQuery), + project.path!, + databaseId, + projectEnvVars, + ); queryData[queryId] = result; }), ); @@ -122,7 +128,7 @@ export async function getStoryQueryData( } } -async function executeRawSql( +export async function executeRawSql( sqlQuery: string, projectFolder: string, databaseId?: string, @@ -297,7 +303,7 @@ function preservesStoryStructure(originalCode: string, candidateCode: string): b function extractStructureTokens(code: string): string[] { const tokenRegex = new RegExp( - String.raw`|<\/grid>||`, + String.raw`|<\/grid>|||`, 'g', ); return code.match(tokenRegex) ?? []; diff --git a/apps/backend/src/services/ping.ts b/apps/backend/src/services/ping.ts index b3a10e046..93c91d5d4 100644 --- a/apps/backend/src/services/ping.ts +++ b/apps/backend/src/services/ping.ts @@ -43,6 +43,7 @@ interface StartupAdditionalInfo { userCount: number | null; betaAutomationsEnabled: boolean; betaContextRecommendationsEnabled: boolean; + betaStoryFiltersEnabled: boolean; smtpConfigured: boolean; loginModesConfigured: { google: boolean; @@ -64,6 +65,7 @@ async function startupAdditionalInfo(): Promise { userCount, betaAutomationsEnabled: env.BETA_AUTOMATIONS_ENABLED, betaContextRecommendationsEnabled: env.BETA_CONTEXT_RECOMMENDATIONS_ENABLED, + betaStoryFiltersEnabled: env.BETA_STORY_FILTERS_ENABLED, smtpConfigured: isSmtpConfigured(), loginModesConfigured: { google: googleConfigured, diff --git a/apps/backend/src/services/story-filters.ts b/apps/backend/src/services/story-filters.ts new file mode 100644 index 000000000..0c6a04b27 --- /dev/null +++ b/apps/backend/src/services/story-filters.ts @@ -0,0 +1,107 @@ +import { + renderSqlTemplate, + type StoryFilterSelections, + type StoryFilterTypeById, + stripSqlFilterBlocks, +} from '@nao/shared/sql-template'; +import { getStoryFiltersFromCode } from '@nao/shared/story-segments'; +import { TRPCError } from '@trpc/server'; + +import * as chatQueries from '../queries/chat.queries'; +import * as projectQueries from '../queries/project.queries'; +import * as storyQueries from '../queries/story.queries'; +import { assertSafeSqlIdentifier } from '../utils/sql-identifiers'; +import { executeRawSql } from './live-story'; + +const FILTER_OPTIONS_LIMIT = 100; + +export async function getStoryFilterOptions( + chatId: string, + storySlug: string, + filterId: string, +): Promise<{ options: string[] }> { + const { code, projectPath, envVars, databaseId } = await loadStoryExecutionContext(chatId, storySlug); + const filter = getStoryFiltersFromCode(code).find((candidate) => candidate.id === filterId); + if (!filter) { + throw new TRPCError({ code: 'NOT_FOUND', message: `Filter "${filterId}" not found in story.` }); + } + + if (filter.options?.length) { + return { options: [...new Set(filter.options)] }; + } + + if (!filter.table || !filter.column) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: `Filter "${filterId}" has no hardcoded options and is missing table/column.`, + }); + } + + const table = assertSafeSqlIdentifier(filter.table, 'table'); + const column = assertSafeSqlIdentifier(filter.column, 'column'); + const sql = `SELECT DISTINCT ${column} AS value FROM ${table} WHERE ${column} IS NOT NULL LIMIT ${FILTER_OPTIONS_LIMIT}`; + const result = await executeRawSql(sql, projectPath, filter.databaseId ?? databaseId, envVars); + const options = result.data + .map((row) => { + if (!row || typeof row !== 'object') { + return null; + } + const value = (row as Record).value; + return value === null || value === undefined ? null : String(value); + }) + .filter((value): value is string => value !== null); + + return { options: [...new Set(options)].sort((a, b) => a.localeCompare(b)) }; +} + +export async function getFilteredStoryQueryData( + chatId: string, + storySlug: string, + selections: StoryFilterSelections, +): Promise> { + const { code, projectPath, envVars } = await loadStoryExecutionContext(chatId, storySlug); + const filters = getStoryFiltersFromCode(code); + const types: StoryFilterTypeById = Object.fromEntries(filters.map((filter) => [filter.id, filter.filterType])); + const sqlQueries = await storyQueries.getSqlQueriesFromCode(chatId, code); + const queryData: Record = {}; + + await Promise.all( + Object.entries(sqlQueries).map(async ([queryId, { sqlQuery, databaseId }]) => { + const renderedSql = + Object.keys(selections).length === 0 + ? stripSqlFilterBlocks(sqlQuery) + : renderSqlTemplate(sqlQuery, selections, types); + queryData[queryId] = await executeRawSql(renderedSql, projectPath, databaseId, envVars); + }), + ); + + return queryData; +} + +async function loadStoryExecutionContext(chatId: string, storySlug: string) { + const version = await storyQueries.getLatestVersionByChatAndSlug(chatId, storySlug); + if (!version) { + throw new TRPCError({ code: 'NOT_FOUND', message: 'Story not found.' }); + } + + const projectId = await chatQueries.getChatProjectId(chatId); + if (!projectId) { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Chat project not found.' }); + } + + const project = await projectQueries.retrieveProjectById(projectId); + if (!project.path) { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Project path not configured.' }); + } + + const envVars = await projectQueries.getEnvVars(projectId); + const sqlQueries = await storyQueries.getSqlQueriesFromCode(chatId, version.code); + const databaseId = Object.values(sqlQueries).find((query) => query.databaseId)?.databaseId; + + return { + code: version.code, + projectPath: project.path, + envVars, + databaseId, + }; +} diff --git a/apps/backend/src/services/update-sql.ts b/apps/backend/src/services/update-sql.ts new file mode 100644 index 000000000..70a6f2909 --- /dev/null +++ b/apps/backend/src/services/update-sql.ts @@ -0,0 +1,103 @@ +import { stripSqlFilterBlocks } from '@nao/shared/sql-template'; +import type { executeSql } from '@nao/shared/tools'; +import { TRPCError } from '@trpc/server'; + +import { executeQuery } from '../agents/tools/execute-sql'; +import { + getExecuteSqlOwnerByQueryId, + getExecuteSqlPartByQueryIdInChat, + updateExecuteSqlPart, +} from '../queries/execute-sql.queries'; +import * as projectQueries from '../queries/project.queries'; +import type { ToolContext } from '../types/tools'; + +export async function updateSqlQueryInChat(opts: { + queryId: string; + sqlQuery: string; + databaseId?: string; + name?: string; + userId: string; +}): Promise<{ input: executeSql.Input; output: executeSql.Output; toolCallId: string; chatId: string }> { + const { existing, owner, context, input } = await prepareSqlEditContext(opts); + const output = await executeQuery({ ...input, query_id: opts.queryId as `query_${string}` }, context); + await updateExecuteSqlPart(existing.toolCallId, input, output); + + return { input, output, toolCallId: existing.toolCallId, chatId: owner.chatId }; +} + +export async function previewSqlQueryInChat(opts: { + queryId: string; + sqlQuery: string; + databaseId?: string; + userId: string; +}): Promise { + const { context, input } = await prepareSqlEditContext(opts); + return executeQuery(input, context); +} + +async function prepareSqlEditContext(opts: { + queryId: string; + sqlQuery: string; + databaseId?: string; + name?: string; + userId: string; +}): Promise<{ + existing: NonNullable>>; + owner: NonNullable>>; + context: ToolContext; + input: executeSql.Input; +}> { + assertSqlQueryEditable(opts.sqlQuery); + + const owner = await getExecuteSqlOwnerByQueryId(opts.queryId); + if (!owner) { + throw new TRPCError({ code: 'NOT_FOUND', message: `Query ${opts.queryId} not found.` }); + } + if (owner.userId !== opts.userId) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'You are not authorized to edit this query.' }); + } + + const existing = await getExecuteSqlPartByQueryIdInChat(owner.chatId, opts.queryId); + if (!existing) { + throw new TRPCError({ code: 'NOT_FOUND', message: `Query ${opts.queryId} not found.` }); + } + + const project = await projectQueries.retrieveProjectById(owner.projectId); + if (!project.path) { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Project path not configured.' }); + } + + const agentSettings = await projectQueries.getAgentSettings(owner.projectId); + const envVars = await projectQueries.getEnvVars(owner.projectId); + const context: ToolContext = { + projectFolder: project.path, + chatId: owner.chatId, + userId: opts.userId, + projectId: owner.projectId, + agentSettings, + envVars, + azureAccessToken: null, + queryResults: new Map(), + generatedArtifacts: { charts: [], stories: [] }, + }; + + const input: executeSql.Input = { + sql_query: opts.sqlQuery, + database_id: opts.databaseId ?? existing.toolInput.database_id, + name: opts.name ?? existing.toolInput.name, + }; + + return { existing, owner, context, input }; +} + +export function assertSqlQueryEditable(sqlQuery: string): void { + if (!sqlQuery.trim()) { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'SQL query cannot be empty.' }); + } + if (stripSqlFilterBlocks(sqlQuery).trim() === '') { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'SQL query cannot be empty after removing filter blocks.', + }); + } +} diff --git a/apps/backend/src/trpc/router.ts b/apps/backend/src/trpc/router.ts index e1d1e98b7..e33871654 100644 --- a/apps/backend/src/trpc/router.ts +++ b/apps/backend/src/trpc/router.ts @@ -27,6 +27,7 @@ import { projectRoutes } from './project.routes'; import { sharedChatRoutes } from './shared-chat.routes'; import { sharedStoryRoutes } from './shared-story.routes'; import { skillRoutes } from './skill.routes'; +import { sqlRoutes } from './sql.routes'; import { storyRoutes } from './story.routes'; import { storyFolderRoutes } from './story-folder.routes'; import { systemRoutes } from './system.routes'; @@ -42,6 +43,7 @@ export const trpcRouter = router({ embed: embedRoutes, chart: chartRoutes, chat: chatRoutes, + sql: sqlRoutes, sharedChat: sharedChatRoutes, automation: automationRoutes, chatFork: chatForkRoutes, diff --git a/apps/backend/src/trpc/shared-story.routes.ts b/apps/backend/src/trpc/shared-story.routes.ts index ca2439d20..3039477c6 100644 --- a/apps/backend/src/trpc/shared-story.routes.ts +++ b/apps/backend/src/trpc/shared-story.routes.ts @@ -2,6 +2,7 @@ import { DOWNLOAD_FORMATS, SHARE_VISIBILITY } from '@nao/shared/types'; import { TRPCError } from '@trpc/server'; import { z } from 'zod/v4'; +import { env } from '../env'; import * as activityQueries from '../queries/activity.queries'; import * as chatQueries from '../queries/chat.queries'; import * as projectQueries from '../queries/project.queries'; @@ -10,6 +11,7 @@ import * as storyQueries from '../queries/story.queries'; import * as storyFolderQueries from '../queries/story-folder.queries'; import { logActivity } from '../services/activity'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; +import { getFilteredStoryQueryData, getStoryFilterOptions } from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { notifySharedItemRecipients } from '../utils/email'; import { buildDownloadResponse } from '../utils/story-download'; @@ -34,6 +36,12 @@ const shareAccessProcedure = resourceProjectProcedure( sharedStoryQueries.canUserAccessSharedStory(item.id, userId), ); +function assertStoryFiltersEnabled() { + if (!env.BETA_STORY_FILTERS_ENABLED) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'Story filters are disabled on this instance.' }); + } +} + export const sharedStoryRoutes = { list: protectedProcedure.input(z.object({ projectId: z.string() })).query(async ({ input, ctx }) => { const projects = await projectQueries.listUserProjects(ctx.user.id); @@ -166,6 +174,33 @@ export const sharedStoryRoutes = { return executeLiveQuery(input.chatId, input.queryId); }), + getFilterOptions: shareAccessProcedure + .input(z.object({ shareId: z.string(), filterId: z.string() })) + .query(async ({ input, ctx }) => { + assertStoryFiltersEnabled(); + const shared = ctx.resource; + if (!shared.chatId) { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'Shared story has no chat.' }); + } + return getStoryFilterOptions(shared.chatId, shared.slug, input.filterId); + }), + + getFilteredQueryData: shareAccessProcedure + .input( + z.object({ + shareId: z.string(), + selections: z.record(z.string(), z.union([z.string(), z.array(z.string())])), + }), + ) + .query(async ({ input, ctx }) => { + assertStoryFiltersEnabled(); + const shared = ctx.resource; + if (!shared.chatId) { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'Shared story has no chat.' }); + } + return getFilteredStoryQueryData(shared.chatId, shared.slug, input.selections); + }), + refreshData: shareAccessProcedure.input(z.object({ shareId: z.string() })).mutation(async ({ ctx }) => { const shared = ctx.resource; const story = await storyQueries.getStoryByChatAndSlug(shared.chatId!, shared.slug); diff --git a/apps/backend/src/trpc/sql.routes.ts b/apps/backend/src/trpc/sql.routes.ts new file mode 100644 index 000000000..45d17379e --- /dev/null +++ b/apps/backend/src/trpc/sql.routes.ts @@ -0,0 +1,32 @@ +import { z } from 'zod/v4'; + +import { previewSqlQueryInChat, updateSqlQueryInChat } from '../services/update-sql'; +import { protectedProcedure } from './trpc'; + +const sqlEditInput = z.object({ + queryId: z.string().regex(/^query_.+$/), + sql_query: z.string().min(1), + database_id: z.string().optional(), + name: z.string().optional(), +}); + +export const sqlRoutes = { + previewQuery: protectedProcedure.input(sqlEditInput).mutation(async ({ input, ctx }) => { + return previewSqlQueryInChat({ + queryId: input.queryId, + sqlQuery: input.sql_query, + databaseId: input.database_id, + userId: ctx.user.id, + }); + }), + + updateQuery: protectedProcedure.input(sqlEditInput).mutation(async ({ input, ctx }) => { + return updateSqlQueryInChat({ + queryId: input.queryId, + sqlQuery: input.sql_query, + databaseId: input.database_id, + name: input.name, + userId: ctx.user.id, + }); + }), +}; diff --git a/apps/backend/src/trpc/story.routes.ts b/apps/backend/src/trpc/story.routes.ts index dfe66ba6c..0e5c7cd02 100644 --- a/apps/backend/src/trpc/story.routes.ts +++ b/apps/backend/src/trpc/story.routes.ts @@ -4,6 +4,7 @@ import { DOWNLOAD_FORMATS } from '@nao/shared/types'; import { TRPCError } from '@trpc/server'; import { z } from 'zod/v4'; +import { env } from '../env'; import { STORY_REFRESH_JOB_NAME } from '../handlers/story-refresh.handler'; import * as activityQueries from '../queries/activity.queries'; import * as chatQueries from '../queries/chat.queries'; @@ -15,6 +16,7 @@ import * as storyFolderQueries from '../queries/story-folder.queries'; import { naturalLanguageToCron } from '../services/cron-nlp'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; import { nextCronTick } from '../services/scheduler.service'; +import { getFilteredStoryQueryData, getStoryFilterOptions } from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { buildDownloadResponse } from '../utils/story-download'; import { extractStorySummary } from '../utils/story-summary'; @@ -23,6 +25,12 @@ import { canSendProcedure, ownedResourceProcedure, projectProtectedProcedure, pr const chatOwnerProcedure = ownedResourceProcedure(chatQueries.getChatOwnerId, 'chat'); const storyOwnerProcedure = ownedResourceProcedure(storyQueries.getStoryOwnerId, 'story'); +function assertStoryFiltersEnabled() { + if (!env.BETA_STORY_FILTERS_ENABLED) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'Story filters are disabled on this instance.' }); + } +} + const bulkStoryItemsInput = z.object({ items: z .array( @@ -288,6 +296,26 @@ export const storyRoutes = { return executeLiveQuery(input.chatId, input.queryId); }), + getFilterOptions: chatOwnerProcedure + .input(z.object({ chatId: z.string(), storySlug: z.string(), filterId: z.string() })) + .query(async ({ input }) => { + assertStoryFiltersEnabled(); + return getStoryFilterOptions(input.chatId, input.storySlug, input.filterId); + }), + + getFilteredQueryData: chatOwnerProcedure + .input( + z.object({ + chatId: z.string(), + storySlug: z.string(), + selections: z.record(z.string(), z.union([z.string(), z.array(z.string())])), + }), + ) + .query(async ({ input }) => { + assertStoryFiltersEnabled(); + return getFilteredStoryQueryData(input.chatId, input.storySlug, input.selections); + }), + parseCronFromText: projectProtectedProcedure .input(z.object({ text: z.string().min(1) })) .mutation(async ({ input, ctx }) => { diff --git a/apps/backend/src/trpc/system.routes.ts b/apps/backend/src/trpc/system.routes.ts index e66801bab..cff588510 100644 --- a/apps/backend/src/trpc/system.routes.ts +++ b/apps/backend/src/trpc/system.routes.ts @@ -10,6 +10,7 @@ export const systemRoutes = { enableUserSignup: await isUserSignupAvailable(), betaAutomationsEnabled: env.BETA_AUTOMATIONS_ENABLED, betaContextRecommendationsEnabled: env.BETA_CONTEXT_RECOMMENDATIONS_ENABLED, + betaStoryFiltersEnabled: env.BETA_STORY_FILTERS_ENABLED, })), version: adminProtectedProcedure.query(() => ({ diff --git a/apps/backend/src/utils/sql-identifiers.ts b/apps/backend/src/utils/sql-identifiers.ts new file mode 100644 index 000000000..745f97753 --- /dev/null +++ b/apps/backend/src/utils/sql-identifiers.ts @@ -0,0 +1,14 @@ +import { TRPCError } from '@trpc/server'; + +export function assertSafeSqlIdentifier(value: string, kind: 'table' | 'column'): string { + const unquoted = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$/; + const doubleQuoted = /^"[^"]+"(\."[^"]+")*$/; + const backtickQuoted = /^`[^`]+`(\.`[^`]+`)*$/; + if (!unquoted.test(value) && !doubleQuoted.test(value) && !backtickQuoted.test(value)) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: `Invalid filter ${kind} identifier "${value}". Use unquoted names, "double quotes", or \`backticks\` (required for hyphens).`, + }); + } + return value; +} diff --git a/apps/backend/src/utils/story-html.tsx b/apps/backend/src/utils/story-html.tsx index 5781652e9..b3d8e4bf0 100644 --- a/apps/backend/src/utils/story-html.tsx +++ b/apps/backend/src/utils/story-html.tsx @@ -88,6 +88,8 @@ function StorySegment({ segment, queryData }: { segment: Segment; queryData: Que return ; case 'table': return ; + case 'filter': + return null; case 'grid': return ; } diff --git a/apps/backend/tests/license.test.ts b/apps/backend/tests/license.test.ts index 7f8e8f30f..113ee267f 100644 --- a/apps/backend/tests/license.test.ts +++ b/apps/backend/tests/license.test.ts @@ -177,6 +177,7 @@ describe('license.service', () => { process.env.NAO_MODE = 'self-hosted'; process.env.BETA_AUTOMATIONS_ENABLED = 'false'; process.env.BETA_CONTEXT_RECOMMENDATIONS_ENABLED = 'true'; + process.env.BETA_STORY_FILTERS_ENABLED = 'true'; process.env.SMTP_HOST = 'smtp.example.com'; process.env.SMTP_MAIL_FROM = 'hello@example.com'; process.env.SMTP_PASSWORD = 'secret'; @@ -202,6 +203,7 @@ describe('license.service', () => { additionalInfo: expect.objectContaining({ betaAutomationsEnabled: false, betaContextRecommendationsEnabled: true, + betaStoryFiltersEnabled: true, smtpConfigured: true, loginModesConfigured: { google: false, diff --git a/apps/backend/tests/story-filters.test.ts b/apps/backend/tests/story-filters.test.ts new file mode 100644 index 000000000..40ed3bf01 --- /dev/null +++ b/apps/backend/tests/story-filters.test.ts @@ -0,0 +1,51 @@ +import { renderSqlTemplate, stripSqlFilterBlocks } from '@nao/shared/sql-template'; +import { describe, expect, it } from 'vitest'; + +import { assertSafeSqlIdentifier } from '../src/utils/sql-identifiers'; + +describe('story filter SQL templates', () => { + it('strips filter blocks for chat / live baseline execution', () => { + const sql = ` +SELECT SUM(revenue) AS revenue +FROM orders +WHERE 1 = 1 +{% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %} +`.trim(); + + expect(stripSqlFilterBlocks(sql)).toContain('WHERE 1 = 1'); + expect(stripSqlFilterBlocks(sql)).not.toContain('{% filter'); + expect(stripSqlFilterBlocks(sql)).not.toContain('country IN'); + }); + + it('renders active filter selections into executable SQL', () => { + const sql = ` +SELECT SUM(revenue) AS revenue +FROM orders +WHERE 1 = 1 +{% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %} +`.trim(); + + expect(renderSqlTemplate(sql, { country: ['US', 'FR'] }, { country: 'multi_select' })).toContain( + "AND country IN ('US', 'FR')", + ); + }); +}); + +describe('assertSafeSqlIdentifier', () => { + it('accepts dotted and quoted identifiers', () => { + expect(assertSafeSqlIdentifier('orders', 'table')).toBe('orders'); + expect(assertSafeSqlIdentifier('public.orders', 'table')).toBe('public.orders'); + expect(assertSafeSqlIdentifier('"Order Status"', 'column')).toBe('"Order Status"'); + expect(assertSafeSqlIdentifier('`nao-production`.`prod_silver`.`dim_products`', 'table')).toBe( + '`nao-production`.`prod_silver`.`dim_products`', + ); + }); + + it('rejects unsafe identifiers', () => { + expect(() => assertSafeSqlIdentifier('orders; DROP TABLE x', 'table')).toThrow(/Invalid filter table/); + expect(() => assertSafeSqlIdentifier('col-name', 'column')).toThrow(/Invalid filter column/); + expect(() => assertSafeSqlIdentifier('nao-production.prod_silver.dim_products', 'table')).toThrow( + /Invalid filter table/, + ); + }); +}); diff --git a/apps/frontend/src/components/fix-in-chat-button.tsx b/apps/frontend/src/components/fix-in-chat-button.tsx new file mode 100644 index 000000000..7b77986ea --- /dev/null +++ b/apps/frontend/src/components/fix-in-chat-button.tsx @@ -0,0 +1,26 @@ +import { MessageSquare } from 'lucide-react'; + +import { useOptionalAgentContext } from '@/contexts/agent.provider'; +import { Button } from '@/components/ui/button'; + +export function FixInChatButton({ message, className }: { message: string; className?: string }) { + const agent = useOptionalAgentContext(); + if (!agent || agent.isReadonly || !message.trim()) { + return null; + } + + return ( + + ); +} diff --git a/apps/frontend/src/components/mcp-app/story-app-view.tsx b/apps/frontend/src/components/mcp-app/story-app-view.tsx index 64b79714b..7ed27d337 100644 --- a/apps/frontend/src/components/mcp-app/story-app-view.tsx +++ b/apps/frontend/src/components/mcp-app/story-app-view.tsx @@ -38,16 +38,58 @@ export function StoryAppView({ title, code, queryData, naoUrl, onDownload }: Sto function StoryBody({ code, queryData }: { code: string; queryData: QueryDataMap | null }) { const renderChart = useCallback( - (chart: ParsedChartBlock) => , - [queryData], + ( + chart: ParsedChartBlock, + { + queryData: data, + baselineQueryData, + hasActiveFilters, + isRefreshing, + }: { + queryData: QueryDataMap | null; + baselineQueryData: QueryDataMap | null; + hasActiveFilters: boolean; + isRefreshing: boolean; + }, + ) => ( + + ), + [], ); const renderTable = useCallback( - (table: ParsedTableBlock) => , - [queryData], + ( + table: ParsedTableBlock, + { + queryData: data, + hasActiveFilters, + isRefreshing, + }: { queryData: QueryDataMap | null; hasActiveFilters: boolean; isRefreshing: boolean }, + ) => ( + + ), + [], ); - return ; + return ( + + ); } function StoryDownloadButton({ onDownload }: { onDownload: (format: DownloadFormat) => Promise }) { diff --git a/apps/frontend/src/components/side-panel/sql-editor.tsx b/apps/frontend/src/components/side-panel/sql-editor.tsx index beef9cb4b..fc1fa22dd 100644 --- a/apps/frontend/src/components/side-panel/sql-editor.tsx +++ b/apps/frontend/src/components/side-panel/sql-editor.tsx @@ -1,32 +1,223 @@ +import { useCallback, useRef, useState } from 'react'; import { Editor } from '@monaco-editor/react'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { Loader2, Play, Save } from 'lucide-react'; +import { usePanelRef } from 'react-resizable-panels'; import { ResizableSeparator, ResizablePanel, ResizablePanelGroup } from '../ui/resizable'; +import type { Monaco } from '@monaco-editor/react'; import type { executeSql } from '@nao/shared/tools'; +import type { editor } from 'monaco-editor'; + +import { useOptionalAgentContext } from '@/contexts/agent.provider'; import { SidePanelHeader } from '@/components/side-panel/side-panel-header'; import { TableDisplay } from '@/components/tool-calls/display-table'; +import { Button } from '@/components/ui/button'; +import { applyExecuteSqlResultToMessages } from '@/lib/execute-sql-messages'; import { formatSQL } from '@/lib/sql-formatter'; +import { trpc } from '@/main'; + +const RESULTS_MIN_HEIGHT = 240; +const SQL_MIN_HEIGHT = 100; +const SEPARATOR_HEIGHT = 12; +const SQL_CONTENT_PADDING = 8; + +export const SidePanelContent = ({ + input, + output, + editable = false, +}: { + input: executeSql.Input; + output: executeSql.Output; + editable?: boolean; +}) => { + const queryClient = useQueryClient(); + const agent = useOptionalAgentContext(); + const [sqlQuery, setSqlQuery] = useState(input.sql_query); + const [savedSql, setSavedSql] = useState(input.sql_query); + const [previewOutput, setPreviewOutput] = useState(output); + const [error, setError] = useState(null); + const handleRunRef = useRef<() => void>(() => {}); + const handleSaveAndRunRef = useRef<() => void>(() => {}); + const sqlPanelRef = usePanelRef(); + const groupElementRef = useRef(null); + + const canEdit = Boolean(editable && output.id && agent && !agent.isReadonly); + const isDirty = sqlQuery !== savedSql; + + const previewMutation = useMutation( + trpc.sql.previewQuery.mutationOptions({ + onSuccess: (result) => { + setPreviewOutput(result); + setError(null); + }, + onError: (err) => { + setError(err.message); + }, + }), + ); + + const saveMutation = useMutation( + trpc.sql.updateQuery.mutationOptions({ + onSuccess: (result) => { + setPreviewOutput(result.output); + setSqlQuery(result.input.sql_query); + setSavedSql(result.input.sql_query); + setError(null); + if (agent) { + agent.setMessages( + applyExecuteSqlResultToMessages(agent.messages, result.output.id, result.input, result.output), + ); + } + queryClient.invalidateQueries({ queryKey: [['chat', 'get']] }); + }, + onError: (err) => { + setError(err.message); + }, + }), + ); + + const isBusy = previewMutation.isPending || saveMutation.isPending; + + const handleRun = () => { + if (!output.id || !sqlQuery.trim() || isBusy) { + return; + } + setError(null); + previewMutation.mutate({ + queryId: output.id, + sql_query: sqlQuery, + database_id: input.database_id, + name: input.name, + }); + }; + + const handleSaveAndRun = () => { + if (!output.id || !sqlQuery.trim() || isBusy) { + return; + } + setError(null); + saveMutation.mutate({ + queryId: output.id, + sql_query: sqlQuery, + database_id: input.database_id, + name: input.name, + }); + }; + + handleRunRef.current = handleRun; + handleSaveAndRunRef.current = handleSaveAndRun; + + const fitSqlPanelToContent = useCallback( + (instance: editor.IStandaloneCodeEditor) => { + const groupHeight = groupElementRef.current?.clientHeight ?? 0; + const panel = sqlPanelRef.current; + if (!panel || groupHeight <= 0) { + return false; + } + + instance.layout(); + const contentHeight = instance.getContentHeight() + SQL_CONTENT_PADDING; + const maxSqlHeight = Math.max(SQL_MIN_HEIGHT, groupHeight - RESULTS_MIN_HEIGHT - SEPARATOR_HEIGHT); + const sqlHeight = Math.min(Math.max(contentHeight, SQL_MIN_HEIGHT), maxSqlHeight); + panel.resize(sqlHeight); + return true; + }, + [sqlPanelRef], + ); + + const handleEditorMount = useCallback( + (instance: editor.IStandaloneCodeEditor, monaco: Monaco) => { + if (canEdit) { + instance.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => { + handleSaveAndRunRef.current(); + }); + instance.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, () => { + handleRunRef.current(); + }); + } + + let fitted = false; + const tryFit = () => { + if (fitted || !fitSqlPanelToContent(instance)) { + return; + } + fitted = true; + disposable.dispose(); + }; + const disposable = instance.onDidContentSizeChange(tryFit); + requestAnimationFrame(tryFit); + }, + [canEdit, fitSqlPanelToContent], + ); -export const SidePanelContent = ({ input, output }: { input: executeSql.Input; output: executeSql.Output }) => { return (
- - + {canEdit && ( +
+
+ {isDirty ? 'Unsaved changes' : 'Edit SQL, then run or save'} +
+
+ + +
+
+ )} + + {error && ( +
{error}
+ )} + + +
setSqlQuery(value ?? '') : undefined} + onMount={handleEditorMount} language='sql' theme='light' options={{ + readOnly: !canEdit, minimap: { enabled: false, }, folding: false, - lineNumbers: 'off', + lineNumbers: canEdit ? 'on' : 'off', scrollbar: { horizontal: 'hidden', vertical: 'hidden', @@ -44,10 +235,10 @@ export const SidePanelContent = ({ input, output }: { input: executeSql.Input; o - + []} - columns={output.columns} + data={previewOutput.data as Record[]} + columns={previewOutput.columns} className='h-full' tableContainerClassName='flex-1 rounded-none border-0' emptyLabel='No rows returned' diff --git a/apps/frontend/src/components/side-panel/story-code-view.tsx b/apps/frontend/src/components/side-panel/story-code-view.tsx index 6e1eb8a8f..06c402ad1 100644 --- a/apps/frontend/src/components/side-panel/story-code-view.tsx +++ b/apps/frontend/src/components/side-panel/story-code-view.tsx @@ -5,6 +5,7 @@ import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { StoryValidationError } from '@nao/shared/story-validation'; import type { Monaco } from '@monaco-editor/react'; import type { editor } from 'monaco-editor'; +import { FixInChatButton } from '@/components/fix-in-chat-button'; import { useEditorTheme } from '@/hooks/use-editor-theme'; const MONACO_OPTIONS = { @@ -154,22 +155,33 @@ export const StoryCodeView = memo(function StoryCodeView({ }); function ValidationErrorBanner({ errors }: { errors: StoryValidationError[] }) { + const fixMessage = [ + "I'm seeing validation errors in the story code:", + ...errors.slice(0, 10).map((error) => `- L${error.line}: ${error.message}`), + ...(errors.length > 10 ? [`- and ${errors.length - 10} more...`] : []), + '', + 'Please fix these errors in the story.', + ].join('\n'); + return ( -
-
- - - {errors.length} validation {errors.length === 1 ? 'error' : 'errors'} - +
+
+
+ + + {errors.length} validation {errors.length === 1 ? 'error' : 'errors'} + +
+
    + {errors.slice(0, 5).map((error, i) => ( +
  • + L{error.line}: {error.message} +
    • + ))} + {errors.length > 5 &&
  • and {errors.length - 5} more...
    • } +
-
    - {errors.slice(0, 5).map((error, i) => ( -
  • - L{error.line}: {error.message} -
    • - ))} - {errors.length > 5 &&
  • and {errors.length - 5} more...
    • } -
+
); } diff --git a/apps/frontend/src/components/side-panel/story-preview.tsx b/apps/frontend/src/components/side-panel/story-preview.tsx index b3f31ec93..c5e80368a 100644 --- a/apps/frontend/src/components/side-panel/story-preview.tsx +++ b/apps/frontend/src/components/side-panel/story-preview.tsx @@ -5,68 +5,104 @@ import type { ParsedChartBlock, ParsedTableBlock } from '@nao/shared/story-segme import type { QueryDataMap } from '@/components/story-embeds'; import { StoryChartEmbed as LiveChartEmbed, StoryTableEmbed as LiveTableEmbed } from '@/components/story-embeds'; +import { StoryFilterBar } from '@/components/story-filter-bar'; import { SegmentList } from '@/components/story-rendering'; import { StoryChartEmbed as StaticChartEmbed } from '@/components/side-panel/story-chart-embed'; import { StoryTableEmbed as StaticTableEmbed } from '@/components/side-panel/story-table-embed'; +import { useStoryFilters } from '@/hooks/use-story-filters'; import { trpc } from '@/main'; interface StoryPreviewProps { code: string; + fullCode?: string; cacheSchedule: string | null; queryData: QueryDataMap | null; chatId: string; + storySlug: string; versionKey?: string | number; } export const StoryPreview = memo(function StoryPreview({ code, + fullCode, cacheSchedule, queryData, chatId, + storySlug, versionKey, }: StoryPreviewProps) { const segments = useMemo(() => splitCodeIntoSegments(code), [code]); const isNoCacheMode = cacheSchedule === NO_CACHE_SCHEDULE; + const filterApi = useMemo(() => ({ kind: 'owned' as const, chatId, storySlug }), [chatId, storySlug]); + const storyFilters = useStoryFilters({ + code: fullCode ?? code, + baselineQueryData: queryData, + api: filterApi, + }); const noCacheQuery = useMemo( () => (isNoCacheMode ? { queryOptions: trpc.story.getLiveQueryData.queryOptions, chatId } : undefined), [isNoCacheMode, chatId], ); + const effectiveQueryData = storyFilters.queryData; + + const useLiveUnfiltered = isNoCacheMode && !storyFilters.hasActiveFilters; + const renderChart = useCallback( (chart: ParsedChartBlock) => { - if (!queryData && !isNoCacheMode) { + if (!effectiveQueryData && !useLiveUnfiltered) { return ; } return ( ); }, - [queryData, isNoCacheMode, noCacheQuery], + [ + effectiveQueryData, + useLiveUnfiltered, + noCacheQuery, + storyFilters.hasActiveFilters, + storyFilters.isFiltering, + queryData, + ], ); const renderTable = useCallback( (table: ParsedTableBlock) => { - if (!queryData && !isNoCacheMode) { + if (!effectiveQueryData && !useLiveUnfiltered) { return ; } return ( ); }, - [queryData, isNoCacheMode, noCacheQuery], + [effectiveQueryData, useLiveUnfiltered, noCacheQuery, storyFilters.hasActiveFilters, storyFilters.isFiltering], ); return (
+ ) diff --git a/apps/frontend/src/components/story-embeds.tsx b/apps/frontend/src/components/story-embeds.tsx index 504938240..afafc7a0d 100644 --- a/apps/frontend/src/components/story-embeds.tsx +++ b/apps/frontend/src/components/story-embeds.tsx @@ -1,13 +1,15 @@ import { Loader2 } from 'lucide-react'; -import { memo } from 'react'; +import { memo, useMemo } from 'react'; import { useQuery } from '@tanstack/react-query'; +import { displayChart } from '@nao/shared/tools'; import type { ParsedChartBlock, ParsedTableBlock } from '@nao/shared/story-segments'; -import type { displayChart } from '@nao/shared/tools'; import { StoryChartEmbedShell } from '@/components/side-panel/story-chart-embed'; import { StoryTableEditControls } from '@/components/side-panel/story-table-embed'; import { ChartDisplay } from '@/components/tool-calls/display-chart'; import { DataTableCard } from '@/components/data-table-card'; +import { alignChartDataToBaselineX } from '@/lib/charts.utils'; +import { cn } from '@/lib/utils'; export type QueryDataMap = Record[]; columns: string[] }>; @@ -35,6 +37,19 @@ function EmbedLoading() { ); } +function EmbedRefreshing({ children, isRefreshing }: { children: React.ReactNode; isRefreshing: boolean }) { + return ( +
+
{children}
+ {isRefreshing && ( +
+ +
+ )} +
+ ); +} + function useLiveQueryData(queryId: string, liveQuery?: LiveQueryConfig) { return useQuery({ queryKey: ['noop', queryId], @@ -48,45 +63,81 @@ function useLiveQueryData(queryId: string, liveQuery?: LiveQueryConfig) { export const StoryChartEmbed = memo(function StoryChartEmbed({ chart, queryData, + baselineQueryData, liveQuery, + hasActiveFilters = false, + isRefreshing = false, }: { chart: ParsedChartBlock; queryData?: QueryDataMap | null; + baselineQueryData?: QueryDataMap | null; liveQuery?: LiveQueryConfig; + hasActiveFilters?: boolean; + isRefreshing?: boolean; }) { const noCacheFetch = useLiveQueryData(chart.queryId, liveQuery); const resolved = liveQuery ? (noCacheFetch.data as { data: Record[]; columns: string[] } | undefined) : queryData?.[chart.queryId]; + const baseline = baselineQueryData?.[chart.queryId]; const resolvedData = resolved?.data; - const resolvedColumns = resolved?.columns ?? []; + const resolvedColumns = resolved?.columns ?? baseline?.columns ?? []; + const keepsXDomain = + hasActiveFilters && + Boolean(baseline?.data?.length) && + !displayChart.isPieChart(chart.chartType as displayChart.ChartType) && + chart.chartType !== 'kpi_card'; + const showRefreshing = isRefreshing || Boolean(liveQuery && noCacheFetch.isFetching); + + const displayData = useMemo(() => { + if (!keepsXDomain || !baseline?.data) { + return resolvedData ?? []; + } + return alignChartDataToBaselineX( + baseline.data, + resolvedData ?? [], + chart.xAxisKey, + chart.series.map((series) => series.data_key), + ); + }, [keepsXDomain, baseline?.data, resolvedData, chart.xAxisKey, chart.series]); if (liveQuery && noCacheFetch.isLoading) { return ; } - if (!resolvedData || resolvedData.length === 0) { + if (!resolved && !keepsXDomain) { return Chart data unavailable; } + if (displayData.length === 0) { + return ( + + {hasActiveFilters ? 'No results match the selected filters' : 'No data to display'} + + ); + } + if (chart.series.length === 0) { return No series configured for chart; } return ( - + + + ); }); @@ -95,33 +146,48 @@ export const StoryTableEmbed = memo(function StoryTableEmbed({ table, queryData, liveQuery, + hasActiveFilters = false, + isRefreshing = false, }: { table: ParsedTableBlock; queryData?: QueryDataMap | null; liveQuery?: LiveQueryConfig; + hasActiveFilters?: boolean; + isRefreshing?: boolean; }) { const noCacheFetch = useLiveQueryData(table.queryId, liveQuery); const resolvedResult = liveQuery ? (noCacheFetch.data as { data: Record[]; columns: string[] } | undefined) : queryData?.[table.queryId]; + const showRefreshing = isRefreshing || Boolean(liveQuery && noCacheFetch.isFetching); if (liveQuery && noCacheFetch.isLoading) { return ; } - if (!resolvedResult?.data) { + if (!resolvedResult) { return Table data unavailable; } + if (!resolvedResult.data || resolvedResult.data.length === 0) { + return ( + + {hasActiveFilters ? 'No results match the selected filters' : 'No data to display'} + + ); + } + const columns = resolvedResult.columns ?? []; return ( - } - /> + + } + /> + ); }); diff --git a/apps/frontend/src/components/story-filter-bar.tsx b/apps/frontend/src/components/story-filter-bar.tsx new file mode 100644 index 000000000..ae39fd4a1 --- /dev/null +++ b/apps/frontend/src/components/story-filter-bar.tsx @@ -0,0 +1,308 @@ +import { AlertTriangle, FilterX, ListFilter, Loader2 } from 'lucide-react'; +import { useCallback, useEffect, useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import type { StoryFilterSelection } from '@nao/shared/sql-template'; +import type { ParsedFilterBlock } from '@nao/shared/story-segments'; + +import type { StoryFilterApi } from '@/hooks/use-story-filters'; +import { FixInChatButton } from '@/components/fix-in-chat-button'; +import { Button } from '@/components/ui/button'; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { Input } from '@/components/ui/input'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; +import { trpc } from '@/main'; + +export function StoryFilterBar({ + filters, + selections, + onSelectionChange, + onClear, + api, + isFiltering, +}: { + filters: ParsedFilterBlock[]; + selections: Record; + onSelectionChange: (filterId: string, selection: StoryFilterSelection) => void; + onClear: () => void; + api?: StoryFilterApi | null; + isFiltering?: boolean; +}) { + const [optionErrors, setOptionErrors] = useState>({}); + + useEffect(() => { + const filterIds = new Set(filters.map((filter) => filter.id)); + setOptionErrors((current) => { + const next = Object.fromEntries(Object.entries(current).filter(([id]) => filterIds.has(id))); + return Object.keys(next).length === Object.keys(current).length ? current : next; + }); + }, [filters]); + + const reportOptionError = useCallback((filterId: string, error: string | null) => { + setOptionErrors((current) => { + if (error) { + if (current[filterId] === error) { + return current; + } + return { ...current, [filterId]: error }; + } + if (!(filterId in current)) { + return current; + } + const next = { ...current }; + delete next[filterId]; + return next; + }); + }, []); + + if (filters.length === 0) { + return null; + } + + const hasActiveFilters = Object.values(selections).some((selection) => + typeof selection === 'string' ? Boolean(selection) : selection.some(Boolean), + ); + + return ( +
+
+ {filters.map((filter) => ( + onSelectionChange(filter.id, selection)} + onOptionError={reportOptionError} + api={api} + /> + ))} + {hasActiveFilters && ( + + )} +
+ +
+ ); +} + +function StoryFilterControl({ + filter, + selection, + onChange, + onOptionError, + api, +}: { + filter: ParsedFilterBlock; + selection: StoryFilterSelection | undefined; + onChange: (selection: StoryFilterSelection) => void; + onOptionError: (filterId: string, error: string | null) => void; + api?: StoryFilterApi | null; +}) { + const { options, error, isLoading } = useFilterOptions(filter, api); + + useEffect(() => { + onOptionError(filter.id, error); + }, [filter.id, error, onOptionError]); + + return ( + + ); +} + +function FilterOptionsLoading() { + return ( +
+ + Loading… +
+ ); +} + +function MultiSelectFilter({ + options, + value, + onChange, +}: { + options: string[]; + value: string[]; + onChange: (value: string[]) => void; +}) { + return ( + + + + + + {options.map((option) => ( + event.preventDefault()} + onCheckedChange={(checked) => + onChange(checked ? [...value, option] : value.filter((candidate) => candidate !== option)) + } + > + {option} + + ))} + + + ); +} + +function RangeFilter({ value, onChange }: { value: string[]; onChange: (value: string[]) => void }) { + return ( +
+ onChange([event.target.value, value[1] ?? ''])} + aria-label='From' + /> + to + onChange([value[0] ?? '', event.target.value])} + aria-label='To' + /> +
+ ); +} + +function FilterOptionsErrorBanner({ + filters, + errors, +}: { + filters: ParsedFilterBlock[]; + errors: Record; +}) { + const errorItems = filters + .map((filter) => { + const message = errors[filter.id]; + return message ? { id: filter.id, label: filter.label, message } : null; + }) + .filter((item): item is { id: string; label: string; message: string } => item !== null); + + if (errorItems.length === 0) { + return null; + } + + const fixMessage = [ + "I'm seeing errors loading story filter options:", + ...errorItems.map((error) => `- Filter "${error.label}" (id: ${error.id}): ${error.message}`), + '', + 'Please fix the filter definition. For table+column select filters with multiple databases, set database_id on the tag. Use a valid table/column identifier for the warehouse dialect (backticks for BigQuery hyphens).', + ].join('\n'); + + return ( +
+
+
+ + Failed to load filter options +
+
    + {errorItems.map((error) => ( +
  • + {error.label}: {error.message} +
    • + ))} +
+
+ +
+ ); +} + +function useFilterOptions( + filter: ParsedFilterBlock, + api?: StoryFilterApi | null, +): { options: string[]; error: string | null; isLoading: boolean } { + const usesHardcodedOptions = Boolean(filter.options?.length); + const needsRemoteOptions = + !usesHardcodedOptions && (filter.filterType === 'select' || filter.filterType === 'multi_select'); + + const ownedQuery = useQuery({ + ...trpc.story.getFilterOptions.queryOptions({ + chatId: api?.kind === 'owned' ? api.chatId : '', + storySlug: api?.kind === 'owned' ? api.storySlug : '', + filterId: filter.id, + }), + enabled: Boolean(api?.kind === 'owned' && needsRemoteOptions), + }); + + const sharedQuery = useQuery({ + ...trpc.storyShare.getFilterOptions.queryOptions({ + shareId: api?.kind === 'shared' ? api.shareId : '', + filterId: filter.id, + }), + enabled: Boolean(api?.kind === 'shared' && needsRemoteOptions), + }); + + if (usesHardcodedOptions) { + return { options: filter.options ?? [], error: null, isLoading: false }; + } + + const query = api?.kind === 'shared' ? sharedQuery : ownedQuery; + return { + options: query.data?.options ?? [], + error: query.error?.message ?? null, + isLoading: needsRemoteOptions && Boolean(api) && query.isPending, + }; +} diff --git a/apps/frontend/src/components/story-rendering.tsx b/apps/frontend/src/components/story-rendering.tsx index 3334ea42d..2b96c3f43 100644 --- a/apps/frontend/src/components/story-rendering.tsx +++ b/apps/frontend/src/components/story-rendering.tsx @@ -43,6 +43,8 @@ export const SegmentList = memo(function SegmentList({ return {renderChart(segment.chart, i)}; case 'table': return {renderTable(segment.table, i)}; + case 'filter': + return null; case 'grid': return ( React.ReactNode; - renderTable: (table: ParsedTableBlock, key: number) => React.ReactNode; + renderChart: (chart: ParsedChartBlock, options: StoryEmbedRenderOptions) => React.ReactNode; + renderTable: (table: ParsedTableBlock, options: StoryEmbedRenderOptions) => React.ReactNode; contentClassName?: string; + baselineQueryData?: QueryDataMap | null; + filterApi?: StoryFilterApi | null; } export function StoryTabbedContent({ @@ -18,6 +32,8 @@ export function StoryTabbedContent({ renderChart, renderTable, contentClassName = 'max-w-5xl mx-auto p-4 md:p-8 flex flex-col gap-4', + baselineQueryData, + filterApi, }: StoryTabbedContentProps) { const tabs = useMemo(() => parseStoryTabs(code), [code]); const [activeIndex, setActiveIndex] = useState(0); @@ -25,6 +41,12 @@ export function StoryTabbedContent({ const activeTabIndex = tabs?.length ? Math.min(activeIndex, tabs.length - 1) : 0; const activeCode = isTabbed && tabs ? tabs[activeTabIndex].innerCode : stripStoryTabsMarkup(code); const segments = useMemo(() => splitCodeIntoSegments(activeCode), [activeCode]); + const storyFilters = useStoryFilters({ + code, + baselineQueryData, + api: filterApi, + enabled: Boolean(filterApi), + }); return (
@@ -38,7 +60,35 @@ export function StoryTabbedContent({ )}
- + + + renderChart(chart, { + queryData: storyFilters.queryData, + baselineQueryData: baselineQueryData ?? null, + hasActiveFilters: storyFilters.hasActiveFilters, + isRefreshing: storyFilters.isFiltering, + key, + }) + } + renderTable={(table, key) => + renderTable(table, { + queryData: storyFilters.queryData, + baselineQueryData: baselineQueryData ?? null, + hasActiveFilters: storyFilters.hasActiveFilters, + isRefreshing: storyFilters.isFiltering, + key, + }) + } + />
diff --git a/apps/frontend/src/components/tool-calls/display-chart.tsx b/apps/frontend/src/components/tool-calls/display-chart.tsx index 043f02e18..dce906c5a 100644 --- a/apps/frontend/src/components/tool-calls/display-chart.tsx +++ b/apps/frontend/src/components/tool-calls/display-chart.tsx @@ -380,6 +380,7 @@ export interface ChartDisplayProps { yAxisMin?: number; yAxisMax?: number; showDataLabels?: boolean; + animate?: boolean; } export const ChartDisplay = memo(function ChartDisplay({ @@ -394,6 +395,7 @@ export const ChartDisplay = memo(function ChartDisplay({ yAxisMin, yAxisMax, showDataLabels, + animate = false, }: ChartDisplayProps) { const dateFormat = useDateFormat(); @@ -495,6 +497,7 @@ export const ChartDisplay = memo(function ChartDisplay({ labelFormatter, showGrid, showDataLabels, + animate, margin: { top: 0, right: 0, bottom: 0, left: 0 }, yAxisMin, yAxisMax, @@ -541,6 +544,7 @@ export const ChartDisplay = memo(function ChartDisplay({ yAxisMin, yAxisMax, showDataLabels, + animate, legendPayload, handleToggleSeriesVisibility, title, diff --git a/apps/frontend/src/components/tool-calls/execute-sql.tsx b/apps/frontend/src/components/tool-calls/execute-sql.tsx index 4da63fc62..5e30b0dfa 100644 --- a/apps/frontend/src/components/tool-calls/execute-sql.tsx +++ b/apps/frontend/src/components/tool-calls/execute-sql.tsx @@ -1,6 +1,6 @@ import { useState } from 'react'; import { useMutation } from '@tanstack/react-query'; -import { ArrowUpRight, Code, Copy, Download, Palette, Table as TableIcon } from 'lucide-react'; +import { ArrowUpRight, Code, Copy, Download, Palette, Pencil, Table as TableIcon } from 'lucide-react'; import { ToolCallWrapper } from './tool-call-wrapper'; import { TableFormatEditDialog } from './display-table-edit-dialog'; import { SqlQueryDisplay } from './sql-query-display'; @@ -24,9 +24,20 @@ export const ExecuteSqlToolCall = ({ const [isFormatOpen, setIsFormatOpen] = useState(false); const { isSettled } = useToolCallContext(); const { open: openSidePanel } = useSidePanel(); - const chatId = useOptionalAgentContext()?.chatId; + const agent = useOptionalAgentContext(); + const chatId = agent?.chatId; + const isEditable = Boolean(agent && !agent.isReadonly && !agent.isRunning && output?.id && input?.sql_query); const logDownload = useMutation(trpc.analyticsEvent.logChatDownload.mutationOptions()); + const openEditor = (editable: boolean) => { + if (state === 'input-streaming' || !output || !input) { + return; + } + openSidePanel( + , + ); + }; + const actions = [ { id: 'results', @@ -82,15 +93,10 @@ export const ExecuteSqlToolCall = ({ title: 'Download results as CSV', }, { - id: 'expand', - label: , - onClick: () => { - if (state === 'input-streaming' || !output || !input) { - return; - } - openSidePanel(); - }, - title: 'Open in side panel', + id: 'edit', + label: , + onClick: () => openEditor(isEditable), + title: isEditable ? 'Edit in side panel' : 'Open in side panel', }, ]; diff --git a/apps/frontend/src/hooks/use-story-filters.ts b/apps/frontend/src/hooks/use-story-filters.ts new file mode 100644 index 000000000..f1f58d854 --- /dev/null +++ b/apps/frontend/src/hooks/use-story-filters.ts @@ -0,0 +1,206 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { keepPreviousData, useQuery } from '@tanstack/react-query'; +import { isFilterSelectionActive } from '@nao/shared/sql-template'; +import { getStoryFiltersFromCode } from '@nao/shared/story-segments'; +import type { StoryFilterSelection, StoryFilterSelections } from '@nao/shared/sql-template'; +import type { ParsedFilterBlock } from '@nao/shared/story-segments'; + +import type { QueryDataMap } from '@/components/story-embeds'; +import { useDebouncedValue } from '@/hooks/use-debounced-value'; +import { trpc } from '@/main'; + +export type StoryFilterApi = { kind: 'owned'; chatId: string; storySlug: string } | { kind: 'shared'; shareId: string }; + +const FILTER_PARAM_PREFIX = 'story_filter_'; + +export function useStoryFilters({ + code, + baselineQueryData, + api, + enabled = true, +}: { + code: string; + baselineQueryData?: QueryDataMap | null; + api?: StoryFilterApi | null; + enabled?: boolean; +}) { + const config = useQuery(trpc.system.getPublicConfig.queryOptions()); + const betaEnabled = config.data?.betaStoryFiltersEnabled === true; + const filtersEnabled = enabled && betaEnabled; + + const filters = useMemo( + () => (filtersEnabled ? dedupeFilters(getStoryFiltersFromCode(code)) : []), + [code, filtersEnabled], + ); + const [selections, setSelections] = useState(() => readSelectionsFromUrl(filters)); + + useEffect(() => { + setSelections(readSelectionsFromUrl(filters)); + }, [filters]); + + useEffect(() => { + writeSelectionsToUrl(selections, filters); + }, [selections, filters]); + + const setSelection = useCallback((filterId: string, selection: StoryFilterSelection) => { + setSelections((current) => { + const next = { ...current }; + if (isEmptySelection(selection)) { + delete next[filterId]; + } else { + next[filterId] = selection; + } + return next; + }); + }, []); + + const clearSelections = useCallback(() => setSelections({}), []); + + const activeSelections = useMemo(() => { + const active: StoryFilterSelections = {}; + for (const filter of filters) { + const selection = selections[filter.id]; + if (isFilterSelectionActive(filter.filterType, selection)) { + active[filter.id] = selection; + } + } + return active; + }, [filters, selections]); + + const hasActiveFilters = Object.keys(activeSelections).length > 0; + const debouncedSelections = useDebouncedValue(activeSelections, 300); + const debouncedHasActive = Object.keys(debouncedSelections).length > 0; + + const ownedQuery = useQuery({ + ...trpc.story.getFilteredQueryData.queryOptions({ + chatId: api?.kind === 'owned' ? api.chatId : '', + storySlug: api?.kind === 'owned' ? api.storySlug : '', + selections: debouncedSelections, + }), + enabled: Boolean(filtersEnabled && api?.kind === 'owned' && debouncedHasActive), + placeholderData: keepPreviousData, + }); + + const sharedQuery = useQuery({ + ...trpc.storyShare.getFilteredQueryData.queryOptions({ + shareId: api?.kind === 'shared' ? api.shareId : '', + selections: debouncedSelections, + }), + enabled: Boolean(filtersEnabled && api?.kind === 'shared' && debouncedHasActive), + placeholderData: keepPreviousData, + }); + + const filteredQueryData = (api?.kind === 'shared' ? sharedQuery.data : ownedQuery.data) as QueryDataMap | undefined; + const isFetching = api?.kind === 'shared' ? sharedQuery.isFetching : ownedQuery.isFetching; + const isSelectionsPending = !areSelectionsEqual(activeSelections, debouncedSelections); + const isFiltering = isSelectionsPending || isFetching; + + return { + filters, + selections, + setSelection, + clearSelections, + hasActiveFilters, + isFiltering, + queryData: debouncedHasActive ? (filteredQueryData ?? baselineQueryData ?? null) : (baselineQueryData ?? null), + }; +} + +function areSelectionsEqual(a: StoryFilterSelections, b: StoryFilterSelections): boolean { + const aKeys = Object.keys(a); + const bKeys = Object.keys(b); + if (aKeys.length !== bKeys.length) { + return false; + } + for (const key of aKeys) { + if (!(key in b)) { + return false; + } + const left = a[key]; + const right = b[key]; + if (Array.isArray(left) && Array.isArray(right)) { + if (left.length !== right.length || left.some((value, index) => value !== right[index])) { + return false; + } + continue; + } + if (left !== right) { + return false; + } + } + return true; +} + +function dedupeFilters(filters: ParsedFilterBlock[]): ParsedFilterBlock[] { + const seen = new Set(); + return filters.filter((filter) => { + if (seen.has(filter.id)) { + return false; + } + seen.add(filter.id); + return true; + }); +} + +function isEmptySelection(selection: StoryFilterSelection): boolean { + return typeof selection === 'string' + ? selection.trim() === '' + : selection.length === 0 || selection.every((value) => value.trim() === ''); +} + +function readSelectionsFromUrl(filters: ParsedFilterBlock[]): StoryFilterSelections { + if (typeof window === 'undefined') { + return {}; + } + + const params = new URLSearchParams(window.location.search); + const selections: StoryFilterSelections = {}; + for (const filter of filters) { + const raw = params.get(`${FILTER_PARAM_PREFIX}${filter.id}`); + if (raw === null) { + continue; + } + if (filter.filterType === 'select' || filter.filterType === 'search') { + selections[filter.id] = raw; + continue; + } + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed) && parsed.every((item) => typeof item === 'string')) { + selections[filter.id] = parsed; + } + } catch { + // ignore malformed URL values + } + } + return selections; +} + +function writeSelectionsToUrl(selections: StoryFilterSelections, filters: ParsedFilterBlock[]) { + if (typeof window === 'undefined') { + return; + } + + const url = new URL(window.location.href); + for (const key of [...url.searchParams.keys()]) { + if (key.startsWith(FILTER_PARAM_PREFIX)) { + url.searchParams.delete(key); + } + } + + const filterById = new Map(filters.map((filter) => [filter.id, filter])); + for (const [filterId, selection] of Object.entries(selections)) { + const filter = filterById.get(filterId); + if (!filter || isEmptySelection(selection)) { + continue; + } + const key = `${FILTER_PARAM_PREFIX}${filterId}`; + if (filter.filterType === 'select' || filter.filterType === 'search') { + url.searchParams.set(key, selection as string); + } else { + url.searchParams.set(key, JSON.stringify(selection)); + } + } + + window.history.replaceState(window.history.state, '', `${url.pathname}${url.search}${url.hash}`); +} diff --git a/apps/frontend/src/lib/charts.utils.test.ts b/apps/frontend/src/lib/charts.utils.test.ts index aba0bd568..a0b146d41 100644 --- a/apps/frontend/src/lib/charts.utils.test.ts +++ b/apps/frontend/src/lib/charts.utils.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { resolvePieTooltipLabel } from './charts.utils'; +import { alignChartDataToBaselineX, resolvePieTooltipLabel } from './charts.utils'; describe('resolvePieTooltipLabel', () => { it('returns the slice category name from the payload', () => { @@ -24,3 +24,20 @@ describe('resolvePieTooltipLabel', () => { expect(resolvePieTooltipLabel(undefined)).toBe(''); }); }); + +describe('alignChartDataToBaselineX', () => { + it('keeps baseline x categories and zero-fills missing filtered rows', () => { + const baseline = [ + { month: 'Jan', revenue: 10 }, + { month: 'Feb', revenue: 20 }, + { month: 'Mar', revenue: 30 }, + ]; + const filtered = [{ month: 'Feb', revenue: 8 }]; + + expect(alignChartDataToBaselineX(baseline, filtered, 'month', ['revenue'])).toEqual([ + { month: 'Jan', revenue: 0 }, + { month: 'Feb', revenue: 8 }, + { month: 'Mar', revenue: 0 }, + ]); + }); +}); diff --git a/apps/frontend/src/lib/charts.utils.ts b/apps/frontend/src/lib/charts.utils.ts index abded0f20..a8adf96ff 100644 --- a/apps/frontend/src/lib/charts.utils.ts +++ b/apps/frontend/src/lib/charts.utils.ts @@ -112,6 +112,51 @@ export function resolveDataKey(data: Record[], key: string): st return match ?? key; } +/** + * Keeps the baseline x-axis categories when filtered SQL drops rows, zero-filling + * series values for missing categories so bars/lines animate in place instead of + * collapsing the axis. + */ +export function alignChartDataToBaselineX( + baseline: Record[], + filtered: Record[], + xAxisKey: string, + seriesKeys: string[], +): Record[] { + if (baseline.length === 0) { + return filtered; + } + + const baselineXKey = resolveDataKey(baseline, xAxisKey); + const filteredXKey = resolveDataKey(filtered, xAxisKey); + const resolvedSeriesKeys = seriesKeys.map((key) => resolveDataKey(baseline, key)); + + const filteredByX = new Map>(); + for (const row of filtered) { + filteredByX.set(String(row[filteredXKey] ?? row[baselineXKey]), row); + } + + return baseline.map((baseRow) => { + const xValue = String(baseRow[baselineXKey]); + const match = filteredByX.get(xValue); + if (!match) { + const emptyRow = { ...baseRow }; + for (const key of resolvedSeriesKeys) { + emptyRow[key] = 0; + } + return emptyRow; + } + + const next = { ...baseRow }; + for (const key of resolvedSeriesKeys) { + const matchKey = resolveDataKey([match], key); + const value = match[matchKey] ?? match[key]; + next[key] = typeof value === 'number' || value === null ? value : value === undefined ? 0 : value; + } + return next; + }); +} + /** Counts the successfully rendered `display_chart` tool calls across a conversation. */ export function countDisplayCharts(messages: UIMessage[]): number { let count = 0; diff --git a/apps/frontend/src/lib/execute-sql-messages.ts b/apps/frontend/src/lib/execute-sql-messages.ts new file mode 100644 index 000000000..83470817b --- /dev/null +++ b/apps/frontend/src/lib/execute-sql-messages.ts @@ -0,0 +1,47 @@ +import type { executeSql } from '@nao/shared/tools'; +import type { UIMessage, UIToolPart } from '@nao/backend/chat'; + +export function applyExecuteSqlInputToMessages( + messages: UIMessage[], + queryId: string, + input: executeSql.Input, +): UIMessage[] { + return messages.map((message) => { + let changed = false; + const parts = message.parts.map((part) => { + if (part.type !== 'tool-execute_sql') { + return part; + } + const toolPart = part as UIToolPart<'execute_sql'>; + if (toolPart.output?.id !== queryId) { + return part; + } + changed = true; + return { ...toolPart, input } as typeof part; + }); + return changed ? { ...message, parts } : message; + }); +} + +export function applyExecuteSqlResultToMessages( + messages: UIMessage[], + queryId: string, + input: executeSql.Input, + output: executeSql.Output, +): UIMessage[] { + return messages.map((message) => { + let changed = false; + const parts = message.parts.map((part) => { + if (part.type !== 'tool-execute_sql') { + return part; + } + const toolPart = part as UIToolPart<'execute_sql'>; + if (toolPart.output?.id !== queryId) { + return part; + } + changed = true; + return { ...toolPart, input, output } as typeof part; + }); + return changed ? { ...message, parts } : message; + }); +} diff --git a/apps/frontend/src/routes/_sidebar-layout.stories.preview.$chatId.$storySlug.tsx b/apps/frontend/src/routes/_sidebar-layout.stories.preview.$chatId.$storySlug.tsx index 31b8fe246..ee51066cb 100644 --- a/apps/frontend/src/routes/_sidebar-layout.stories.preview.$chatId.$storySlug.tsx +++ b/apps/frontend/src/routes/_sidebar-layout.stories.preview.$chatId.$storySlug.tsx @@ -155,6 +155,7 @@ function StoryPreviewPage() { code={editor.code} queryData={story.queryData as QueryDataMap | null} chatId={chatId} + storySlug={storySlug} cacheSchedule={story.cacheSchedule} />, )} @@ -223,14 +224,17 @@ function PreviewContent({ code, queryData, chatId, + storySlug, cacheSchedule, }: { code: string; queryData: QueryDataMap | null; chatId: string; + storySlug: string; cacheSchedule?: string | null; }) { const isNoCacheMode = cacheSchedule === 'no-cache'; + const filterApi = useMemo(() => ({ kind: 'owned' as const, chatId, storySlug }), [chatId, storySlug]); const noCacheQuery = useMemo( () => (isNoCacheMode ? { queryOptions: trpc.story.getLiveQueryData.queryOptions, chatId } : undefined), @@ -238,18 +242,59 @@ function PreviewContent({ ); const renderChart = useCallback( - (chart: ParsedChartBlock) => ( - + ( + chart: ParsedChartBlock, + { + queryData: data, + baselineQueryData, + hasActiveFilters, + isRefreshing, + }: { + queryData: QueryDataMap | null; + baselineQueryData: QueryDataMap | null; + hasActiveFilters: boolean; + isRefreshing: boolean; + }, + ) => ( + ), - [isNoCacheMode, queryData, noCacheQuery], + [isNoCacheMode, noCacheQuery], ); const renderTable = useCallback( - (table: ParsedTableBlock) => ( - + ( + table: ParsedTableBlock, + { + queryData: data, + hasActiveFilters, + isRefreshing, + }: { queryData: QueryDataMap | null; hasActiveFilters: boolean; isRefreshing: boolean }, + ) => ( + ), - [isNoCacheMode, queryData, noCacheQuery], + [isNoCacheMode, noCacheQuery], ); - return ; + return ( + + ); } diff --git a/apps/frontend/src/routes/_sidebar-layout.stories.shared.$shareId.tsx b/apps/frontend/src/routes/_sidebar-layout.stories.shared.$shareId.tsx index c5431df0b..6aeddd61a 100644 --- a/apps/frontend/src/routes/_sidebar-layout.stories.shared.$shareId.tsx +++ b/apps/frontend/src/routes/_sidebar-layout.stories.shared.$shareId.tsx @@ -168,6 +168,7 @@ function SharedStoryPage() { code={editor.code} queryData={story.queryData as QueryDataMap | null} chatId={story.chatId!} + shareId={shareId} cacheSchedule={story.cacheSchedule} /> } @@ -281,14 +282,17 @@ function SharedStoryContent({ code, queryData, chatId, + shareId, cacheSchedule, }: { code: string; queryData: QueryDataMap | null; chatId: string; + shareId: string; cacheSchedule?: string | null; }) { const isNoCacheMode = cacheSchedule === 'no-cache'; + const filterApi = useMemo(() => ({ kind: 'shared' as const, shareId }), [shareId]); const noCacheQuery = useMemo( () => (isNoCacheMode ? { queryOptions: trpc.storyShare.getLiveQueryData.queryOptions, chatId } : undefined), @@ -296,22 +300,61 @@ function SharedStoryContent({ ); const renderChart = useCallback( - (chart: ParsedChartBlock) => ( - + ( + chart: ParsedChartBlock, + { + queryData: data, + baselineQueryData, + hasActiveFilters, + isRefreshing, + }: { + queryData: QueryDataMap | null; + baselineQueryData: QueryDataMap | null; + hasActiveFilters: boolean; + isRefreshing: boolean; + }, + ) => ( + ), - [isNoCacheMode, queryData, noCacheQuery], + [isNoCacheMode, noCacheQuery], ); const renderTable = useCallback( - (table: ParsedTableBlock) => ( - + ( + table: ParsedTableBlock, + { + queryData: data, + hasActiveFilters, + isRefreshing, + }: { queryData: QueryDataMap | null; hasActiveFilters: boolean; isRefreshing: boolean }, + ) => ( + ), - [isNoCacheMode, queryData, noCacheQuery], + [isNoCacheMode, noCacheQuery], ); return (
- +
); } diff --git a/apps/frontend/src/routes/_sidebar-layout.stories.standalone.$storyId.tsx b/apps/frontend/src/routes/_sidebar-layout.stories.standalone.$storyId.tsx index 54a4c18da..655107db4 100644 --- a/apps/frontend/src/routes/_sidebar-layout.stories.standalone.$storyId.tsx +++ b/apps/frontend/src/routes/_sidebar-layout.stories.standalone.$storyId.tsx @@ -108,7 +108,12 @@ function StandaloneStoryPage() { /> - + - + } /> @@ -246,16 +256,71 @@ function StandaloneEditableStory({ ); } -function StandaloneStoryContent({ code, queryData }: { code: string; queryData: QueryDataMap | null }) { +function StandaloneStoryContent({ + code, + queryData, + chatId, + storySlug, +}: { + code: string; + queryData: QueryDataMap | null; + chatId?: string | null; + storySlug?: string; +}) { + const filterApi = chatId && storySlug ? { kind: 'owned' as const, chatId, storySlug } : null; + const renderChart = useCallback( - (chart: ParsedChartBlock) => , - [queryData], + ( + chart: ParsedChartBlock, + { + queryData: data, + baselineQueryData, + hasActiveFilters, + isRefreshing, + }: { + queryData: QueryDataMap | null; + baselineQueryData: QueryDataMap | null; + hasActiveFilters: boolean; + isRefreshing: boolean; + }, + ) => ( + + ), + [], ); const renderTable = useCallback( - (table: ParsedTableBlock) => , - [queryData], + ( + table: ParsedTableBlock, + { + queryData: data, + hasActiveFilters, + isRefreshing, + }: { queryData: QueryDataMap | null; hasActiveFilters: boolean; isRefreshing: boolean }, + ) => ( + + ), + [], ); - return ; + return ( + + ); } diff --git a/apps/shared/package.json b/apps/shared/package.json index bc7eaff45..0aab17ad8 100644 --- a/apps/shared/package.json +++ b/apps/shared/package.json @@ -12,7 +12,8 @@ "./story-segments": "./src/story-segments.ts", "./story-table-utils": "./src/story-table-utils.ts", "./story-tabs": "./src/story-tabs.ts", - "./story-validation": "./src/story-validation.ts" + "./story-validation": "./src/story-validation.ts", + "./sql-template": "./src/sql-template.ts" }, "scripts": { "test": "vitest run", diff --git a/apps/shared/src/chart-block.ts b/apps/shared/src/chart-block.ts index b9638e465..a0361359f 100644 --- a/apps/shared/src/chart-block.ts +++ b/apps/shared/src/chart-block.ts @@ -1,3 +1,4 @@ +import type { StoryFilterType } from './sql-template'; import type * as displayChart from './tools/display-chart'; export type { McpChartEmbedStoredConfig } from './mcp-embed'; @@ -47,3 +48,22 @@ export function buildStoryTableBlock(input: StoryTableBlockInput): string { : ''; return `
`; } + +export interface StoryFilterBlockInput { + id: string; + column?: string; + label?: string; + type: StoryFilterType; + table?: string; + database_id?: string; + options?: string[]; +} + +export function buildStoryFilterBlock(input: StoryFilterBlockInput): string { + const labelAttr = input.label ? ` label="${escapeDoubleQuotedStoryAttr(input.label)}"` : ''; + const columnAttr = input.column ? ` column="${escapeDoubleQuotedStoryAttr(input.column)}"` : ''; + const tableAttr = input.table ? ` table="${escapeDoubleQuotedStoryAttr(input.table)}"` : ''; + const databaseIdAttr = input.database_id ? ` database_id="${escapeDoubleQuotedStoryAttr(input.database_id)}"` : ''; + const optionsAttr = input.options ? ` options='${escapeSingleQuotedStoryAttr(JSON.stringify(input.options))}'` : ''; + return ``; +} diff --git a/apps/shared/src/chart-builder.tsx b/apps/shared/src/chart-builder.tsx index b5d052018..78d236744 100644 --- a/apps/shared/src/chart-builder.tsx +++ b/apps/shared/src/chart-builder.tsx @@ -165,8 +165,12 @@ export interface BuildChartProps { /** Chart background color, used as the separator between stacked segments. Pass a concrete color on surfaces where CSS vars do not resolve (backend PNG/HTML export). */ backgroundColor?: string; showDataLabels?: boolean; + /** When true, Recharts animates series as data changes (e.g. story filters). */ + animate?: boolean; } +const CHART_ANIMATION_DURATION_MS = 400; + /** * Builds a Recharts element tree from a display_chart tool config. * @@ -475,7 +479,8 @@ function buildBarChart(props: ResolvedProps) { stackId={isStacked ? 'stack' : undefined} radius={isStacked ? undefined : [4, 4, 4, 4]} shape={isStacked ? renderStackedBarShape(seriesKeys, s.data_key, separatorColor) : undefined} - isAnimationActive={false} + isAnimationActive={Boolean(props.animate)} + animationDuration={CHART_ANIMATION_DURATION_MS} > {showDataLabels && !isStacked && ( @@ -589,7 +594,8 @@ function buildAreaChart(props: ResolvedProps) { stroke={colorFor(s.data_key, i)} fill={`url(#grad-${i})`} stackId={isStacked ? 'stack' : undefined} - isAnimationActive={false} + isAnimationActive={Boolean(props.animate)} + animationDuration={CHART_ANIMATION_DURATION_MS} > {showDataLabels && !isStacked && } {stackTotalLabel && i === stackTotalLabelIndex && } @@ -632,7 +638,8 @@ function buildScatterChart(props: ResolvedProps) { key={s.data_key} dataKey={s.data_key} fill={colorFor(s.data_key, i)} - isAnimationActive={false} + isAnimationActive={Boolean(props.animate)} + animationDuration={CHART_ANIMATION_DURATION_MS} /> ))} @@ -655,7 +662,8 @@ function buildRadarChart(props: ResolvedProps) { stroke={colorFor(s.data_key, i)} fill={colorFor(s.data_key, i)} fillOpacity={0.3} - isAnimationActive={false} + isAnimationActive={Boolean(props.animate)} + animationDuration={CHART_ANIMATION_DURATION_MS} /> ))} @@ -687,7 +695,8 @@ function buildPieChart(props: ResolvedProps) { labelLine={false} stroke={backgroundColor} strokeWidth={1} - isAnimationActive={false} + isAnimationActive={Boolean(props.animate)} + animationDuration={CHART_ANIMATION_DURATION_MS} /> {children} diff --git a/apps/shared/src/index.ts b/apps/shared/src/index.ts index 8760ed94e..3b4ba7f54 100644 --- a/apps/shared/src/index.ts +++ b/apps/shared/src/index.ts @@ -7,5 +7,6 @@ export * from './mcp'; export * from './mcp-embed'; export * from './mentions'; export * from './pluralize'; +export * from './sql-template'; export * from './types'; export * from './utils'; diff --git a/apps/shared/src/sql-template.ts b/apps/shared/src/sql-template.ts new file mode 100644 index 000000000..e2f7f6ef5 --- /dev/null +++ b/apps/shared/src/sql-template.ts @@ -0,0 +1,121 @@ +export const STORY_FILTER_TYPES = ['select', 'multi_select', 'search', 'date_range'] as const; + +export type StoryFilterType = (typeof STORY_FILTER_TYPES)[number]; + +export type StoryFilterSelection = string | string[]; + +export type StoryFilterSelections = Record; + +export type StoryFilterTypeById = Record; + +const FILTER_BLOCK_REGEX = /\{%\s*filter\s+([A-Za-z_][A-Za-z0-9_]*)\s*%\}([\s\S]*?)\{%\s*endfilter\s*%\}/g; +const FILTER_PLACEHOLDER_REGEX = /\{\{\s*filters\.([A-Za-z_][A-Za-z0-9_]*)\.sql\s*\}\}/g; + +export function stripSqlFilterBlocks(sql: string): string { + return sql + .replace(FILTER_BLOCK_REGEX, '') + .replace(/[ \t]+\n/g, '\n') + .replace(/\n{2,}/g, '\n'); +} + +export function extractSqlFilterIds(sql: string): string[] { + const ids: string[] = []; + const seen = new Set(); + const regex = new RegExp(FILTER_BLOCK_REGEX.source, 'g'); + let match: RegExpExecArray | null; + while ((match = regex.exec(sql)) !== null) { + const id = match[1]; + if (!seen.has(id)) { + seen.add(id); + ids.push(id); + } + } + return ids; +} + +export function renderSqlTemplate(sql: string, selections: StoryFilterSelections, types: StoryFilterTypeById): string { + const rendered = sql.replace(FILTER_BLOCK_REGEX, (_full, filterId: string, inner: string) => { + const filterType = types[filterId]; + if (!filterType) { + throw new Error(`Unknown story filter "${filterId}" referenced in SQL template.`); + } + + const selection = selections[filterId]; + const sqlValue = renderFilterSqlValue(filterType, selection); + if (sqlValue === null) { + return ''; + } + + return inner.replace(FILTER_PLACEHOLDER_REGEX, (_placeholder, placeholderId: string) => { + if (placeholderId !== filterId) { + throw new Error( + `Filter block "${filterId}" contains placeholder for unrelated filter "${placeholderId}".`, + ); + } + return sqlValue; + }); + }); + + const leftover = rendered.match(/\{\{\s*filters\.([A-Za-z_][A-Za-z0-9_]*)\.sql\s*\}\}/); + if (leftover) { + throw new Error( + `SQL template placeholder {{ filters.${leftover[1]}.sql }} must appear inside a {% filter %} block.`, + ); + } + + return rendered.replace(/[ \t]+\n/g, '\n').replace(/\n{2,}/g, '\n'); +} + +export function renderFilterSqlValue( + type: StoryFilterType, + selection: StoryFilterSelection | undefined, +): string | null { + if (!isFilterSelectionActive(type, selection)) { + return null; + } + + switch (type) { + case 'select': + return quoteSqlString(selection as string); + case 'multi_select': + return (selection as string[]).filter(Boolean).map(quoteSqlString).join(', '); + case 'search': + return quoteSqlString(`%${escapeLikePattern(selection as string)}%`); + case 'date_range': { + const [from, to] = selection as string[]; + return `${quoteSqlString(from)} AND ${quoteSqlString(to)}`; + } + } +} + +export function isFilterSelectionActive(type: StoryFilterType, selection: StoryFilterSelection | undefined): boolean { + if (selection === undefined) { + return false; + } + + switch (type) { + case 'select': + case 'search': + return typeof selection === 'string' && selection.trim() !== ''; + case 'multi_select': + return Array.isArray(selection) && selection.some((value) => value.trim() !== ''); + case 'date_range': + return ( + Array.isArray(selection) && + selection.length >= 2 && + Boolean(selection[0]?.trim()) && + Boolean(selection[1]?.trim()) + ); + } +} + +function quoteSqlString(value: string): string { + if (value.includes('\0')) { + throw new Error('Filter values cannot contain null bytes.'); + } + return `'${value.replace(/'/g, "''")}'`; +} + +function escapeLikePattern(value: string): string { + return value.replace(/([%_\\])/g, '\\$1'); +} diff --git a/apps/shared/src/story-segments.ts b/apps/shared/src/story-segments.ts index 0fbeaf2ae..06566a234 100644 --- a/apps/shared/src/story-segments.ts +++ b/apps/shared/src/story-segments.ts @@ -1,5 +1,6 @@ import { buildStoryTableBlock } from './chart-block'; import { type ColumnConditionalFormats, sanitizeConditionalFormats } from './conditional-formatting'; +import { STORY_FILTER_TYPES, type StoryFilterType } from './sql-template'; export interface ParsedChartBlock { queryId: string; @@ -23,10 +24,25 @@ export interface ParsedTableBlock { rawTag?: string; } +export interface ParsedFilterBlock { + id: string; + column?: string; + label: string; + filterType: StoryFilterType; + table?: string; + /** Database to use when loading options via SELECT DISTINCT on `table.column`. */ + databaseId?: string; + /** Hardcoded dropdown values; when set, options are not loaded from `table.column`. */ + options?: string[]; + /** The original `` tag this block was parsed from, when available. */ + rawTag?: string; +} + export type Segment = | { type: 'markdown'; content: string } | { type: 'chart'; chart: ParsedChartBlock } | { type: 'table'; table: ParsedTableBlock } + | { type: 'filter'; filter: ParsedFilterBlock } | { type: 'grid'; cols: number; children: Segment[] }; export const TAG_ATTRS = String.raw`(?:[^>"']|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')*?`; @@ -41,7 +57,7 @@ export function tableTagRegex(flags = ''): RegExp { export function storyBlockRegex(): RegExp { return new RegExp( - String.raw`([\s\S]*?)<\/grid>||`, + String.raw`([\s\S]*?)<\/grid>|||`, 'g', ); } @@ -109,6 +125,59 @@ export function parseTableBlock(attrString: string): ParsedTableBlock | null { }; } +export function parseFilterBlock(attrString: string): ParsedFilterBlock | null { + const attrs = parseChartAttributes(attrString); + if (!attrs.id || !STORY_FILTER_TYPES.includes(attrs.type as StoryFilterType)) { + return null; + } + + const filterType = attrs.type as StoryFilterType; + const options = parseStringArrayAttr(attrs.options); + const needsOptionsSource = filterType === 'select' || filterType === 'multi_select'; + const hasHardcodedOptions = Boolean(options?.length); + const hasTableSource = Boolean(attrs.table && attrs.column); + if (needsOptionsSource && !hasHardcodedOptions && !hasTableSource) { + return null; + } + + return { + id: attrs.id, + ...(attrs.column && { column: attrs.column }), + label: attrs.label || attrs.column || attrs.id, + filterType, + ...(attrs.table && { table: attrs.table }), + ...(attrs.database_id && { databaseId: attrs.database_id }), + ...(hasHardcodedOptions && { options }), + }; +} + +function parseStringArrayAttr(value: string | undefined): string[] | undefined { + if (!value) { + return undefined; + } + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) && parsed.every((item) => typeof item === 'string') ? parsed : undefined; + } catch { + return undefined; + } +} + +export function getStoryFiltersFromCode(code: string): ParsedFilterBlock[] { + const filters: ParsedFilterBlock[] = []; + const filterRegex = new RegExp(String.raw``, 'g'); + let match: RegExpExecArray | null; + + while ((match = filterRegex.exec(code)) !== null) { + const filter = parseFilterBlock(match[1]); + if (filter) { + filters.push({ ...filter, rawTag: match[0] }); + } + } + + return filters; +} + function parseConditionalFormats(value: string | undefined): ColumnConditionalFormats | undefined { if (!value) { return undefined; @@ -245,6 +314,11 @@ export function splitCodeIntoSegments(code: string): Segment[] { if (table) { segments.push({ type: 'table', table: { ...table, rawTag: match[0] } }); } + } else if (match[5] !== undefined) { + const filter = parseFilterBlock(match[5]); + if (filter) { + segments.push({ type: 'filter', filter: { ...filter, rawTag: match[0] } }); + } } lastIndex = match.index + match[0].length; diff --git a/apps/shared/src/story-validation.ts b/apps/shared/src/story-validation.ts index aadd20d73..88b1a8085 100644 --- a/apps/shared/src/story-validation.ts +++ b/apps/shared/src/story-validation.ts @@ -1,3 +1,4 @@ +import { STORY_FILTER_TYPES } from './sql-template'; import { parseChartAttributes, parseSeriesJsonArray, TAG_ATTRS } from './story-segments'; export interface StoryValidationError { @@ -9,6 +10,7 @@ export interface StoryValidationError { const REQUIRED_CHART_ATTRS = ['query_id', 'chart_type', 'x_axis_key'] as const; const REQUIRED_TABLE_ATTRS = ['query_id'] as const; +const REQUIRED_FILTER_ATTRS = ['id', 'type'] as const; const VALID_CHART_TYPES = new Set([ 'bar', @@ -40,6 +42,7 @@ export function validateStoryCode(code: string): StoryValidationError[] { errors.push(...validateGridBlocks(code)); errors.push(...validateChartBlocks(code)); errors.push(...validateTableBlocks(code)); + errors.push(...validateFilterBlocks(code)); errors.push(...validateTabsBlocks(code)); errors.push(...validateUnterminatedTags(code)); @@ -259,6 +262,98 @@ function validateTableBlocks(code: string): StoryValidationError[] { return errors; } +function validateFilterBlocks(code: string): StoryValidationError[] { + const errors: StoryValidationError[] = []; + const filterRegex = new RegExp(String.raw``, 'g'); + const filterIds = new Set(); + let match: RegExpExecArray | null; + + while ((match = filterRegex.exec(code)) !== null) { + const [fullMatch, attrString, slash] = match; + const position = getPosition(code, match.index); + const attrs = parseChartAttributes(attrString ?? ''); + + if (slash !== '/') { + errors.push({ + message: ' tag must be self-closing — use "/>" instead of ">".', + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + + const missing = REQUIRED_FILTER_ATTRS.filter((attr) => !attrs[attr]); + if (missing.length > 0) { + errors.push({ + message: `Filter is missing required attribute${missing.length === 1 ? '' : 's'}: ${missing.join(', ')}.`, + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + + if (attrs.type && !STORY_FILTER_TYPES.includes(attrs.type as (typeof STORY_FILTER_TYPES)[number])) { + errors.push({ + message: `Invalid filter type "${attrs.type}". Valid types: ${STORY_FILTER_TYPES.join(', ')}.`, + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + + const options = parseFilterOptionsAttr(attrs.options); + if (attrs.options !== undefined && options === undefined) { + errors.push({ + message: 'Filter `options` attribute must be a valid JSON array of strings.', + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + + const needsOptionsSource = attrs.type === 'select' || attrs.type === 'multi_select'; + if (needsOptionsSource) { + const hasHardcodedOptions = Boolean(options?.length); + const hasTableSource = Boolean(attrs.table && attrs.column); + if (!hasHardcodedOptions && !hasTableSource) { + errors.push({ + message: + 'Select filters require either `options=\'["a","b"]\'` or both `table` and `column` attributes.', + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + } + + if (attrs.id && filterIds.has(attrs.id)) { + errors.push({ + message: `Filter id "${attrs.id}" must be unique within the story.`, + line: position.line, + column: position.column, + length: fullMatch.length, + }); + } + if (attrs.id) { + filterIds.add(attrs.id); + } + } + + return errors; +} + +function parseFilterOptionsAttr(value: string | undefined): string[] | undefined { + if (!value) { + return undefined; + } + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) && parsed.every((item) => typeof item === 'string') ? parsed : undefined; + } catch { + return undefined; + } +} + function isMarkdownTable(code: string, index: number): boolean { const lineStart = code.lastIndexOf('\n', index - 1) + 1; const linePrefix = code.slice(lineStart, index); @@ -334,7 +429,7 @@ function findMatchingClose(code: string, startIndex: number): number { function validateUnterminatedTags(code: string): StoryValidationError[] { const errors: StoryValidationError[] = []; - const tagRegex = /<(chart|table)\b[^>]*$/gm; + const tagRegex = /<(chart|table|filter)\b[^>]*$/gm; let match: RegExpExecArray | null; while ((match = tagRegex.exec(code)) !== null) { diff --git a/apps/shared/src/tools/execute-sql.ts b/apps/shared/src/tools/execute-sql.ts index ab96a7e2c..95d28b4ab 100644 --- a/apps/shared/src/tools/execute-sql.ts +++ b/apps/shared/src/tools/execute-sql.ts @@ -9,6 +9,9 @@ export const InputSchema = z.object({ .optional() .describe('The database name/id to use. Required if multiple databases are configured.'), name: z.string().optional().describe('A descriptive name for the query that will be used to show in the UI.'), + query_id: QueryIdSchema.optional().describe( + 'When set, replace the SQL of this existing query in-place (same query_id) instead of creating a new one. Prefer this when adding story filter templates so chart/table tags keep working.', + ), }); export const OutputSchema = z.object({ diff --git a/apps/shared/tests/sql-template.test.ts b/apps/shared/tests/sql-template.test.ts new file mode 100644 index 000000000..72d773075 --- /dev/null +++ b/apps/shared/tests/sql-template.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; + +import { + extractSqlFilterIds, + renderFilterSqlValue, + renderSqlTemplate, + stripSqlFilterBlocks, +} from '../src/sql-template'; + +const SAMPLE_SQL = ` +SELECT SUM(revenue) AS revenue, country +FROM orders +WHERE 1 = 1 +{% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %} +{% filter q %} AND customer_name ILIKE {{ filters.q.sql }} {% endfilter %} +{% filter period %} AND order_date BETWEEN {{ filters.period.sql }} {% endfilter %} +GROUP BY country +`.trim(); + +describe('stripSqlFilterBlocks', () => { + it('removes all filter blocks so SQL runs without story filters', () => { + expect(stripSqlFilterBlocks(SAMPLE_SQL)).toBe( + ` +SELECT SUM(revenue) AS revenue, country +FROM orders +WHERE 1 = 1 +GROUP BY country +`.trim(), + ); + }); +}); + +describe('extractSqlFilterIds', () => { + it('returns unique filter ids in order of appearance', () => { + expect(extractSqlFilterIds(SAMPLE_SQL)).toEqual(['country', 'q', 'period']); + }); +}); + +describe('renderFilterSqlValue', () => { + it('renders select / multi_select / search / date_range fragments', () => { + expect(renderFilterSqlValue('select', 'US')).toBe("'US'"); + expect(renderFilterSqlValue('multi_select', ['US', "O'Reilly"])).toBe("'US', 'O''Reilly'"); + expect(renderFilterSqlValue('search', 'acme%')).toBe("'%acme\\%%'"); + expect(renderFilterSqlValue('date_range', ['2024-01-01', '2024-12-31'])).toBe("'2024-01-01' AND '2024-12-31'"); + }); + + it('returns null for inactive selections', () => { + expect(renderFilterSqlValue('select', '')).toBeNull(); + expect(renderFilterSqlValue('multi_select', [])).toBeNull(); + expect(renderFilterSqlValue('date_range', ['2024-01-01', ''])).toBeNull(); + }); +}); + +describe('renderSqlTemplate', () => { + const types = { + country: 'multi_select', + q: 'search', + period: 'date_range', + } as const; + + it('keeps only active filter blocks and interpolates .sql placeholders', () => { + const rendered = renderSqlTemplate( + SAMPLE_SQL, + { + country: ['US', 'FR'], + q: 'acme', + }, + types, + ); + + expect(rendered).toContain("AND country IN ('US', 'FR')"); + expect(rendered).toContain("AND customer_name ILIKE '%acme%'"); + expect(rendered).not.toContain('order_date BETWEEN'); + expect(rendered).not.toContain('{% filter'); + expect(rendered).not.toContain('{{ filters'); + }); + + it('strips all blocks when no selections are active', () => { + expect(renderSqlTemplate(SAMPLE_SQL, {}, types)).toBe(stripSqlFilterBlocks(SAMPLE_SQL)); + }); + + it('rejects unknown filter ids and misplaced placeholders', () => { + expect(() => renderSqlTemplate(SAMPLE_SQL, { country: ['US'] }, {})).toThrow(/Unknown story filter/); + expect(() => + renderSqlTemplate('SELECT 1 WHERE x = {{ filters.country.sql }}', { country: 'US' }, { country: 'select' }), + ).toThrow(/must appear inside/); + }); +}); diff --git a/apps/shared/tests/story-segments.test.ts b/apps/shared/tests/story-segments.test.ts index fb0f5ca5d..27a717a75 100644 --- a/apps/shared/tests/story-segments.test.ts +++ b/apps/shared/tests/story-segments.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { buildStoryChartBlock } from '../src/chart-block'; -import { splitCodeIntoSegments } from '../src/story-segments'; +import { buildStoryChartBlock, buildStoryFilterBlock } from '../src/chart-block'; +import { getStoryFiltersFromCode, splitCodeIntoSegments } from '../src/story-segments'; function chartOf(code: string) { const segment = splitCodeIntoSegments(code).find((s) => s.type === 'chart'); @@ -121,3 +121,72 @@ describe('splitCodeIntoSegments slash-in-attribute handling', () => { expect(chart?.series).toEqual([{ data_key: 'rev', color: 'var(--chart-1)', label: undefined }]); }); }); + +describe('story filter tags', () => { + it('parses filter tags from code and as segments', () => { + const code = [ + buildStoryFilterBlock({ + id: 'country', + column: 'country', + label: 'Country', + type: 'multi_select', + table: 'orders', + }), + '', + ].join('\n'); + + expect(getStoryFiltersFromCode(code)).toEqual([ + { + id: 'country', + column: 'country', + label: 'Country', + filterType: 'multi_select', + table: 'orders', + rawTag: expect.any(String), + }, + ]); + expect(splitCodeIntoSegments(code).some((segment) => segment.type === 'filter')).toBe(true); + }); + + it('parses hardcoded options without table/column', () => { + const code = buildStoryFilterBlock({ + id: 'country', + label: 'Country', + type: 'select', + options: ['US', 'FR'], + }); + + expect(getStoryFiltersFromCode(code)).toEqual([ + { + id: 'country', + label: 'Country', + filterType: 'select', + options: ['US', 'FR'], + rawTag: expect.any(String), + }, + ]); + }); + + it('parses database_id for table-backed filters', () => { + const code = buildStoryFilterBlock({ + id: 'product', + column: 'product_name', + label: 'Product', + type: 'select', + table: '`nao-production`.`prod_silver`.`dim_products`', + database_id: 'bigquery', + }); + + expect(getStoryFiltersFromCode(code)).toEqual([ + { + id: 'product', + column: 'product_name', + label: 'Product', + filterType: 'select', + table: '`nao-production`.`prod_silver`.`dim_products`', + databaseId: 'bigquery', + rawTag: expect.any(String), + }, + ]); + }); +}); diff --git a/apps/shared/tests/story-validation.test.ts b/apps/shared/tests/story-validation.test.ts index 42b16f287..f20032de0 100644 --- a/apps/shared/tests/story-validation.test.ts +++ b/apps/shared/tests/story-validation.test.ts @@ -220,6 +220,39 @@ describe('validateStoryCode', () => { }); }); + describe('filter validation', () => { + it('accepts a well-formed filter tag with table source', () => { + const code = ''; + expect(validateStoryCode(code)).toEqual([]); + }); + + it('accepts a filter with hardcoded options', () => { + const code = ``; + expect(validateStoryCode(code)).toEqual([]); + }); + + it('flags select filters missing both options and table/column', () => { + expect( + validateStoryCode('').some((e) => + /require either `options=/.test(e.message), + ), + ).toBe(true); + expect( + validateStoryCode('').some( + (e) => /Invalid filter type/.test(e.message), + ), + ).toBe(true); + }); + + it('flags duplicate filter ids', () => { + const code = [ + '', + '', + ].join('\n'); + expect(validateStoryCode(code).some((e) => /must be unique/.test(e.message))).toBe(true); + }); + }); + it('reports line and column for errors', () => { const code = ['# intro', '', 'some text', '', ''].join('\n'); const errors = validateStoryCode(code); From a03fd6e7656668a1065b162ecf1679d809088b06 Mon Sep 17 00:00:00 2001 From: Christophe Blefari Date: Thu, 23 Jul 2026 12:06:46 +0200 Subject: [PATCH 2/4] Improve the agentic management of filters with stories --- apps/backend/src/agents/tools/execute-sql.ts | 33 +- apps/backend/src/agents/tools/story.ts | 11 + .../components/tool-outputs/execute-sql.tsx | 38 +- .../src/components/tool-outputs/story.tsx | 17 +- apps/backend/src/queries/chart-image.ts | 26 +- apps/backend/src/queries/chat.queries.ts | 50 +- .../src/queries/execute-sql.queries.ts | 129 ++++- .../src/queries/shared-story.queries.ts | 20 +- apps/backend/src/queries/story.queries.ts | 39 +- apps/backend/src/services/agent.ts | 30 +- apps/backend/src/services/story-filters.ts | 36 +- .../src/services/story-template-validation.ts | 33 ++ apps/backend/src/services/update-sql.ts | 58 +-- apps/backend/src/trpc/shared-story.routes.ts | 18 +- apps/backend/src/trpc/sql.routes.ts | 10 +- apps/backend/src/trpc/story.routes.ts | 15 +- apps/backend/src/utils/chat-context-usage.ts | 3 +- apps/frontend/package.json | 2 + .../chat-messages/chat-messages-readonly.tsx | 3 +- .../chat-messages/chat-messages.tsx | 6 +- .../src/components/side-panel/sql-editor.tsx | 221 ++++++--- .../side-panel/story-chart-embed.tsx | 70 +-- .../side-panel/story-chart-query.tsx | 75 +++ .../components/side-panel/story-preview.tsx | 38 +- .../side-panel/story-table-embed.tsx | 15 +- .../src/components/story-filter-bar.tsx | 116 ++++- .../src/components/story-tabbed-content.tsx | 86 ++-- .../tool-calls/display-chart-table.tsx | 10 +- .../components/tool-calls/display-chart.tsx | 11 +- .../src/components/tool-calls/execute-sql.tsx | 6 +- apps/frontend/src/components/ui/button.tsx | 2 +- apps/frontend/src/components/ui/calendar.tsx | 162 +++++++ apps/frontend/src/components/ui/popover.tsx | 40 ++ .../frontend/src/contexts/story-query-sql.tsx | 20 + apps/frontend/src/hooks/use-side-panel.ts | 6 +- apps/frontend/src/hooks/use-story-filters.ts | 1 + apps/frontend/src/lib/execute-sql-messages.ts | 65 ++- apps/shared/package.json | 3 +- apps/shared/src/execute-sql-parts.ts | 85 ++++ apps/shared/src/index.ts | 1 + apps/shared/src/sql-template.ts | 148 ++++++ apps/shared/src/tools/execute-sql.ts | 10 + apps/shared/src/tools/story.ts | 1 + apps/shared/tests/execute-sql-parts.test.ts | 89 ++++ apps/shared/tests/sql-template.test.ts | 47 ++ bun.lock | 131 ++++- package-lock.json | 458 +++++++++++++++++- 47 files changed, 2044 insertions(+), 450 deletions(-) create mode 100644 apps/backend/src/services/story-template-validation.ts create mode 100644 apps/frontend/src/components/side-panel/story-chart-query.tsx create mode 100644 apps/frontend/src/components/ui/calendar.tsx create mode 100644 apps/frontend/src/components/ui/popover.tsx create mode 100644 apps/frontend/src/contexts/story-query-sql.tsx create mode 100644 apps/shared/src/execute-sql-parts.ts create mode 100644 apps/shared/tests/execute-sql-parts.test.ts diff --git a/apps/backend/src/agents/tools/execute-sql.ts b/apps/backend/src/agents/tools/execute-sql.ts index ef97730f7..d3e96195e 100644 --- a/apps/backend/src/agents/tools/execute-sql.ts +++ b/apps/backend/src/agents/tools/execute-sql.ts @@ -1,4 +1,4 @@ -import { stripSqlFilterBlocks } from '@nao/shared/sql-template'; +import { stripSqlFilterBlocks, validateSqlFilterTemplate } from '@nao/shared/sql-template'; import type { executeSql } from '@nao/shared/tools'; import { executeSql as schemas } from '@nao/shared/tools'; @@ -14,6 +14,7 @@ export async function executeQuery( { sql_query, database_id, query_id }: executeSql.Input, context: ToolContext, ): Promise { + const templateWarnings = env.BETA_STORY_FILTERS_ENABLED ? validateSqlFilterTemplate(sql_query) : []; const effectiveSql = stripSqlFilterBlocks(sql_query); const writePermEnabled = context.agentSettings?.sql?.dangerouslyWritePermEnabled ?? false; if (!writePermEnabled && !(await isReadOnlySqlQuery(effectiveSql))) { @@ -24,7 +25,7 @@ export async function executeQuery( } if (context.adminMode) { - return executeAppDbQuery(effectiveSql, context, query_id); + return withTemplateWarnings(await executeAppDbQuery(effectiveSql, context, query_id), templateWarnings); } const naoProjectFolder = context.projectFolder; @@ -55,12 +56,15 @@ export async function executeQuery( const appliedLimit = detectQueryRowLimit(effectiveSql); - return { - _version: '1', - ...data, - id, - ...(appliedLimit !== null && { applied_limit: appliedLimit }), - }; + return withTemplateWarnings( + { + _version: '1', + ...data, + id, + ...(appliedLimit !== null && { applied_limit: appliedLimit }), + }, + templateWarnings, + ); } async function executeAppDbQuery( @@ -82,6 +86,13 @@ async function executeAppDbQuery( }; } +function withTemplateWarnings(output: executeSql.Output, templateWarnings: string[]): executeSql.Output { + if (templateWarnings.length === 0) { + return output; + } + return { ...output, template_warnings: templateWarnings }; +} + async function updateExistingQuery( input: executeSql.Input & { query_id: `query_${string}` }, context: ToolContext, @@ -109,8 +120,10 @@ function buildExecuteSqlToolDescription() { 'Execute a SQL query against the connected database and return the results. If multiple databases are configured, specify the database_id.', ...(env.BETA_STORY_FILTERS_ENABLED ? [ - 'Story filters may be embedded as SQL template blocks that are stripped when this tool runs in chat:', - 'WHERE 1 = 1 {% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %}.', + 'Story filters may be embedded as SQL template blocks that are stripped when this tool runs in chat.', + 'Correct syntax: WHERE 1 = 1 {% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %}.', + "For date_range filters, {{ filters..sql }} already expands to 'start' AND 'end', so write: {% filter period %} AND order_date BETWEEN {{ filters.period.sql }} {% endfilter %}.", + 'Never use filters..start, .end, .value, or placeholders outside {% filter %} blocks — invalid templates return template_warnings.', 'Prefer query_id when adding story filter templates so existing /
tags keep working.', ] : []), diff --git a/apps/backend/src/agents/tools/story.ts b/apps/backend/src/agents/tools/story.ts index 83be66777..553ed57da 100644 --- a/apps/backend/src/agents/tools/story.ts +++ b/apps/backend/src/agents/tools/story.ts @@ -7,6 +7,7 @@ import { env } from '../../env'; import { getDisplayChartTableFormatsForChat } from '../../queries/chart-image'; import * as storyQueries from '../../queries/story.queries'; import * as storyFolderQueries from '../../queries/story-folder.queries'; +import { getStoryTemplateWarnings } from '../../services/story-template-validation'; import type { ToolContext } from '../../types/tools'; import { createTool } from '../../utils/tools'; @@ -15,7 +16,9 @@ const STORY_FILTER_DESCRIPTION = [ 'For select/multi_select, provide either table+column (options from SELECT DISTINCT) or options=\'["a","b"]\' (hardcoded values).', 'When using table+column and multiple databases are configured, set database_id on the filter so option loading targets the correct database.', 'Matching SQL must use template blocks that reference the same filter id, e.g. WHERE 1 = 1 {% filter country %} AND country IN ({{ filters.country.sql }}) {% endfilter %}.', + "For date_range, {{ filters..sql }} already expands to 'start' AND 'end' — write {% filter period %} AND order_date BETWEEN {{ filters.period.sql }} {% endfilter %}. Never use .start/.end/.value.", 'Chat and live refresh strip unset filter blocks so the query still runs; active story filter selections re-render and re-execute SQL.', + 'Invalid filter templates are reported as template_warnings in the tool result — fix them before finishing.', 'When adding filters to existing charts, prefer execute_sql with query_id set to the existing query so chart/table tags keep the same query_id.', ].join(' '); @@ -90,6 +93,7 @@ export default createTool({ version: version.version, code: version.code, title: version.title, + ...(await storyTemplateWarnings(chatId, version.code)), }; } @@ -128,6 +132,7 @@ export default createTool({ version: version.version, code: version.code, title: version.title, + ...(await storyTemplateWarnings(chatId, version.code)), }; } @@ -154,6 +159,7 @@ export default createTool({ version: version.version, code: version.code, title: version.title, + ...(await storyTemplateWarnings(chatId, version.code)), }; }, @@ -165,6 +171,11 @@ async function carryOverTableFormatting(code: string, chatId: string): Promise { + const warnings = await getStoryTemplateWarnings(chatId, code); + return warnings.length > 0 ? { template_warnings: warnings } : {}; +} + function rememberStoryArtifact(context: ToolContext, id: string, title: string): void { const existing = context.generatedArtifacts.stories.find((story) => story.id === id); if (existing) { diff --git a/apps/backend/src/components/tool-outputs/execute-sql.tsx b/apps/backend/src/components/tool-outputs/execute-sql.tsx index fb8b9f62a..a09efc5dc 100644 --- a/apps/backend/src/components/tool-outputs/execute-sql.tsx +++ b/apps/backend/src/components/tool-outputs/execute-sql.tsx @@ -1,14 +1,30 @@ import { pluralize } from '@nao/shared'; import type { executeSql } from '@nao/shared/tools'; -import { Block, ListItem, Span, Title, TitledList } from '../../lib/markdown'; +import { Block, List, ListItem, Span, Title, TitledList } from '../../lib/markdown'; import { QueryRows } from './query-rows'; const MAX_ROWS = 40; export const ExecuteSqlOutput = ({ output, maxRows = MAX_ROWS }: { output: executeSql.Output; maxRows?: number }) => { + const templateWarnings = output.template_warnings ?? []; + + if (output.superseded) { + return ( + + Stale result: query {output.id} was re-run later in this conversation. Refer to the latest execute_sql + result with this query id (or call read_query_result) for the current SQL and rows. + + ); + } + if (output.data.length === 0) { - return The query was successfully executed and returned no rows.; + return ( + + The query was successfully executed and returned no rows. + {templateWarnings.length > 0 && } + + ); } const isTruncated = output.data.length > maxRows; @@ -41,6 +57,8 @@ export const ExecuteSqlOutput = ({ output, maxRows = MAX_ROWS }: { output: execu )} + {templateWarnings.length > 0 && } + {remainingRows > 0 && ( @@ -52,3 +70,19 @@ export const ExecuteSqlOutput = ({ output, maxRows = MAX_ROWS }: { output: execu ); }; + +function TemplateWarnings({ warnings }: { warnings: string[] }) { + return ( + + + Story filter template warnings — the baseline query ran (filter blocks stripped), but story filters will + not apply correctly until you fix the SQL with execute_sql (prefer query_id): + + + {warnings.map((warning) => ( + {warning} + ))} + + + ); +} diff --git a/apps/backend/src/components/tool-outputs/story.tsx b/apps/backend/src/components/tool-outputs/story.tsx index 7332da9ff..6ea6f4a69 100644 --- a/apps/backend/src/components/tool-outputs/story.tsx +++ b/apps/backend/src/components/tool-outputs/story.tsx @@ -1,6 +1,6 @@ import type { story } from '@nao/shared/tools'; -import { Block } from '../../lib/markdown'; +import { Block, List, ListItem, Span } from '../../lib/markdown'; export type StoryModelOutput = story.Output & { _stale?: boolean; @@ -21,6 +21,8 @@ export function StoryOutput({ output }: { output: StoryModelOutput }) { ); } + const templateWarnings = output.template_warnings ?? []; + return ( Story "{output.title}" (v{output.version}) — {output.id} @@ -30,6 +32,19 @@ export function StoryOutput({ output }: { output: StoryModelOutput }) { current version. Base any further changes on this content. )} + {templateWarnings.length > 0 && ( + + + Story filter template warnings — fix the referenced SQL with execute_sql (prefer query_id) + and/or the story filter tags before considering this story complete: + + + {templateWarnings.map((warning) => ( + {warning} + ))} + + + )} {output.code} ); diff --git a/apps/backend/src/queries/chart-image.ts b/apps/backend/src/queries/chart-image.ts index 066bea5ab..baf2d2ddd 100644 --- a/apps/backend/src/queries/chart-image.ts +++ b/apps/backend/src/queries/chart-image.ts @@ -1,11 +1,12 @@ import { displayChart, executeSql } from '@nao/shared/tools'; -import { and, asc, eq, isNull, sql } from 'drizzle-orm'; +import { and, asc, eq, isNull } from 'drizzle-orm'; import s from '../db/abstractSchema'; import { db } from '../db/db'; -import dbConfig, { Dialect } from '../db/dbConfig'; +import { HandlerError } from '../utils/error'; import { takeFirstOrThrow } from '../utils/queries'; import { selectLatestDisplayChartTableFormats } from './chart-image.utils'; +import { getLatestExecuteSqlByQueryId } from './execute-sql.queries'; const DISPLAY_CHART_TOOL_TYPE = 'tool-display_chart'; @@ -34,22 +35,13 @@ export const getDisplayConfigByToolCallId = async (toolCallId: string): Promise< export const getExecuteSqlPartByQueryId = async ( queryId: string, ): Promise<{ toolInput: executeSql.Input; toolOutput: executeSql.Output }> => { - const jsonIdFilter = - dbConfig.dialect === Dialect.Postgres - ? sql`${s.messagePart.toolOutput}->>'id' = ${queryId}` - : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; - - const result = await takeFirstOrThrow( - db - .select({ toolInput: s.messagePart.toolInput, toolOutput: s.messagePart.toolOutput }) - .from(s.messagePart) - .where(jsonIdFilter) - .execute(), - ); - + const result = await getLatestExecuteSqlByQueryId(queryId); + if (!result) { + throw new HandlerError('NOT_FOUND', `Query ${queryId} not found.`); + } return { - toolInput: executeSql.InputSchema.parse(result.toolInput), - toolOutput: executeSql.OutputSchema.parse(result.toolOutput), + toolInput: result.toolInput, + toolOutput: result.toolOutput, }; }; diff --git a/apps/backend/src/queries/chat.queries.ts b/apps/backend/src/queries/chat.queries.ts index 7396b7e1f..13ca7d2e7 100644 --- a/apps/backend/src/queries/chat.queries.ts +++ b/apps/backend/src/queries/chat.queries.ts @@ -29,6 +29,7 @@ import { import { applyChatFilters, buildChatGroups, type EnrichedChat, type SourcePlatform } from '../utils/chat-list'; import { convertDBPartToUIPart, mapUIPartsToDBParts } from '../utils/chat-message-part-mappings'; import { getErrorMessage } from '../utils/utils'; +import * as executeSqlQueries from './execute-sql.queries'; const chatCreatedAtMs = dbConfig.dialect === Dialect.Postgres @@ -853,20 +854,8 @@ export const getChatProjectId = async (chatId: string): Promise => { - const jsonIdFilter = - dbConfig.dialect === Dialect.Postgres - ? sql`${s.messagePart.toolOutput}->>'id' = ${queryId}` - : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; - - const [result] = await db - .select({ projectId: s.chat.projectId }) - .from(s.messagePart) - .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) - .innerJoin(s.chat, eq(s.chatMessage.chatId, s.chat.id)) - .where(jsonIdFilter) - .execute(); - - return result?.projectId; + const owner = await executeSqlQueries.getExecuteSqlOwnerByQueryId(queryId); + return owner?.projectId; }; /** @@ -878,24 +867,8 @@ export const getQueryResultByQueryId = async ( chatId: string, queryId: string, ): Promise<{ columns: string[]; data: Record[] } | null> => { - const jsonIdFilter = buildQueryIdJsonFilter(queryId); - - const [result] = await db - .select({ toolOutput: s.messagePart.toolOutput }) - .from(s.messagePart) - .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) - .where( - and( - eq(s.chatMessage.chatId, chatId), - isNull(s.chatMessage.supersededAt), - eq(s.messagePart.toolName, 'execute_sql'), - jsonIdFilter, - ), - ) - .limit(1) - .execute(); - - return extractQueryResultFromToolOutput(result?.toolOutput); + const part = await executeSqlQueries.getExecuteSqlPartByQueryIdInChat(chatId, queryId); + return part ? extractQueryResultFromToolOutput(part.toolOutput) : null; }; export const getQueryResultByQueryIdInProject = async ( @@ -903,8 +876,6 @@ export const getQueryResultByQueryIdInProject = async ( userId: string, queryId: string, ): Promise<{ columns: string[]; data: Record[]; chatId: string } | null> => { - const jsonIdFilter = buildQueryIdJsonFilter(queryId); - const [result] = await db .select({ toolOutput: s.messagePart.toolOutput, chatId: s.chat.id }) .from(s.messagePart) @@ -915,10 +886,11 @@ export const getQueryResultByQueryIdInProject = async ( eq(s.chat.projectId, projectId), eq(s.chat.userId, userId), isNull(s.chatMessage.supersededAt), - eq(s.messagePart.toolName, 'execute_sql'), - jsonIdFilter, + eq(s.messagePart.toolName, executeSqlQueries.EXECUTE_SQL_TOOL_NAME), + executeSqlQueries.messagePartToolOutputIdEquals(queryId), ), ) + .orderBy(desc(s.chatMessage.createdAt), desc(s.messagePart.order)) .limit(1) .execute(); @@ -929,12 +901,6 @@ export const getQueryResultByQueryIdInProject = async ( return { ...extracted, chatId: result.chatId }; }; -function buildQueryIdJsonFilter(queryId: string) { - return dbConfig.dialect === Dialect.Postgres - ? sql`${s.messagePart.toolOutput}->>'id' = ${queryId}` - : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; -} - function extractQueryResultFromToolOutput( toolOutput: unknown, ): { columns: string[]; data: Record[] } | null { diff --git a/apps/backend/src/queries/execute-sql.queries.ts b/apps/backend/src/queries/execute-sql.queries.ts index b5a942d72..ff1fb5059 100644 --- a/apps/backend/src/queries/execute-sql.queries.ts +++ b/apps/backend/src/queries/execute-sql.queries.ts @@ -1,39 +1,74 @@ import { executeSql } from '@nao/shared/tools'; -import { and, eq, sql } from 'drizzle-orm'; +import { and, asc, desc, eq, isNull, sql } from 'drizzle-orm'; import s from '../db/abstractSchema'; import { db } from '../db/db'; import dbConfig, { Dialect } from '../db/dbConfig'; import { takeFirstOrThrow } from '../utils/queries'; -const EXECUTE_SQL_TOOL_NAME = 'execute_sql'; +export const EXECUTE_SQL_TOOL_NAME = 'execute_sql'; -function queryIdFilter(queryId: string) { +export function messagePartToolOutputIdEquals(queryId: string) { return dbConfig.dialect === Dialect.Postgres ? sql`${s.messagePart.toolOutput}->>'id' = ${queryId}` : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; } -export async function getExecuteSqlOwnerByQueryId( - queryId: string, -): Promise<{ projectId: string; userId: string; chatId: string; toolCallId: string } | null> { +export type LatestExecuteSqlRow = { + projectId: string; + userId: string; + chatId: string; + toolCallId: string; + toolInput: executeSql.Input; + toolOutput: executeSql.Output; +}; + +/** Latest non-superseded execute_sql part for a query id (global). */ +export async function getLatestExecuteSqlByQueryId(queryId: string): Promise { const [row] = await db .select({ projectId: s.chat.projectId, userId: s.chat.userId, chatId: s.chat.id, toolCallId: s.messagePart.toolCallId, + toolInput: s.messagePart.toolInput, + toolOutput: s.messagePart.toolOutput, }) .from(s.messagePart) .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) .innerJoin(s.chat, eq(s.chatMessage.chatId, s.chat.id)) - .where(and(eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), queryIdFilter(queryId))) + .where( + and( + eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), + isNull(s.chatMessage.supersededAt), + messagePartToolOutputIdEquals(queryId), + ), + ) + .orderBy(desc(s.chatMessage.createdAt), desc(s.messagePart.order)) + .limit(1) .execute(); - if (!row?.toolCallId) { + if (!row?.toolCallId || !row.toolInput || !row.toolOutput) { return null; } + return { + projectId: row.projectId, + userId: row.userId, + chatId: row.chatId, + toolCallId: row.toolCallId, + toolInput: executeSql.InputSchema.parse(row.toolInput), + toolOutput: executeSql.OutputSchema.parse(row.toolOutput), + }; +} + +export async function getExecuteSqlOwnerByQueryId( + queryId: string, +): Promise<{ projectId: string; userId: string; chatId: string; toolCallId: string } | null> { + const row = await getLatestExecuteSqlByQueryId(queryId); + if (!row) { + return null; + } return { projectId: row.projectId, userId: row.userId, @@ -42,6 +77,7 @@ export async function getExecuteSqlOwnerByQueryId( }; } +/** Latest non-superseded execute_sql part for a query id within a chat. */ export async function getExecuteSqlPartByQueryIdInChat( chatId: string, queryId: string, @@ -57,10 +93,13 @@ export async function getExecuteSqlPartByQueryIdInChat( .where( and( eq(s.chatMessage.chatId, chatId), + isNull(s.chatMessage.supersededAt), eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), - queryIdFilter(queryId), + messagePartToolOutputIdEquals(queryId), ), ) + .orderBy(desc(s.chatMessage.createdAt), desc(s.messagePart.order)) + .limit(1) .execute(); if (!row?.toolCallId || !row.toolInput || !row.toolOutput) { @@ -88,3 +127,75 @@ export async function updateExecuteSqlPart( .execute(), ); } + +async function loadLatestExecuteSqlParts(chatId: string) { + return db + .select({ + toolInput: s.messagePart.toolInput, + toolOutput: s.messagePart.toolOutput, + }) + .from(s.messagePart) + .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) + .where( + and( + eq(s.chatMessage.chatId, chatId), + isNull(s.chatMessage.supersededAt), + eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), + ), + ) + .orderBy(asc(s.chatMessage.createdAt), asc(s.messagePart.order)) + .execute(); +} + +/** + * Load SQL templates for the given query ids from a chat. + * When duplicates exist, the latest non-superseded part wins. + */ +export async function getLatestSqlQueriesByIds( + chatId: string, + queryIds: Set, +): Promise> { + if (queryIds.size === 0) { + return {}; + } + + const parts = await loadLatestExecuteSqlParts(chatId); + const queries: Record = {}; + for (const part of parts) { + const output = part.toolOutput as { id?: string } | null; + const input = part.toolInput as { sql_query?: string; database_id?: string } | null; + if (output?.id && queryIds.has(output.id) && input?.sql_query) { + queries[output.id] = { + sqlQuery: input.sql_query, + ...(input.database_id && { databaseId: input.database_id }), + }; + } + } + return queries; +} + +/** + * Load cached query result rows for the given query ids from a chat. + * When duplicates exist, the latest non-superseded part wins. + */ +export async function getLatestSqlQueryDataByIds( + chatId: string, + queryIds: Set, +): Promise> { + if (queryIds.size === 0) { + return {}; + } + + const parts = await loadLatestExecuteSqlParts(chatId); + const data: Record = {}; + for (const part of parts) { + const output = part.toolOutput as { id?: string; data?: unknown[]; columns?: string[] } | null; + if (output?.id && queryIds.has(output.id)) { + data[output.id] = { + data: output.data ?? [], + columns: output.columns ?? [], + }; + } + } + return data; +} diff --git a/apps/backend/src/queries/shared-story.queries.ts b/apps/backend/src/queries/shared-story.queries.ts index dd0860bcd..890248b75 100644 --- a/apps/backend/src/queries/shared-story.queries.ts +++ b/apps/backend/src/queries/shared-story.queries.ts @@ -2,6 +2,7 @@ import { and, count, desc, eq, isNull, max, or, type SQL, sql } from 'drizzle-or import s, { type DBSharedStory } from '../db/abstractSchema'; import { db } from '../db/db'; +import * as executeSqlQueries from './execute-sql.queries'; export type SharedStoryWithLatest = DBSharedStory & { updatedAt: Date; @@ -141,24 +142,7 @@ export async function getQueryDataFromCode( return null; } - const parts = await db - .select({ toolOutput: s.messagePart.toolOutput }) - .from(s.messagePart) - .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) - .where(and(eq(s.chatMessage.chatId, chatId), eq(s.messagePart.toolName, 'execute_sql'))) - .execute(); - - const data: Record = {}; - for (const part of parts) { - const output = part.toolOutput as { id?: string; data?: unknown[]; columns?: string[] } | null; - if (output?.id && queryIds.has(output.id)) { - data[output.id] = { - data: output.data ?? [], - columns: output.columns ?? [], - }; - } - } - + const data = await executeSqlQueries.getLatestSqlQueryDataByIds(chatId, queryIds); return Object.keys(data).length > 0 ? data : null; } diff --git a/apps/backend/src/queries/story.queries.ts b/apps/backend/src/queries/story.queries.ts index ea9da345d..b41cb041d 100644 --- a/apps/backend/src/queries/story.queries.ts +++ b/apps/backend/src/queries/story.queries.ts @@ -3,6 +3,7 @@ import { and, asc, desc, eq, inArray, isNull, max, or, type SQL, sql } from 'dri import s, { type DBStory, type DBStoryDataCache, type DBStoryVersion } from '../db/abstractSchema'; import { db, type DBExecutor } from '../db/db'; +import * as executeSqlQueries from './execute-sql.queries'; export type UserStoryRow = Pick< DBStory, @@ -568,15 +569,21 @@ export async function getSqlQueriesFromCode( return {}; } - return getSqlQueriesByIds(chatId, queryIds); + return executeSqlQueries.getLatestSqlQueriesByIds(chatId, queryIds); } export async function getSqlQueryById( chatId: string, queryId: string, ): Promise<{ sqlQuery: string; databaseId?: string } | null> { - const result = await getSqlQueriesByIds(chatId, new Set([queryId])); - return result[queryId] ?? null; + const part = await executeSqlQueries.getExecuteSqlPartByQueryIdInChat(chatId, queryId); + if (!part) { + return null; + } + return { + sqlQuery: part.toolInput.sql_query, + ...(part.toolInput.database_id && { databaseId: part.toolInput.database_id }), + }; } async function queryStoriesWithLatestVersion( @@ -665,32 +672,6 @@ async function getOrCreateStandaloneStory(data: { return row; } -async function getSqlQueriesByIds( - chatId: string, - queryIds: Set, -): Promise> { - const parts = await db - .select({ toolInput: s.messagePart.toolInput, toolOutput: s.messagePart.toolOutput }) - .from(s.messagePart) - .innerJoin(s.chatMessage, eq(s.messagePart.messageId, s.chatMessage.id)) - .where(and(eq(s.chatMessage.chatId, chatId), eq(s.messagePart.toolName, 'execute_sql'))) - .execute(); - - const queries: Record = {}; - for (const part of parts) { - const output = part.toolOutput as { id?: string } | null; - const input = part.toolInput as { sql_query?: string; database_id?: string } | null; - if (output?.id && queryIds.has(output.id) && input?.sql_query) { - queries[output.id] = { - sqlQuery: input.sql_query, - ...(input.database_id && { databaseId: input.database_id }), - }; - } - } - - return queries; -} - async function getStoryVersion( whereCondition: SQL, options?: { latest?: boolean }, diff --git a/apps/backend/src/services/agent.ts b/apps/backend/src/services/agent.ts index 8532ad17d..91c29e064 100644 --- a/apps/backend/src/services/agent.ts +++ b/apps/backend/src/services/agent.ts @@ -1,3 +1,4 @@ +import { markSupersededExecuteSqlParts } from '@nao/shared/execute-sql-parts'; import { story } from '@nao/shared/tools'; import type { LlmProvider, LlmSelectedModel } from '@nao/shared/types'; import { @@ -72,6 +73,7 @@ import { mcpService } from './mcp'; import { memoryService } from './memory'; import { getAzureAccessTokenForUser } from './microsoft-auth.service'; import { skillService } from './skill'; +import { getStoryTemplateWarnings } from './story-template-validation'; export interface AgentRunResult { text: string; @@ -535,7 +537,8 @@ class AgentManager { chatUrl?: string, ): Promise { const settledUiMessages = settleInterruptedToolParts(uiMessages); - const uiMessagesWithStories = await this._syncStoryToolOutputs(settledUiMessages); + const uiMessagesWithoutStaleQueries = markSupersededExecuteSqlParts(settledUiMessages); + const uiMessagesWithStories = await this._syncStoryToolOutputs(uiMessagesWithoutStaleQueries); const uiMessagesWithStoryMode = this._addStoryMode(uiMessagesWithStories, mentions); const uiMessagesWithSkills = this._addSkills(uiMessagesWithStoryMode, mentions); const uiMessagesWithCitation = this._addCitationContext(uiMessagesWithSkills); @@ -621,16 +624,23 @@ class AgentManager { } try { - const latestVersions = new Map< + const latestStories = new Map< string, - Awaited> + { + version: NonNullable>>; + templateWarnings: string[]; + } >(); await Promise.all( [...lastToolCallByStory.keys()].map(async (storyId) => { - latestVersions.set( - storyId, - await storyQueries.getLatestVersionByChatAndSlug(this.chat.id, storyId), - ); + const version = await storyQueries.getLatestVersionByChatAndSlug(this.chat.id, storyId); + if (!version) { + return; + } + latestStories.set(storyId, { + version, + templateWarnings: await getStoryTemplateWarnings(this.chat.id, version.code), + }); }), ); @@ -647,10 +657,11 @@ class AgentManager { return { ...part, output: { ...part.output, _stale: true, code: '' } }; } - const latest = latestVersions.get(storyId); - if (!latest) { + const latestStory = latestStories.get(storyId); + if (!latestStory) { return part; } + const { version: latest, templateWarnings } = latestStory; return { ...part, @@ -660,6 +671,7 @@ class AgentManager { code: latest.code, title: latest.title, _editedByUser: latest.source === 'user', + template_warnings: templateWarnings.length > 0 ? templateWarnings : undefined, }, }; }), diff --git a/apps/backend/src/services/story-filters.ts b/apps/backend/src/services/story-filters.ts index 0c6a04b27..b384e9263 100644 --- a/apps/backend/src/services/story-filters.ts +++ b/apps/backend/src/services/story-filters.ts @@ -60,17 +60,13 @@ export async function getFilteredStoryQueryData( selections: StoryFilterSelections, ): Promise> { const { code, projectPath, envVars } = await loadStoryExecutionContext(chatId, storySlug); - const filters = getStoryFiltersFromCode(code); - const types: StoryFilterTypeById = Object.fromEntries(filters.map((filter) => [filter.id, filter.filterType])); + const types = filterTypesFromCode(code); const sqlQueries = await storyQueries.getSqlQueriesFromCode(chatId, code); const queryData: Record = {}; await Promise.all( Object.entries(sqlQueries).map(async ([queryId, { sqlQuery, databaseId }]) => { - const renderedSql = - Object.keys(selections).length === 0 - ? stripSqlFilterBlocks(sqlQuery) - : renderSqlTemplate(sqlQuery, selections, types); + const renderedSql = renderStorySql(sqlQuery, selections, types); queryData[queryId] = await executeRawSql(renderedSql, projectPath, databaseId, envVars); }), ); @@ -78,6 +74,34 @@ export async function getFilteredStoryQueryData( return queryData; } +export async function getStoryQuerySql( + chatId: string, + storySlug: string, + queryId: string, + selections: StoryFilterSelections = {}, +): Promise<{ sqlQuery: string; renderedSql: string }> { + const { code } = await loadStoryExecutionContext(chatId, storySlug); + const query = await storyQueries.getSqlQueryById(chatId, queryId); + if (!query) { + throw new TRPCError({ code: 'NOT_FOUND', message: `Query "${queryId}" not found.` }); + } + + return { + sqlQuery: query.sqlQuery, + renderedSql: renderStorySql(query.sqlQuery, selections, filterTypesFromCode(code)), + }; +} + +function filterTypesFromCode(code: string): StoryFilterTypeById { + return Object.fromEntries(getStoryFiltersFromCode(code).map((filter) => [filter.id, filter.filterType])); +} + +function renderStorySql(sqlQuery: string, selections: StoryFilterSelections, types: StoryFilterTypeById): string { + return Object.keys(selections).length === 0 + ? stripSqlFilterBlocks(sqlQuery) + : renderSqlTemplate(sqlQuery, selections, types); +} + async function loadStoryExecutionContext(chatId: string, storySlug: string) { const version = await storyQueries.getLatestVersionByChatAndSlug(chatId, storySlug); if (!version) { diff --git a/apps/backend/src/services/story-template-validation.ts b/apps/backend/src/services/story-template-validation.ts new file mode 100644 index 000000000..08b9b53a7 --- /dev/null +++ b/apps/backend/src/services/story-template-validation.ts @@ -0,0 +1,33 @@ +import { findUnreferencedStoryFilters, validateSqlFilterTemplate } from '@nao/shared/sql-template'; +import { getStoryFiltersFromCode } from '@nao/shared/story-segments'; + +import { env } from '../env'; +import * as storyQueries from '../queries/story.queries'; + +export async function getStoryTemplateWarnings(chatId: string, code: string): Promise { + if (!env.BETA_STORY_FILTERS_ENABLED) { + return []; + } + + const filters = getStoryFiltersFromCode(code); + const knownFilterIds = filters.map((filter) => filter.id); + const sqlQueries = await storyQueries.getSqlQueriesFromCode(chatId, code); + const warnings: string[] = []; + + for (const [queryId, { sqlQuery }] of Object.entries(sqlQueries)) { + for (const issue of validateSqlFilterTemplate(sqlQuery, { knownFilterIds })) { + warnings.push(`[${queryId}] ${issue}`); + } + } + + if (knownFilterIds.length > 0) { + const sqlList = Object.values(sqlQueries).map((query) => query.sqlQuery); + for (const filterId of findUnreferencedStoryFilters(knownFilterIds, sqlList)) { + warnings.push( + `Story filter "${filterId}" is declared but not referenced in any chart/table SQL. Add {% filter ${filterId} %} ... {{ filters.${filterId}.sql }} ... {% endfilter %} to the relevant queries, or remove the unused tag.`, + ); + } + } + + return warnings; +} diff --git a/apps/backend/src/services/update-sql.ts b/apps/backend/src/services/update-sql.ts index 70a6f2909..f6feaae10 100644 --- a/apps/backend/src/services/update-sql.ts +++ b/apps/backend/src/services/update-sql.ts @@ -3,13 +3,8 @@ import type { executeSql } from '@nao/shared/tools'; import { TRPCError } from '@trpc/server'; import { executeQuery } from '../agents/tools/execute-sql'; -import { - getExecuteSqlOwnerByQueryId, - getExecuteSqlPartByQueryIdInChat, - updateExecuteSqlPart, -} from '../queries/execute-sql.queries'; -import * as projectQueries from '../queries/project.queries'; -import type { ToolContext } from '../types/tools'; +import { getLatestExecuteSqlByQueryId, updateExecuteSqlPart } from '../queries/execute-sql.queries'; +import { buildToolContext } from './agent'; export async function updateSqlQueryInChat(opts: { queryId: string; @@ -18,11 +13,11 @@ export async function updateSqlQueryInChat(opts: { name?: string; userId: string; }): Promise<{ input: executeSql.Input; output: executeSql.Output; toolCallId: string; chatId: string }> { - const { existing, owner, context, input } = await prepareSqlEditContext(opts); + const { existing, context, input } = await prepareSqlEditContext(opts); const output = await executeQuery({ ...input, query_id: opts.queryId as `query_${string}` }, context); await updateExecuteSqlPart(existing.toolCallId, input, output); - return { input, output, toolCallId: existing.toolCallId, chatId: owner.chatId }; + return { input, output, toolCallId: existing.toolCallId, chatId: existing.chatId }; } export async function previewSqlQueryInChat(opts: { @@ -41,53 +36,30 @@ async function prepareSqlEditContext(opts: { databaseId?: string; name?: string; userId: string; -}): Promise<{ - existing: NonNullable>>; - owner: NonNullable>>; - context: ToolContext; - input: executeSql.Input; -}> { +}) { assertSqlQueryEditable(opts.sqlQuery); - const owner = await getExecuteSqlOwnerByQueryId(opts.queryId); - if (!owner) { - throw new TRPCError({ code: 'NOT_FOUND', message: `Query ${opts.queryId} not found.` }); - } - if (owner.userId !== opts.userId) { - throw new TRPCError({ code: 'FORBIDDEN', message: 'You are not authorized to edit this query.' }); - } - - const existing = await getExecuteSqlPartByQueryIdInChat(owner.chatId, opts.queryId); + const existing = await getLatestExecuteSqlByQueryId(opts.queryId); if (!existing) { throw new TRPCError({ code: 'NOT_FOUND', message: `Query ${opts.queryId} not found.` }); } - - const project = await projectQueries.retrieveProjectById(owner.projectId); - if (!project.path) { - throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Project path not configured.' }); + if (existing.userId !== opts.userId) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'You are not authorized to edit this query.' }); } - const agentSettings = await projectQueries.getAgentSettings(owner.projectId); - const envVars = await projectQueries.getEnvVars(owner.projectId); - const context: ToolContext = { - projectFolder: project.path, - chatId: owner.chatId, + const context = await buildToolContext({ + projectId: existing.projectId, userId: opts.userId, - projectId: owner.projectId, - agentSettings, - envVars, - azureAccessToken: null, - queryResults: new Map(), - generatedArtifacts: { charts: [], stories: [] }, - }; + chatId: existing.chatId, + }); const input: executeSql.Input = { sql_query: opts.sqlQuery, - database_id: opts.databaseId ?? existing.toolInput.database_id, - name: opts.name ?? existing.toolInput.name, + database_id: opts.databaseId ?? existing.toolInput.database_id ?? undefined, + name: opts.name ?? existing.toolInput.name ?? undefined, }; - return { existing, owner, context, input }; + return { existing, context, input }; } export function assertSqlQueryEditable(sqlQuery: string): void { diff --git a/apps/backend/src/trpc/shared-story.routes.ts b/apps/backend/src/trpc/shared-story.routes.ts index 3039477c6..4a2a8118f 100644 --- a/apps/backend/src/trpc/shared-story.routes.ts +++ b/apps/backend/src/trpc/shared-story.routes.ts @@ -11,7 +11,7 @@ import * as storyQueries from '../queries/story.queries'; import * as storyFolderQueries from '../queries/story-folder.queries'; import { logActivity } from '../services/activity'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; -import { getFilteredStoryQueryData, getStoryFilterOptions } from '../services/story-filters'; +import { getFilteredStoryQueryData, getStoryFilterOptions, getStoryQuerySql } from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { notifySharedItemRecipients } from '../utils/email'; import { buildDownloadResponse } from '../utils/story-download'; @@ -201,6 +201,22 @@ export const sharedStoryRoutes = { return getFilteredStoryQueryData(shared.chatId, shared.slug, input.selections); }), + getQuerySql: shareAccessProcedure + .input( + z.object({ + shareId: z.string(), + queryId: z.string(), + selections: z.record(z.string(), z.union([z.string(), z.array(z.string())])).default({}), + }), + ) + .query(async ({ input, ctx }) => { + const shared = ctx.resource; + if (!shared.chatId) { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'Shared story has no chat.' }); + } + return getStoryQuerySql(shared.chatId, shared.slug, input.queryId, input.selections); + }), + refreshData: shareAccessProcedure.input(z.object({ shareId: z.string() })).mutation(async ({ ctx }) => { const shared = ctx.resource; const story = await storyQueries.getStoryByChatAndSlug(shared.chatId!, shared.slug); diff --git a/apps/backend/src/trpc/sql.routes.ts b/apps/backend/src/trpc/sql.routes.ts index 45d17379e..b1709ab07 100644 --- a/apps/backend/src/trpc/sql.routes.ts +++ b/apps/backend/src/trpc/sql.routes.ts @@ -6,8 +6,8 @@ import { protectedProcedure } from './trpc'; const sqlEditInput = z.object({ queryId: z.string().regex(/^query_.+$/), sql_query: z.string().min(1), - database_id: z.string().optional(), - name: z.string().optional(), + database_id: z.string().nullish(), + name: z.string().nullish(), }); export const sqlRoutes = { @@ -15,7 +15,7 @@ export const sqlRoutes = { return previewSqlQueryInChat({ queryId: input.queryId, sqlQuery: input.sql_query, - databaseId: input.database_id, + databaseId: input.database_id ?? undefined, userId: ctx.user.id, }); }), @@ -24,8 +24,8 @@ export const sqlRoutes = { return updateSqlQueryInChat({ queryId: input.queryId, sqlQuery: input.sql_query, - databaseId: input.database_id, - name: input.name, + databaseId: input.database_id ?? undefined, + name: input.name ?? undefined, userId: ctx.user.id, }); }), diff --git a/apps/backend/src/trpc/story.routes.ts b/apps/backend/src/trpc/story.routes.ts index 0e5c7cd02..53352a364 100644 --- a/apps/backend/src/trpc/story.routes.ts +++ b/apps/backend/src/trpc/story.routes.ts @@ -16,7 +16,7 @@ import * as storyFolderQueries from '../queries/story-folder.queries'; import { naturalLanguageToCron } from '../services/cron-nlp'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; import { nextCronTick } from '../services/scheduler.service'; -import { getFilteredStoryQueryData, getStoryFilterOptions } from '../services/story-filters'; +import { getFilteredStoryQueryData, getStoryFilterOptions, getStoryQuerySql } from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { buildDownloadResponse } from '../utils/story-download'; import { extractStorySummary } from '../utils/story-summary'; @@ -316,6 +316,19 @@ export const storyRoutes = { return getFilteredStoryQueryData(input.chatId, input.storySlug, input.selections); }), + getQuerySql: chatOwnerProcedure + .input( + z.object({ + chatId: z.string(), + storySlug: z.string(), + queryId: z.string(), + selections: z.record(z.string(), z.union([z.string(), z.array(z.string())])).default({}), + }), + ) + .query(async ({ input }) => { + return getStoryQuerySql(input.chatId, input.storySlug, input.queryId, input.selections); + }), + parseCronFromText: projectProtectedProcedure .input(z.object({ text: z.string().min(1) })) .mutation(async ({ input, ctx }) => { diff --git a/apps/backend/src/utils/chat-context-usage.ts b/apps/backend/src/utils/chat-context-usage.ts index b87febce6..404741468 100644 --- a/apps/backend/src/utils/chat-context-usage.ts +++ b/apps/backend/src/utils/chat-context-usage.ts @@ -1,3 +1,4 @@ +import { markSupersededExecuteSqlParts } from '@nao/shared/execute-sql-parts'; import type { LlmProvider } from '@nao/shared/types'; import { convertToModelMessages, type ModelMessage, type Tool } from 'ai'; @@ -35,7 +36,7 @@ export async function getChatAsModelMessages(opts: { projectId: string; tools: Record; }): Promise { - const uiMessages = await chatQueries.getChatMessages(opts.chatId); + const uiMessages = markSupersededExecuteSqlParts(await chatQueries.getChatMessages(opts.chatId)); const uiMessagesWithCompaction = compactionService.useLastCompaction(uiMessages); const memories = await memoryService.safeGetUserMemories(opts.userId, opts.projectId, opts.chatId); const systemPrompt = renderToMarkdown(SystemPrompt({ memories })); diff --git a/apps/frontend/package.json b/apps/frontend/package.json index cee04b892..315d3c928 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -25,6 +25,7 @@ "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-menubar": "^1.1.16", + "@radix-ui/react-popover": "^1.1.21", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.4", "@radix-ui/react-tooltip": "^1.2.8", @@ -60,6 +61,7 @@ "radix-ui": "^1.4.3", "react": "^19.2.0", "react-datepicker": "^9.1.0", + "react-day-picker": "^10.0.1", "react-dom": "^19.2.0", "react-resizable-panels": "^4", "react-syntax-highlighter": "^16.1.0", diff --git a/apps/frontend/src/components/chat-messages/chat-messages-readonly.tsx b/apps/frontend/src/components/chat-messages/chat-messages-readonly.tsx index 151b0ba73..7f5733727 100644 --- a/apps/frontend/src/components/chat-messages/chat-messages-readonly.tsx +++ b/apps/frontend/src/components/chat-messages/chat-messages-readonly.tsx @@ -1,5 +1,6 @@ import { memo, useMemo } from 'react'; import { ThumbsDown, ThumbsUp } from 'lucide-react'; +import { filterSupersededExecuteSqlParts } from '@nao/shared/execute-sql-parts'; import { UserMessageBubble } from './user-message'; import type { ForkMetadata, UIMessage } from '@nao/backend/chat'; import { SelectionCitationExcerpt } from '@/components/selection-citation-excerpt'; @@ -25,7 +26,7 @@ export function ChatMessagesReadonly({ className?: string; forkMetadata?: ForkMetadata; }) { - const messageGroups = useMemo(() => groupMessages(messages), [messages]); + const messageGroups = useMemo(() => groupMessages(filterSupersededExecuteSqlParts(messages)), [messages]); const citation = useMemo(() => { if (!forkMetadata?.selectionText || forkMetadata.selectionStart == null || forkMetadata.selectionEnd == null) { diff --git a/apps/frontend/src/components/chat-messages/chat-messages.tsx b/apps/frontend/src/components/chat-messages/chat-messages.tsx index 078a21e97..6fa12cafd 100644 --- a/apps/frontend/src/components/chat-messages/chat-messages.tsx +++ b/apps/frontend/src/components/chat-messages/chat-messages.tsx @@ -1,6 +1,7 @@ import { memo, useMemo, useRef } from 'react'; import { useQuery } from '@tanstack/react-query'; import { useParams, useRouterState } from '@tanstack/react-router'; +import { filterSupersededExecuteSqlParts } from '@nao/shared/execute-sql-parts'; import { TextShimmer } from '../ui/text-shimmer'; import { ChatError } from './chat-error'; import { FollowUpSuggestions } from './follow-up-suggestions'; @@ -74,7 +75,10 @@ export const ChatMessagesContent = memo(() => { const followUpSuggestionsToolCall = useMemo(() => getLastFollowUpSuggestionsToolCall(messages), [messages]); const extraComponentsRef = useRef(null); const extraComponentsHeight = useHeight(extraComponentsRef); - const visibleMessages = useMemo(() => messages.filter((m) => !m.isForked), [messages]); + const visibleMessages = useMemo( + () => filterSupersededExecuteSqlParts(messages.filter((m) => !m.isForked)), + [messages], + ); const messageGroups = useMemo(() => groupMessages(visibleMessages), [visibleMessages]); const forkMetadata = useQuery({ diff --git a/apps/frontend/src/components/side-panel/sql-editor.tsx b/apps/frontend/src/components/side-panel/sql-editor.tsx index fc1fa22dd..85854d2c0 100644 --- a/apps/frontend/src/components/side-panel/sql-editor.tsx +++ b/apps/frontend/src/components/side-panel/sql-editor.tsx @@ -1,14 +1,17 @@ -import { useCallback, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Editor } from '@monaco-editor/react'; +import { validateSqlFilterTemplate } from '@nao/shared/sql-template'; import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { Loader2, Play, Save } from 'lucide-react'; +import { AlertTriangle, Loader2, Play, Save } from 'lucide-react'; import { usePanelRef } from 'react-resizable-panels'; import { ResizableSeparator, ResizablePanel, ResizablePanelGroup } from '../ui/resizable'; import type { Monaco } from '@monaco-editor/react'; +import type { UIMessage } from '@nao/backend/chat'; import type { executeSql } from '@nao/shared/tools'; import type { editor } from 'monaco-editor'; import { useOptionalAgentContext } from '@/contexts/agent.provider'; +import { FixInChatButton } from '@/components/fix-in-chat-button'; import { SidePanelHeader } from '@/components/side-panel/side-panel-header'; import { TableDisplay } from '@/components/tool-calls/display-table'; import { Button } from '@/components/ui/button'; @@ -32,80 +35,134 @@ export const SidePanelContent = ({ }) => { const queryClient = useQueryClient(); const agent = useOptionalAgentContext(); + const agentRef = useRef(agent); + agentRef.current = agent; + const queryId = output.id; const [sqlQuery, setSqlQuery] = useState(input.sql_query); const [savedSql, setSavedSql] = useState(input.sql_query); const [previewOutput, setPreviewOutput] = useState(output); const [error, setError] = useState(null); - const handleRunRef = useRef<() => void>(() => {}); - const handleSaveAndRunRef = useRef<() => void>(() => {}); + const rootRef = useRef(null); + const editorRef = useRef(null); const sqlPanelRef = usePanelRef(); const groupElementRef = useRef(null); + const sqlQueryRef = useRef(sqlQuery); + sqlQueryRef.current = sqlQuery; - const canEdit = Boolean(editable && output.id && agent && !agent.isReadonly); + useEffect(() => { + setSqlQuery(input.sql_query); + setSavedSql(input.sql_query); + setPreviewOutput(output); + setError(null); + }, [queryId]); // eslint-disable-line react-hooks/exhaustive-deps -- reset only when switching queries + + const canEdit = Boolean(editable && queryId && agent && !agent.isReadonly); const isDirty = sqlQuery !== savedSql; + const validationIssues = useMemo(() => (canEdit ? validateSqlFilterTemplate(sqlQuery) : []), [canEdit, sqlQuery]); + + const previewMutation = useMutation(trpc.sql.previewQuery.mutationOptions()); + const saveMutation = useMutation(trpc.sql.updateQuery.mutationOptions()); + const isBusy = previewMutation.isPending || saveMutation.isPending; + const isBusyRef = useRef(isBusy); + isBusyRef.current = isBusy; + + const buildMutationInput = useCallback( + (sql: string) => ({ + queryId: queryId!, + sql_query: sql, + database_id: input.database_id ?? undefined, + name: input.name ?? undefined, + }), + [queryId, input.database_id, input.name], + ); + + const beginMutation = useCallback(() => { + const sql = editorRef.current?.getValue() ?? sqlQueryRef.current; + if (!queryId || !sql.trim() || isBusyRef.current) { + return null; + } + setSqlQuery(sql); + setError(null); + return sql; + }, [queryId]); - const previewMutation = useMutation( - trpc.sql.previewQuery.mutationOptions({ + const handleRun = useCallback(() => { + const sql = beginMutation(); + if (!sql) { + return; + } + previewMutation.mutate(buildMutationInput(sql), { onSuccess: (result) => { setPreviewOutput(result); setError(null); }, - onError: (err) => { - setError(err.message); - }, - }), - ); + onError: (err) => setError(getMutationErrorMessage(err)), + }); + }, [beginMutation, buildMutationInput, previewMutation]); - const saveMutation = useMutation( - trpc.sql.updateQuery.mutationOptions({ + const handleSaveAndRun = useCallback(() => { + const sql = beginMutation(); + if (!sql) { + return; + } + saveMutation.mutate(buildMutationInput(sql), { onSuccess: (result) => { setPreviewOutput(result.output); setSqlQuery(result.input.sql_query); setSavedSql(result.input.sql_query); setError(null); - if (agent) { - agent.setMessages( - applyExecuteSqlResultToMessages(agent.messages, result.output.id, result.input, result.output), - ); - } - queryClient.invalidateQueries({ queryKey: [['chat', 'get']] }); - }, - onError: (err) => { - setError(err.message); - }, - }), - ); - const isBusy = previewMutation.isPending || saveMutation.isPending; + const applyUpdate = (messages: UIMessage[]) => + applyExecuteSqlResultToMessages(messages, result.output.id, result.input, result.output); - const handleRun = () => { - if (!output.id || !sqlQuery.trim() || isBusy) { - return; - } - setError(null); - previewMutation.mutate({ - queryId: output.id, - sql_query: sqlQuery, - database_id: input.database_id, - name: input.name, + const currentAgent = agentRef.current; + if (currentAgent) { + currentAgent.setMessages((messages) => applyUpdate(messages)); + if (currentAgent.chatId) { + queryClient.setQueryData(trpc.chat.get.queryKey({ chatId: currentAgent.chatId }), (previous) => + previous ? { ...previous, messages: applyUpdate(previous.messages) } : previous, + ); + } + } + + void queryClient.invalidateQueries(trpc.story.getQuerySql.pathFilter()); + void queryClient.invalidateQueries(trpc.story.getFilteredQueryData.pathFilter()); + void queryClient.invalidateQueries(trpc.storyShare.getQuerySql.pathFilter()); + void queryClient.invalidateQueries(trpc.storyShare.getFilteredQueryData.pathFilter()); + }, + onError: (err) => setError(getMutationErrorMessage(err)), }); - }; + }, [beginMutation, buildMutationInput, saveMutation, queryClient]); - const handleSaveAndRun = () => { - if (!output.id || !sqlQuery.trim() || isBusy) { + useEffect(() => { + if (!canEdit) { return; } - setError(null); - saveMutation.mutate({ - queryId: output.id, - sql_query: sqlQuery, - database_id: input.database_id, - name: input.name, - }); - }; - handleRunRef.current = handleRun; - handleSaveAndRunRef.current = handleSaveAndRun; + const isFocusInsidePanel = (target: EventTarget | null) => + target instanceof Node && Boolean(rootRef.current?.contains(target)); + + const onKeyDown = (event: KeyboardEvent) => { + if (!isFocusInsidePanel(event.target)) { + return; + } + const withModifier = (event.metaKey || event.ctrlKey) && !event.altKey && !event.shiftKey; + if (withModifier && event.key.toLowerCase() === 's') { + event.preventDefault(); + event.stopPropagation(); + handleSaveAndRun(); + return; + } + if (withModifier && event.key === 'Enter') { + event.preventDefault(); + event.stopPropagation(); + handleRun(); + } + }; + + document.addEventListener('keydown', onKeyDown, true); + return () => document.removeEventListener('keydown', onKeyDown, true); + }, [canEdit, handleSaveAndRun, handleRun]); const fitSqlPanelToContent = useCallback( (instance: editor.IStandaloneCodeEditor) => { @@ -126,15 +183,8 @@ export const SidePanelContent = ({ ); const handleEditorMount = useCallback( - (instance: editor.IStandaloneCodeEditor, monaco: Monaco) => { - if (canEdit) { - instance.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => { - handleSaveAndRunRef.current(); - }); - instance.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, () => { - handleRunRef.current(); - }); - } + (instance: editor.IStandaloneCodeEditor, _monaco: Monaco) => { + editorRef.current = instance; let fitted = false; const tryFit = () => { @@ -147,12 +197,12 @@ export const SidePanelContent = ({ const disposable = instance.onDidContentSizeChange(tryFit); requestAnimationFrame(tryFit); }, - [canEdit, fitSqlPanelToContent], + [fitSqlPanelToContent], ); return ( -
- +
+ {canEdit && (
@@ -161,6 +211,7 @@ export const SidePanelContent = ({
)} + {validationIssues.length > 0 && } ); }; + +function SqlValidationBanner({ queryId, issues }: { queryId: string | undefined; issues: string[] }) { + const fixMessage = [ + `I'm seeing SQL template validation warnings${queryId ? ` for query ${queryId}` : ''}:`, + ...issues.map((issue) => `- ${issue}`), + '', + 'Please fix the SQL filter template while preserving the query id.', + ].join('\n'); + + return ( +
+
+
+ + + {issues.length} SQL template {issues.length === 1 ? 'warning' : 'warnings'} + +
+
    + {issues.slice(0, 5).map((issue) => ( +
  • + {issue} +
    • + ))} + {issues.length > 5 &&
  • and {issues.length - 5} more...
    • } +
+
+ +
+ ); +} + +function getMutationErrorMessage(err: unknown): string { + if (err instanceof Error && err.message) { + return err.message; + } + if (err && typeof err === 'object' && 'shape' in err) { + const shape = (err as { shape?: { message?: string } }).shape; + if (shape?.message) { + return shape.message; + } + } + return 'Failed to run query'; +} diff --git a/apps/frontend/src/components/side-panel/story-chart-embed.tsx b/apps/frontend/src/components/side-panel/story-chart-embed.tsx index bcc17407c..b8ee57246 100644 --- a/apps/frontend/src/components/side-panel/story-chart-embed.tsx +++ b/apps/frontend/src/components/side-panel/story-chart-embed.tsx @@ -1,15 +1,18 @@ -import { Pencil } from 'lucide-react'; +import { Code, Pencil } from 'lucide-react'; import { memo, useMemo, useState } from 'react'; -import type { UIMessage } from '@nao/backend/chat'; import type { displayChart } from '@nao/shared/tools'; +import { StoryChartQueryView } from '@/components/side-panel/story-chart-query'; import { ChartDisplay } from '@/components/tool-calls/display-chart'; import { ChartConfigEditDialog } from '@/components/tool-calls/display-chart-edit-dialog'; import { Button } from '@/components/ui/button'; import { useOptionalAgentContext } from '@/contexts/agent.provider'; import { useStoryChartEdit } from '@/contexts/story-chart-edit'; import { useStoryEmbedData } from '@/contexts/story-embed-data'; +import { useStoryQuerySql } from '@/contexts/story-query-sql'; import { sortByDateKey } from '@/lib/charts.utils'; +import { findLatestExecuteSqlInMessages } from '@/lib/execute-sql-messages'; +import { cn } from '@/lib/utils'; interface ChartBlock { queryId: string; @@ -34,18 +37,7 @@ export const StoryChartEmbed = memo(function StoryChartEmbed({ chart }: { chart: return fromEmbedData; } - const findInMessages = (messages: UIMessage[]) => { - for (const message of messages) { - for (const part of message.parts) { - if (part.type === 'tool-execute_sql' && part.output?.id === chart.queryId) { - return part.output; - } - } - } - return null; - }; - - return findInMessages(agent?.messages ?? []); + return findLatestExecuteSqlInMessages(agent?.messages ?? [], chart.queryId)?.output ?? null; }, [embedData, agent?.messages, chart.queryId]); const data = useMemo( @@ -103,8 +95,12 @@ interface StoryChartEmbedShellProps { */ export function StoryChartEmbedShell({ chart, availableColumns, children }: StoryChartEmbedShellProps) { const edit = useStoryChartEdit(); + const querySqlSource = useStoryQuerySql(); const [isEditOpen, setIsEditOpen] = useState(false); + const [showQuery, setShowQuery] = useState(false); const canEdit = Boolean(edit && chart.rawTag); + const canViewQuery = Boolean(querySqlSource); + const showHeader = canEdit || canViewQuery || (chart.chartType != 'kpi_card' && chart.title); const config = useMemo( () => ({ @@ -128,7 +124,7 @@ export function StoryChartEmbedShell({ chart, availableColumns, children }: Stor return (
- {(canEdit || (chart.chartType != 'kpi_card' && chart.title)) && ( + {showHeader && (
{chart.chartType != 'kpi_card' && chart.title ? ( @@ -137,20 +133,40 @@ export function StoryChartEmbedShell({ chart, availableColumns, children }: Stor ) : (
)} - {canEdit && ( - - )} +
+ {canViewQuery && ( + + )} + {canEdit && ( + + )} +
)} -
{children}
+ {showQuery && querySqlSource ? ( + + ) : ( +
{children}
+ )} {canEdit && edit && chart.rawTag && ( + + Loading query… +
+ ); + } + + if (query.isError || !data) { + return ( +
+ {query.error?.message ?? 'Query unavailable'} +
+ ); + } + + const showBoth = data.sqlQuery.trim() !== data.renderedSql.trim(); + + return ( +
+ {showBoth ? ( + <> +
+

+ Template query +

+ +
+
+

+ Executed query +

+ +
+ + ) : ( +
+

Query

+ +
+ )} +
+ ); +} diff --git a/apps/frontend/src/components/side-panel/story-preview.tsx b/apps/frontend/src/components/side-panel/story-preview.tsx index c5e80368a..8a5700441 100644 --- a/apps/frontend/src/components/side-panel/story-preview.tsx +++ b/apps/frontend/src/components/side-panel/story-preview.tsx @@ -9,6 +9,7 @@ import { StoryFilterBar } from '@/components/story-filter-bar'; import { SegmentList } from '@/components/story-rendering'; import { StoryChartEmbed as StaticChartEmbed } from '@/components/side-panel/story-chart-embed'; import { StoryTableEmbed as StaticTableEmbed } from '@/components/side-panel/story-table-embed'; +import { StoryQuerySqlProvider } from '@/contexts/story-query-sql'; import { useStoryFilters } from '@/hooks/use-story-filters'; import { trpc } from '@/main'; @@ -48,6 +49,10 @@ export const StoryPreview = memo(function StoryPreview({ const effectiveQueryData = storyFilters.queryData; const useLiveUnfiltered = isNoCacheMode && !storyFilters.hasActiveFilters; + const querySqlSource = useMemo( + () => ({ api: filterApi, selections: storyFilters.activeSelections }), + [filterApi, storyFilters.activeSelections], + ); const renderChart = useCallback( (chart: ParsedChartBlock) => { @@ -94,21 +99,22 @@ export const StoryPreview = memo(function StoryPreview({ ); return ( -
- - -
+ +
+ + +
+ ); }); diff --git a/apps/frontend/src/components/side-panel/story-table-embed.tsx b/apps/frontend/src/components/side-panel/story-table-embed.tsx index 14418d42e..5d73680fe 100644 --- a/apps/frontend/src/components/side-panel/story-table-embed.tsx +++ b/apps/frontend/src/components/side-panel/story-table-embed.tsx @@ -1,7 +1,6 @@ import { sanitizeConditionalFormats } from '@nao/shared/conditional-formatting'; import { Pencil } from 'lucide-react'; import { memo, useMemo, useState } from 'react'; -import type { UIMessage } from '@nao/backend/chat'; import type { ParsedTableBlock } from '@nao/shared/story-segments'; import { DataTableCard } from '@/components/data-table-card'; @@ -10,6 +9,7 @@ import { Button } from '@/components/ui/button'; import { useOptionalAgentContext } from '@/contexts/agent.provider'; import { useStoryEmbedData } from '@/contexts/story-embed-data'; import { useStoryTableEdit } from '@/contexts/story-table-edit'; +import { findLatestExecuteSqlInMessages } from '@/lib/execute-sql-messages'; export const StoryTableEmbed = memo(function StoryTableEmbed({ table }: { table: ParsedTableBlock }) { const agent = useOptionalAgentContext(); @@ -21,18 +21,7 @@ export const StoryTableEmbed = memo(function StoryTableEmbed({ table }: { table: return fromEmbedData; } - const findInMessages = (messages: UIMessage[]) => { - for (const message of messages) { - for (const part of message.parts) { - if (part.type === 'tool-execute_sql' && part.output?.id === table.queryId) { - return part.output; - } - } - } - return null; - }; - - return findInMessages(agent?.messages ?? []); + return findLatestExecuteSqlInMessages(agent?.messages ?? [], table.queryId)?.output ?? null; }, [embedData, agent?.messages, table.queryId]); if (!sourceData?.data || !Array.isArray(sourceData.data)) { diff --git a/apps/frontend/src/components/story-filter-bar.tsx b/apps/frontend/src/components/story-filter-bar.tsx index ae39fd4a1..18f86baff 100644 --- a/apps/frontend/src/components/story-filter-bar.tsx +++ b/apps/frontend/src/components/story-filter-bar.tsx @@ -1,12 +1,16 @@ -import { AlertTriangle, FilterX, ListFilter, Loader2 } from 'lucide-react'; +import { AlertTriangle, CalendarIcon, FilterX, Loader2, X } from 'lucide-react'; +import { format } from 'date-fns'; import { useCallback, useEffect, useState } from 'react'; import { useQuery } from '@tanstack/react-query'; +import { isFilterSelectionActive } from '@nao/shared/sql-template'; import type { StoryFilterSelection } from '@nao/shared/sql-template'; +import type { DateRange } from 'react-day-picker'; import type { ParsedFilterBlock } from '@nao/shared/story-segments'; import type { StoryFilterApi } from '@/hooks/use-story-filters'; import { FixInChatButton } from '@/components/fix-in-chat-button'; import { Button } from '@/components/ui/button'; +import { Calendar } from '@/components/ui/calendar'; import { DropdownMenu, DropdownMenuCheckboxItem, @@ -14,7 +18,9 @@ import { DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; import { Input } from '@/components/ui/input'; +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; +import { cn } from '@/lib/utils'; import { trpc } from '@/main'; export function StoryFilterBar({ @@ -23,14 +29,12 @@ export function StoryFilterBar({ onSelectionChange, onClear, api, - isFiltering, }: { filters: ParsedFilterBlock[]; selections: Record; onSelectionChange: (filterId: string, selection: StoryFilterSelection) => void; onClear: () => void; api?: StoryFilterApi | null; - isFiltering?: boolean; }) { const [optionErrors, setOptionErrors] = useState>({}); @@ -111,9 +115,23 @@ function StoryFilterControl({ onOptionError(filter.id, error); }, [filter.id, error, onOptionError]); + const isActive = isFilterSelectionActive(filter.filterType, selection); + return ( -
+ ); + }, + ...components, + }} + {...props} + /> + ); +} + +function CalendarDayButton({ className, day, modifiers, ...props }: React.ComponentProps) { + const defaultClassNames = getDefaultClassNames(); + + const ref = React.useRef(null); + React.useEffect(() => { + if (modifiers.focused) ref.current?.focus(); + }, [modifiers.focused]); + + return ( +
+
+ {children} +
+
tags keep working.', ] : []), diff --git a/apps/backend/src/agents/tools/story.ts b/apps/backend/src/agents/tools/story.ts index 553ed57da..291ae17d6 100644 --- a/apps/backend/src/agents/tools/story.ts +++ b/apps/backend/src/agents/tools/story.ts @@ -171,9 +171,15 @@ async function carryOverTableFormatting(code: string, chatId: string): Promise { - const warnings = await getStoryTemplateWarnings(chatId, code); - return warnings.length > 0 ? { template_warnings: warnings } : {}; + try { + const warnings = await getStoryTemplateWarnings(chatId, code); + return warnings.length > 0 ? { template_warnings: warnings } : {}; + } catch (error) { + console.error('Failed to compute story template warnings', error); + return {}; + } } function rememberStoryArtifact(context: ToolContext, id: string, title: string): void { diff --git a/apps/backend/src/queries/execute-sql.queries.ts b/apps/backend/src/queries/execute-sql.queries.ts index 12ef568a4..1b75614ea 100644 --- a/apps/backend/src/queries/execute-sql.queries.ts +++ b/apps/backend/src/queries/execute-sql.queries.ts @@ -1,5 +1,5 @@ import { executeSql } from '@nao/shared/tools'; -import { and, asc, desc, eq, isNull, sql } from 'drizzle-orm'; +import { and, asc, desc, eq, isNull, or, sql } from 'drizzle-orm'; import s from '../db/abstractSchema'; import { db } from '../db/db'; @@ -14,6 +14,10 @@ export function messagePartToolOutputIdEquals(queryId: string) { : sql`json_extract(${s.messagePart.toolOutput}, '$.id') = ${queryId}`; } +function messagePartToolOutputIdIn(queryIds: Set) { + return or(...[...queryIds].map(messagePartToolOutputIdEquals)); +} + export type LatestExecuteSqlRow = { projectId: string; userId: string; @@ -46,7 +50,7 @@ export async function getLatestExecuteSqlByQueryId(queryId: string): Promise) { return db .select({ toolInput: s.messagePart.toolInput, @@ -144,9 +148,10 @@ async function loadLatestExecuteSqlParts(chatId: string) { eq(s.chatMessage.chatId, chatId), isNull(s.chatMessage.supersededAt), eq(s.messagePart.toolName, EXECUTE_SQL_TOOL_NAME), + messagePartToolOutputIdIn(queryIds), ), ) - .orderBy(asc(s.chatMessage.createdAt), asc(s.messagePart.order)) + .orderBy(asc(s.chatMessage.createdAt), asc(s.messagePart.createdAt), asc(s.messagePart.order)) .execute(); } @@ -162,7 +167,7 @@ export async function getLatestSqlQueriesByIds( return {}; } - const parts = await loadLatestExecuteSqlParts(chatId); + const parts = await loadLatestExecuteSqlParts(chatId, queryIds); const queries: Record = {}; for (const part of parts) { const output = part.toolOutput as { id?: string } | null; @@ -189,7 +194,7 @@ export async function getLatestSqlQueryDataByIds( return {}; } - const parts = await loadLatestExecuteSqlParts(chatId); + const parts = await loadLatestExecuteSqlParts(chatId, queryIds); const data: Record = {}; for (const part of parts) { const output = part.toolOutput as { id?: string; data?: unknown[]; columns?: string[] } | null; diff --git a/apps/backend/src/services/story-filters.ts b/apps/backend/src/services/story-filters.ts index 54cc2703e..1206a4f14 100644 --- a/apps/backend/src/services/story-filters.ts +++ b/apps/backend/src/services/story-filters.ts @@ -7,6 +7,7 @@ import { import { getStoryFiltersFromCode } from '@nao/shared/story-segments'; import { TRPCError } from '@trpc/server'; +import { env } from '../env'; import * as chatQueries from '../queries/chat.queries'; import * as projectQueries from '../queries/project.queries'; import * as storyQueries from '../queries/story.queries'; @@ -15,6 +16,12 @@ import { executeRawSql } from './live-story'; const FILTER_OPTIONS_LIMIT = 100; +export function assertStoryFiltersEnabled() { + if (!env.BETA_STORY_FILTERS_ENABLED) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'Story filters are disabled on this instance.' }); + } +} + export async function getStoryFilterOptions( chatId: string, storySlug: string, diff --git a/apps/backend/src/services/story-template-validation.ts b/apps/backend/src/services/story-template-validation.ts index 08b9b53a7..b73334358 100644 --- a/apps/backend/src/services/story-template-validation.ts +++ b/apps/backend/src/services/story-template-validation.ts @@ -14,6 +14,12 @@ export async function getStoryTemplateWarnings(chatId: string, code: string): Pr const sqlQueries = await storyQueries.getSqlQueriesFromCode(chatId, code); const warnings: string[] = []; + for (const duplicateId of findDuplicateFilterIds(knownFilterIds)) { + warnings.push( + `Story declares multiple tags with id "${duplicateId}". Filter ids must be unique — rename or remove the duplicates so selections and SQL rendering use the same definition.`, + ); + } + for (const [queryId, { sqlQuery }] of Object.entries(sqlQueries)) { for (const issue of validateSqlFilterTemplate(sqlQuery, { knownFilterIds })) { warnings.push(`[${queryId}] ${issue}`); @@ -31,3 +37,15 @@ export async function getStoryTemplateWarnings(chatId: string, code: string): Pr return warnings; } + +function findDuplicateFilterIds(filterIds: string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const filterId of filterIds) { + if (seen.has(filterId)) { + duplicates.add(filterId); + } + seen.add(filterId); + } + return [...duplicates]; +} diff --git a/apps/backend/src/trpc/shared-story.routes.ts b/apps/backend/src/trpc/shared-story.routes.ts index 4a2a8118f..49f58db04 100644 --- a/apps/backend/src/trpc/shared-story.routes.ts +++ b/apps/backend/src/trpc/shared-story.routes.ts @@ -2,7 +2,6 @@ import { DOWNLOAD_FORMATS, SHARE_VISIBILITY } from '@nao/shared/types'; import { TRPCError } from '@trpc/server'; import { z } from 'zod/v4'; -import { env } from '../env'; import * as activityQueries from '../queries/activity.queries'; import * as chatQueries from '../queries/chat.queries'; import * as projectQueries from '../queries/project.queries'; @@ -11,7 +10,12 @@ import * as storyQueries from '../queries/story.queries'; import * as storyFolderQueries from '../queries/story-folder.queries'; import { logActivity } from '../services/activity'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; -import { getFilteredStoryQueryData, getStoryFilterOptions, getStoryQuerySql } from '../services/story-filters'; +import { + assertStoryFiltersEnabled, + getFilteredStoryQueryData, + getStoryFilterOptions, + getStoryQuerySql, +} from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { notifySharedItemRecipients } from '../utils/email'; import { buildDownloadResponse } from '../utils/story-download'; @@ -36,12 +40,6 @@ const shareAccessProcedure = resourceProjectProcedure( sharedStoryQueries.canUserAccessSharedStory(item.id, userId), ); -function assertStoryFiltersEnabled() { - if (!env.BETA_STORY_FILTERS_ENABLED) { - throw new TRPCError({ code: 'FORBIDDEN', message: 'Story filters are disabled on this instance.' }); - } -} - export const sharedStoryRoutes = { list: protectedProcedure.input(z.object({ projectId: z.string() })).query(async ({ input, ctx }) => { const projects = await projectQueries.listUserProjects(ctx.user.id); @@ -210,6 +208,7 @@ export const sharedStoryRoutes = { }), ) .query(async ({ input, ctx }) => { + assertStoryFiltersEnabled(); const shared = ctx.resource; if (!shared.chatId) { throw new TRPCError({ code: 'BAD_REQUEST', message: 'Shared story has no chat.' }); diff --git a/apps/backend/src/trpc/story.routes.ts b/apps/backend/src/trpc/story.routes.ts index 53352a364..aaf6e50ab 100644 --- a/apps/backend/src/trpc/story.routes.ts +++ b/apps/backend/src/trpc/story.routes.ts @@ -4,7 +4,6 @@ import { DOWNLOAD_FORMATS } from '@nao/shared/types'; import { TRPCError } from '@trpc/server'; import { z } from 'zod/v4'; -import { env } from '../env'; import { STORY_REFRESH_JOB_NAME } from '../handlers/story-refresh.handler'; import * as activityQueries from '../queries/activity.queries'; import * as chatQueries from '../queries/chat.queries'; @@ -16,7 +15,12 @@ import * as storyFolderQueries from '../queries/story-folder.queries'; import { naturalLanguageToCron } from '../services/cron-nlp'; import { executeLiveQuery, getStoryQueryData, refreshStoryData } from '../services/live-story'; import { nextCronTick } from '../services/scheduler.service'; -import { getFilteredStoryQueryData, getStoryFilterOptions, getStoryQuerySql } from '../services/story-filters'; +import { + assertStoryFiltersEnabled, + getFilteredStoryQueryData, + getStoryFilterOptions, + getStoryQuerySql, +} from '../services/story-filters'; import { logAnalyticsEvent } from '../utils/analytics-event'; import { buildDownloadResponse } from '../utils/story-download'; import { extractStorySummary } from '../utils/story-summary'; @@ -25,12 +29,6 @@ import { canSendProcedure, ownedResourceProcedure, projectProtectedProcedure, pr const chatOwnerProcedure = ownedResourceProcedure(chatQueries.getChatOwnerId, 'chat'); const storyOwnerProcedure = ownedResourceProcedure(storyQueries.getStoryOwnerId, 'story'); -function assertStoryFiltersEnabled() { - if (!env.BETA_STORY_FILTERS_ENABLED) { - throw new TRPCError({ code: 'FORBIDDEN', message: 'Story filters are disabled on this instance.' }); - } -} - const bulkStoryItemsInput = z.object({ items: z .array( @@ -326,6 +324,7 @@ export const storyRoutes = { }), ) .query(async ({ input }) => { + assertStoryFiltersEnabled(); return getStoryQuerySql(input.chatId, input.storySlug, input.queryId, input.selections); }), diff --git a/apps/frontend/src/components/side-panel/story-preview.tsx b/apps/frontend/src/components/side-panel/story-preview.tsx index 49fa5f4ca..bec8e706d 100644 --- a/apps/frontend/src/components/side-panel/story-preview.tsx +++ b/apps/frontend/src/components/side-panel/story-preview.tsx @@ -53,8 +53,8 @@ export const StoryPreview = memo(function StoryPreview({ const useLiveUnfiltered = isNoCacheMode && !storyFilters.hasActiveFilters; const querySqlSource = useMemo( - () => (filtersEnabled ? { api: filterApi, selections: storyFilters.activeSelections } : null), - [filterApi, filtersEnabled, storyFilters.activeSelections], + () => (storyFilters.filtersEnabled ? { api: filterApi, selections: storyFilters.debouncedSelections } : null), + [filterApi, storyFilters.filtersEnabled, storyFilters.debouncedSelections], ); const renderChart = useCallback( diff --git a/apps/frontend/src/components/story-filter-bar.tsx b/apps/frontend/src/components/story-filter-bar.tsx index 92ddfc9ee..bc5fd3898 100644 --- a/apps/frontend/src/components/story-filter-bar.tsx +++ b/apps/frontend/src/components/story-filter-bar.tsx @@ -287,7 +287,9 @@ function parseDateString(value: string | undefined): Date | undefined { if (!year || !month || !day) { return undefined; } - return new Date(year, month - 1, day); + const date = new Date(year, month - 1, day); + const isSameCalendarDate = date.getFullYear() === year && date.getMonth() === month - 1 && date.getDate() === day; + return isSameCalendarDate ? date : undefined; } function formatDateRangeLabel(range: DateRange | undefined): string { diff --git a/apps/frontend/src/components/story-tabbed-content.tsx b/apps/frontend/src/components/story-tabbed-content.tsx index 604ed762f..2c424729b 100644 --- a/apps/frontend/src/components/story-tabbed-content.tsx +++ b/apps/frontend/src/components/story-tabbed-content.tsx @@ -49,8 +49,11 @@ export function StoryTabbedContent({ enabled: Boolean(filterApi), }); const querySqlSource = useMemo( - () => (filterApi ? { api: filterApi, selections: storyFilters.activeSelections } : null), - [filterApi, storyFilters.activeSelections], + () => + filterApi && storyFilters.filtersEnabled + ? { api: filterApi, selections: storyFilters.debouncedSelections } + : null, + [filterApi, storyFilters.filtersEnabled, storyFilters.debouncedSelections], ); return ( diff --git a/apps/frontend/src/hooks/use-story-filters.ts b/apps/frontend/src/hooks/use-story-filters.ts index a634e7a72..93219b86c 100644 --- a/apps/frontend/src/hooks/use-story-filters.ts +++ b/apps/frontend/src/hooks/use-story-filters.ts @@ -108,9 +108,11 @@ export function useStoryFilters({ const isFiltering = isSelectionsPending || isFetching; return { + filtersEnabled, filters, selections, activeSelections, + debouncedSelections, setSelection, clearSelections, hasActiveFilters, diff --git a/apps/shared/src/sql-template.ts b/apps/shared/src/sql-template.ts index d00852eed..c98b171f0 100644 --- a/apps/shared/src/sql-template.ts +++ b/apps/shared/src/sql-template.ts @@ -12,7 +12,7 @@ export const STORY_FILTER_ID_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/; const FILTER_BLOCK_REGEX = /\{%\s*filter\s+([A-Za-z_][A-Za-z0-9_]*)\s*%\}([\s\S]*?)\{%\s*endfilter\s*%\}/g; const FILTER_PLACEHOLDER_REGEX = /\{\{\s*filters\.([A-Za-z_][A-Za-z0-9_]*)\.sql\s*\}\}/g; -const ANY_FILTER_PLACEHOLDER_REGEX = /\{\{\s*filters\.([^\s.{}]+)(?:\.([^\s{}]+))?\s*\}\}/g; +const ANY_FILTER_PLACEHOLDER_REGEX = /\{\{\s*filters\.([^\s.{}]+)(?:\.([^\s{}]*))?\s*\}\}/g; const FILTER_OPEN_REGEX = /\{%\s*filter\b[^%}]*%\}/g; const FILTER_CLOSE_REGEX = /\{%\s*endfilter\s*%\}/g; const FILTER_DELIMITER_REGEX = /\{%\s*(?:filter\b([^%}]*)|(endfilter))\s*%\}/g; diff --git a/apps/shared/tests/sql-template.test.ts b/apps/shared/tests/sql-template.test.ts index 1e7a19549..6636268b8 100644 --- a/apps/shared/tests/sql-template.test.ts +++ b/apps/shared/tests/sql-template.test.ts @@ -152,6 +152,11 @@ SELECT 1 FROM orders WHERE 1 = 1 ); expect(delimiterIssues.some((issue) => /Unexpected "\{% endfilter %\}"/.test(issue))).toBe(true); }); + + it('flags placeholders with a trailing dot and no property', () => { + const issues = validateSqlFilterTemplate('SELECT 1 WHERE country = {{ filters.country. }}'); + expect(issues.some((issue) => /property "\.sql" is required/.test(issue))).toBe(true); + }); }); describe('findUnreferencedStoryFilters', () => {