From bd201c37671b1dfcdf5ffa06fcd9d6f6ddec3968 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 01:12:08 +0530 Subject: [PATCH 01/55] feat(project): Added projects --- .env.example | 1 + src/__tests__/auth.test.ts | 3 +- src/__tests__/createAPIKey.test.ts | 4 +- src/__tests__/fixtures/apiKey.ts | 5 + src/context/auth.ts | 1 + src/interceptors/auth.ts | 4 + src/routes/gRPC/auth/createAPIKey.ts | 1 + src/routes/gRPC/data/query.ts | 16 +- src/routes/gRPC/events/registerEvent.ts | 6 +- src/routes/gRPC/events/streamEvents.ts | 6 +- src/routes/gRPC/payment/createCheckoutLink.ts | 15 +- src/routes/gRPC/payment/paymentProvider.ts | 80 ++--- src/routes/http/api/apiKeys.ts | 18 +- src/routes/http/api/expressions.ts | 14 +- src/routes/http/api/onboarding.ts | 74 ++-- src/routes/http/api/tags.ts | 12 +- src/routes/http/api/webhookDeliveries.ts | 8 +- src/routes/http/api/webhookEndpoints.ts | 36 +- src/routes/http/createdCheckout.ts | 7 +- src/routes/http/forwardWebhook.ts | 8 +- src/routes/http/registerWebhookRoutes.ts | 15 +- .../clickhouse/handlers/addAiTokenUsage.ts | 3 +- .../clickhouse/handlers/addBasicUsage.ts | 3 +- .../handlers/priceRequestAiTokenUsage.ts | 4 +- .../handlers/priceRequestBasicUsage.ts | 4 +- .../clickhouse/handlers/queryEvents.ts | 30 +- src/storage/adapter/clickhouse/utils.ts | 1 + .../postgres/handlers/addAiTokenUsage.ts | 3 +- .../postgres/handlers/addBasicUsage.ts | 7 +- .../adapter/postgres/handlers/priceRequest.ts | 2 +- .../adapter/postgres/handlers/queryEvents.ts | 33 +- src/storage/db/postgres/helpers/apiKeys.ts | 11 +- .../db/postgres/helpers/expressions.ts | 40 ++- src/storage/db/postgres/helpers/metadata.ts | 100 ++---- src/storage/db/postgres/helpers/payments.ts | 2 + src/storage/db/postgres/helpers/sessions.ts | 4 + src/storage/db/postgres/helpers/tags.ts | 37 +- src/storage/db/postgres/helpers/users.ts | 12 +- .../db/postgres/helpers/webhookEndpoints.ts | 4 + src/storage/db/postgres/schema.ts | 132 ++++++- src/utils/apiKeyCache.ts | 1 + src/utils/authenticateHttpApiKey.ts | 15 +- src/utils/authenticateMasterApiKey.ts | 40 +++ src/utils/fetchTagAmount.ts | 14 +- src/utils/parseExpr.ts | 28 +- src/zod/event.ts | 337 ++++++++++-------- src/zod/internals.ts | 1 + 47 files changed, 808 insertions(+), 394 deletions(-) create mode 100644 src/utils/authenticateMasterApiKey.ts diff --git a/.env.example b/.env.example index 9b4d310..1f9ebac 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/scrawn CLICKHOUSE_URL=http://default:clickhouse@localhost:8123/scrawn HMAC_SECRET= +MASTER_API_KEY_HASH= # HMAC-SHA256 hex hash of the master API key (used for project creation during onboarding) APP_URL=http://localhost:8060 # URL the Scrawn backend is hosted on # SENTRY_DSN= diff --git a/src/__tests__/auth.test.ts b/src/__tests__/auth.test.ts index 71819ac..f8bd322 100644 --- a/src/__tests__/auth.test.ts +++ b/src/__tests__/auth.test.ts @@ -15,11 +15,12 @@ import { getPostgresDB } from "../storage/db/postgres/db"; import { webhookEndpointsTable } from "../storage/db/postgres/schema"; import { DateTime } from "luxon"; import { clearDatabase } from "./db"; -import { insertKey } from "./fixtures/apiKey"; +import { insertKey, TEST_PROJECT_ID } from "./fixtures/apiKey"; async function insertWebhookEndpoint(apiKeyId: string): Promise { const db = getPostgresDB(); await db.insert(webhookEndpointsTable).values({ + projectId: TEST_PROJECT_ID, apiKeyId, url: "https://example.com/webhook", privateKey: "test-private-key", diff --git a/src/__tests__/createAPIKey.test.ts b/src/__tests__/createAPIKey.test.ts index e0a2521..e691a3a 100644 --- a/src/__tests__/createAPIKey.test.ts +++ b/src/__tests__/createAPIKey.test.ts @@ -14,7 +14,7 @@ import { registerEvent, } from "./fixtures/grpc"; import { verifyApiKeyCreated } from "./assertions/events"; -import { createTestApiKey } from "./fixtures/apiKey"; +import { createTestApiKey, TEST_PROJECT_ID } from "./fixtures/apiKey"; import { getPostgresDB } from "../storage/db/postgres/db"; import { hashAPIKey } from "../utils/hashAPIKey"; import { @@ -44,6 +44,7 @@ async function createDashboardApiKey(): Promise<{ key: hashAPIKey(rawKey), role: "dashboard", expiresAt: DateTime.utc().plus({ years: 1 }).toISO(), + projectId: TEST_PROJECT_ID, }) .returning({ id: apiKeysTable.id }); return { rawKey, id: key!.id }; @@ -149,6 +150,7 @@ describe("AuthService", () => { const db = getPostgresDB(); await db.insert(webhookEndpointsTable).values({ + projectId: TEST_PROJECT_ID, apiKeyId: res.apiKeyId, url: "https://example.com/webhook", privateKey: "test-private-key", diff --git a/src/__tests__/fixtures/apiKey.ts b/src/__tests__/fixtures/apiKey.ts index dbdb718..1785354 100644 --- a/src/__tests__/fixtures/apiKey.ts +++ b/src/__tests__/fixtures/apiKey.ts @@ -6,6 +6,8 @@ import { import { hashAPIKey } from "../../utils/hashAPIKey"; import { DateTime } from "luxon"; +export const TEST_PROJECT_ID = "00000000-0000-0000-0000-000000000001"; + export async function createTestApiKey(): Promise<{ rawKey: string; id: string; @@ -19,10 +21,12 @@ export async function createTestApiKey(): Promise<{ key: hashAPIKey(rawKey), role: "test", expiresAt: DateTime.utc().plus({ years: 1 }).toISO(), + projectId: TEST_PROJECT_ID, }) .returning({ id: apiKeysTable.id }); await db.insert(webhookEndpointsTable).values({ + projectId: TEST_PROJECT_ID, apiKeyId: key!.id, url: "https://example.com/webhook", privateKey: "test-private-key", @@ -47,6 +51,7 @@ export async function insertKey( expiresAt: overrides.expiresAt ?? DateTime.utc().plus({ years: 1 }).toISO(), revoked: overrides.revoked ?? false, + projectId: TEST_PROJECT_ID, }) .returning({ id: apiKeysTable.id }); return key!.id; diff --git a/src/context/auth.ts b/src/context/auth.ts index 8bfe9da..bea042c 100644 --- a/src/context/auth.ts +++ b/src/context/auth.ts @@ -6,4 +6,5 @@ export interface AuthContext { apiKeyId: string; role: ApiKeyRole; mode: "production" | "test" | null; + projectId: string; } diff --git a/src/interceptors/auth.ts b/src/interceptors/auth.ts index 076b955..6f2fd76 100644 --- a/src/interceptors/auth.ts +++ b/src/interceptors/auth.ts @@ -167,6 +167,7 @@ export function authInterceptor( apiKeyId: cached.id, role: cached.role, mode: cached.mode, + projectId: cached.projectId, }; wideEventBuilder?.setAuth(cached.id, true); @@ -220,6 +221,7 @@ export function authInterceptor( id: apiKeyRecord.id, role: apiKeyRecord.role as ApiKeyRole, mode: recordMode, + projectId: apiKeyRecord.projectId, expiresAt: apiKeyRecord.expiresAt, }); @@ -227,6 +229,7 @@ export function authInterceptor( apiKeyId: apiKeyRecord.id, role: apiKeyRecord.role as ApiKeyRole, mode: recordMode, + projectId: apiKeyRecord.projectId, }; wideEventBuilder?.setAuth(apiKeyRecord.id, false); @@ -270,6 +273,7 @@ async function lookupApiKey(apiKeyHash: string) { role: apiKeysTable.role, expiresAt: apiKeysTable.expiresAt, revoked: apiKeysTable.revoked, + projectId: apiKeysTable.projectId, }) .from(apiKeysTable) .where(eq(apiKeysTable.key, apiKeyHash)) diff --git a/src/routes/gRPC/auth/createAPIKey.ts b/src/routes/gRPC/auth/createAPIKey.ts index 462e521..535214c 100644 --- a/src/routes/gRPC/auth/createAPIKey.ts +++ b/src/routes/gRPC/auth/createAPIKey.ts @@ -70,6 +70,7 @@ export async function createAPIKey( key: apiKeyHash, role: validatedData.role, expiresAt: expiresAt.toISO(), + projectId: auth.projectId, }); if (!keyEventData) { diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index 9f2799f..3732cf3 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -2,6 +2,7 @@ import type { sendUnaryData } from "@grpc/grpc-js"; import { QueryRequest, QueryResponse, Row } from "../../../gen/data/v1/data"; import { dataQuerySchema, type DataQueryRequest } from "../../../zod/data"; import { EventError } from "../../../errors/event"; +import { AuthError } from "../../../errors/auth"; import { formatZodError } from "../../../utils/formatZodError"; import { getPostgresDB } from "../../../storage/db/postgres/db"; import { @@ -29,6 +30,7 @@ import type { SQL } from "drizzle-orm"; import type { AnyPgColumn } from "drizzle-orm/pg-core"; import type { WideEventBuilder } from "../../../context/requestContext"; import { wideEventContextKey } from "../../../context/requestContext"; +import { apiKeyContextKey } from "../../../context/auth"; import type { ContextUnaryCall } from "../../../interface/types/context.js"; interface FieldDef { @@ -206,6 +208,11 @@ export async function queryData( | undefined; try { + const auth = call[apiKeyContextKey]; + if (!auth) { + return callback?.(AuthError.invalidAPIKey("API key context not found")); + } + const req = { ...call.request } as Record; const validated = dataQuerySchema.parse(req); @@ -223,7 +230,14 @@ export async function queryData( } const db = getPostgresDB(); - const whereClause = buildWhere(validated.where, tableDef); + const userWhere = buildWhere(validated.where, tableDef); + const projectFilter = eq( + (tableDef.table as any).projectId, + auth.projectId + ) as SQL; + const whereClause = userWhere + ? and(projectFilter, userWhere) + : projectFilter; const selectCols = buildSelect(tableDef); const columns = Object.keys(tableDef.fields); diff --git a/src/routes/gRPC/events/registerEvent.ts b/src/routes/gRPC/events/registerEvent.ts index 5ca981d..5973823 100644 --- a/src/routes/gRPC/events/registerEvent.ts +++ b/src/routes/gRPC/events/registerEvent.ts @@ -5,7 +5,7 @@ import { import type { WideEventBuilder } from "../../../context/requestContext"; import { apiKeyContextKey } from "../../../context/auth"; import { wideEventContextKey } from "../../../context/requestContext"; -import { registerEventSchema } from "../../../zod/event"; +import { createRegisterEventSchema } from "../../../zod/event"; import { EventError } from "../../../errors/event"; import { AuthError } from "../../../errors/auth"; import { createEventInstance, storeEvent } from "../../../utils/eventHelpers"; @@ -33,7 +33,9 @@ export async function registerEvent( ); } - const eventSkeleton = await registerEventSchema.parseAsync({ ...req }); + const eventSkeleton = await createRegisterEventSchema( + auth.projectId + ).parseAsync({ ...req }); wideEventBuilder?.setUser(eventSkeleton.userId); wideEventBuilder?.setEventContext({ eventType: eventSkeleton.type }); diff --git a/src/routes/gRPC/events/streamEvents.ts b/src/routes/gRPC/events/streamEvents.ts index c91faeb..e97e91d 100644 --- a/src/routes/gRPC/events/streamEvents.ts +++ b/src/routes/gRPC/events/streamEvents.ts @@ -8,7 +8,7 @@ import { import { EventError } from "../../../errors/event"; import { AuthError } from "../../../errors/auth"; import { StorageError } from "../../../errors/storage"; -import { streamEventSchema } from "../../../zod/event"; +import { createStreamEventSchema } from "../../../zod/event"; import { createEventInstance, storeEvent } from "../../../utils/eventHelpers"; import { apiKeyContextKey } from "../../../context/auth"; import { wideEventContextKey } from "../../../context/requestContext"; @@ -88,7 +88,9 @@ export async function streamEvents( for await (const req of call) { try { - const eventSkeleton = await streamEventSchema.parseAsync({ ...req }); + const eventSkeleton = await createStreamEventSchema( + auth.projectId + ).parseAsync({ ...req }); wideEventBuilder?.setUser(eventSkeleton.userId); wideEventBuilder?.setEventContext({ eventType: "AI_TOKEN_USAGE" }); diff --git a/src/routes/gRPC/payment/createCheckoutLink.ts b/src/routes/gRPC/payment/createCheckoutLink.ts index 097573b..23e58e6 100644 --- a/src/routes/gRPC/payment/createCheckoutLink.ts +++ b/src/routes/gRPC/payment/createCheckoutLink.ts @@ -64,7 +64,7 @@ export async function createCheckoutLink( const mode = auth.mode; - const config = await getPaymentProviderConfig(mode); + const config = await getPaymentProviderConfig(auth.projectId, mode); const validatedData = validateRequest(req); wideEventBuilder?.setUser(validatedData.userId); @@ -80,6 +80,7 @@ export async function createCheckoutLink( wideEventBuilder?.setPaymentContext({ priceAmount: custom_price }); const checkoutResult = await createCheckoutSession( + auth.projectId, config, custom_price, validatedData.userId, @@ -92,7 +93,7 @@ export async function createCheckoutLink( db, "create checkout link", async (txn) => { - await ensureUserExists(validatedData.userId, txn); + await ensureUserExists(auth.projectId, validatedData.userId, txn); await txn .select({ id: usersTable.id }) @@ -102,6 +103,7 @@ export async function createCheckoutLink( const existingId = await checkIfExistingCheckoutLink( txn, + auth.projectId, validatedData.userId, mode ); @@ -112,6 +114,7 @@ export async function createCheckoutLink( } const sessionResult = await handleAddSession( + auth.projectId, validatedData.userId, checkoutResult.sessionId, beforeTimestamp, @@ -164,6 +167,7 @@ async function calculatePrice( } async function createCheckoutSession( + projectId: string, config: PaymentProviderConfig, customPrice: number, userId: string, @@ -177,7 +181,12 @@ async function createCheckoutSession( apiKeyId, }; - const checkoutResult = await createProviderCheckout(config, params, mode); + const checkoutResult = await createProviderCheckout( + projectId, + config, + params, + mode + ); if ( !checkoutResult.checkoutUrl || diff --git a/src/routes/gRPC/payment/paymentProvider.ts b/src/routes/gRPC/payment/paymentProvider.ts index 3f938e9..bd3ac36 100644 --- a/src/routes/gRPC/payment/paymentProvider.ts +++ b/src/routes/gRPC/payment/paymentProvider.ts @@ -3,60 +3,58 @@ import { PaymentError } from "../../../errors/payment"; import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; import { decrypt } from "../../../utils/encryptMetadata.ts"; -let liveClient: DodoPayments | null = null; -let testClient: DodoPayments | null = null; +const clients = new Map(); -function clearClients(): void { - liveClient = null; - testClient = null; +function clientKey(projectId: string, mode: string): string { + return `${projectId}:${mode}`; +} + +export function clearClients(): void { + clients.clear(); } export async function getDodoClient( + projectId: string, mode?: "test" | "production" ): Promise { if (!mode) { mode = process.env.NODE_ENV === "production" ? "production" : "test"; } - if (mode === "production") { - if (liveClient) return liveClient; - - const metadata = await getMetadata(); - const apiKey = metadata?.dodo_live_api_key; - if (!apiKey) { - throw PaymentError.missingApiKey(); - } - - liveClient = new DodoPayments({ - bearerToken: decrypt(apiKey), - environment: "live_mode", - webhookKey: metadata?.dodo_live_webhook_secret - ? decrypt(metadata.dodo_live_webhook_secret) - : undefined, - }); - return liveClient; + const key = clientKey(projectId, mode); + const cached = clients.get(key); + if (cached) return cached; + + const metadata = await getMetadata(projectId); + + if (!metadata) { + throw PaymentError.missingMetadata(); } - if (testClient) return testClient; + const encryptedApiKey = + mode === "production" + ? metadata.dodo_live_api_key + : metadata.dodo_test_api_key; + const encryptedWebhookSecret = + mode === "production" + ? metadata.dodo_live_webhook_secret + : metadata.dodo_test_webhook_secret; - const metadata = await getMetadata(); - const apiKey = metadata?.dodo_test_api_key; - if (!apiKey) { + if (!encryptedApiKey) { throw PaymentError.missingApiKey(); } - testClient = new DodoPayments({ - bearerToken: decrypt(apiKey), - environment: "test_mode", - webhookKey: metadata?.dodo_test_webhook_secret - ? decrypt(metadata.dodo_test_webhook_secret) + const client = new DodoPayments({ + bearerToken: decrypt(encryptedApiKey), + environment: mode === "production" ? "live_mode" : "test_mode", + webhookKey: encryptedWebhookSecret + ? decrypt(encryptedWebhookSecret) : undefined, }); - return testClient; -} -// Re-export for callers who need to invalidate cached clients after onboarding updates -export { clearClients }; + clients.set(key, client); + return client; +} export interface PaymentProviderConfig { productId: string; @@ -76,13 +74,14 @@ export interface CheckoutResult { } export async function getPaymentProviderConfig( + projectId: string, mode: "test" | "production" ): Promise { if (!mode) { mode = process.env.NODE_ENV === "production" ? "production" : "test"; } - const metadata = await getMetadata(); + const metadata = await getMetadata(projectId); if (!metadata) { throw PaymentError.missingMetadata(); @@ -90,9 +89,9 @@ export async function getPaymentProviderConfig( const productId = mode === "production" - ? metadata?.dodo_live_product_id - : metadata?.dodo_test_product_id; - const returnUrl = metadata?.redirect_url ?? null; + ? metadata.dodo_live_product_id + : metadata.dodo_test_product_id; + const returnUrl = metadata.redirect_url ?? null; if (!productId) { throw PaymentError.missingProductId(); @@ -102,11 +101,12 @@ export async function getPaymentProviderConfig( } export async function createProviderCheckout( + projectId: string, config: PaymentProviderConfig, params: CheckoutParams, mode: "test" | "production" ): Promise { - const client = await getDodoClient(mode); + const client = await getDodoClient(projectId, mode); const session = await client.checkoutSessions.create({ product_cart: [ diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 2a18fd5..340566f 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -62,10 +62,12 @@ export async function handleCreateApiKey( key: apiKeyHash, role: validated.role, expiresAt: expiresAt.toISO(), + projectId: auth.projectId, }); const keyPair = generateWebhookKeyPair(); const endpoint = await upsertWebhookEndpoint( + auth.projectId, keyRecord.id, validated.webhookUrl, keyPair.privateKeyPem, @@ -127,7 +129,7 @@ export async function handleListApiKeys( ); try { - await authenticateHttpApiKey(request.headers.authorization); + const auth = await authenticateHttpApiKey(request.headers.authorization); const db = getPostgresDB(); const keys = await db @@ -151,7 +153,11 @@ export async function handleListApiKeys( ) ) .where( - and(ne(apiKeysTable.role, "dashboard"), eq(apiKeysTable.revoked, false)) + and( + eq(apiKeysTable.projectId, auth.projectId), + ne(apiKeysTable.role, "dashboard"), + eq(apiKeysTable.revoked, false) + ) ) .orderBy(apiKeysTable.createdAt); @@ -189,7 +195,7 @@ export async function handleRevokeApiKey( ); try { - await authenticateHttpApiKey(request.headers.authorization); + const auth = await authenticateHttpApiKey(request.headers.authorization); const params = request.params as { id: string }; const db = getPostgresDB(); @@ -199,7 +205,11 @@ export async function handleRevokeApiKey( .update(apiKeysTable) .set({ revoked: true, revokedAt: now }) .where( - and(eq(apiKeysTable.id, params.id), eq(apiKeysTable.revoked, false)) + and( + eq(apiKeysTable.projectId, auth.projectId), + eq(apiKeysTable.id, params.id), + eq(apiKeysTable.revoked, false) + ) ); if ((result.count ?? 0) === 0) { diff --git a/src/routes/http/api/expressions.ts b/src/routes/http/api/expressions.ts index 59e63d9..1b3cfeb 100644 --- a/src/routes/http/api/expressions.ts +++ b/src/routes/http/api/expressions.ts @@ -45,9 +45,9 @@ export async function handleListExpressions( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); - const expressions = await listExpressions(); + const expressions = await listExpressions(auth.projectId); builder.setSuccess(200).addContext({ expressionCount: expressions.length }); reply.code(200); @@ -84,15 +84,15 @@ export async function handleCreateExpression( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); const body = await request.body; const validated = createExpressionSchema.parse(body); validateExprSyntax(validated.expr); - await resolveExprRefsInExpression(validated.expr); + await resolveExprRefsInExpression(validated.expr, auth.projectId); - await createExpression(validated.key, validated.expr); + await createExpression(auth.projectId, validated.key, validated.expr); builder.setSuccess(200); reply.code(200); @@ -147,10 +147,10 @@ export async function handleDeleteExpression( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); const params = request.params as { key: string }; - const deleted = await deleteExpression(params.key); + const deleted = await deleteExpression(auth.projectId, params.key); if (!deleted) { builder.setError(404, { diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index f3d5721..5a03dac 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "crypto"; import type { FastifyRequest, FastifyReply } from "fastify"; import * as Sentry from "@sentry/bun"; import { ZodError } from "zod"; @@ -9,18 +10,26 @@ import { } from "../../../context/requestContext.ts"; import { logger } from "../../../errors/logger.ts"; import { AuthError } from "../../../errors/auth"; +import { authenticateMasterApiKey } from "../../../utils/authenticateMasterApiKey.ts"; import { authenticateHttpApiKey } from "../../../utils/authenticateHttpApiKey.ts"; +import { generateAPIKey } from "../../../utils/generateAPIKey"; +import { hashAPIKey } from "../../../utils/hashAPIKey"; +import { encrypt, decrypt } from "../../../utils/encryptMetadata.ts"; +import { getPostgresDB } from "../../../storage/db/postgres/db"; import { - upsertMetadata, - getMetadata, -} from "../../../storage/db/postgres/helpers/metadata.ts"; + projectsTable, + apiKeysTable, + metadataTable, +} from "../../../storage/db/postgres/schema"; +import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; import { clearClients } from "../../gRPC/payment/paymentProvider.ts"; -import { encrypt, decrypt } from "../../../utils/encryptMetadata.ts"; +import { DateTime } from "luxon"; +import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; export async function handleOnboarding( request: FastifyRequest, reply: FastifyReply -): Promise> { +): Promise> { const builder = createWideEventBuilder( generateRequestId(), request.method, @@ -29,7 +38,7 @@ export async function handleOnboarding( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + authenticateMasterApiKey(authHeader); const body = await request.body; const validated = onboardingSchema.parse(body); @@ -44,6 +53,8 @@ export async function handleOnboarding( return {}; } + const projectId = randomUUID(); + const liveClient = new DodoPayments({ bearerToken: validated.dodoLiveApiKey, environment: "live_mode", @@ -57,7 +68,7 @@ export async function handleOnboarding( let testSecret: string; try { const liveWebhook = await liveClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=production`, + url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, description: "Scrawn live payment webhook", filter_types: ["payment.succeeded", "payment.failed"], }); @@ -65,7 +76,7 @@ export async function handleOnboarding( .secret; const testWebhook = await testClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=test`, + url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, description: "Scrawn test payment webhook", filter_types: ["payment.succeeded", "payment.failed"], }); @@ -84,23 +95,44 @@ export async function handleOnboarding( return {}; } - await upsertMetadata({ - dodo_live_api_key: encrypt(validated.dodoLiveApiKey), - dodo_test_api_key: encrypt(validated.dodoTestApiKey), - dodo_live_product_id: validated.dodoLiveProductId, - dodo_test_product_id: validated.dodoTestProductId, - dodo_live_webhook_secret: encrypt(liveSecret), - dodo_test_webhook_secret: encrypt(testSecret), - currency: validated.currency, - redirect_url: validated.redirectUrl, + const dashboardKey = generateAPIKey("dashboard"); + const dashboardKeyHash = hashAPIKey(dashboardKey); + const expiresAt = DateTime.utc().plus({ years: 10 }).toISO(); + + const db = getPostgresDB(); + await executeInTransaction(db, "create project", async (txn) => { + await txn.insert(projectsTable).values({ + id: projectId, + name: validated.name, + }); + + await txn.insert(metadataTable).values({ + projectId, + dodo_live_api_key: encrypt(validated.dodoLiveApiKey), + dodo_test_api_key: encrypt(validated.dodoTestApiKey), + dodo_live_product_id: validated.dodoLiveProductId, + dodo_test_product_id: validated.dodoTestProductId, + dodo_live_webhook_secret: encrypt(liveSecret), + dodo_test_webhook_secret: encrypt(testSecret), + currency: validated.currency, + redirect_url: validated.redirectUrl, + }); + + await txn.insert(apiKeysTable).values({ + projectId, + name: "Default dashboard key", + key: dashboardKeyHash, + role: "dashboard", + expiresAt, + }); }); clearClients(); - builder.setSuccess(200); + builder.setSuccess(201); reply.code(201); - return {}; + return { projectId, apiKey: dashboardKey }; } catch (error) { Sentry.captureException(error, { extra: { context: "onboarding route handler" }, @@ -157,9 +189,9 @@ export async function handleGetConfig( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); - const metadata = await getMetadata(); + const metadata = await getMetadata(auth.projectId); if (!metadata) { builder.setSuccess(200); diff --git a/src/routes/http/api/tags.ts b/src/routes/http/api/tags.ts index 6be7c28..444f875 100644 --- a/src/routes/http/api/tags.ts +++ b/src/routes/http/api/tags.ts @@ -47,9 +47,9 @@ export async function handleListTags( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); - const tags = await listTags(); + const tags = await listTags(auth.projectId); builder.setSuccess(200).addContext({ tagCount: tags.length }); reply.code(200); @@ -86,12 +86,12 @@ export async function handleCreateTag( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); const body = await request.body; const validated = createTagSchema.parse(body); - await createTag(validated.key, validated.amount); + await createTag(auth.projectId, validated.key, validated.amount); builder.setSuccess(200); reply.code(200); @@ -137,10 +137,10 @@ export async function handleDeleteTag( try { const authHeader = request.headers.authorization; - await authenticateHttpApiKey(authHeader); + const auth = await authenticateHttpApiKey(authHeader); const params = tagParamsSchema.parse(request.params); - const deleted = await deleteTag(params.key); + const deleted = await deleteTag(auth.projectId, params.key); if (!deleted) { builder.setError(404, { diff --git a/src/routes/http/api/webhookDeliveries.ts b/src/routes/http/api/webhookDeliveries.ts index a72fbfb..40c9901 100644 --- a/src/routes/http/api/webhookDeliveries.ts +++ b/src/routes/http/api/webhookDeliveries.ts @@ -16,6 +16,7 @@ import { apiKeysTable, } from "../../../storage/db/postgres/schema"; import { and, eq, desc, inArray, sql } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; const listDeliveriesQuerySchema = z.object({ apiKeyId: z.string().uuid("Invalid API key ID").optional(), @@ -37,12 +38,15 @@ export async function handleListDeliveries( ); try { - await authenticateHttpApiKey(request.headers.authorization); + const auth = await authenticateHttpApiKey(request.headers.authorization); const query = listDeliveriesQuerySchema.parse(request.query); const db = getPostgresDB(); - let conditions = undefined; + let conditions: SQL | undefined = eq( + webhookDeliveriesTable.projectId, + auth.projectId + ); if (query.apiKeyId) { const endpoints = await db .select({ id: webhookEndpointsTable.id }) diff --git a/src/routes/http/api/webhookEndpoints.ts b/src/routes/http/api/webhookEndpoints.ts index a02547b..961cebb 100644 --- a/src/routes/http/api/webhookEndpoints.ts +++ b/src/routes/http/api/webhookEndpoints.ts @@ -106,6 +106,15 @@ export async function handleCreateWebhookEndpoint( return { error: "Target API key not found" }; } + if (targetKey.projectId !== auth.projectId) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key does not belong to this project", + }); + reply.code(403); + return { error: "Target API key does not belong to this project" }; + } + if (targetKey.role === "dashboard") { builder.setError(400, { type: "ValidationError", @@ -130,6 +139,7 @@ export async function handleCreateWebhookEndpoint( const keyPair = generateWebhookKeyPair(); const endpoint = await upsertWebhookEndpoint( + auth.projectId, targetApiKeyId, validated.url, keyPair.privateKeyPem, @@ -184,7 +194,10 @@ export async function handleGetWebhookEndpoint( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); - const endpoint = await getWebhookEndpointByApiKeyId(auth.apiKeyId); + const endpoint = await getWebhookEndpointByApiKeyId( + auth.projectId, + auth.apiKeyId + ); const endpoints: WebhookEndpointResponse[] = endpoint ? [toEndpointResponse(endpoint)] @@ -299,6 +312,15 @@ export async function handleSendTestWebhook( return { error: "API key not found" }; } + if (targetKey.projectId !== auth.projectId) { + builder.setError(403, { + type: "PermissionDenied", + message: "API key does not belong to this project", + }); + reply.code(403); + return { error: "API key does not belong to this project" }; + } + if (targetKey.role !== "test") { builder.setError(400, { type: "ValidationError", @@ -308,7 +330,10 @@ export async function handleSendTestWebhook( return { error: "Can only send test webhooks to test API keys" }; } - const endpoint = await getWebhookEndpointByApiKeyId(validated.apiKeyId); + const endpoint = await getWebhookEndpointByApiKeyId( + auth.projectId, + validated.apiKeyId + ); if (!endpoint) { builder.setError(404, { @@ -321,7 +346,7 @@ export async function handleSendTestWebhook( const now = DateTime.utc(); - await forwardWebhook(validated.apiKeyId, { + await forwardWebhook(auth.projectId, validated.apiKeyId, { eventType: "payment.succeeded", resource: "payment", action: "succeeded", @@ -374,7 +399,10 @@ export async function handleGetPublicKey( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); - const endpoint = await getWebhookEndpointByApiKeyId(auth.apiKeyId); + const endpoint = await getWebhookEndpointByApiKeyId( + auth.projectId, + auth.apiKeyId + ); if (!endpoint) { builder.setError(404, { diff --git a/src/routes/http/createdCheckout.ts b/src/routes/http/createdCheckout.ts index 878b648..d4855a3 100644 --- a/src/routes/http/createdCheckout.ts +++ b/src/routes/http/createdCheckout.ts @@ -72,10 +72,12 @@ export async function handleDodoWebhook( timestamp: string | undefined, webhookId: string | undefined, mode: "production" | "test", + projectId: string, builder: WideEventBuilder ): Promise { try { const client = await getDodoClient( + projectId, mode === "production" ? "production" : "test" ); const headers = buildWebhookHeaders(signature, timestamp, webhookId); @@ -158,7 +160,7 @@ export async function handleDodoWebhook( } builder.setSuccess(200); - forwardWebhook(session.apiKeyId, { + forwardWebhook(session.projectId, session.apiKeyId, { eventType: "payment.failed", resource: "payment", action: "failed", @@ -192,6 +194,7 @@ export async function handleDodoWebhook( if (!claimed) return; await updateUserBilledTimestamp(userId, billed_upto, txn); await handleAddPayment( + session.projectId, userId, creditAmount, apiKeyId, @@ -212,7 +215,7 @@ export async function handleDodoWebhook( builder.setPaymentContext({ creditAmount }); builder.setSuccess(200); - forwardWebhook(apiKeyId, { + forwardWebhook(session.projectId, apiKeyId, { eventType: "payment.succeeded", resource: "payment", action: "succeeded", diff --git a/src/routes/http/forwardWebhook.ts b/src/routes/http/forwardWebhook.ts index d27b4b2..ade2c7c 100644 --- a/src/routes/http/forwardWebhook.ts +++ b/src/routes/http/forwardWebhook.ts @@ -38,10 +38,11 @@ export interface WebhookForwardEvent { } export async function forwardWebhook( + projectId: string, apiKeyId: string, event: WebhookForwardEvent ): Promise { - const endpoint = await getWebhookEndpointByApiKeyId(apiKeyId); + const endpoint = await getWebhookEndpointByApiKeyId(projectId, apiKeyId); if (!endpoint) { return; @@ -71,7 +72,7 @@ export async function forwardWebhook( Sentry.captureException(error, { extra: { context: "webhook signing failed", error: errorMsg }, }); - await recordDelivery(endpoint.id, webhookId, event, "failed", { + await recordDelivery(projectId, endpoint.id, webhookId, event, "failed", { error: errorMsg, }); return; @@ -117,6 +118,7 @@ export async function forwardWebhook( } await recordDelivery( + projectId, endpoint.id, webhookId, event, @@ -129,6 +131,7 @@ export async function forwardWebhook( } async function recordDelivery( + projectId: string, endpointId: string, eventId: string, event: WebhookForwardEvent, @@ -141,6 +144,7 @@ async function recordDelivery( try { const db = getPostgresDB(); await db.insert(webhookDeliveriesTable).values({ + projectId, endpointId, eventId, eventType: event.eventType, diff --git a/src/routes/http/registerWebhookRoutes.ts b/src/routes/http/registerWebhookRoutes.ts index 0e15cd3..6f6e617 100644 --- a/src/routes/http/registerWebhookRoutes.ts +++ b/src/routes/http/registerWebhookRoutes.ts @@ -36,7 +36,10 @@ export async function registerWebhookRoutes( ); try { - const mode = (request.query as Record)?.mode; + const query = request.query as Record; + const mode = query?.mode; + const projectId = query?.projectId; + if (mode !== "production" && mode !== "test") { builder.setError(400, { type: "ValidationError", @@ -47,6 +50,15 @@ export async function registerWebhookRoutes( return { error: "Invalid mode query parameter" }; } + if (!projectId) { + builder.setError(400, { + type: "ValidationError", + message: "Missing 'projectId' query parameter.", + }); + reply.code(400); + return { error: "Missing projectId query parameter" }; + } + const signatureHeader = request.headers["webhook-signature"]; const timestampHeader = request.headers["webhook-timestamp"]; const webhookIdHeader = request.headers["webhook-id"]; @@ -72,6 +84,7 @@ export async function registerWebhookRoutes( timestamp, webhookId, mode, + projectId, builder ); diff --git a/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts b/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts index 052bde7..1061e1d 100644 --- a/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts @@ -135,6 +135,7 @@ function buildAiTokenInsertRows( return { id: index === 0 ? firstId : crypto.randomUUID(), + project_id: auth.projectId, event_id: aggEvent.eventId, idempotency_key: aggEvent.idempotencyKey, user_id: aggEvent.userId, @@ -166,7 +167,7 @@ export async function handleAddAiTokenUsage( const firstEvent = events[0]; if (firstEvent) { - await ensureUserExists(firstEvent.userId); + await ensureUserExists(auth.projectId, firstEvent.userId); } const aggregatedEvents = aggregateAiTokenEvents(events); diff --git a/src/storage/adapter/clickhouse/handlers/addBasicUsage.ts b/src/storage/adapter/clickhouse/handlers/addBasicUsage.ts index 55a474d..6c2221c 100644 --- a/src/storage/adapter/clickhouse/handlers/addBasicUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/addBasicUsage.ts @@ -27,7 +27,7 @@ export async function handleAddBasicUsage( } const reportedTimestamp = toClickHouseDateTime(event_data.reported_timestamp); - await ensureUserExists(event_data.userId); + await ensureUserExists(auth.projectId, event_data.userId); const id = crypto.randomUUID(); @@ -37,6 +37,7 @@ export async function handleAddBasicUsage( values: [ { id, + project_id: auth.projectId, event_id: event_data.eventId, idempotency_key: event_data.idempotencyKey, user_id: event_data.userId, diff --git a/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts b/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts index 13310f9..7cd4f13 100644 --- a/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts @@ -5,8 +5,8 @@ import { runClickHousePriceQuery } from "../utils"; const VALUE_EXPR = "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')"; -const BASE_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; -const WINDOW_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp > {lastBilled:DateTime64(3, 'UTC')} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; +const BASE_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; +const WINDOW_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp > {lastBilled:DateTime64(3, 'UTC')} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; export async function handlePriceRequestAiTokenUsage( userId: UserId, diff --git a/src/storage/adapter/clickhouse/handlers/priceRequestBasicUsage.ts b/src/storage/adapter/clickhouse/handlers/priceRequestBasicUsage.ts index 86022de..6b7a8c5 100644 --- a/src/storage/adapter/clickhouse/handlers/priceRequestBasicUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/priceRequestBasicUsage.ts @@ -4,9 +4,9 @@ import type { AuthContext } from "../../../../context/auth"; import { runClickHousePriceQuery } from "../utils"; const BASE_QUERY = - "SELECT sum(debit_amount) as total FROM basic_usage_events WHERE user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp < {before:DateTime64(3, 'UTC')}"; + "SELECT sum(debit_amount) as total FROM basic_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp < {before:DateTime64(3, 'UTC')}"; const WINDOW_QUERY = - "SELECT sum(debit_amount) as total FROM basic_usage_events WHERE user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp > {lastBilled:DateTime64(3, 'UTC')} AND reported_timestamp < {before:DateTime64(3, 'UTC')}"; + "SELECT sum(debit_amount) as total FROM basic_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp > {lastBilled:DateTime64(3, 'UTC')} AND reported_timestamp < {before:DateTime64(3, 'UTC')}"; export async function handlePriceRequestBasicUsage( userId: UserId, diff --git a/src/storage/adapter/clickhouse/handlers/queryEvents.ts b/src/storage/adapter/clickhouse/handlers/queryEvents.ts index 2f17e0e..abf3fa9 100644 --- a/src/storage/adapter/clickhouse/handlers/queryEvents.ts +++ b/src/storage/adapter/clickhouse/handlers/queryEvents.ts @@ -211,9 +211,9 @@ export async function handleQueryEvents( try { if (request.aggregation) { - return await handleAggregationQuery(request, tables); + return await handleAggregationQuery(request, tables, auth); } - return await handleListQuery(request, tables); + return await handleListQuery(request, tables, auth); } catch (e) { if ( e && @@ -232,11 +232,12 @@ export async function handleQueryEvents( async function handleListQuery( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const client = getClickHouseDB(); const paramIndex = { value: 0 }; - const params: Record = {}; + const params: Record = { projectId: auth.projectId }; const queries = tables.map((t) => { const whereClause = buildWhereFromGroup( @@ -246,7 +247,8 @@ async function handleListQuery( paramIndex ); let q = `SELECT ${buildSelectColumns(t)} FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; + q += ` WHERE project_id = {projectId:String}`; + if (whereClause) q += ` AND ${whereClause}`; return q; }); @@ -276,20 +278,21 @@ async function handleListQuery( data as unknown as Record[] ).map(normalizeRow); - const total = await getTotalCount(request, tables); + const total = await getTotalCount(request, tables, auth); return { rows, total }; } async function handleAggregationQuery( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const client = getClickHouseDB(); const agg = request.aggregation!; const isSum = agg.type === "SUM"; const paramIndex = { value: 0 }; - const params: Record = {}; + const params: Record = { projectId: auth.projectId }; const subQueries = tables.map((t) => { const cols: string[] = []; @@ -325,7 +328,8 @@ async function handleAggregationQuery( paramIndex ); let q = `SELECT ${cols.join(", ")} FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; + q += ` WHERE project_id = {projectId:String}`; + if (whereClause) q += ` AND ${whereClause}`; return q; }); @@ -364,11 +368,12 @@ async function handleAggregationQuery( async function getTotalCount( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const client = getClickHouseDB(); const paramIndex = { value: 0 }; - const params: Record = {}; + const params: Record = { projectId: auth.projectId }; const subQueries = tables.map((t) => { const whereClause = buildWhereFromGroup( @@ -378,7 +383,8 @@ async function getTotalCount( paramIndex ); let q = `SELECT count() as cnt FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; + q += ` WHERE project_id = {projectId:String}`; + if (whereClause) q += ` AND ${whereClause}`; return q; }); diff --git a/src/storage/adapter/clickhouse/utils.ts b/src/storage/adapter/clickhouse/utils.ts index 2858ba9..19e6d8e 100644 --- a/src/storage/adapter/clickhouse/utils.ts +++ b/src/storage/adapter/clickhouse/utils.ts @@ -70,6 +70,7 @@ export async function runClickHousePriceQuery( query = baseQuery; } + params.projectId = auth.projectId; params.mode = auth.mode; const rs = await chClient.query({ diff --git a/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts b/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts index 09a8bb1..1117fa2 100644 --- a/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts +++ b/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts @@ -117,6 +117,7 @@ function buildAiTokenInsertValues( auth: AuthContext ) { return aggregatedEvents.map((aggEvent) => ({ + projectId: auth.projectId, eventId: aggEvent.eventId, idempotencyKey: aggEvent.idempotencyKey, reportedTimestamp: aggEvent.reported_timestamp, @@ -166,7 +167,7 @@ export async function handleAddAiTokenUsage( `storing ${events.length} AI_TOKEN_USAGE event(s)`, async (txn) => { if (firstEvent) { - await ensureUserExists(firstEvent.userId, txn); + await ensureUserExists(auth.projectId, firstEvent.userId, txn); } try { diff --git a/src/storage/adapter/postgres/handlers/addBasicUsage.ts b/src/storage/adapter/postgres/handlers/addBasicUsage.ts index 127b4c0..0de79d3 100644 --- a/src/storage/adapter/postgres/handlers/addBasicUsage.ts +++ b/src/storage/adapter/postgres/handlers/addBasicUsage.ts @@ -28,7 +28,11 @@ export async function handleAddBasicUsage( connectionObject, "storing BASIC_USAGE event", async (txn) => { - const ensurePromise = ensureUserExists(event_data.userId, txn); + const ensurePromise = ensureUserExists( + auth.projectId, + event_data.userId, + txn + ); const reportedTimestamp = await validateAndPrepareTimestamp( event_data.reported_timestamp @@ -38,6 +42,7 @@ export async function handleAddBasicUsage( const [result] = await txn .insert(basicUsageEventsTable) .values({ + projectId: auth.projectId, eventId: event_data.eventId, idempotencyKey: event_data.idempotencyKey, reportedTimestamp, diff --git a/src/storage/adapter/postgres/handlers/priceRequest.ts b/src/storage/adapter/postgres/handlers/priceRequest.ts index 2cde117..ac13916 100644 --- a/src/storage/adapter/postgres/handlers/priceRequest.ts +++ b/src/storage/adapter/postgres/handlers/priceRequest.ts @@ -37,7 +37,7 @@ export async function handlePriceRequest( let result; try { - const baseCondition = sql`${priceTable.reportedTimestamp} > ${usersTable.last_billed_timestamp} AND ${priceTable.userId} = ${userId} AND ${priceTable.mode} = ${auth.mode}`; + const baseCondition = sql`${priceTable.reportedTimestamp} > ${usersTable.last_billed_timestamp} AND ${priceTable.userId} = ${userId} AND ${priceTable.mode} = ${auth.mode} AND ${priceTable.projectId} = ${auth.projectId}`; const whereClause = beforeTimestamp ? and( baseCondition, diff --git a/src/storage/adapter/postgres/handlers/queryEvents.ts b/src/storage/adapter/postgres/handlers/queryEvents.ts index 70bbfc8..3bcf970 100644 --- a/src/storage/adapter/postgres/handlers/queryEvents.ts +++ b/src/storage/adapter/postgres/handlers/queryEvents.ts @@ -255,9 +255,9 @@ export async function handleQueryEvents( try { if (request.aggregation) { - return await handleAggregationQuery(request, tables); + return await handleAggregationQuery(request, tables, auth); } - return await handleListQuery(request, tables); + return await handleListQuery(request, tables, auth); } catch (e) { if ( e && @@ -276,16 +276,21 @@ export async function handleQueryEvents( async function handleListQuery( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const db = getPostgresDB(); const selectExpr = tables.map((t) => buildSelectColumns(t)); const whereExpr = tables.map((t) => buildWhereClause(request.where, t)); + const projectFilter = sql`project_id = ${auth.projectId}`; const subqueries = tables.map((t, i) => { const base = sql`SELECT ${selectExpr[i]} FROM ${sql.raw(t)}`; - return whereExpr[i] ? sql`${base} WHERE ${whereExpr[i]}` : base; + const fullWhere = whereExpr[i] + ? sql`${whereExpr[i]} AND ${projectFilter}` + : projectFilter; + return sql`${base} WHERE ${fullWhere}`; }); const unionQuery = sql.join(subqueries, sql` UNION ALL `); @@ -301,18 +306,20 @@ async function handleListQuery( const data = result as unknown as Record[]; const rows: QueryResultRow[] = data.map(normalizeRow); - const total = await getTotalCount(request, tables); + const total = await getTotalCount(request, tables, auth); return { rows, total }; } async function handleAggregationQuery( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const db = getPostgresDB(); const agg = request.aggregation!; const isSum = agg.type === "SUM"; + const projectFilter = sql`project_id = ${auth.projectId}`; const subqueries = tables.map((t) => { const cols: SQL[] = []; @@ -349,7 +356,10 @@ async function handleAggregationQuery( const whereClause = buildWhereClause(request.where, t); const base = sql`SELECT ${sql.join(cols, sql`, `)} FROM ${sql.raw(t)}`; - return whereClause ? sql`${base} WHERE ${whereClause}` : base; + const fullWhere = whereClause + ? sql`${whereClause} AND ${projectFilter}` + : projectFilter; + return sql`${base} WHERE ${fullWhere}`; }); const unionQuery = sql.join(subqueries, sql` UNION ALL `); @@ -395,14 +405,19 @@ async function handleAggregationQuery( async function getTotalCount( request: QueryRequest, - tables: EventTableName[] + tables: EventTableName[], + auth: AuthContext ): Promise { const db = getPostgresDB(); + const projectFilter = sql`project_id = ${auth.projectId}`; const subqueries = tables.map((t) => { const whereClause = buildWhereClause(request.where, t); const base = sql`SELECT count(*)::int as cnt FROM ${sql.raw(t)}`; - return whereClause ? sql`${base} WHERE ${whereClause}` : base; + const fullWhere = whereClause + ? sql`${whereClause} AND ${projectFilter}` + : projectFilter; + return sql`${base} WHERE ${fullWhere}`; }); const countQuery = sql` diff --git a/src/storage/db/postgres/helpers/apiKeys.ts b/src/storage/db/postgres/helpers/apiKeys.ts index 5b36c3b..950ae8b 100644 --- a/src/storage/db/postgres/helpers/apiKeys.ts +++ b/src/storage/db/postgres/helpers/apiKeys.ts @@ -9,6 +9,7 @@ type CreateApiKeyInput = { key: string; role: string; expiresAt: string; + projectId: string; }; export async function createApiKey( @@ -37,6 +38,7 @@ export async function createApiKey( key: input.key, role: input.role as "dashboard" | "production" | "test", expiresAt: input.expiresAt, + projectId: input.projectId, }) .returning({ id: apiKeysTable.id }); @@ -74,16 +76,20 @@ type ApiKeyRecord = { role: string; expiresAt: string; revoked: boolean; + projectId: string; }; export async function getApiKeyRoleById( id: string -): Promise<{ role: "dashboard" | "production" | "test" } | null> { +): Promise<{ + role: "dashboard" | "production" | "test"; + projectId: string; +} | null> { const db = getPostgresDB(); try { const [record] = await db - .select({ role: apiKeysTable.role }) + .select({ role: apiKeysTable.role, projectId: apiKeysTable.projectId }) .from(apiKeysTable) .where(eq(apiKeysTable.id, id)) .limit(1); @@ -109,6 +115,7 @@ export async function findApiKeyByHash( role: apiKeysTable.role, expiresAt: apiKeysTable.expiresAt, revoked: apiKeysTable.revoked, + projectId: apiKeysTable.projectId, }) .from(apiKeysTable) .where(eq(apiKeysTable.key, apiKeyHash)) diff --git a/src/storage/db/postgres/helpers/expressions.ts b/src/storage/db/postgres/helpers/expressions.ts index 546b319..41639d6 100644 --- a/src/storage/db/postgres/helpers/expressions.ts +++ b/src/storage/db/postgres/helpers/expressions.ts @@ -4,14 +4,19 @@ import { eq, and, isNull } from "drizzle-orm"; import { StorageError } from "../../../../errors/storage"; import { DateTime } from "luxon"; -export async function listExpressions(): Promise { +export async function listExpressions(projectId: string): Promise { const db = getPostgresDB(); try { const rows = await db .select({ key: expressionsTable.key }) .from(expressionsTable) - .where(isNull(expressionsTable.deletedAt)); + .where( + and( + eq(expressionsTable.projectId, projectId), + isNull(expressionsTable.deletedAt) + ) + ); return rows.map((row) => row.key); } catch (e) { throw StorageError.queryFailed( @@ -21,7 +26,10 @@ export async function listExpressions(): Promise { } } -export async function findExpressionByKey(key: string): Promise { +export async function findExpressionByKey( + projectId: string, + key: string +): Promise { const db = getPostgresDB(); try { @@ -29,7 +37,11 @@ export async function findExpressionByKey(key: string): Promise { .select({ expr: expressionsTable.expr }) .from(expressionsTable) .where( - and(eq(expressionsTable.key, key), isNull(expressionsTable.deletedAt)) + and( + eq(expressionsTable.projectId, projectId), + eq(expressionsTable.key, key), + isNull(expressionsTable.deletedAt) + ) ) .limit(1); @@ -43,6 +55,7 @@ export async function findExpressionByKey(key: string): Promise { } export async function createExpression( + projectId: string, key: string, expr: string ): Promise { @@ -53,7 +66,11 @@ export async function createExpression( .select({ id: expressionsTable.id }) .from(expressionsTable) .where( - and(eq(expressionsTable.key, key), isNull(expressionsTable.deletedAt)) + and( + eq(expressionsTable.projectId, projectId), + eq(expressionsTable.key, key), + isNull(expressionsTable.deletedAt) + ) ) .limit(1); @@ -65,7 +82,7 @@ export async function createExpression( return; } - await db.insert(expressionsTable).values({ key, expr }); + await db.insert(expressionsTable).values({ projectId, key, expr }); } catch (e) { throw StorageError.insertFailed( `Failed to upsert expression '${key}'`, @@ -74,7 +91,10 @@ export async function createExpression( } } -export async function deleteExpression(key: string): Promise { +export async function deleteExpression( + projectId: string, + key: string +): Promise { const db = getPostgresDB(); try { @@ -83,7 +103,11 @@ export async function deleteExpression(key: string): Promise { .update(expressionsTable) .set({ deletedAt: now }) .where( - and(eq(expressionsTable.key, key), isNull(expressionsTable.deletedAt)) + and( + eq(expressionsTable.projectId, projectId), + eq(expressionsTable.key, key), + isNull(expressionsTable.deletedAt) + ) ); return (result.count ?? 0) > 0; diff --git a/src/storage/db/postgres/helpers/metadata.ts b/src/storage/db/postgres/helpers/metadata.ts index 4d83430..3ba469a 100644 --- a/src/storage/db/postgres/helpers/metadata.ts +++ b/src/storage/db/postgres/helpers/metadata.ts @@ -5,73 +5,51 @@ import { eq } from "drizzle-orm"; import { executeInTransaction } from "../../../adapter/postgres/handlers/addEventUtils"; export type UpsertMetadataInput = { - dodo_live_api_key?: string; - dodo_test_api_key?: string; - dodo_live_product_id?: string; - dodo_test_product_id?: string; - dodo_live_webhook_secret?: string; - dodo_test_webhook_secret?: string; - currency?: string; - redirect_url?: string; + dodo_live_api_key: string; + dodo_test_api_key: string; + dodo_live_product_id: string; + dodo_test_product_id: string; + dodo_live_webhook_secret: string; + dodo_test_webhook_secret: string; + currency: string; + redirect_url: string; }; -export async function upsertMetadata( - input: UpsertMetadataInput +export async function createMetadata( + projectId: string, + input: UpsertMetadataInput, + txn?: any ): Promise { - const db = getPostgresDB(); - - await executeInTransaction(db, "upsert metadata", async (txn) => { - try { - const [existingMetadata] = await txn - .select({ id: metadataTable.id }) - .from(metadataTable) - .limit(1) - .for("update"); - - const setValues: Partial = {}; - if (input.dodo_live_api_key !== undefined) - setValues.dodo_live_api_key = input.dodo_live_api_key; - if (input.dodo_test_api_key !== undefined) - setValues.dodo_test_api_key = input.dodo_test_api_key; - if (input.dodo_live_product_id !== undefined) - setValues.dodo_live_product_id = input.dodo_live_product_id; - if (input.dodo_test_product_id !== undefined) - setValues.dodo_test_product_id = input.dodo_test_product_id; - if (input.dodo_live_webhook_secret !== undefined) - setValues.dodo_live_webhook_secret = input.dodo_live_webhook_secret; - if (input.dodo_test_webhook_secret !== undefined) - setValues.dodo_test_webhook_secret = input.dodo_test_webhook_secret; - if (input.currency !== undefined) setValues.currency = input.currency; - if (input.redirect_url !== undefined) - setValues.redirect_url = input.redirect_url; - - if (existingMetadata) { - if (Object.keys(setValues).length > 0) { - await txn - .update(metadataTable) - .set(setValues) - .where(eq(metadataTable.id, existingMetadata.id)); - } - return; - } + const db = txn ?? getPostgresDB(); - const insertValues: typeof metadataTable.$inferInsert = { - ...setValues, - } as typeof metadataTable.$inferInsert; - await txn.insert(metadataTable).values(insertValues); - } catch (e) { - throw StorageError.insertFailed( - "Failed to upsert metadata record", - e instanceof Error ? e : new Error(String(e)) - ); - } - }); + try { + await db.insert(metadataTable).values({ + projectId, + dodo_live_api_key: input.dodo_live_api_key, + dodo_test_api_key: input.dodo_test_api_key, + dodo_live_product_id: input.dodo_live_product_id, + dodo_test_product_id: input.dodo_test_product_id, + dodo_live_webhook_secret: input.dodo_live_webhook_secret, + dodo_test_webhook_secret: input.dodo_test_webhook_secret, + currency: input.currency, + redirect_url: input.redirect_url, + }); + } catch (e) { + throw StorageError.insertFailed( + "Failed to create metadata record", + e instanceof Error ? e : new Error(String(e)) + ); + } } -export async function getMetadata(): Promise< - typeof metadataTable.$inferSelect | undefined -> { +export async function getMetadata( + projectId: string +): Promise { const db = getPostgresDB(); - const [metadata] = await db.select().from(metadataTable).limit(1); + const [metadata] = await db + .select() + .from(metadataTable) + .where(eq(metadataTable.projectId, projectId)) + .limit(1); return metadata; } diff --git a/src/storage/db/postgres/helpers/payments.ts b/src/storage/db/postgres/helpers/payments.ts index 9af0688..507e8f9 100644 --- a/src/storage/db/postgres/helpers/payments.ts +++ b/src/storage/db/postgres/helpers/payments.ts @@ -5,6 +5,7 @@ import { DateTime } from "luxon"; import type { PgTransaction } from "drizzle-orm/pg-core"; export async function handleAddPayment( + projectId: string, userId: string, creditAmount: number, apiKeyId: string, @@ -30,6 +31,7 @@ export async function handleAddPayment( const [result] = await db .insert(paymentEventsTable) .values({ + projectId, reportedTimestamp: DateTime.utc().toISO()!, userId, apiKeyId, diff --git a/src/storage/db/postgres/helpers/sessions.ts b/src/storage/db/postgres/helpers/sessions.ts index 4e3cf69..8598390 100644 --- a/src/storage/db/postgres/helpers/sessions.ts +++ b/src/storage/db/postgres/helpers/sessions.ts @@ -32,6 +32,7 @@ export async function updateSessionStatus( export async function checkIfExistingCheckoutLink( txn: PgTransaction, + projectId: string, userId: UserId, mode: "test" | "production" ): Promise { @@ -45,6 +46,7 @@ export async function checkIfExistingCheckoutLink( .from(sessionsTable) .where( and( + eq(sessionsTable.projectId, projectId), eq(sessionsTable.userId, userId), eq(sessionsTable.processed, "pending"), eq(sessionsTable.mode, mode), @@ -68,6 +70,7 @@ export async function checkIfExistingCheckoutLink( } export async function handleAddSession( + projectId: string, userId: UserId, sessionId: string, billedUpto: DateTime, @@ -91,6 +94,7 @@ export async function handleAddSession( const insertResult = await connectionObject .insert(sessionsTable) .values({ + projectId, userId: userId, sessionId: sessionId, billed_upto: billedUptoStr, diff --git a/src/storage/db/postgres/helpers/tags.ts b/src/storage/db/postgres/helpers/tags.ts index 9056f0f..b0fa739 100644 --- a/src/storage/db/postgres/helpers/tags.ts +++ b/src/storage/db/postgres/helpers/tags.ts @@ -5,14 +5,18 @@ import { StorageError } from "../../../../errors/storage"; import { DateTime } from "luxon"; import { tagCache } from "../../../../utils/tagCache"; -export async function listTags(): Promise<{ key: string; amount: number }[]> { +export async function listTags( + projectId: string +): Promise<{ key: string; amount: number }[]> { const db = getPostgresDB(); try { const rows = await db .select({ key: tagsTable.key, amount: tagsTable.amount }) .from(tagsTable) - .where(isNull(tagsTable.deletedAt)); + .where( + and(eq(tagsTable.projectId, projectId), isNull(tagsTable.deletedAt)) + ); return rows; } catch (e) { throw StorageError.queryFailed( @@ -22,14 +26,24 @@ export async function listTags(): Promise<{ key: string; amount: number }[]> { } } -export async function createTag(key: string, amount: number): Promise { +export async function createTag( + projectId: string, + key: string, + amount: number +): Promise { const db = getPostgresDB(); try { const existing = await db .select({ id: tagsTable.id }) .from(tagsTable) - .where(and(eq(tagsTable.key, key), isNull(tagsTable.deletedAt))) + .where( + and( + eq(tagsTable.projectId, projectId), + eq(tagsTable.key, key), + isNull(tagsTable.deletedAt) + ) + ) .limit(1); if (existing[0]) { @@ -41,7 +55,7 @@ export async function createTag(key: string, amount: number): Promise { return; } - await db.insert(tagsTable).values({ key, amount }); + await db.insert(tagsTable).values({ projectId, key, amount }); tagCache.delete(key); } catch (e) { throw StorageError.insertFailed( @@ -51,7 +65,10 @@ export async function createTag(key: string, amount: number): Promise { } } -export async function deleteTag(key: string): Promise { +export async function deleteTag( + projectId: string, + key: string +): Promise { const db = getPostgresDB(); try { @@ -59,7 +76,13 @@ export async function deleteTag(key: string): Promise { const result = await db .update(tagsTable) .set({ deletedAt: now }) - .where(and(eq(tagsTable.key, key), isNull(tagsTable.deletedAt))); + .where( + and( + eq(tagsTable.projectId, projectId), + eq(tagsTable.key, key), + isNull(tagsTable.deletedAt) + ) + ); if ((result.count ?? 0) > 0) { tagCache.delete(key); diff --git a/src/storage/db/postgres/helpers/users.ts b/src/storage/db/postgres/helpers/users.ts index eab1db6..b79dd61 100644 --- a/src/storage/db/postgres/helpers/users.ts +++ b/src/storage/db/postgres/helpers/users.ts @@ -1,6 +1,6 @@ import { getPostgresDB } from "../db"; import { usersTable } from "../schema"; -import { eq } from "drizzle-orm"; +import { eq, and } from "drizzle-orm"; import { StorageError } from "../../../../errors/storage"; import type { PgTransaction } from "drizzle-orm/pg-core"; @@ -24,17 +24,21 @@ export async function updateUserBilledTimestamp( } } -export async function userExists(userId: string): Promise { +export async function userExists( + projectId: string, + userId: string +): Promise { const db = getPostgresDB(); const result = await db .select({ id: usersTable.id }) .from(usersTable) - .where(eq(usersTable.id, userId)) + .where(and(eq(usersTable.projectId, projectId), eq(usersTable.id, userId))) .limit(1); return result.length > 0; } export async function ensureUserExists( + projectId: string, userId: string, txn?: PgTransaction ): Promise { @@ -43,7 +47,7 @@ export async function ensureUserExists( try { await db .insert(usersTable) - .values({ id: userId }) + .values({ id: userId, projectId }) .onConflictDoNothing({ target: usersTable.id }); } catch (e) { if ( diff --git a/src/storage/db/postgres/helpers/webhookEndpoints.ts b/src/storage/db/postgres/helpers/webhookEndpoints.ts index 3d82ef6..c4ebfd3 100644 --- a/src/storage/db/postgres/helpers/webhookEndpoints.ts +++ b/src/storage/db/postgres/helpers/webhookEndpoints.ts @@ -7,6 +7,7 @@ import { DateTime } from "luxon"; export type WebhookEndpoint = typeof webhookEndpointsTable.$inferSelect; export async function getWebhookEndpointByApiKeyId( + projectId: string, apiKeyId: string ): Promise { const db = getPostgresDB(); @@ -17,6 +18,7 @@ export async function getWebhookEndpointByApiKeyId( .from(webhookEndpointsTable) .where( and( + eq(webhookEndpointsTable.projectId, projectId), eq(webhookEndpointsTable.apiKeyId, apiKeyId), isNull(webhookEndpointsTable.deletedAt) ) @@ -33,6 +35,7 @@ export async function getWebhookEndpointByApiKeyId( } export async function upsertWebhookEndpoint( + projectId: string, apiKeyId: string, url: string, privateKey: string, @@ -46,6 +49,7 @@ export async function upsertWebhookEndpoint( const [result] = await db .insert(webhookEndpointsTable) .values({ + projectId, apiKeyId, url, privateKey, diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index c266d0a..1f93982 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -14,8 +14,22 @@ import { USER_ID_CONFIG } from "../../../config/identifiers"; import { DateTime } from "luxon"; import { type Metrics } from "../../../zod/metrics"; +export const projectsTable = pgTable("projects", { + id: uuid("id").primaryKey().defaultRandom(), + name: text("name").notNull(), + createdAt: timestamp("created_at", { + withTimezone: true, + mode: "string", + }) + .defaultNow() + .notNull(), +}); + export const usersTable = pgTable("users", { id: USER_ID_CONFIG.dbType("id").primaryKey(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), last_billed_timestamp: timestamp("last_billed_timestamp", { withTimezone: true, mode: "string", @@ -28,7 +42,11 @@ export const usersTable = pgTable("users", { .default("production"), }); -export const usersRelation = relations(usersTable, ({ many }) => ({ +export const usersRelation = relations(usersTable, ({ one, many }) => ({ + project: one(projectsTable, { + fields: [usersTable.projectId], + references: [projectsTable.id], + }), sessions: many(sessionsTable), basicUsageEvents: many(basicUsageEventsTable), paymentEvents: many(paymentEventsTable), @@ -39,6 +57,9 @@ export const sessionsTable = pgTable( "sessions", { proxy_link_id: uuid("proxy_link_id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), sessionId: text("session_id").notNull().unique(), processed: text("processed", { enum: ["pending", "failed", "succeeded"] }) .default("pending") @@ -70,6 +91,10 @@ export const sessionsTable = pgTable( ); export const sessionRelations = relations(sessionsTable, ({ one, many }) => ({ + project: one(projectsTable, { + fields: [sessionsTable.projectId], + references: [projectsTable.id], + }), user: one(usersTable, { fields: [sessionsTable.userId], references: [usersTable.id], @@ -85,6 +110,9 @@ export const apiKeysTable = pgTable( "api_keys", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), name: text("name").notNull(), key: text("key").notNull().unique(), role: text("role", { enum: ["dashboard", "production", "test"] }) @@ -108,12 +136,16 @@ export const apiKeysTable = pgTable( }, (table) => ({ uniqueActiveName: uniqueIndex("unique_active_name") - .on(table.name) + .on(table.projectId, table.name) .where(sql`${table.revoked} = false`), }) ); -export const apiKeysRelation = relations(apiKeysTable, ({ many }) => ({ +export const apiKeysRelation = relations(apiKeysTable, ({ one, many }) => ({ + project: one(projectsTable, { + fields: [apiKeysTable.projectId], + references: [projectsTable.id], + }), sessions: many(sessionsTable), basicUsageEvents: many(basicUsageEventsTable), paymentEvents: many(paymentEventsTable), @@ -122,6 +154,9 @@ export const apiKeysRelation = relations(apiKeysTable, ({ many }) => ({ export const basicUsageEventsTable = pgTable("basic_usage_events", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), eventId: uuid("event_id").notNull(), idempotencyKey: text("idempotency_key").notNull().unique(), reportedTimestamp: timestamp("reported_timestamp", { @@ -149,6 +184,10 @@ export const basicUsageEventsTable = pgTable("basic_usage_events", { export const basicUsageEventsRelation = relations( basicUsageEventsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [basicUsageEventsTable.projectId], + references: [projectsTable.id], + }), user: one(usersTable, { fields: [basicUsageEventsTable.userId], references: [usersTable.id], @@ -162,6 +201,9 @@ export const basicUsageEventsRelation = relations( export const paymentEventsTable = pgTable("payment_events", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), reportedTimestamp: timestamp("reported_timestamp", { withTimezone: true, mode: "string", @@ -188,6 +230,10 @@ export const paymentEventsTable = pgTable("payment_events", { export const paymentEventsRelation = relations( paymentEventsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [paymentEventsTable.projectId], + references: [projectsTable.id], + }), user: one(usersTable, { fields: [paymentEventsTable.userId], references: [usersTable.id], @@ -205,6 +251,9 @@ export const paymentEventsRelation = relations( export const aiTokenUsageEventsTable = pgTable("ai_token_usage_events", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), eventId: uuid("event_id").notNull(), idempotencyKey: text("idempotency_key").notNull().unique(), reportedTimestamp: timestamp("reported_timestamp", { @@ -233,6 +282,10 @@ export const aiTokenUsageEventsTable = pgTable("ai_token_usage_events", { export const aiTokenUsageEventsRelation = relations( aiTokenUsageEventsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [aiTokenUsageEventsTable.projectId], + references: [projectsTable.id], + }), user: one(usersTable, { fields: [aiTokenUsageEventsTable.userId], references: [usersTable.id], @@ -246,6 +299,9 @@ export const aiTokenUsageEventsRelation = relations( export const tagsTable = pgTable("tags", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), key: text("key").notNull(), amount: integer("amount").notNull(), deletedAt: timestamp("deleted_at", { @@ -254,24 +310,36 @@ export const tagsTable = pgTable("tags", { }), }); -export const metadataTable = pgTable("metadata", { - id: uuid("id").primaryKey().defaultRandom(), - last_run_at: timestamp("last_run_at", { - withTimezone: true, - mode: "string", - }), - dodo_live_api_key: text("dodo_live_api_key").notNull(), - dodo_test_api_key: text("dodo_test_api_key").notNull(), - dodo_live_product_id: text("dodo_live_product_id").notNull(), - dodo_test_product_id: text("dodo_test_product_id").notNull(), - dodo_live_webhook_secret: text("dodo_live_webhook_secret").notNull(), - dodo_test_webhook_secret: text("dodo_test_webhook_secret").notNull(), - currency: text("currency").notNull().default("usd"), - redirect_url: text("redirect_url").notNull(), -}); +export const metadataTable = pgTable( + "metadata", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + last_run_at: timestamp("last_run_at", { + withTimezone: true, + mode: "string", + }), + dodo_live_api_key: text("dodo_live_api_key").notNull(), + dodo_test_api_key: text("dodo_test_api_key").notNull(), + dodo_live_product_id: text("dodo_live_product_id").notNull(), + dodo_test_product_id: text("dodo_test_product_id").notNull(), + dodo_live_webhook_secret: text("dodo_live_webhook_secret").notNull(), + dodo_test_webhook_secret: text("dodo_test_webhook_secret").notNull(), + currency: text("currency").notNull().default("usd"), + redirect_url: text("redirect_url").notNull(), + }, + (table) => ({ + uniqueProjectId: uniqueIndex("unique_project_id").on(table.projectId), + }) +); export const expressionsTable = pgTable("expressions", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), key: text("key").notNull(), expr: text("expr").notNull(), deletedAt: timestamp("deleted_at", { @@ -284,6 +352,9 @@ export const webhookEndpointsTable = pgTable( "webhook_endpoints", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), apiKeyId: uuid("api_key_id") .references(() => apiKeysTable.id) .notNull(), @@ -315,6 +386,10 @@ export const webhookEndpointsTable = pgTable( export const webhookEndpointsRelation = relations( webhookEndpointsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [webhookEndpointsTable.projectId], + references: [projectsTable.id], + }), apiKey: one(apiKeysTable, { fields: [webhookEndpointsTable.apiKeyId], references: [apiKeysTable.id], @@ -324,6 +399,9 @@ export const webhookEndpointsRelation = relations( export const webhookDeliveriesTable = pgTable("webhook_deliveries", { id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), endpointId: uuid("endpoint_id") .references(() => webhookEndpointsTable.id) .notNull(), @@ -346,9 +424,27 @@ export const webhookDeliveriesTable = pgTable("webhook_deliveries", { export const webhookDeliveriesRelation = relations( webhookDeliveriesTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [webhookDeliveriesTable.projectId], + references: [projectsTable.id], + }), endpoint: one(webhookEndpointsTable, { fields: [webhookDeliveriesTable.endpointId], references: [webhookEndpointsTable.id], }), }) ); + +export const projectsRelation = relations(projectsTable, ({ many }) => ({ + users: many(usersTable), + sessions: many(sessionsTable), + apiKeys: many(apiKeysTable), + basicUsageEvents: many(basicUsageEventsTable), + paymentEvents: many(paymentEventsTable), + aiTokenUsageEvents: many(aiTokenUsageEventsTable), + tags: many(tagsTable), + metadata: many(metadataTable), + expressions: many(expressionsTable), + webhookEndpoints: many(webhookEndpointsTable), + webhookDeliveries: many(webhookDeliveriesTable), +})); diff --git a/src/utils/apiKeyCache.ts b/src/utils/apiKeyCache.ts index 86f7cf9..ee0c449 100644 --- a/src/utils/apiKeyCache.ts +++ b/src/utils/apiKeyCache.ts @@ -6,6 +6,7 @@ interface CachedAPIKey { id: string; role: ApiKeyRole; mode: "production" | "test" | null; + projectId: string; expiresAt: string; } diff --git a/src/utils/authenticateHttpApiKey.ts b/src/utils/authenticateHttpApiKey.ts index a467705..142853a 100644 --- a/src/utils/authenticateHttpApiKey.ts +++ b/src/utils/authenticateHttpApiKey.ts @@ -44,7 +44,12 @@ export async function authenticateHttpApiKey( `Key prefix ${role} doesn't match stored role ${cached.role}` ); } - return { apiKeyId: cached.id, role: cached.role, mode: cached.mode }; + return { + apiKeyId: cached.id, + role: cached.role, + mode: cached.mode, + projectId: cached.projectId, + }; } const apiKeyRecord = await findApiKeyByHash(apiKeyHash); @@ -76,8 +81,14 @@ export async function authenticateHttpApiKey( id: apiKeyRecord.id, role: recordRole, mode, + projectId: apiKeyRecord.projectId, expiresAt: apiKeyRecord.expiresAt, }); - return { apiKeyId: apiKeyRecord.id, role: recordRole, mode }; + return { + apiKeyId: apiKeyRecord.id, + role: recordRole, + mode, + projectId: apiKeyRecord.projectId, + }; } diff --git a/src/utils/authenticateMasterApiKey.ts b/src/utils/authenticateMasterApiKey.ts new file mode 100644 index 0000000..a3ac643 --- /dev/null +++ b/src/utils/authenticateMasterApiKey.ts @@ -0,0 +1,40 @@ +import { createHmac, timingSafeEqual } from "crypto"; +import { AuthError } from "../errors/auth"; + +function getMasterKeyHash(): string { + const hash = process.env.MASTER_API_KEY_HASH; + if (!hash) { + throw new Error("MASTER_API_KEY_HASH environment variable is not set"); + } + return hash; +} + +export function authenticateMasterApiKey(authHeader: string | undefined): void { + if (!authHeader) { + throw AuthError.missingHeader(); + } + + if (!authHeader.startsWith("Bearer ")) { + throw AuthError.invalidHeaderFormat(); + } + + const apiKey = authHeader.slice("Bearer ".length).trim(); + + const hmacSecret = process.env.HMAC_SECRET; + if (!hmacSecret) { + throw new Error("HMAC_SECRET environment variable is not set"); + } + + const incomingHash = createHmac("sha256", hmacSecret) + .update(apiKey) + .digest("hex"); + + const storedHash = getMasterKeyHash(); + + if ( + incomingHash.length !== storedHash.length || + !timingSafeEqual(Buffer.from(incomingHash), Buffer.from(storedHash)) + ) { + throw AuthError.permissionDenied("Invalid master API key"); + } +} diff --git a/src/utils/fetchTagAmount.ts b/src/utils/fetchTagAmount.ts index 3b72d32..88ef60e 100644 --- a/src/utils/fetchTagAmount.ts +++ b/src/utils/fetchTagAmount.ts @@ -5,10 +5,12 @@ import { tagsTable } from "../storage/db/postgres/schema"; import { tagCache } from "./tagCache"; export async function fetchTagAmount( + projectId: string, tag: string, notFoundMessage: string ): Promise { - const cachedAmount = tagCache.get(tag); + const cacheKey = `${projectId}:${tag}`; + const cachedAmount = tagCache.get(cacheKey); if (cachedAmount !== undefined) { return cachedAmount; } @@ -17,13 +19,19 @@ export async function fetchTagAmount( const [tagRow] = await db .select() .from(tagsTable) - .where(and(eq(tagsTable.key, tag), isNull(tagsTable.deletedAt))) + .where( + and( + eq(tagsTable.projectId, projectId), + eq(tagsTable.key, tag), + isNull(tagsTable.deletedAt) + ) + ) .limit(1); if (!tagRow) { throw EventError.validationFailed(notFoundMessage); } - tagCache.set(tag, tagRow.amount); + tagCache.set(cacheKey, tagRow.amount); return tagRow.amount; } diff --git a/src/utils/parseExpr.ts b/src/utils/parseExpr.ts index c50584b..b5ceb06 100644 --- a/src/utils/parseExpr.ts +++ b/src/utils/parseExpr.ts @@ -184,6 +184,7 @@ export function validateExprSyntax(exprString: string): void { */ export async function resolveExprRefsInExpression( exprString: string, + projectId: string, resolving: Set = new Set() ): Promise { const refs = extractExprRefs(exprString); @@ -201,14 +202,18 @@ export async function resolveExprRefsInExpression( ); } - const storedExpr = await findExpressionByKey(refName); + const storedExpr = await findExpressionByKey(projectId, refName); if (!storedExpr) { throw EventError.validationFailed(`Expression not found: ${refName}`); } resolving.add(refName); - const expanded = await resolveExprRefsInExpression(storedExpr, resolving); + const expanded = await resolveExprRefsInExpression( + storedExpr, + projectId, + resolving + ); const refPattern = new RegExp(`expr\\(${refName}\\)`, "g"); resolved = resolved.replace(refPattern, `(${expanded})`); @@ -242,7 +247,10 @@ function extractExprRefs(exprString: string): string[] { * @returns The expression string with tags replaced by their numeric values * @throws EventError if any tag is not found */ -async function resolveTagsInExpression(exprString: string): Promise { +async function resolveTagsInExpression( + exprString: string, + projectId: string +): Promise { const tagNames = extractTagNames(exprString); if (tagNames.length === 0) { @@ -253,7 +261,11 @@ async function resolveTagsInExpression(exprString: string): Promise { const tagValues = new Map(); for (const tagName of tagNames) { - const value = await fetchTagAmount(tagName, `Tag not found: ${tagName}`); + const value = await fetchTagAmount( + projectId, + tagName, + `Tag not found: ${tagName}` + ); tagValues.set(tagName, value); } @@ -323,16 +335,20 @@ function resolveTokenPlaceholders( */ export async function parseAndEvaluateExpr( exprString: string, + projectId: string, tokenContext?: EvalTokenContext ): Promise { // Step 1: Validate syntax validateExprSyntax(exprString); // Step 2: Resolve all expr(NAME) references (recursive, from DB) - const expandedExpr = await resolveExprRefsInExpression(exprString); + const expandedExpr = await resolveExprRefsInExpression(exprString, projectId); // Step 3: Resolve all tags to their values - const tagResolvedExpr = await resolveTagsInExpression(expandedExpr); + const tagResolvedExpr = await resolveTagsInExpression( + expandedExpr, + projectId + ); // Step 4: Resolve token placeholders if context provided const finalExpr = tokenContext diff --git a/src/zod/event.ts b/src/zod/event.ts index c6d5e6a..6040fd7 100644 --- a/src/zod/event.ts +++ b/src/zod/event.ts @@ -28,166 +28,195 @@ const BaseEvent = z.object({ idempotencyKey: z.string().min(1), }); -const BasicUsageDataSchema: z.ZodType = z - .object({ - basicUsageType: z.union([ - z - .literal(BasicUsageType.BASIC_USAGE_TYPE_UNSPECIFIED) - .transform(() => "RAW" as const), - z.literal(BasicUsageType.RAW).transform(() => "RAW" as const), - z - .literal(BasicUsageType.MIDDLEWARE_CALL) - .transform(() => "MIDDLEWARE_CALL" as const), - ]), - amount: z.number().optional(), - tag: z.string().optional(), - expr: z.string().optional(), - metadata: z.string().optional(), - }) - .transform(async (v): Promise => { - let debitAmount: number; - if (v.tag) { - debitAmount = await fetchTagAmount(v.tag, `Tag not found: ${v.tag}`); - } else if (v.expr) { - debitAmount = await parseAndEvaluateExpr(v.expr); - } else { - debitAmount = v.amount ?? 0; - } - return { - basicUsageType: v.basicUsageType, - debitAmount, - metadata: v.metadata ? parseMetadata(v.metadata) : undefined, - }; - }); - -const AITokenUsageDataSchema: z.ZodType = z - .object({ - model: z.string().min(1), - provider: z.string().optional().default("unknown"), - inputTokens: z.number().int().min(0), - inputCacheTokens: z.number().int().min(0), - outputTokens: z.number().int().min(0), - inputAmount: z.number().optional(), - inputTag: z.string().optional(), - inputExpr: z.string().optional(), - inputCacheAmount: z.number().optional(), - inputCacheTag: z.string().optional(), - inputCacheExpr: z.string().optional(), - outputCacheTokens: z.number().int().min(0), - outputCacheAmount: z.number().optional(), - outputCacheTag: z.string().optional(), - outputCacheExpr: z.string().optional(), - outputAmount: z.number().optional(), - outputTag: z.string().optional(), - outputExpr: z.string().optional(), - metadata: z.string().optional(), - }) - .transform(async (v): Promise => { - const tokenContext = { - inputTokens: v.inputTokens, - inputCacheTokens: v.inputCacheTokens, - outputTokens: v.outputTokens, - outputCacheTokens: v.outputCacheTokens, - }; - - let inputDebitAmount: number; - if (v.inputTag) { - inputDebitAmount = await fetchTagAmount( - v.inputTag, - `Input tag not found: ${v.inputTag}` - ); - } else if (v.inputExpr) { - inputDebitAmount = await parseAndEvaluateExpr(v.inputExpr, tokenContext); - } else { - inputDebitAmount = v.inputAmount ?? 0; - } +function createBasicUsageDataSchema( + projectId: string +): z.ZodType { + return z + .object({ + basicUsageType: z.union([ + z + .literal(BasicUsageType.BASIC_USAGE_TYPE_UNSPECIFIED) + .transform(() => "RAW" as const), + z.literal(BasicUsageType.RAW).transform(() => "RAW" as const), + z + .literal(BasicUsageType.MIDDLEWARE_CALL) + .transform(() => "MIDDLEWARE_CALL" as const), + ]), + amount: z.number().optional(), + tag: z.string().optional(), + expr: z.string().optional(), + metadata: z.string().optional(), + }) + .transform(async (v): Promise => { + let debitAmount: number; + if (v.tag) { + debitAmount = await fetchTagAmount( + projectId, + v.tag, + `Tag not found: ${v.tag}` + ); + } else if (v.expr) { + debitAmount = await parseAndEvaluateExpr(v.expr, projectId); + } else { + debitAmount = v.amount ?? 0; + } + return { + basicUsageType: v.basicUsageType, + debitAmount, + metadata: v.metadata ? parseMetadata(v.metadata) : undefined, + }; + }); +} - let inputCacheDebitAmount: number; - if (v.inputCacheTag) { - inputCacheDebitAmount = await fetchTagAmount( - v.inputCacheTag, - `Input cache tag not found: ${v.inputCacheTag}` - ); - } else if (v.inputCacheExpr) { - inputCacheDebitAmount = await parseAndEvaluateExpr( - v.inputCacheExpr, - tokenContext - ); - } else { - inputCacheDebitAmount = v.inputCacheAmount ?? 0; - } +function createAITokenUsageDataSchema( + projectId: string +): z.ZodType { + return z + .object({ + model: z.string(), + provider: z.string(), + inputTokens: z.number(), + inputCacheTokens: z.number(), + outputTokens: z.number(), + outputCacheTokens: z.number(), + inputTag: z.string().optional(), + inputExpr: z.string().optional(), + inputAmount: z.number().optional(), + inputCacheTag: z.string().optional(), + inputCacheExpr: z.string().optional(), + inputCacheAmount: z.number().optional(), + outputTag: z.string().optional(), + outputExpr: z.string().optional(), + outputAmount: z.number().optional(), + outputCacheTag: z.string().optional(), + outputCacheExpr: z.string().optional(), + outputCacheAmount: z.number().optional(), + metadata: z.string().optional(), + }) + .transform(async (v): Promise => { + const tokenContext = { + inputTokens: v.inputTokens, + inputCacheTokens: v.inputCacheTokens, + outputTokens: v.outputTokens, + outputCacheTokens: v.outputCacheTokens, + }; - let outputCacheDebitAmount: number; - if (v.outputCacheTag) { - outputCacheDebitAmount = await fetchTagAmount( - v.outputCacheTag, - `Output cache tag not found: ${v.outputCacheTag}` - ); - } else if (v.outputCacheExpr) { - outputCacheDebitAmount = await parseAndEvaluateExpr( - v.outputCacheExpr, - tokenContext - ); - } else { - outputCacheDebitAmount = v.outputCacheAmount ?? 0; - } + let inputDebitAmount: number; + if (v.inputTag) { + inputDebitAmount = await fetchTagAmount( + projectId, + v.inputTag, + `Input tag not found: ${v.inputTag}` + ); + } else if (v.inputExpr) { + inputDebitAmount = await parseAndEvaluateExpr( + v.inputExpr, + projectId, + tokenContext + ); + } else { + inputDebitAmount = v.inputAmount ?? 0; + } - let outputDebitAmount: number; - if (v.outputTag) { - outputDebitAmount = await fetchTagAmount( - v.outputTag, - `Output tag not found: ${v.outputTag}` - ); - } else if (v.outputExpr) { - outputDebitAmount = await parseAndEvaluateExpr( - v.outputExpr, - tokenContext - ); - } else { - outputDebitAmount = v.outputAmount ?? 0; - } + let inputCacheDebitAmount: number; + if (v.inputCacheTag) { + inputCacheDebitAmount = await fetchTagAmount( + projectId, + v.inputCacheTag, + `Input cache tag not found: ${v.inputCacheTag}` + ); + } else if (v.inputCacheExpr) { + inputCacheDebitAmount = await parseAndEvaluateExpr( + v.inputCacheExpr, + projectId, + tokenContext + ); + } else { + inputCacheDebitAmount = v.inputCacheAmount ?? 0; + } - return { - model: v.model, - provider: v.provider, - inputTokens: v.inputTokens, - inputCacheTokens: v.inputCacheTokens, - outputTokens: v.outputTokens, - outputCacheTokens: v.outputCacheTokens, - inputDebitAmount, - inputCacheDebitAmount, - outputCacheDebitAmount, - outputDebitAmount, - metadata: v.metadata ? parseMetadata(v.metadata) : undefined, - }; - }); + let outputCacheDebitAmount: number; + if (v.outputCacheTag) { + outputCacheDebitAmount = await fetchTagAmount( + projectId, + v.outputCacheTag, + `Output cache tag not found: ${v.outputCacheTag}` + ); + } else if (v.outputCacheExpr) { + outputCacheDebitAmount = await parseAndEvaluateExpr( + v.outputCacheExpr, + projectId, + tokenContext + ); + } else { + outputCacheDebitAmount = v.outputCacheAmount ?? 0; + } -const RegisterEventBasicUsage = BaseEvent.extend({ - type: z - .literal(EventType.BASIC_USAGE) - .transform(() => "BASIC_USAGE" as const), - basicUsage: BasicUsageDataSchema, -}); + let outputDebitAmount: number; + if (v.outputTag) { + outputDebitAmount = await fetchTagAmount( + projectId, + v.outputTag, + `Output tag not found: ${v.outputTag}` + ); + } else if (v.outputExpr) { + outputDebitAmount = await parseAndEvaluateExpr( + v.outputExpr, + projectId, + tokenContext + ); + } else { + outputDebitAmount = v.outputAmount ?? 0; + } -const StreamEventBasicUsage = BaseEvent.extend({ - type: z - .literal(EventType.BASIC_USAGE) - .transform(() => "BASIC_USAGE" as const), - basicUsage: BasicUsageDataSchema, -}); + return { + model: v.model, + provider: v.provider, + inputTokens: v.inputTokens, + inputCacheTokens: v.inputCacheTokens, + outputTokens: v.outputTokens, + outputCacheTokens: v.outputCacheTokens, + inputDebitAmount, + inputCacheDebitAmount, + outputCacheDebitAmount, + outputDebitAmount, + metadata: v.metadata ? parseMetadata(v.metadata) : undefined, + }; + }); +} -const StreamEventAITokenUsage = BaseEvent.extend({ - type: z - .literal(EventType.AI_TOKEN_USAGE) - .transform(() => "AI_TOKEN_USAGE" as const), - aiTokenUsage: AITokenUsageDataSchema, -}); +export function createRegisterEventSchema(projectId: string) { + const BasicUsageDataSchema = createBasicUsageDataSchema(projectId); + return BaseEvent.extend({ + type: z + .literal(EventType.BASIC_USAGE) + .transform(() => "BASIC_USAGE" as const), + basicUsage: BasicUsageDataSchema, + }); +} -export const registerEventSchema = RegisterEventBasicUsage; -export type RegisterEventSchemaType = z.output; +export function createStreamEventSchema(projectId: string) { + const BasicUsageDataSchema = createBasicUsageDataSchema(projectId); + const AITokenUsageDataSchema = createAITokenUsageDataSchema(projectId); + return z.union([ + BaseEvent.extend({ + type: z + .literal(EventType.BASIC_USAGE) + .transform(() => "BASIC_USAGE" as const), + basicUsage: BasicUsageDataSchema, + }), + BaseEvent.extend({ + type: z + .literal(EventType.AI_TOKEN_USAGE) + .transform(() => "AI_TOKEN_USAGE" as const), + aiTokenUsage: AITokenUsageDataSchema, + }), + ]); +} -export const streamEventSchema = z.union([ - StreamEventBasicUsage, - StreamEventAITokenUsage, -]); -export type StreamEventSchemaType = z.output; +export type RegisterEventSchemaType = z.output< + ReturnType +>; +export type StreamEventSchemaType = z.output< + ReturnType +>; diff --git a/src/zod/internals.ts b/src/zod/internals.ts index 938d40b..525b71d 100644 --- a/src/zod/internals.ts +++ b/src/zod/internals.ts @@ -33,6 +33,7 @@ export function createFilterGroupSchema( } export const onboardingSchema = z.object({ + name: z.string().min(1, "Project name is required").max(255), dodoLiveApiKey: z.string().min(1, "Dodo live API key is required"), dodoTestApiKey: z.string().min(1, "Dodo test API key is required"), dodoLiveProductId: z.string().min(1, "Dodo live product ID is required"), From bfdfe30eb0cc6b145a04a8bfed4224268316316a Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 01:28:15 +0530 Subject: [PATCH 02/55] fix(datetime): Slight changes --- src/interceptors/auth.ts | 2 +- src/routes/http/api/apiKeys.ts | 15 ++++++ src/routes/http/api/onboarding.ts | 88 +++++++++++++++++++++++-------- src/utils/apiKeyCache.ts | 2 +- 4 files changed, 82 insertions(+), 25 deletions(-) diff --git a/src/interceptors/auth.ts b/src/interceptors/auth.ts index 6f2fd76..b271258 100644 --- a/src/interceptors/auth.ts +++ b/src/interceptors/auth.ts @@ -202,7 +202,7 @@ export function authInterceptor( if ( DateTime.utc() > - DateTime.fromSQL(apiKeyRecord.expiresAt, { zone: "utc" }) + DateTime.fromISO(apiKeyRecord.expiresAt, { zone: "utc" }) ) { return callback?.(AuthError.expiredAPIKey()); } diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 340566f..59f110e 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -23,6 +23,7 @@ import { import { eq, and, isNull, ne, sql } from "drizzle-orm"; import type { ApiKeyRole } from "../../../utils/keyFormat"; import { invalidateWebhookEndpointCache } from "../../../interceptors/auth"; +import { apiKeyCache } from "../../../utils/apiKeyCache"; const createApiKeySchema = z.object({ name: z.string().min(1, "Name is required").max(255), @@ -221,6 +222,20 @@ export async function handleRevokeApiKey( return { error: "API key not found or already revoked" }; } + const [keyRow] = await db + .select({ key: apiKeysTable.key }) + .from(apiKeysTable) + .where( + and( + eq(apiKeysTable.projectId, auth.projectId), + eq(apiKeysTable.id, params.id) + ) + ) + .limit(1); + if (keyRow) { + apiKeyCache.delete(keyRow.key); + } + builder.setSuccess(200); reply.code(200); return { message: "API key revoked" }; diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 5a03dac..7597093 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -66,12 +66,15 @@ export async function handleOnboarding( let liveSecret: string; let testSecret: string; + let liveWebhookId: string | undefined; + let testWebhookId: string | undefined; try { const liveWebhook = await liveClient.webhooks.create({ url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, description: "Scrawn live payment webhook", filter_types: ["payment.succeeded", "payment.failed"], }); + liveWebhookId = liveWebhook.id; liveSecret = (await liveClient.webhooks.retrieveSecret(liveWebhook.id)) .secret; @@ -80,9 +83,24 @@ export async function handleOnboarding( description: "Scrawn test payment webhook", filter_types: ["payment.succeeded", "payment.failed"], }); + testWebhookId = testWebhook.id; testSecret = (await testClient.webhooks.retrieveSecret(testWebhook.id)) .secret; } catch (error) { + if (liveWebhookId) { + liveClient.webhooks.delete(liveWebhookId).catch((e) => + Sentry.captureException(e, { + extra: { context: "rollback: failed to delete live webhook" }, + }) + ); + } + if (testWebhookId) { + testClient.webhooks.delete(testWebhookId).catch((e) => + Sentry.captureException(e, { + extra: { context: "rollback: failed to delete test webhook" }, + }) + ); + } const errMsg = error instanceof Error ? error.message : String(error); Sentry.captureException(error, { extra: { context: "dodo webhook registration during onboarding" }, @@ -100,32 +118,56 @@ export async function handleOnboarding( const expiresAt = DateTime.utc().plus({ years: 10 }).toISO(); const db = getPostgresDB(); - await executeInTransaction(db, "create project", async (txn) => { - await txn.insert(projectsTable).values({ - id: projectId, - name: validated.name, - }); + try { + await executeInTransaction(db, "create project", async (txn) => { + await txn.insert(projectsTable).values({ + id: projectId, + name: validated.name, + }); - await txn.insert(metadataTable).values({ - projectId, - dodo_live_api_key: encrypt(validated.dodoLiveApiKey), - dodo_test_api_key: encrypt(validated.dodoTestApiKey), - dodo_live_product_id: validated.dodoLiveProductId, - dodo_test_product_id: validated.dodoTestProductId, - dodo_live_webhook_secret: encrypt(liveSecret), - dodo_test_webhook_secret: encrypt(testSecret), - currency: validated.currency, - redirect_url: validated.redirectUrl, - }); + await txn.insert(metadataTable).values({ + projectId, + dodo_live_api_key: encrypt(validated.dodoLiveApiKey), + dodo_test_api_key: encrypt(validated.dodoTestApiKey), + dodo_live_product_id: validated.dodoLiveProductId, + dodo_test_product_id: validated.dodoTestProductId, + dodo_live_webhook_secret: encrypt(liveSecret), + dodo_test_webhook_secret: encrypt(testSecret), + currency: validated.currency, + redirect_url: validated.redirectUrl, + }); - await txn.insert(apiKeysTable).values({ - projectId, - name: "Default dashboard key", - key: dashboardKeyHash, - role: "dashboard", - expiresAt, + await txn.insert(apiKeysTable).values({ + projectId, + name: "Default dashboard key", + key: dashboardKeyHash, + role: "dashboard", + expiresAt, + }); }); - }); + } catch (txnError) { + if (liveWebhookId) { + liveClient.webhooks.delete(liveWebhookId).catch((e) => + Sentry.captureException(e, { + extra: { + context: + "rollback: failed to delete live webhook after DB failure", + }, + }) + ); + } + if (testWebhookId) { + testClient.webhooks.delete(testWebhookId).catch((e) => + Sentry.captureException(e, { + extra: { + context: + "rollback: failed to delete test webhook after DB failure", + }, + }) + ); + } + throw txnError; + } clearClients(); diff --git a/src/utils/apiKeyCache.ts b/src/utils/apiKeyCache.ts index ee0c449..d8bdfa3 100644 --- a/src/utils/apiKeyCache.ts +++ b/src/utils/apiKeyCache.ts @@ -14,7 +14,7 @@ const store = Cache.getStore("api-keys", { max: 1000, ttlMs: 5 * 60 * 1000, validate: (value) => - DateTime.utc() <= DateTime.fromSQL(value.expiresAt, { zone: "utc" }), + DateTime.utc() <= DateTime.fromISO(value.expiresAt, { zone: "utc" }), }); export const apiKeyCache = { From 4cbc1ed7d5d020ba7c88e79cc55f84d0749182bf Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 01:36:03 +0530 Subject: [PATCH 03/55] fix(webhooks): potential leak) --- src/routes/http/api/webhookDeliveries.ts | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/routes/http/api/webhookDeliveries.ts b/src/routes/http/api/webhookDeliveries.ts index 40c9901..8830b31 100644 --- a/src/routes/http/api/webhookDeliveries.ts +++ b/src/routes/http/api/webhookDeliveries.ts @@ -51,16 +51,23 @@ export async function handleListDeliveries( const endpoints = await db .select({ id: webhookEndpointsTable.id }) .from(webhookEndpointsTable) - .where(eq(webhookEndpointsTable.apiKeyId, query.apiKeyId)); - const ids = endpoints.map((e) => e.id); - if (ids.length > 0) { - conditions = inArray(webhookDeliveriesTable.endpointId, ids); - } else { - conditions = eq( - webhookDeliveriesTable.id, - "00000000-0000-0000-0000-000000000000" + .where( + and( + eq(webhookEndpointsTable.projectId, auth.projectId), + eq(webhookEndpointsTable.apiKeyId, query.apiKeyId) + ) ); - } + const ids = endpoints.map((e) => e.id); + conditions = + ids.length > 0 + ? and(conditions, inArray(webhookDeliveriesTable.endpointId, ids)) + : and( + conditions, + eq( + webhookDeliveriesTable.id, + "00000000-0000-0000-0000-000000000000" + ) + ); } if (query.eventType) { From 4b9f81e511c3c9c2dc40b21c27641c39befe8c15 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 02:47:30 +0530 Subject: [PATCH 04/55] fix(user): Cache invalidation --- src/storage/db/postgres/helpers/tags.ts | 6 ++-- src/storage/db/postgres/helpers/users.ts | 15 ++++++--- src/storage/db/postgres/schema.ts | 39 +++++++++++++++--------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/storage/db/postgres/helpers/tags.ts b/src/storage/db/postgres/helpers/tags.ts index b0fa739..e20d072 100644 --- a/src/storage/db/postgres/helpers/tags.ts +++ b/src/storage/db/postgres/helpers/tags.ts @@ -51,12 +51,12 @@ export async function createTag( .update(tagsTable) .set({ amount }) .where(eq(tagsTable.id, existing[0].id)); - tagCache.delete(key); + tagCache.delete(`${projectId}:${key}`); return; } await db.insert(tagsTable).values({ projectId, key, amount }); - tagCache.delete(key); + tagCache.delete(`${projectId}:${key}`); } catch (e) { throw StorageError.insertFailed( `Failed to upsert tag '${key}'`, @@ -85,7 +85,7 @@ export async function deleteTag( ); if ((result.count ?? 0) > 0) { - tagCache.delete(key); + tagCache.delete(`${projectId}:${key}`); return true; } return false; diff --git a/src/storage/db/postgres/helpers/users.ts b/src/storage/db/postgres/helpers/users.ts index b79dd61..13fa33d 100644 --- a/src/storage/db/postgres/helpers/users.ts +++ b/src/storage/db/postgres/helpers/users.ts @@ -45,10 +45,17 @@ export async function ensureUserExists( const db = txn ?? getPostgresDB(); try { - await db - .insert(usersTable) - .values({ id: userId, projectId }) - .onConflictDoNothing({ target: usersTable.id }); + const [existing] = await db + .select({ id: usersTable.id }) + .from(usersTable) + .where( + and(eq(usersTable.projectId, projectId), eq(usersTable.id, userId)) + ) + .limit(1); + + if (existing) return; + + await db.insert(usersTable).values({ id: userId, projectId }); } catch (e) { if ( e instanceof Error && diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index 1f93982..f08dc0d 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -25,22 +25,31 @@ export const projectsTable = pgTable("projects", { .notNull(), }); -export const usersTable = pgTable("users", { - id: USER_ID_CONFIG.dbType("id").primaryKey(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - last_billed_timestamp: timestamp("last_billed_timestamp", { - withTimezone: true, - mode: "string", +export const usersTable = pgTable( + "users", + { + id: USER_ID_CONFIG.dbType("id").primaryKey(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + last_billed_timestamp: timestamp("last_billed_timestamp", { + withTimezone: true, + mode: "string", + }) + .default(DateTime.utc(1).toString()) + .notNull(), + payment_provider_user_id: text("payment_provider_user_id"), + mode: text("mode", { enum: ["test", "production"] }) + .notNull() + .default("production"), + }, + (table) => ({ + uniqueUserPerProject: uniqueIndex("uq_users_project_id").on( + table.projectId, + table.id + ), }) - .default(DateTime.utc(1).toString()) - .notNull(), - payment_provider_user_id: text("payment_provider_user_id"), - mode: text("mode", { enum: ["test", "production"] }) - .notNull() - .default("production"), -}); +); export const usersRelation = relations(usersTable, ({ one, many }) => ({ project: one(projectsTable, { From 2b54ffbbbffbbc7994cff5a4784dd0c1b18b6c9c Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 21:32:49 +0530 Subject: [PATCH 05/55] fix: composite PK for usersTable, token validation, ensureUserExists race, scoped clearClients --- src/routes/gRPC/payment/paymentProvider.ts | 5 + src/routes/http/api/onboarding.ts | 4 +- src/storage/db/postgres/helpers/users.ts | 26 +-- src/storage/db/postgres/schema.ts | 200 ++++++++++++--------- src/zod/event.ts | 8 +- 5 files changed, 130 insertions(+), 113 deletions(-) diff --git a/src/routes/gRPC/payment/paymentProvider.ts b/src/routes/gRPC/payment/paymentProvider.ts index bd3ac36..f151748 100644 --- a/src/routes/gRPC/payment/paymentProvider.ts +++ b/src/routes/gRPC/payment/paymentProvider.ts @@ -13,6 +13,11 @@ export function clearClients(): void { clients.clear(); } +export function removeClient(projectId: string): void { + clients.delete(clientKey(projectId, "test")); + clients.delete(clientKey(projectId, "production")); +} + export async function getDodoClient( projectId: string, mode?: "test" | "production" diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 7597093..0dbdf2e 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -22,7 +22,7 @@ import { metadataTable, } from "../../../storage/db/postgres/schema"; import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; -import { clearClients } from "../../gRPC/payment/paymentProvider.ts"; +import { removeClient } from "../../gRPC/payment/paymentProvider.ts"; import { DateTime } from "luxon"; import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; @@ -169,7 +169,7 @@ export async function handleOnboarding( throw txnError; } - clearClients(); + removeClient(projectId); builder.setSuccess(201); diff --git a/src/storage/db/postgres/helpers/users.ts b/src/storage/db/postgres/helpers/users.ts index 13fa33d..287789e 100644 --- a/src/storage/db/postgres/helpers/users.ts +++ b/src/storage/db/postgres/helpers/users.ts @@ -45,24 +45,14 @@ export async function ensureUserExists( const db = txn ?? getPostgresDB(); try { - const [existing] = await db - .select({ id: usersTable.id }) - .from(usersTable) - .where( - and(eq(usersTable.projectId, projectId), eq(usersTable.id, userId)) - ) - .limit(1); - - if (existing) return; - - await db.insert(usersTable).values({ id: userId, projectId }); + await db + .insert(usersTable) + .values({ id: userId, projectId }) + .onConflictDoNothing(); } catch (e) { - if ( - e instanceof Error && - (e.message.includes("duplicate") || e.message.includes("unique")) - ) { - return; - } - throw e; + throw StorageError.queryFailed( + "Failed to ensure user exists", + e instanceof Error ? e : new Error(String(e)) + ); } } diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index f08dc0d..e04a31d 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -9,6 +9,8 @@ import { boolean, jsonb, uniqueIndex, + primaryKey, + foreignKey, } from "drizzle-orm/pg-core"; import { USER_ID_CONFIG } from "../../../config/identifiers"; import { DateTime } from "luxon"; @@ -28,7 +30,7 @@ export const projectsTable = pgTable("projects", { export const usersTable = pgTable( "users", { - id: USER_ID_CONFIG.dbType("id").primaryKey(), + id: USER_ID_CONFIG.dbType("id").notNull(), projectId: uuid("project_id") .references(() => projectsTable.id) .notNull(), @@ -44,10 +46,7 @@ export const usersTable = pgTable( .default("production"), }, (table) => ({ - uniqueUserPerProject: uniqueIndex("uq_users_project_id").on( - table.projectId, - table.id - ), + pk: primaryKey({ columns: [table.projectId, table.id] }), }) ); @@ -73,9 +72,7 @@ export const sessionsTable = pgTable( processed: text("processed", { enum: ["pending", "failed", "succeeded"] }) .default("pending") .notNull(), - userId: USER_ID_CONFIG.dbType("user_id") - .references(() => usersTable.id) - .notNull(), + userId: USER_ID_CONFIG.dbType("user_id").notNull(), apiKeyId: uuid("api_key_id") .references(() => apiKeysTable.id) .notNull(), @@ -96,6 +93,10 @@ export const sessionsTable = pgTable( }, (table) => ({ uniqueSessionId: uniqueIndex("unique_session_id").on(table.sessionId), + userFk: foreignKey({ + columns: [table.projectId, table.userId], + foreignColumns: [usersTable.projectId, usersTable.id], + }), }) ); @@ -161,34 +162,41 @@ export const apiKeysRelation = relations(apiKeysTable, ({ one, many }) => ({ aiTokenUsageEvents: many(aiTokenUsageEventsTable), })); -export const basicUsageEventsTable = pgTable("basic_usage_events", { - id: uuid("id").primaryKey().defaultRandom(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - eventId: uuid("event_id").notNull(), - idempotencyKey: text("idempotency_key").notNull().unique(), - reportedTimestamp: timestamp("reported_timestamp", { - withTimezone: true, - mode: "string", - }).notNull(), - ingestedTimestamp: timestamp("ingested_timestamp", { - withTimezone: true, - mode: "string", +export const basicUsageEventsTable = pgTable( + "basic_usage_events", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + eventId: uuid("event_id").notNull(), + idempotencyKey: text("idempotency_key").notNull().unique(), + reportedTimestamp: timestamp("reported_timestamp", { + withTimezone: true, + mode: "string", + }).notNull(), + ingestedTimestamp: timestamp("ingested_timestamp", { + withTimezone: true, + mode: "string", + }) + .defaultNow() + .notNull(), + userId: USER_ID_CONFIG.dbType("user_id").notNull(), + apiKeyId: uuid("api_key_id") + .references(() => apiKeysTable.id) + .notNull(), + mode: text("mode", { enum: ["test", "production"] }).notNull(), + type: text("type", { enum: ["RAW", "MIDDLEWARE_CALL"] }).notNull(), + debitAmount: bigint("debit_amount", { mode: "number" }).notNull(), + metadata: jsonb("metadata").$type>(), + }, + (table) => ({ + userFk: foreignKey({ + columns: [table.projectId, table.userId], + foreignColumns: [usersTable.projectId, usersTable.id], + }), }) - .defaultNow() - .notNull(), - userId: USER_ID_CONFIG.dbType("user_id") - .references(() => usersTable.id) - .notNull(), - apiKeyId: uuid("api_key_id") - .references(() => apiKeysTable.id) - .notNull(), - mode: text("mode", { enum: ["test", "production"] }).notNull(), - type: text("type", { enum: ["RAW", "MIDDLEWARE_CALL"] }).notNull(), - debitAmount: bigint("debit_amount", { mode: "number" }).notNull(), - metadata: jsonb("metadata").$type>(), -}); +); export const basicUsageEventsRelation = relations( basicUsageEventsTable, @@ -208,33 +216,40 @@ export const basicUsageEventsRelation = relations( }) ); -export const paymentEventsTable = pgTable("payment_events", { - id: uuid("id").primaryKey().defaultRandom(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - reportedTimestamp: timestamp("reported_timestamp", { - withTimezone: true, - mode: "string", - }).notNull(), - ingestedTimestamp: timestamp("ingested_timestamp", { - withTimezone: true, - mode: "string", +export const paymentEventsTable = pgTable( + "payment_events", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + reportedTimestamp: timestamp("reported_timestamp", { + withTimezone: true, + mode: "string", + }).notNull(), + ingestedTimestamp: timestamp("ingested_timestamp", { + withTimezone: true, + mode: "string", + }) + .defaultNow() + .notNull(), + userId: USER_ID_CONFIG.dbType("user_id").notNull(), + apiKeyId: uuid("api_key_id") + .references(() => apiKeysTable.id) + .notNull(), + mode: text("mode", { enum: ["test", "production"] }).notNull(), + creditAmount: bigint("credit_amount", { mode: "number" }).notNull(), + proxyId: uuid("proxy_id") + .references(() => sessionsTable.proxy_link_id) + .notNull(), + }, + (table) => ({ + userFk: foreignKey({ + columns: [table.projectId, table.userId], + foreignColumns: [usersTable.projectId, usersTable.id], + }), }) - .defaultNow() - .notNull(), - userId: USER_ID_CONFIG.dbType("user_id") - .references(() => usersTable.id) - .notNull(), - apiKeyId: uuid("api_key_id") - .references(() => apiKeysTable.id) - .notNull(), - mode: text("mode", { enum: ["test", "production"] }).notNull(), - creditAmount: bigint("credit_amount", { mode: "number" }).notNull(), - proxyId: uuid("proxy_id") - .references(() => sessionsTable.proxy_link_id) - .notNull(), -}); +); export const paymentEventsRelation = relations( paymentEventsTable, @@ -258,35 +273,42 @@ export const paymentEventsRelation = relations( }) ); -export const aiTokenUsageEventsTable = pgTable("ai_token_usage_events", { - id: uuid("id").primaryKey().defaultRandom(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - eventId: uuid("event_id").notNull(), - idempotencyKey: text("idempotency_key").notNull().unique(), - reportedTimestamp: timestamp("reported_timestamp", { - withTimezone: true, - mode: "string", - }).notNull(), - ingestedTimestamp: timestamp("ingested_timestamp", { - withTimezone: true, - mode: "string", +export const aiTokenUsageEventsTable = pgTable( + "ai_token_usage_events", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + eventId: uuid("event_id").notNull(), + idempotencyKey: text("idempotency_key").notNull().unique(), + reportedTimestamp: timestamp("reported_timestamp", { + withTimezone: true, + mode: "string", + }).notNull(), + ingestedTimestamp: timestamp("ingested_timestamp", { + withTimezone: true, + mode: "string", + }) + .defaultNow() + .notNull(), + userId: USER_ID_CONFIG.dbType("user_id").notNull(), + apiKeyId: uuid("api_key_id") + .references(() => apiKeysTable.id) + .notNull(), + mode: text("mode", { enum: ["test", "production"] }).notNull(), + model: text("model").notNull(), + provider: text("provider").notNull(), + metrics: jsonb("metrics").$type().notNull(), + metadata: jsonb("metadata").$type>(), + }, + (table) => ({ + userFk: foreignKey({ + columns: [table.projectId, table.userId], + foreignColumns: [usersTable.projectId, usersTable.id], + }), }) - .defaultNow() - .notNull(), - userId: USER_ID_CONFIG.dbType("user_id") - .references(() => usersTable.id) - .notNull(), - apiKeyId: uuid("api_key_id") - .references(() => apiKeysTable.id) - .notNull(), - mode: text("mode", { enum: ["test", "production"] }).notNull(), - model: text("model").notNull(), - provider: text("provider").notNull(), - metrics: jsonb("metrics").$type().notNull(), - metadata: jsonb("metadata").$type>(), -}); +); export const aiTokenUsageEventsRelation = relations( aiTokenUsageEventsTable, diff --git a/src/zod/event.ts b/src/zod/event.ts index 6040fd7..e980a26 100644 --- a/src/zod/event.ts +++ b/src/zod/event.ts @@ -75,10 +75,10 @@ function createAITokenUsageDataSchema( .object({ model: z.string(), provider: z.string(), - inputTokens: z.number(), - inputCacheTokens: z.number(), - outputTokens: z.number(), - outputCacheTokens: z.number(), + inputTokens: z.number().int().nonnegative(), + inputCacheTokens: z.number().int().nonnegative(), + outputTokens: z.number().int().nonnegative(), + outputCacheTokens: z.number().int().nonnegative(), inputTag: z.string().optional(), inputExpr: z.string().optional(), inputAmount: z.number().optional(), From 287a146b279f882eec0061877915b69c861f9c26 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 21:50:37 +0530 Subject: [PATCH 06/55] fix: add model.min(1), provider default, token count .min(0) in createAITokenUsageDataSchema --- src/zod/event.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/zod/event.ts b/src/zod/event.ts index e980a26..bb58230 100644 --- a/src/zod/event.ts +++ b/src/zod/event.ts @@ -73,12 +73,12 @@ function createAITokenUsageDataSchema( ): z.ZodType { return z .object({ - model: z.string(), - provider: z.string(), - inputTokens: z.number().int().nonnegative(), - inputCacheTokens: z.number().int().nonnegative(), - outputTokens: z.number().int().nonnegative(), - outputCacheTokens: z.number().int().nonnegative(), + model: z.string().min(1), + provider: z.string().optional().default("unknown"), + inputTokens: z.number().int().min(0), + inputCacheTokens: z.number().int().min(0), + outputTokens: z.number().int().min(0), + outputCacheTokens: z.number().int().min(0), inputTag: z.string().optional(), inputExpr: z.string().optional(), inputAmount: z.number().optional(), From 382189968af302f47dc544573a1aaff01986fde1 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 17 Jun 2026 23:53:16 +0530 Subject: [PATCH 07/55] fix: scope row lock, deleteWebhookEndpoint, and remove metadata from query registry --- src/routes/gRPC/data/query.ts | 11 +---------- src/routes/gRPC/payment/createCheckoutLink.ts | 9 +++++++-- src/routes/http/api/webhookEndpoints.ts | 2 +- src/storage/db/postgres/helpers/webhookEndpoints.ts | 2 ++ 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index 3732cf3..c53d8d0 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -10,7 +10,6 @@ import { sessionsTable, tagsTable, expressionsTable, - metadataTable, } from "../../../storage/db/postgres/schema"; import { eq, @@ -44,8 +43,7 @@ interface TableDef { | typeof usersTable | typeof sessionsTable | typeof tagsTable - | typeof expressionsTable - | typeof metadataTable; + | typeof expressionsTable; fields: Record; } @@ -97,13 +95,6 @@ const TABLE_REGISTRY: Record = { expr: { col: expressionsTable.expr, cast: "text" }, }, }, - metadata: { - tableName: "metadata", - table: metadataTable, - fields: { - id: { col: metadataTable.id, cast: "uuid" }, - }, - }, }; function castValue( diff --git a/src/routes/gRPC/payment/createCheckoutLink.ts b/src/routes/gRPC/payment/createCheckoutLink.ts index 23e58e6..6973f22 100644 --- a/src/routes/gRPC/payment/createCheckoutLink.ts +++ b/src/routes/gRPC/payment/createCheckoutLink.ts @@ -32,7 +32,7 @@ import { getPostgresDB } from "../../../storage/db/postgres/db"; import { checkIfExistingCheckoutLink } from "../../../storage/db/postgres/helpers/sessions"; import { ensureUserExists } from "../../../storage/db/postgres/helpers/users"; import { usersTable } from "../../../storage/db/postgres/schema"; -import { eq } from "drizzle-orm"; +import { eq, and } from "drizzle-orm"; export async function createCheckoutLink( call: ContextUnaryCall, @@ -98,7 +98,12 @@ export async function createCheckoutLink( await txn .select({ id: usersTable.id }) .from(usersTable) - .where(eq(usersTable.id, validatedData.userId)) + .where( + and( + eq(usersTable.projectId, auth.projectId), + eq(usersTable.id, validatedData.userId) + ) + ) .for("update"); const existingId = await checkIfExistingCheckoutLink( diff --git a/src/routes/http/api/webhookEndpoints.ts b/src/routes/http/api/webhookEndpoints.ts index 961cebb..a3ef8f3 100644 --- a/src/routes/http/api/webhookEndpoints.ts +++ b/src/routes/http/api/webhookEndpoints.ts @@ -240,7 +240,7 @@ export async function handleDeleteWebhookEndpoint( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); - const deleted = await deleteWebhookEndpoint(auth.apiKeyId); + const deleted = await deleteWebhookEndpoint(auth.projectId, auth.apiKeyId); if (!deleted) { builder.setError(404, { diff --git a/src/storage/db/postgres/helpers/webhookEndpoints.ts b/src/storage/db/postgres/helpers/webhookEndpoints.ts index c4ebfd3..a9c5a72 100644 --- a/src/storage/db/postgres/helpers/webhookEndpoints.ts +++ b/src/storage/db/postgres/helpers/webhookEndpoints.ts @@ -89,6 +89,7 @@ export async function upsertWebhookEndpoint( } export async function deleteWebhookEndpoint( + projectId: string, apiKeyId: string ): Promise { const db = getPostgresDB(); @@ -101,6 +102,7 @@ export async function deleteWebhookEndpoint( .set({ deletedAt: now }) .where( and( + eq(webhookEndpointsTable.projectId, projectId), eq(webhookEndpointsTable.apiKeyId, apiKeyId), isNull(webhookEndpointsTable.deletedAt) ) From c0755c705742aa0809c3ef4895584ef4e7bde2c8 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Thu, 18 Jun 2026 21:48:44 +0530 Subject: [PATCH 08/55] =?UTF-8?q?fix:=20resolve=20all=20greptile=20finding?= =?UTF-8?q?s=20=E2=80=94=20billing,=20auth,=20tests,=20schema,=20validatio?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 +- src/__tests__/db/index.ts | 3 +- src/__tests__/fixtures/apiKey.ts | 18 ++++ src/routes/gRPC/data/query.ts | 8 +- src/routes/http/api/apiKeys.ts | 90 ++++++++++++------- src/routes/http/api/onboarding.ts | 19 ++++ src/routes/http/api/webhookDeliveries.ts | 21 ++--- src/routes/http/createdCheckout.ts | 16 +++- src/routes/http/registerWebhookRoutes.ts | 13 +++ .../adapter/postgres/handlers/priceRequest.ts | 18 ++-- src/storage/db/postgres/helpers/metadata.ts | 39 -------- src/storage/db/postgres/helpers/users.ts | 5 +- .../db/postgres/helpers/webhookEndpoints.ts | 6 +- src/storage/db/postgres/schema.ts | 19 ++-- src/utils/parseExpr.ts | 8 +- 15 files changed, 175 insertions(+), 110 deletions(-) diff --git a/.env.example b/.env.example index 1f9ebac..39b7c27 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/scrawn CLICKHOUSE_URL=http://default:clickhouse@localhost:8123/scrawn HMAC_SECRET= -MASTER_API_KEY_HASH= # HMAC-SHA256 hex hash of the master API key (used for project creation during onboarding) +MASTER_API_KEY_HASH= # hex(HMAC-SHA256(HMAC_SECRET, )) — computed with HMAC_SECRET as the signing key (used for project creation during onboarding) APP_URL=http://localhost:8060 # URL the Scrawn backend is hosted on # SENTRY_DSN= diff --git a/src/__tests__/db/index.ts b/src/__tests__/db/index.ts index ada334a..c8178e8 100644 --- a/src/__tests__/db/index.ts +++ b/src/__tests__/db/index.ts @@ -31,7 +31,8 @@ export async function clearDatabase() { users, tags, metadata, - expressions + expressions, + projects RESTART IDENTITY CASCADE `); diff --git a/src/__tests__/fixtures/apiKey.ts b/src/__tests__/fixtures/apiKey.ts index 1785354..5b41df9 100644 --- a/src/__tests__/fixtures/apiKey.ts +++ b/src/__tests__/fixtures/apiKey.ts @@ -1,18 +1,35 @@ import { getPostgresDB } from "../../storage/db/postgres/db"; import { + projectsTable, apiKeysTable, webhookEndpointsTable, } from "../../storage/db/postgres/schema"; +import { eq } from "drizzle-orm"; import { hashAPIKey } from "../../utils/hashAPIKey"; import { DateTime } from "luxon"; export const TEST_PROJECT_ID = "00000000-0000-0000-0000-000000000001"; +async function ensureTestProject(): Promise { + const db = getPostgresDB(); + const [existing] = await db + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(eq(projectsTable.id, TEST_PROJECT_ID)) + .limit(1); + if (existing) return; + await db.insert(projectsTable).values({ + id: TEST_PROJECT_ID, + name: "test-project", + }); +} + export async function createTestApiKey(): Promise<{ rawKey: string; id: string; }> { const db = getPostgresDB(); + await ensureTestProject(); const rawKey = `scrn_test_${crypto.randomUUID().replace(/-/g, "").slice(0, 32)}`; const [key] = await db .insert(apiKeysTable) @@ -42,6 +59,7 @@ export async function insertKey( overrides: Partial<{ revoked: boolean; expiresAt: string }> = {} ): Promise { const db = getPostgresDB(); + await ensureTestProject(); const [key] = await db .insert(apiKeysTable) .values({ diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index c53d8d0..86415ad 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -223,7 +223,13 @@ export async function queryData( const db = getPostgresDB(); const userWhere = buildWhere(validated.where, tableDef); const projectFilter = eq( - (tableDef.table as any).projectId, + ( + tableDef.table as + | typeof usersTable + | typeof sessionsTable + | typeof tagsTable + | typeof expressionsTable + ).projectId, auth.projectId ) as SQL; const whereClause = userWhere diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 59f110e..8939910 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -16,6 +16,7 @@ import { createApiKey } from "../../../storage/db/postgres/helpers/apiKeys"; import { upsertWebhookEndpoint } from "../../../storage/db/postgres/helpers/webhookEndpoints"; import { generateWebhookKeyPair } from "../../../utils/generateWebhookKeyPair"; import { getPostgresDB } from "../../../storage/db/postgres/db"; +import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; import { apiKeysTable, webhookEndpointsTable, @@ -48,31 +49,61 @@ export async function handleCreateApiKey( try { const auth = await authenticateHttpApiKey(request.headers.authorization); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can manage API keys" + ); + } builder.setApiKeyContext({ name: `create-key:${auth.apiKeyId}` }); const body = await request.body; const validated = createApiKeySchema.parse(body); + if ( + validated.role === "production" && + !validated.webhookUrl.startsWith("https://") + ) { + builder.setError(400, { + type: "ValidationError", + message: "Production webhook URLs must use HTTPS", + }); + reply.code(400); + return { error: "Production webhook URLs must use HTTPS" }; + } + const apiKey = generateAPIKey(validated.role as ApiKeyRole); const apiKeyHash = hashAPIKey(apiKey); const now = DateTime.utc(); const expiresAt = now.plus({ seconds: validated.expiresIn }); - const keyRecord = await createApiKey({ - name: validated.name, - key: apiKeyHash, - role: validated.role, - expiresAt: expiresAt.toISO(), - projectId: auth.projectId, - }); + const db = getPostgresDB(); + const { keyRecord, endpoint } = await executeInTransaction( + db, + "create API key", + async (txn) => { + const rec = await createApiKey( + { + name: validated.name, + key: apiKeyHash, + role: validated.role, + expiresAt: expiresAt.toISO(), + projectId: auth.projectId, + }, + txn + ); + + const keyPair = generateWebhookKeyPair(); + const ep = await upsertWebhookEndpoint( + auth.projectId, + rec.id, + validated.webhookUrl, + keyPair.privateKeyPem, + keyPair.publicKeyPrefixed, + txn + ); - const keyPair = generateWebhookKeyPair(); - const endpoint = await upsertWebhookEndpoint( - auth.projectId, - keyRecord.id, - validated.webhookUrl, - keyPair.privateKeyPem, - keyPair.publicKeyPrefixed + return { keyRecord: rec, endpoint: ep }; + } ); invalidateWebhookEndpointCache(keyRecord.id); @@ -131,6 +162,11 @@ export async function handleListApiKeys( try { const auth = await authenticateHttpApiKey(request.headers.authorization); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can manage API keys" + ); + } const db = getPostgresDB(); const keys = await db @@ -197,12 +233,17 @@ export async function handleRevokeApiKey( try { const auth = await authenticateHttpApiKey(request.headers.authorization); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can manage API keys" + ); + } const params = request.params as { id: string }; const db = getPostgresDB(); const now = DateTime.utc().toISO(); - const result = await db + const [revokedRow] = await db .update(apiKeysTable) .set({ revoked: true, revokedAt: now }) .where( @@ -211,9 +252,10 @@ export async function handleRevokeApiKey( eq(apiKeysTable.id, params.id), eq(apiKeysTable.revoked, false) ) - ); + ) + .returning({ key: apiKeysTable.key }); - if ((result.count ?? 0) === 0) { + if (!revokedRow) { builder.setError(404, { type: "NotFoundError", message: "API key not found or already revoked", @@ -222,19 +264,7 @@ export async function handleRevokeApiKey( return { error: "API key not found or already revoked" }; } - const [keyRow] = await db - .select({ key: apiKeysTable.key }) - .from(apiKeysTable) - .where( - and( - eq(apiKeysTable.projectId, auth.projectId), - eq(apiKeysTable.id, params.id) - ) - ) - .limit(1); - if (keyRow) { - apiKeyCache.delete(keyRow.key); - } + apiKeyCache.delete(revokedRow.key); builder.setSuccess(200); reply.code(200); diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 0dbdf2e..429322a 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -24,6 +24,7 @@ import { import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; import { removeClient } from "../../gRPC/payment/paymentProvider.ts"; import { DateTime } from "luxon"; +import { eq } from "drizzle-orm"; import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; export async function handleOnboarding( @@ -55,6 +56,21 @@ export async function handleOnboarding( const projectId = randomUUID(); + const existing = await getPostgresDB() + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(eq(projectsTable.name, validated.name)) + .limit(1); + + if (existing.length > 0) { + builder.setError(409, { + type: "ConflictError", + message: `Project with name '${validated.name}' already exists`, + }); + reply.code(409); + return {}; + } + const liveClient = new DodoPayments({ bearerToken: validated.dodoLiveApiKey, environment: "live_mode", @@ -232,6 +248,9 @@ export async function handleGetConfig( try { const authHeader = request.headers.authorization; const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied("Only dashboard keys can read config"); + } const metadata = await getMetadata(auth.projectId); diff --git a/src/routes/http/api/webhookDeliveries.ts b/src/routes/http/api/webhookDeliveries.ts index 8830b31..0bca7fb 100644 --- a/src/routes/http/api/webhookDeliveries.ts +++ b/src/routes/http/api/webhookDeliveries.ts @@ -71,24 +71,21 @@ export async function handleListDeliveries( } if (query.eventType) { - conditions = conditions - ? and(conditions, eq(webhookDeliveriesTable.eventType, query.eventType)) - : eq(webhookDeliveriesTable.eventType, query.eventType); + conditions = and( + conditions, + eq(webhookDeliveriesTable.eventType, query.eventType) + ); } if (query.status) { - conditions = conditions - ? and( - conditions, - sql`${webhookDeliveriesTable.status} = ${query.status}` - ) - : sql`${webhookDeliveriesTable.status} = ${query.status}`; + conditions = and( + conditions, + sql`${webhookDeliveriesTable.status} = ${query.status}` + ); } if (query.role) { - conditions = conditions - ? and(conditions, sql`${apiKeysTable.role} = ${query.role}`) - : sql`${apiKeysTable.role} = ${query.role}`; + conditions = and(conditions, sql`${apiKeysTable.role} = ${query.role}`); } const rows = await db diff --git a/src/routes/http/createdCheckout.ts b/src/routes/http/createdCheckout.ts index d4855a3..84700d3 100644 --- a/src/routes/http/createdCheckout.ts +++ b/src/routes/http/createdCheckout.ts @@ -135,6 +135,15 @@ export async function handleDodoWebhook( ); } + if (session.projectId !== projectId) { + return errorResponse( + 404, + "NotFoundError", + `Session not found for checkout_session_id: ${checkout_session_id}`, + builder + ); + } + if (session.processed !== "pending") { Sentry.captureMessage( `Webhook received for session ${checkout_session_id} with non-pending status: ${session.processed}`, @@ -192,7 +201,12 @@ export async function handleDodoWebhook( txn ); if (!claimed) return; - await updateUserBilledTimestamp(userId, billed_upto, txn); + await updateUserBilledTimestamp( + session.projectId, + userId, + billed_upto, + txn + ); await handleAddPayment( session.projectId, userId, diff --git a/src/routes/http/registerWebhookRoutes.ts b/src/routes/http/registerWebhookRoutes.ts index 6f6e617..84502e8 100644 --- a/src/routes/http/registerWebhookRoutes.ts +++ b/src/routes/http/registerWebhookRoutes.ts @@ -59,6 +59,19 @@ export async function registerWebhookRoutes( return { error: "Missing projectId query parameter" }; } + if ( + !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + projectId + ) + ) { + builder.setError(400, { + type: "ValidationError", + message: "Invalid 'projectId' query parameter.", + }); + reply.code(400); + return { error: "Invalid projectId query parameter" }; + } + const signatureHeader = request.headers["webhook-signature"]; const timestampHeader = request.headers["webhook-timestamp"]; const webhookIdHeader = request.headers["webhook-id"]; diff --git a/src/storage/adapter/postgres/handlers/priceRequest.ts b/src/storage/adapter/postgres/handlers/priceRequest.ts index ac13916..5a2bfe2 100644 --- a/src/storage/adapter/postgres/handlers/priceRequest.ts +++ b/src/storage/adapter/postgres/handlers/priceRequest.ts @@ -50,7 +50,13 @@ export async function handlePriceRequest( price: sum(priceColumn), }) .from(priceTable) - .innerJoin(usersTable, eq(priceTable.userId, usersTable.id)) + .innerJoin( + usersTable, + and( + eq(priceTable.userId, usersTable.id), + eq(priceTable.projectId, usersTable.projectId) + ) + ) .where(whereClause) .groupBy(priceTable.userId); } catch (e) { @@ -82,15 +88,7 @@ export async function handlePriceRequest( return 0; } - let parsedPrice: number; - try { - parsedPrice = parseInt(priceValue); - } catch (e) { - throw StorageError.priceCalculationFailed( - userId, - new Error(`Failed to parse price value: ${priceValue}`) - ); - } + const parsedPrice = parseInt(priceValue, 10); if (isNaN(parsedPrice)) { throw StorageError.priceCalculationFailed( diff --git a/src/storage/db/postgres/helpers/metadata.ts b/src/storage/db/postgres/helpers/metadata.ts index 3ba469a..f696125 100644 --- a/src/storage/db/postgres/helpers/metadata.ts +++ b/src/storage/db/postgres/helpers/metadata.ts @@ -2,45 +2,6 @@ import { getPostgresDB } from "../db"; import { metadataTable } from "../schema"; import { StorageError } from "../../../../errors/storage"; import { eq } from "drizzle-orm"; -import { executeInTransaction } from "../../../adapter/postgres/handlers/addEventUtils"; - -export type UpsertMetadataInput = { - dodo_live_api_key: string; - dodo_test_api_key: string; - dodo_live_product_id: string; - dodo_test_product_id: string; - dodo_live_webhook_secret: string; - dodo_test_webhook_secret: string; - currency: string; - redirect_url: string; -}; - -export async function createMetadata( - projectId: string, - input: UpsertMetadataInput, - txn?: any -): Promise { - const db = txn ?? getPostgresDB(); - - try { - await db.insert(metadataTable).values({ - projectId, - dodo_live_api_key: input.dodo_live_api_key, - dodo_test_api_key: input.dodo_test_api_key, - dodo_live_product_id: input.dodo_live_product_id, - dodo_test_product_id: input.dodo_test_product_id, - dodo_live_webhook_secret: input.dodo_live_webhook_secret, - dodo_test_webhook_secret: input.dodo_test_webhook_secret, - currency: input.currency, - redirect_url: input.redirect_url, - }); - } catch (e) { - throw StorageError.insertFailed( - "Failed to create metadata record", - e instanceof Error ? e : new Error(String(e)) - ); - } -} export async function getMetadata( projectId: string diff --git a/src/storage/db/postgres/helpers/users.ts b/src/storage/db/postgres/helpers/users.ts index 287789e..624a5a5 100644 --- a/src/storage/db/postgres/helpers/users.ts +++ b/src/storage/db/postgres/helpers/users.ts @@ -5,6 +5,7 @@ import { StorageError } from "../../../../errors/storage"; import type { PgTransaction } from "drizzle-orm/pg-core"; export async function updateUserBilledTimestamp( + projectId: string, userId: string, billedUpto: string, txn?: PgTransaction @@ -15,7 +16,9 @@ export async function updateUserBilledTimestamp( await db .update(usersTable) .set({ last_billed_timestamp: billedUpto }) - .where(eq(usersTable.id, userId)); + .where( + and(eq(usersTable.projectId, projectId), eq(usersTable.id, userId)) + ); } catch (e) { throw StorageError.queryFailed( "Failed to update user billed timestamp", diff --git a/src/storage/db/postgres/helpers/webhookEndpoints.ts b/src/storage/db/postgres/helpers/webhookEndpoints.ts index a9c5a72..341ae20 100644 --- a/src/storage/db/postgres/helpers/webhookEndpoints.ts +++ b/src/storage/db/postgres/helpers/webhookEndpoints.ts @@ -1,6 +1,7 @@ import { getPostgresDB } from "../db"; import { webhookEndpointsTable } from "../schema"; import { eq, and, isNull } from "drizzle-orm"; +import type { PgTransaction } from "drizzle-orm/pg-core"; import { StorageError } from "../../../../errors/storage"; import { DateTime } from "luxon"; @@ -39,9 +40,10 @@ export async function upsertWebhookEndpoint( apiKeyId: string, url: string, privateKey: string, - publicKey: string + publicKey: string, + txn?: PgTransaction ): Promise { - const db = getPostgresDB(); + const db = txn ?? getPostgresDB(); try { const now = DateTime.utc().toISO(); diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index e04a31d..3eea693 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -106,8 +106,8 @@ export const sessionRelations = relations(sessionsTable, ({ one, many }) => ({ references: [projectsTable.id], }), user: one(usersTable, { - fields: [sessionsTable.userId], - references: [usersTable.id], + fields: [sessionsTable.projectId, sessionsTable.userId], + references: [usersTable.projectId, usersTable.id], }), apiKey: one(apiKeysTable, { fields: [sessionsTable.apiKeyId], @@ -206,8 +206,8 @@ export const basicUsageEventsRelation = relations( references: [projectsTable.id], }), user: one(usersTable, { - fields: [basicUsageEventsTable.userId], - references: [usersTable.id], + fields: [basicUsageEventsTable.projectId, basicUsageEventsTable.userId], + references: [usersTable.projectId, usersTable.id], }), apiKey: one(apiKeysTable, { fields: [basicUsageEventsTable.apiKeyId], @@ -259,8 +259,8 @@ export const paymentEventsRelation = relations( references: [projectsTable.id], }), user: one(usersTable, { - fields: [paymentEventsTable.userId], - references: [usersTable.id], + fields: [paymentEventsTable.projectId, paymentEventsTable.userId], + references: [usersTable.projectId, usersTable.id], }), apiKey: one(apiKeysTable, { fields: [paymentEventsTable.apiKeyId], @@ -318,8 +318,11 @@ export const aiTokenUsageEventsRelation = relations( references: [projectsTable.id], }), user: one(usersTable, { - fields: [aiTokenUsageEventsTable.userId], - references: [usersTable.id], + fields: [ + aiTokenUsageEventsTable.projectId, + aiTokenUsageEventsTable.userId, + ], + references: [usersTable.projectId, usersTable.id], }), apiKey: one(apiKeysTable, { fields: [aiTokenUsageEventsTable.apiKeyId], diff --git a/src/utils/parseExpr.ts b/src/utils/parseExpr.ts index b5ceb06..8a5559a 100644 --- a/src/utils/parseExpr.ts +++ b/src/utils/parseExpr.ts @@ -37,10 +37,10 @@ const ALLOWED_FUNCTIONS = new Set([ "div", "tag", "expr", - "inputTokens", - "outputTokens", - "inputCacheTokens", - "outputCacheTokens", + "inputtokens", + "outputtokens", + "inputcachetokens", + "outputcachetokens", ]); /** From 7b6179f22d365ec2fb26a300cd35376e61dfa14b Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Fri, 19 Jun 2026 16:06:43 +0530 Subject: [PATCH 09/55] fix: prevent dashboard self-revoke, remove unsafe cast in query.ts --- src/routes/gRPC/data/query.ts | 23 ++++++++--------------- src/routes/http/api/apiKeys.ts | 3 ++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index 86415ad..f52c852 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -37,13 +37,15 @@ interface FieldDef { cast: "text" | "integer" | "uuid" | "timestamptz" | "boolean"; } +type ScopedTable = + | typeof usersTable + | typeof sessionsTable + | typeof tagsTable + | typeof expressionsTable; + interface TableDef { tableName: string; - table: - | typeof usersTable - | typeof sessionsTable - | typeof tagsTable - | typeof expressionsTable; + table: ScopedTable; fields: Record; } @@ -222,16 +224,7 @@ export async function queryData( const db = getPostgresDB(); const userWhere = buildWhere(validated.where, tableDef); - const projectFilter = eq( - ( - tableDef.table as - | typeof usersTable - | typeof sessionsTable - | typeof tagsTable - | typeof expressionsTable - ).projectId, - auth.projectId - ) as SQL; + const projectFilter = eq(tableDef.table.projectId, auth.projectId) as SQL; const whereClause = userWhere ? and(projectFilter, userWhere) : projectFilter; diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 8939910..2dc084d 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -250,7 +250,8 @@ export async function handleRevokeApiKey( and( eq(apiKeysTable.projectId, auth.projectId), eq(apiKeysTable.id, params.id), - eq(apiKeysTable.revoked, false) + eq(apiKeysTable.revoked, false), + ne(apiKeysTable.role, "dashboard") ) ) .returning({ key: apiKeysTable.key }); From 91b36962cbef8bc29bda1ecb0aa6617ad3b8de0c Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 22 Jun 2026 03:50:46 +0530 Subject: [PATCH 10/55] fix(schema.ts): added project_id to clickhouse db schema --- src/storage/adapter/clickhouse/schema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storage/adapter/clickhouse/schema.ts b/src/storage/adapter/clickhouse/schema.ts index ae12dcd..6eb31fd 100644 --- a/src/storage/adapter/clickhouse/schema.ts +++ b/src/storage/adapter/clickhouse/schema.ts @@ -5,6 +5,7 @@ import { innerProduct } from "drizzle-orm"; const BASIC_USAGE_EVENTS_TABLE = ` CREATE TABLE IF NOT EXISTS basic_usage_events ( id UUID DEFAULT generateUUIDv4(), + project_id String, event_id String, idempotency_key String, user_id String, @@ -22,6 +23,7 @@ ORDER BY (idempotency_key, user_id) const AI_TOKEN_USAGE_EVENTS_TABLE = ` CREATE TABLE IF NOT EXISTS ai_token_usage_events ( id UUID DEFAULT generateUUIDv4(), + project_id String, event_id String, idempotency_key String, user_id String, From b6410a758a316f71f2922093c53c3710cced1717 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Wed, 24 Jun 2026 02:52:44 +0530 Subject: [PATCH 11/55] fix(generateInitialAPIKey): now API key gets inserted into the db --- src/utils/generateInitialAPIKey.ts | 52 ++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/utils/generateInitialAPIKey.ts b/src/utils/generateInitialAPIKey.ts index ecff084..7adc257 100644 --- a/src/utils/generateInitialAPIKey.ts +++ b/src/utils/generateInitialAPIKey.ts @@ -1,6 +1,8 @@ import { createHmac, randomUUID } from "crypto"; import { generateAPIKey } from "./generateAPIKey"; import { DateTime } from "luxon"; +import { getPostgresDB } from "../storage/db/postgres/db"; +import { projectsTable, apiKeysTable } from "../storage/db/postgres/schema"; const HMAC_SECRET = process.env.HMAC_SECRET; @@ -17,18 +19,19 @@ function hashAPIKey(apiKey: string): string { } export type InitialApiKeyData = { + projectId: string; apiKeyId: string; apiKey: string; apiKeyHash: string; name: string; - role: string; + role: "dashboard" | "production" | "test"; createdAt: string; expiresAt: string; - insertSql: string; authorizationHeader: string; }; export function generateInitialApiKeyData(): InitialApiKeyData { + const projectId = randomUUID(); const apiKeyId = randomUUID(); const apiKey = generateAPIKey("dashboard"); const apiKeyHash = hashAPIKey(apiKey); @@ -37,20 +40,8 @@ export function generateInitialApiKeyData(): InitialApiKeyData { const createdAt = DateTime.utc().toISO(); const expiresAt = DateTime.utc().plus({ days: 365 }).toISO(); - const insertSql = - "INSERT INTO api_keys (id, name, key, role, created_at, expires_at, revoked, revoked_at)\n" + - "VALUES (\n" + - ` '${apiKeyId}',\n` + - ` '${name}',\n` + - ` '${apiKeyHash}',\n` + - ` '${role}',\n` + - ` '${createdAt}',\n` + - ` '${expiresAt}',\n` + - " false,\n" + - " NULL\n" + - ");"; - return { + projectId, apiKeyId, apiKey, apiKeyHash, @@ -58,9 +49,36 @@ export function generateInitialApiKeyData(): InitialApiKeyData { role, createdAt, expiresAt, - insertSql, authorizationHeader: `Authorization: Bearer ${apiKey}`, }; } -console.log(generateInitialApiKeyData()); +async function insertInitialData(data: InitialApiKeyData) { + const db = getPostgresDB(process.env.DATABASE_URL); + + await db.insert(projectsTable).values({ + id: data.projectId, + name: "Default Project", + createdAt: data.createdAt, + }); + + await db.insert(apiKeysTable).values({ + id: data.apiKeyId, + projectId: data.projectId, + name: data.name, + key: data.apiKeyHash, + role: data.role, + createdAt: data.createdAt, + expiresAt: data.expiresAt, + revoked: false, + revokedAt: null, + }); +} + +const data = generateInitialApiKeyData(); + +await insertInitialData(data); + +console.log("Initial API key generation was successful.."); +console.log(data); +process.exit(0); From d998a50862faef90511dd75d86f2690b710cca0b Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Wed, 24 Jun 2026 06:14:33 +0530 Subject: [PATCH 12/55] fix(generateInitialAPIKey): idempotency gaurd added --- src/utils/generateInitialAPIKey.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/utils/generateInitialAPIKey.ts b/src/utils/generateInitialAPIKey.ts index 7adc257..7815baf 100644 --- a/src/utils/generateInitialAPIKey.ts +++ b/src/utils/generateInitialAPIKey.ts @@ -1,3 +1,4 @@ +import { sql } from "drizzle-orm"; import { createHmac, randomUUID } from "crypto"; import { generateAPIKey } from "./generateAPIKey"; import { DateTime } from "luxon"; @@ -56,6 +57,19 @@ export function generateInitialApiKeyData(): InitialApiKeyData { async function insertInitialData(data: InitialApiKeyData) { const db = getPostgresDB(process.env.DATABASE_URL); + const existing = await db + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(sql`name = 'Default Project'`) + .limit(1); + + if (existing.length > 0) { + console.log( + `Default Project already exists (id=${existing[0]!.id}). Skipping seed.` + ); + return; + } + await db.insert(projectsTable).values({ id: data.projectId, name: "Default Project", From 97cb89efbd7d1f7a1b8ae63925a30cc50e6ebddd Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Thu, 25 Jun 2026 02:13:39 +0530 Subject: [PATCH 13/55] fix(project): schema fixes --- src/routes/http/api/onboarding.ts | 28 +++++++++++++++++---- src/storage/adapter/clickhouse/schema.ts | 4 +-- src/storage/adapter/clickhouse/utils.ts | 13 +++++++--- src/storage/db/postgres/helpers/metadata.ts | 8 ++++++ src/storage/db/postgres/schema.ts | 21 ++++++++++++++++ 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 429322a..b347638 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -21,7 +21,10 @@ import { apiKeysTable, metadataTable, } from "../../../storage/db/postgres/schema"; -import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; +import { + getMetadata, + getAnyMetadata, +} from "../../../storage/db/postgres/helpers/metadata"; import { removeClient } from "../../gRPC/payment/paymentProvider.ts"; import { DateTime } from "luxon"; import { eq } from "drizzle-orm"; @@ -247,12 +250,27 @@ export async function handleGetConfig( try { const authHeader = request.headers.authorization; - const auth = await authenticateHttpApiKey(authHeader); - if (auth.role !== "dashboard") { - throw AuthError.permissionDenied("Only dashboard keys can read config"); + + let projectId: string | undefined; + let isMasterKey = false; + try { + authenticateMasterApiKey(authHeader); + isMasterKey = true; + } catch (masterErr) { + if (!(masterErr instanceof AuthError)) { + throw masterErr; + } + + const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied("Only dashboard keys can read config"); + } + projectId = auth.projectId; } - const metadata = await getMetadata(auth.projectId); + const metadata = isMasterKey + ? await getAnyMetadata() + : await getMetadata(projectId!); if (!metadata) { builder.setSuccess(200); diff --git a/src/storage/adapter/clickhouse/schema.ts b/src/storage/adapter/clickhouse/schema.ts index 6eb31fd..653c662 100644 --- a/src/storage/adapter/clickhouse/schema.ts +++ b/src/storage/adapter/clickhouse/schema.ts @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS basic_usage_events ( debit_amount Int64, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (idempotency_key, user_id) +ORDER BY (project_id, idempotency_key, user_id) `; const AI_TOKEN_USAGE_EVENTS_TABLE = ` @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS ai_token_usage_events ( metrics String, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (idempotency_key, user_id) +ORDER BY (project_id, idempotency_key, user_id) `; export async function runClickHouseMigrations(): Promise { diff --git a/src/storage/adapter/clickhouse/utils.ts b/src/storage/adapter/clickhouse/utils.ts index 19e6d8e..4873556 100644 --- a/src/storage/adapter/clickhouse/utils.ts +++ b/src/storage/adapter/clickhouse/utils.ts @@ -2,7 +2,7 @@ import type { DateTime } from "luxon"; import { DateTime as LuxonDateTime } from "luxon"; import { getPostgresDB } from "../../db/postgres/db"; import { usersTable } from "../../db/postgres/schema"; -import { eq } from "drizzle-orm"; +import { eq, and } from "drizzle-orm"; import { StorageError } from "../../../errors/storage"; import { getClickHouseDB } from "../../db/clickhouse"; import type { UserId } from "../../../config/identifiers"; @@ -12,13 +12,18 @@ export function toClickHouseDateTime(dt: DateTime): string { return dt.toUTC().toFormat("yyyy-MM-dd HH:mm:ss.SSS"); } -async function fetchLastBilled(userId: string): Promise { +async function fetchLastBilled( + userId: string, + projectId: string +): Promise { const pgDb = getPostgresDB(); try { const [user] = await pgDb .select({ lastBilled: usersTable.last_billed_timestamp }) .from(usersTable) - .where(eq(usersTable.id, userId)) + .where( + and(eq(usersTable.id, userId), eq(usersTable.projectId, projectId)) + ) .limit(1); return user?.lastBilled ?? null; } catch { @@ -49,7 +54,7 @@ export async function runClickHousePriceQuery( } const beforeTs = toClickHouseDateTime(beforeTimestamp); - const lastBilled = await fetchLastBilled(userId); + const lastBilled = await fetchLastBilled(userId, auth.projectId); try { let query: string; diff --git a/src/storage/db/postgres/helpers/metadata.ts b/src/storage/db/postgres/helpers/metadata.ts index f696125..d14389b 100644 --- a/src/storage/db/postgres/helpers/metadata.ts +++ b/src/storage/db/postgres/helpers/metadata.ts @@ -14,3 +14,11 @@ export async function getMetadata( .limit(1); return metadata; } + +export async function getAnyMetadata(): Promise< + typeof metadataTable.$inferSelect | undefined +> { + const db = getPostgresDB(); + const [metadata] = await db.select().from(metadataTable).limit(1); + return metadata; +} diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index 3eea693..510c9ff 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -344,6 +344,13 @@ export const tagsTable = pgTable("tags", { }), }); +export const tagsRelation = relations(tagsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [tagsTable.projectId], + references: [projectsTable.id], + }), +})); + export const metadataTable = pgTable( "metadata", { @@ -369,6 +376,13 @@ export const metadataTable = pgTable( }) ); +export const metadataRelation = relations(metadataTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [metadataTable.projectId], + references: [projectsTable.id], + }), +})); + export const expressionsTable = pgTable("expressions", { id: uuid("id").primaryKey().defaultRandom(), projectId: uuid("project_id") @@ -382,6 +396,13 @@ export const expressionsTable = pgTable("expressions", { }), }); +export const expressionsRelation = relations(expressionsTable, ({ one }) => ({ + project: one(projectsTable, { + fields: [expressionsTable.projectId], + references: [projectsTable.id], + }), +})); + export const webhookEndpointsTable = pgTable( "webhook_endpoints", { From 37d1423e6866ec9543488be87e2f60ce713f3850 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Thu, 25 Jun 2026 23:51:58 +0530 Subject: [PATCH 14/55] test: add integration tests for Query Engine and Data Engine gRPC services --- src/__tests__/dataQuery.test.ts | 609 ++++++++++++++++++++++ src/__tests__/queryEvents.test.ts | 816 ++++++++++++++++++++++++++++++ 2 files changed, 1425 insertions(+) create mode 100644 src/__tests__/dataQuery.test.ts create mode 100644 src/__tests__/queryEvents.test.ts diff --git a/src/__tests__/dataQuery.test.ts b/src/__tests__/dataQuery.test.ts new file mode 100644 index 0000000..2b1e967 --- /dev/null +++ b/src/__tests__/dataQuery.test.ts @@ -0,0 +1,609 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { Metadata } from "@grpc/grpc-js"; +import { + DataQueryServiceClient, + FilterCondition, + FilterGroup, + QueryRequest, + Operator, + LogicalOperator, + OrderBy, +} from "../gen/data/v1/data"; +import { + GRPC_ADDRESS, + grpcInsecureCredentials, + grpcMetadata, +} from "./fixtures/grpc"; +import { createTestApiKey } from "./fixtures/apiKey"; +import { clearDatabase } from "./db"; +import { getPostgresDB } from "../storage/db/postgres/db"; +import { + usersTable, + sessionsTable, + tagsTable, + expressionsTable, +} from "../storage/db/postgres/schema"; +import { DateTime } from "luxon"; +import type { QueryResponse } from "../gen/data/v1/data"; + +function queryData( + client: DataQueryServiceClient, + request: QueryRequest, + metadata: Metadata +): Promise { + return new Promise((resolve, reject) => { + client.query(request, metadata, (error, res) => { + if (error) reject(error); + else if (!res) reject(new Error("empty response")); + else resolve(res); + }); + }); +} + +function makeFilterCondition( + field: string, + operator: Operator, + value: string +): FilterCondition { + return FilterCondition.create({ field, operator, value }); +} + +function makeFilterGroup( + logical: LogicalOperator, + conditions: FilterCondition[], + groups: FilterGroup[] = [] +): FilterGroup { + return FilterGroup.create({ logical, conditions, groups }); +} + +const USER_1 = crypto.randomUUID(); +const USER_2 = crypto.randomUUID(); +let rawKey: string; + +async function seedData() { + const db = getPostgresDB(); + + const key = await createTestApiKey(); + rawKey = key.rawKey; + + await db.insert(usersTable).values([ + { id: USER_1, mode: "test", payment_provider_user_id: "pp_user_1" }, + { id: USER_2, mode: "production", payment_provider_user_id: "pp_user_2" }, + ]); + + await db.insert(sessionsTable).values([ + { + proxy_link_id: crypto.randomUUID(), + sessionId: "session-1", + userId: USER_1, + apiKeyId: key.id, + processed: "pending", + billed_upto: DateTime.utc().toISO(), + checkoutUrl: "https://checkout.example.com/1", + mode: "test", + }, + { + proxy_link_id: crypto.randomUUID(), + sessionId: "session-2", + userId: USER_2, + apiKeyId: key.id, + processed: "succeeded", + billed_upto: DateTime.utc().toISO(), + checkoutUrl: "https://checkout.example.com/2", + mode: "production", + }, + ]); + + await db.insert(tagsTable).values([ + { key: "premium", amount: 100 }, + { key: "basic", amount: 50 }, + ]); + + await db.insert(expressionsTable).values([ + { key: "discount_10", expr: "mul(amount, 0.9)" }, + { key: "premium_price", expr: "add(100, mul(tag('premium'), 2))" }, + ]); +} + +describe("DataQuery", () => { + let client: DataQueryServiceClient; + + beforeAll(async () => { + client = new DataQueryServiceClient(GRPC_ADDRESS, grpcInsecureCredentials); + await seedData(); + }); + + afterAll(async () => { + await clearDatabase(); + client.close(); + }); + + describe("query users table", () => { + it("returns all users", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.columns).toContain("id"); + expect(res.columns).toContain("mode"); + expect(res.rows.length).toBeGreaterThanOrEqual(2); + expect(res.total).toBeGreaterThanOrEqual(2); + }); + + it("filters by mode EQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("mode", Operator.EQ, "test"), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBeGreaterThanOrEqual(1); + const modeIdx = res.columns.indexOf("mode"); + for (const row of res.rows) { + expect(row.values[modeIdx]).toBe("test"); + } + }); + + it("filters by paymentProviderUserId EQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("paymentProviderUserId", Operator.EQ, "pp_user_1"), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const idIdx = res.columns.indexOf("id"); + expect(res.rows[0]!.values[idIdx]).toBe(USER_1); + }); + + it("filters by paymentProviderUserId CONTAINS", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition( + "paymentProviderUserId", + Operator.CONTAINS, + "pp_user" + ), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(2); + }); + + it("filters by NEQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("mode", Operator.NEQ, "test"), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBeGreaterThanOrEqual(1); + const modeIdx = res.columns.indexOf("mode"); + for (const row of res.rows) { + expect(row.values[modeIdx]).not.toBe("test"); + } + }); + }); + + describe("query sessions table", () => { + it("returns sessions with correct columns", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "sessions", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.columns).toContain("sessionId"); + expect(res.columns).toContain("userId"); + expect(res.columns).toContain("processed"); + expect(res.columns).toContain("mode"); + expect(res.rows.length).toBeGreaterThanOrEqual(2); + }); + + it("filters sessions by processed status", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("processed", Operator.EQ, "succeeded"), + ]); + const req = QueryRequest.create({ + table: "sessions", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const processedIdx = res.columns.indexOf("processed"); + expect(res.rows[0]!.values[processedIdx]).toBe("succeeded"); + }); + }); + + describe("query tags table", () => { + it("returns tags with key and amount", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "tags", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.columns).toContain("key"); + expect(res.columns).toContain("amount"); + expect(res.rows.length).toBeGreaterThanOrEqual(2); + }); + + it("filters tags by amount GT", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("amount", Operator.GT, "50"), + ]); + const req = QueryRequest.create({ + table: "tags", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const amountIdx = res.columns.indexOf("amount"); + expect(Number(res.rows[0]!.values[amountIdx])).toBeGreaterThan(50); + }); + + it("filters tags by key EQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("key", Operator.EQ, "premium"), + ]); + const req = QueryRequest.create({ + table: "tags", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const keyIdx = res.columns.indexOf("key"); + expect(res.rows[0]!.values[keyIdx]).toBe("premium"); + }); + }); + + describe("query expressions table", () => { + it("returns expressions", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "expressions", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.columns).toContain("key"); + expect(res.columns).toContain("expr"); + expect(res.rows.length).toBeGreaterThanOrEqual(2); + }); + }); + + describe("query metadata table", () => { + it("returns metadata rows", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "metadata", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.columns).toContain("id"); + expect(res.rows.length).toBeGreaterThanOrEqual(0); + }); + }); + + describe("orderBy", () => { + it("orders by mode descending", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [OrderBy.create({ field: "mode", descending: true })], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + // NOTE: same orderBy issue as ascending test + expect(res.rows.length).toBeGreaterThanOrEqual(2); + }); + + it("orders by mode ascending", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [OrderBy.create({ field: "mode", descending: false })], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + // NOTE: orderBy is silently dropped — Zod field is "orderByList" but proto sends "orderBy" + // This test documents the current buggy behavior (insertion order returned) + expect(res.rows.length).toBeGreaterThanOrEqual(2); + }); + }); + + describe("pagination", () => { + it("respects limit", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 1, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + expect(res.total).toBeGreaterThanOrEqual(2); + }); + + it("respects offset", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const reqAll = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + const resAll = await queryData( + client, + reqAll, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const reqOffset = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 1, + }); + const resOffset = await queryData( + client, + reqOffset, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(resOffset.rows.length).toBe(resAll.rows.length - 1); + }); + }); + + describe("AND/OR filter groups", () => { + it("combines conditions with AND", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("mode", Operator.EQ, "test"), + makeFilterCondition( + "paymentProviderUserId", + Operator.CONTAINS, + "pp_user" + ), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const modeIdx = res.columns.indexOf("mode"); + expect(res.rows[0]!.values[modeIdx]).toBe("test"); + }); + + it("combines filter groups with OR", async () => { + const where = makeFilterGroup( + LogicalOperator.OR, + [], + [ + makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("key", Operator.EQ, "premium"), + ]), + makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("key", Operator.EQ, "basic"), + ]), + ] + ); + const req = QueryRequest.create({ + table: "tags", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + const res = await queryData( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(2); + }); + }); + + describe("error handling", () => { + it("rejects unknown table", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "nonexistent", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + await expect( + queryData(client, req, grpcMetadata(`Bearer ${rawKey}`)) + ).rejects.toThrow("Invalid option"); + }); + + it("rejects unknown field in filter", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("nonexistentField", Operator.EQ, "value"), + ]); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + await expect( + queryData(client, req, grpcMetadata(`Bearer ${rawKey}`)) + ).rejects.toThrow("Unknown field"); + }); + + it("rejects unauthenticated requests", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = QueryRequest.create({ + table: "users", + where, + orderBy: [], + limit: 100, + offset: 0, + }); + + await expect(queryData(client, req, new Metadata())).rejects.toThrow( + "Missing Authorization header" + ); + }); + }); +}); diff --git a/src/__tests__/queryEvents.test.ts b/src/__tests__/queryEvents.test.ts new file mode 100644 index 0000000..06dc1c8 --- /dev/null +++ b/src/__tests__/queryEvents.test.ts @@ -0,0 +1,816 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +const isClickHouse = process.env.STORAGE_ADAPTER === "clickhouse"; +import { Metadata } from "@grpc/grpc-js"; +import type { + QueryEventsResponse, + QueryEventsRequest, + FilterCondition, + FilterGroup, +} from "../gen/query/v1/query"; +import { + QueryServiceClient, + Operator, + LogicalOperator, + AggregationType, +} from "../gen/query/v1/query"; +import { + GRPC_ADDRESS, + grpcInsecureCredentials, + grpcMetadata, +} from "./fixtures/grpc"; +import { createTestApiKey } from "./fixtures/apiKey"; +import { clearDatabase } from "./db"; +import { getPostgresDB } from "../storage/db/postgres/db"; +import { + usersTable, + basicUsageEventsTable, + aiTokenUsageEventsTable, +} from "../storage/db/postgres/schema"; +import { DateTime } from "luxon"; +import { getClickHouseDB } from "../storage/db/clickhouse"; + +function queryEvents( + client: QueryServiceClient, + request: QueryEventsRequest, + metadata: Metadata +): Promise { + return new Promise((resolve, reject) => { + client.queryEvents(request, metadata, (error, res) => { + if (error) reject(error); + else if (!res) reject(new Error("empty response")); + else resolve(res); + }); + }); +} + +function makeFilterCondition( + field: string, + operator: Operator, + value: string +): FilterCondition { + return { field, operator, value }; +} + +function makeFilterGroup( + logical: LogicalOperator, + conditions: FilterCondition[], + groups: FilterGroup[] = [] +): FilterGroup { + return { logical, conditions, groups }; +} + +const USER_1 = crypto.randomUUID(); +const USER_2 = crypto.randomUUID(); + +interface SeededEvent { + eventId: string; + userId: string; + idempotencyKey: string; + debitAmount: number; + type: "BASIC_USAGE" | "AI_TOKEN_USAGE"; + model?: string; + provider?: string; +} + +let seededBasic: SeededEvent[] = []; +let seededAi: SeededEvent[] = []; +let rawKey: string; + +function toClickHouseDT(dt: DateTime): string { + return dt.toUTC().toFormat("yyyy-MM-dd HH:mm:ss.SSS"); +} + +async function seedData() { + const db = getPostgresDB(); + + await db.insert(usersTable).values([ + { id: USER_1, mode: "test" }, + { id: USER_2, mode: "production" }, + ]); + + const ts = DateTime.utc(); + const key1 = await createTestApiKey(); + const key2 = await createTestApiKey(); + const key3 = await createTestApiKey(); + const key4 = await createTestApiKey(); + const key5 = await createTestApiKey(); + + const bu1 = await db + .insert(basicUsageEventsTable) + .values({ + eventId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + reportedTimestamp: ts.toISO(), + userId: USER_1, + apiKeyId: key1.id, + mode: "test", + type: "RAW", + debitAmount: 100, + }) + .returning({ + eventId: basicUsageEventsTable.eventId, + idempotencyKey: basicUsageEventsTable.idempotencyKey, + }); + + const bu2 = await db + .insert(basicUsageEventsTable) + .values({ + eventId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + reportedTimestamp: ts.toISO(), + userId: USER_1, + apiKeyId: key2.id, + mode: "test", + type: "MIDDLEWARE_CALL", + debitAmount: 50, + }) + .returning({ + eventId: basicUsageEventsTable.eventId, + idempotencyKey: basicUsageEventsTable.idempotencyKey, + }); + + const bu3 = await db + .insert(basicUsageEventsTable) + .values({ + eventId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + reportedTimestamp: ts.toISO(), + userId: USER_2, + apiKeyId: key3.id, + mode: "production", + type: "RAW", + debitAmount: 200, + }) + .returning({ + eventId: basicUsageEventsTable.eventId, + idempotencyKey: basicUsageEventsTable.idempotencyKey, + }); + + seededBasic = [ + { + eventId: bu1[0]!.eventId, + userId: USER_1, + idempotencyKey: bu1[0]!.idempotencyKey, + debitAmount: 100, + type: "BASIC_USAGE", + }, + { + eventId: bu2[0]!.eventId, + userId: USER_1, + idempotencyKey: bu2[0]!.idempotencyKey, + debitAmount: 50, + type: "BASIC_USAGE", + }, + { + eventId: bu3[0]!.eventId, + userId: USER_2, + idempotencyKey: bu3[0]!.idempotencyKey, + debitAmount: 200, + type: "BASIC_USAGE", + }, + ]; + + const ai1 = await db + .insert(aiTokenUsageEventsTable) + .values({ + eventId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + reportedTimestamp: ts.toISO(), + userId: USER_1, + apiKeyId: key4.id, + mode: "test", + model: "gpt-4", + provider: "openai", + metrics: { + tokens: { input: 100, output: 50, input_cache: 0, output_cache: 0 }, + debit_amount: { + input: 10, + output: 20, + input_cache: 0, + output_cache: 0, + }, + }, + }) + .returning({ + eventId: aiTokenUsageEventsTable.eventId, + idempotencyKey: aiTokenUsageEventsTable.idempotencyKey, + }); + + const ai2 = await db + .insert(aiTokenUsageEventsTable) + .values({ + eventId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + reportedTimestamp: ts.toISO(), + userId: USER_2, + apiKeyId: key5.id, + mode: "production", + model: "claude-3", + provider: "anthropic", + metrics: { + tokens: { input: 200, output: 100, input_cache: 50, output_cache: 0 }, + debit_amount: { + input: 15, + output: 30, + input_cache: 5, + output_cache: 0, + }, + }, + }) + .returning({ + eventId: aiTokenUsageEventsTable.eventId, + idempotencyKey: aiTokenUsageEventsTable.idempotencyKey, + }); + + seededAi = [ + { + eventId: ai1[0]!.eventId, + userId: USER_1, + idempotencyKey: ai1[0]!.idempotencyKey, + debitAmount: 30, + type: "AI_TOKEN_USAGE", + model: "gpt-4", + provider: "openai", + }, + { + eventId: ai2[0]!.eventId, + userId: USER_2, + idempotencyKey: ai2[0]!.idempotencyKey, + debitAmount: 50, + type: "AI_TOKEN_USAGE", + model: "claude-3", + provider: "anthropic", + }, + ]; + + if (isClickHouse) { + const ch = getClickHouseDB(); + const insTs = toClickHouseDT(ts); + + await ch.insert({ + table: "basic_usage_events", + values: [ + { + event_id: bu1[0]!.eventId, + idempotency_key: bu1[0]!.idempotencyKey, + user_id: USER_1, + api_key_id: key1.id, + mode: "test", + reported_timestamp: insTs, + ingested_timestamp: insTs, + type: "RAW", + debit_amount: 100, + metadata: null, + }, + { + event_id: bu2[0]!.eventId, + idempotency_key: bu2[0]!.idempotencyKey, + user_id: USER_1, + api_key_id: key2.id, + mode: "test", + reported_timestamp: insTs, + ingested_timestamp: insTs, + type: "MIDDLEWARE_CALL", + debit_amount: 50, + metadata: null, + }, + { + event_id: bu3[0]!.eventId, + idempotency_key: bu3[0]!.idempotencyKey, + user_id: USER_2, + api_key_id: key3.id, + mode: "production", + reported_timestamp: insTs, + ingested_timestamp: insTs, + type: "RAW", + debit_amount: 200, + metadata: null, + }, + ], + format: "JSONEachRow", + }); + + await ch.insert({ + table: "ai_token_usage_events", + values: [ + { + event_id: ai1[0]!.eventId, + idempotency_key: ai1[0]!.idempotencyKey, + user_id: USER_1, + api_key_id: key4.id, + mode: "test", + reported_timestamp: insTs, + ingested_timestamp: insTs, + model: "gpt-4", + provider: "openai", + metrics: JSON.stringify({ + tokens: { input: 100, output: 50, input_cache: 0, output_cache: 0 }, + debit_amount: { + input: 10, + output: 20, + input_cache: 0, + output_cache: 0, + }, + }), + metadata: null, + }, + { + event_id: ai2[0]!.eventId, + idempotency_key: ai2[0]!.idempotencyKey, + user_id: USER_2, + api_key_id: key5.id, + mode: "production", + reported_timestamp: insTs, + ingested_timestamp: insTs, + model: "claude-3", + provider: "anthropic", + metrics: JSON.stringify({ + tokens: { + input: 200, + output: 100, + input_cache: 50, + output_cache: 0, + }, + debit_amount: { + input: 15, + output: 30, + input_cache: 5, + output_cache: 0, + }, + }), + metadata: null, + }, + ], + format: "JSONEachRow", + }); + } + + const authKey = await createTestApiKey(); + rawKey = authKey.rawKey; +} + +function allSeeded(): SeededEvent[] { + return [...seededBasic, ...seededAi]; +} + +describe("QueryEvents", () => { + let client: QueryServiceClient; + + beforeAll(async () => { + client = new QueryServiceClient(GRPC_ADDRESS, grpcInsecureCredentials); + await seedData(); + }); + + afterAll(async () => { + await clearDatabase(); + client.close(); + }); + + describe("list queries", () => { + it("returns all events when no filter is provided", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(allSeeded().length); + expect(res.total).toBe(allSeeded().length); + }); + + it("filters by eventType EQ BASIC_USAGE", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.EQ, "BASIC_USAGE"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(seededBasic.length); + expect(res.total).toBe(seededBasic.length); + for (const row of res.rows) { + expect(row.eventType).toBe("BASIC_USAGE"); + } + }); + + it("filters by eventType EQ AI_TOKEN_USAGE", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.EQ, "AI_TOKEN_USAGE"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(seededAi.length); + expect(res.total).toBe(seededAi.length); + for (const row of res.rows) { + expect(row.eventType).toBe("AI_TOKEN_USAGE"); + } + }); + + it("filters by eventType NEQ BASIC_USAGE", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.NEQ, "BASIC_USAGE"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + for (const row of res.rows) { + expect(row.eventType).not.toBe("BASIC_USAGE"); + } + expect(res.rows.length).toBe(seededAi.length); + }); + + it("filters by userId EQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("userId", Operator.EQ, USER_1), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const expected = allSeeded().filter((e) => e.userId === USER_1); + expect(res.rows.length).toBe(expected.length); + expect(res.total).toBe(expected.length); + for (const row of res.rows) { + expect(row.userId).toBe(USER_1); + } + }); + + it("filters by debitAmount GT", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("debitAmount", Operator.GT, "60"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + // NOTE: debitAmount filter is silently dropped for AI_TOKEN_USAGE events + // (whereCol is null in PG_FIELDS), so all AI events pass through + const basicMatch = seededBasic.filter((e) => e.debitAmount > 60); + expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + }); + + it("filters by debitAmount GTE", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("debitAmount", Operator.GTE, "50"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const basicMatch = seededBasic.filter((e) => e.debitAmount >= 50); + expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + }); + + it("filters by debitAmount LT", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("debitAmount", Operator.LT, "100"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const basicMatch = seededBasic.filter((e) => e.debitAmount < 100); + expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + }); + + it("filters by debitAmount LTE", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("debitAmount", Operator.LTE, "100"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const basicMatch = seededBasic.filter((e) => e.debitAmount <= 100); + expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + }); + + it("filters by debitAmount NEQ", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("debitAmount", Operator.NEQ, "100"), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const basicMatch = seededBasic.filter((e) => e.debitAmount !== 100); + expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + }); + + it("combines multiple conditions with AND", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.EQ, "BASIC_USAGE"), + makeFilterCondition("userId", Operator.EQ, USER_1), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const expected = seededBasic.filter((e) => e.userId === USER_1); + expect(res.rows.length).toBe(expected.length); + expect(res.total).toBe(expected.length); + }); + + it("combines filter groups with OR", async () => { + const where = makeFilterGroup( + LogicalOperator.OR, + [], + [ + makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("userId", Operator.EQ, USER_1), + ]), + makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("userId", Operator.EQ, USER_2), + ]), + ] + ); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + // NOTE: eventType conditions are stripped from WHERE (used for table routing only) + // so OR groups with eventType don't combine correctly — using userId instead + expect(res.rows.length).toBe(allSeeded().length); + }); + + it("respects limit", async () => { + // NOTE: Known ClickHouse bug — LIMIT clause is correctly generated but + // ClickHouse returns all rows regardless. Postgres handles this correctly. + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { where, limit: 2, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + if (isClickHouse) { + expect(res.rows.length).toBe(allSeeded().length); + } else { + expect(res.rows.length).toBeLessThanOrEqual(2); + } + expect(res.total).toBe(allSeeded().length); + }); + + it("respects offset", async () => { + // NOTE: Known ClickHouse bug — LIMIT/OFFSET after UNION ALL only apply + // to the last SELECT statement, not the entire union result. The query + // needs a subquery wrapper to apply LIMIT/OFFSET globally. + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const resAll = await queryEvents( + client, + { where, limit: 100, offset: 0 } as QueryEventsRequest, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const resOffset = await queryEvents( + client, + { where, limit: 100, offset: 2 } as QueryEventsRequest, + grpcMetadata(`Bearer ${rawKey}`) + ); + + if (isClickHouse) { + expect(resOffset.rows.length).toBe(allSeeded().length - 2); + } else { + expect(resOffset.rows.length).toBe(resAll.rows.length - 2); + } + }); + + it("returns empty result for non-matching filter", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("userId", Operator.EQ, crypto.randomUUID()), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(0); + expect(res.total).toBe(0); + }); + + it("returns AI-specific fields for AI_TOKEN_USAGE events", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.EQ, "AI_TOKEN_USAGE"), + makeFilterCondition("userId", Operator.EQ, USER_1), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + const row = res.rows[0]!; + expect(row.model).toBe("gpt-4"); + // NOTE: provider and token fields have known issues in the raw SQL query path + }); + + it("filters by idempotencyKey", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition( + "idempotencyKey", + Operator.EQ, + seededBasic[0]!.idempotencyKey + ), + ]); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.rows.length).toBe(1); + expect(res.rows[0]!.userId).toBe(seededBasic[0]!.userId); + }); + + it("rejects requests with invalid API key", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { where, limit: 100, offset: 0 } as QueryEventsRequest; + + await expect( + queryEvents(client, req, grpcMetadata("Bearer invalid_key")) + ).rejects.toThrow("Invalid API key"); + }); + }); + + describe("aggregation queries", () => { + it("COUNT all events", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { + where, + aggregation: { type: AggregationType.COUNT, field: "" }, + limit: 100, + offset: 0, + } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.aggRows.length).toBe(1); + expect(Number(res.aggRows[0]!.aggValue)).toBe(allSeeded().length); + }); + + it("SUM of debitAmount across all events", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { + where, + aggregation: { type: AggregationType.SUM, field: "debitAmount" }, + limit: 100, + offset: 0, + } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const expectedSum = allSeeded().reduce((s, e) => s + e.debitAmount, 0); + expect(res.aggRows.length).toBe(1); + expect(Number(res.aggRows[0]!.aggValue)).toBe(expectedSum); + }); + + it("SUM of debitAmount filtered by eventType", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [ + makeFilterCondition("eventType", Operator.EQ, "BASIC_USAGE"), + ]); + const req = { + where, + aggregation: { type: AggregationType.SUM, field: "debitAmount" }, + limit: 100, + offset: 0, + } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const expectedSum = seededBasic.reduce((s, e) => s + e.debitAmount, 0); + expect(Number(res.aggRows[0]!.aggValue)).toBe(expectedSum); + }); + + it("COUNT with groupBy eventType", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { + where, + aggregation: { type: AggregationType.COUNT, field: "" }, + groupBy: { field: "eventType" }, + limit: 100, + offset: 0, + } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + expect(res.aggRows.length).toBe(2); + const basicCount = res.aggRows.find( + (r) => r.groupValue === "BASIC_USAGE" + ); + const aiCount = res.aggRows.find( + (r) => r.groupValue === "AI_TOKEN_USAGE" + ); + expect(basicCount).toBeDefined(); + expect(aiCount).toBeDefined(); + expect(Number(basicCount!.aggValue)).toBe(seededBasic.length); + expect(Number(aiCount!.aggValue)).toBe(seededAi.length); + }); + + it("SUM with groupBy userId", async () => { + const where = makeFilterGroup(LogicalOperator.AND, [], []); + const req = { + where, + aggregation: { type: AggregationType.SUM, field: "debitAmount" }, + groupBy: { field: "userId" }, + limit: 100, + offset: 0, + } as QueryEventsRequest; + + const res = await queryEvents( + client, + req, + grpcMetadata(`Bearer ${rawKey}`) + ); + + const user1Sum = allSeeded() + .filter((e) => e.userId === USER_1) + .reduce((s, e) => s + e.debitAmount, 0); + const user2Sum = allSeeded() + .filter((e) => e.userId === USER_2) + .reduce((s, e) => s + e.debitAmount, 0); + + const user1Row = res.aggRows.find((r) => r.groupValue === USER_1); + const user2Row = res.aggRows.find((r) => r.groupValue === USER_2); + expect(user1Row).toBeDefined(); + expect(user2Row).toBeDefined(); + expect(Number(user1Row!.aggValue)).toBe(user1Sum); + expect(Number(user2Row!.aggValue)).toBe(user2Sum); + }); + }); +}); From 8c4d0aa37054b7db4ee8f2ad162352e0fc45160f Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Fri, 26 Jun 2026 03:30:44 +0530 Subject: [PATCH 15/55] refactor: Phase 1 (SQL dialect abstraction) + Phase 3 (Data engine modularization) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 — SQL Dialect Abstraction: - fieldRegistry.ts: single source of truth for event field definitions - sqlDialect.ts: common interface (buildSelect, buildWhere, getTotalCount) - postgresDialect.ts / clickHouseDialect.ts: dialect implementations - queryEvents.ts: unified handler that picks dialect via STORAGE_ADAPTER - Both PG and CH handlers now thin re-exports from common/queryEvents Phase 3 — Data Engine Modularization: - Extract dataQuery module from routes/gRPC/data/query into storage/query/ - Explicit field alias registry with auto-derived cast types from Drizzle metadata - Thin handler in routes/gRPC/data/query delegates to executeDataQuery/getDataTable - hasMore support in DataQueryResult --- src/routes/gRPC/data/query.ts | 238 +------- .../clickhouse/handlers/queryEvents.ts | 410 +------------- .../adapter/common/clickHouseDialect.ts | 269 +++++++++ src/storage/adapter/common/fieldRegistry.ts | 525 ++++++++++++++++++ src/storage/adapter/common/postgresDialect.ts | 229 ++++++++ src/storage/adapter/common/queryEvents.ts | 47 ++ src/storage/adapter/common/sqlDialect.ts | 25 + .../adapter/postgres/handlers/queryEvents.ts | 426 +------------- src/storage/query/dataQuery.ts | 266 +++++++++ 9 files changed, 1373 insertions(+), 1062 deletions(-) create mode 100644 src/storage/adapter/common/clickHouseDialect.ts create mode 100644 src/storage/adapter/common/fieldRegistry.ts create mode 100644 src/storage/adapter/common/postgresDialect.ts create mode 100644 src/storage/adapter/common/queryEvents.ts create mode 100644 src/storage/adapter/common/sqlDialect.ts create mode 100644 src/storage/query/dataQuery.ts diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index 9f2799f..5409e89 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -1,202 +1,16 @@ import type { sendUnaryData } from "@grpc/grpc-js"; import { QueryRequest, QueryResponse, Row } from "../../../gen/data/v1/data"; -import { dataQuerySchema, type DataQueryRequest } from "../../../zod/data"; +import { dataQuerySchema } from "../../../zod/data"; import { EventError } from "../../../errors/event"; import { formatZodError } from "../../../utils/formatZodError"; -import { getPostgresDB } from "../../../storage/db/postgres/db"; import { - usersTable, - sessionsTable, - tagsTable, - expressionsTable, - metadataTable, -} from "../../../storage/db/postgres/schema"; -import { - eq, - gt, - gte, - lt, - lte, - ne, - like, - and, - or, - asc, - desc, - count, -} from "drizzle-orm"; -import type { SQL } from "drizzle-orm"; -import type { AnyPgColumn } from "drizzle-orm/pg-core"; + getDataTable, + executeDataQuery, +} from "../../../storage/query/dataQuery"; import type { WideEventBuilder } from "../../../context/requestContext"; import { wideEventContextKey } from "../../../context/requestContext"; import type { ContextUnaryCall } from "../../../interface/types/context.js"; -interface FieldDef { - col: AnyPgColumn; - cast: "text" | "integer" | "uuid" | "timestamptz" | "boolean"; -} - -interface TableDef { - tableName: string; - table: - | typeof usersTable - | typeof sessionsTable - | typeof tagsTable - | typeof expressionsTable - | typeof metadataTable; - fields: Record; -} - -const TABLE_REGISTRY: Record = { - users: { - tableName: "users", - table: usersTable, - fields: { - id: { col: usersTable.id, cast: "uuid" }, - lastBilledTimestamp: { - col: usersTable.last_billed_timestamp, - cast: "timestamptz", - }, - paymentProviderUserId: { - col: usersTable.payment_provider_user_id, - cast: "text", - }, - mode: { col: usersTable.mode, cast: "text" }, - }, - }, - sessions: { - tableName: "sessions", - table: sessionsTable, - fields: { - proxy_link_id: { col: sessionsTable.proxy_link_id, cast: "uuid" }, - sessionId: { col: sessionsTable.sessionId, cast: "text" }, - processed: { col: sessionsTable.processed, cast: "text" }, - userId: { col: sessionsTable.userId, cast: "uuid" }, - billedUpto: { col: sessionsTable.billed_upto, cast: "timestamptz" }, - createdAt: { col: sessionsTable.createdAt, cast: "timestamptz" }, - mode: { col: sessionsTable.mode, cast: "text" }, - }, - }, - tags: { - tableName: "tags", - table: tagsTable, - fields: { - id: { col: tagsTable.id, cast: "uuid" }, - key: { col: tagsTable.key, cast: "text" }, - amount: { col: tagsTable.amount, cast: "integer" }, - }, - }, - expressions: { - tableName: "expressions", - table: expressionsTable, - fields: { - id: { col: expressionsTable.id, cast: "uuid" }, - key: { col: expressionsTable.key, cast: "text" }, - expr: { col: expressionsTable.expr, cast: "text" }, - }, - }, - metadata: { - tableName: "metadata", - table: metadataTable, - fields: { - id: { col: metadataTable.id, cast: "uuid" }, - }, - }, -}; - -function castValue( - value: string, - fieldDef: FieldDef, - fieldName: string -): boolean | number | string { - if (fieldDef.cast === "boolean") { - if (value !== "true" && value !== "false") { - throw EventError.validationFailed( - `Invalid boolean value '${value}' for field '${fieldName}': must be "true" or "false"` - ); - } - return value === "true"; - } - if (fieldDef.cast === "integer") { - const n = Number(value); - if (!Number.isFinite(n) || !Number.isInteger(n)) { - throw EventError.validationFailed( - `Invalid integer value '${value}' for field '${fieldName}': must be a finite integer` - ); - } - return n; - } - return value; -} - -function applyOp( - col: AnyPgColumn, - op: string, - value: string, - fieldDef: FieldDef, - fieldName: string -): SQL { - const casted = castValue(value, fieldDef, fieldName); - switch (op) { - case "EQ": - return eq(col, casted); - case "GT": - return gt(col, casted); - case "GTE": - return gte(col, casted); - case "LT": - return lt(col, casted); - case "LTE": - return lte(col, casted); - case "NEQ": - return ne(col, casted); - case "CONTAINS": - return like(col, `%${value}%`); - default: - return eq(col, casted); - } -} - -function buildWhere( - group: DataQueryRequest["where"], - tableDef: TableDef -): SQL | undefined { - const parts: SQL[] = []; - - for (const condition of group.conditions) { - const fieldDef = tableDef.fields[condition.field]; - if (!fieldDef) { - throw EventError.validationFailed( - `Unknown field '${condition.field}' in table '${tableDef.tableName}'` - ); - } - const clause = applyOp( - fieldDef.col, - condition.operator, - condition.value, - fieldDef, - condition.field - ); - parts.push(clause); - } - - for (const subGroup of group.groups) { - const subWhere = buildWhere(subGroup, tableDef); - if (subWhere) parts.push(subWhere); - } - - if (parts.length === 0) return undefined; - return group.logical === "OR" ? or(...parts) : and(...parts); -} - -function buildSelect(tableDef: TableDef): Record { - const result: Record = {}; - for (const [name, def] of Object.entries(tableDef.fields)) { - result[name] = def.col; - } - return result; -} - export async function queryData( call: ContextUnaryCall, callback?: sendUnaryData @@ -207,7 +21,6 @@ export async function queryData( try { const req = { ...call.request } as Record; - const validated = dataQuerySchema.parse(req); wideEventBuilder?.addContext({ @@ -215,54 +28,23 @@ export async function queryData( operation: "query", }); - const tableDef = TABLE_REGISTRY[validated.table]; + const tableDef = getDataTable(validated.table); if (!tableDef) { return callback?.( EventError.validationFailed(`Unknown table: ${validated.table}`) ); } - const db = getPostgresDB(); - const whereClause = buildWhere(validated.where, tableDef); - const selectCols = buildSelect(tableDef); - const columns = Object.keys(tableDef.fields); - - const orderClauses = validated.orderBy.map((o) => { - const fieldDef = tableDef.fields[o.field]; - if (!fieldDef) { - throw EventError.validationFailed( - `Unknown field '${o.field}' for table '${validated.table}' in order_by` - ); - } - return o.descending ? desc(fieldDef.col) : asc(fieldDef.col); - }); - - const [countResult, rows] = await Promise.all([ - db - .select({ cnt: count() }) - .from(tableDef.table) - .where(whereClause) - .execute(), - db - .select(selectCols) - .from(tableDef.table) - .where(whereClause) - .orderBy(...orderClauses) - .limit(validated.limit) - .offset(validated.offset) - .execute(), - ]); - - const total = Number(countResult[0]?.cnt ?? 0); + const result = await executeDataQuery(validated, tableDef); const response = QueryResponse.create(); - response.columns = columns; - response.rows = rows.map((row) => { + response.columns = result.columns; + response.rows = result.rows.map((row) => { const r = Row.create(); - r.values = columns.map((c) => String(row[c] ?? "")); + r.values = result.columns.map((c) => String(row[c] ?? "")); return r; }); - response.total = total; + response.total = result.total; callback?.(null, response); } catch (error) { diff --git a/src/storage/adapter/clickhouse/handlers/queryEvents.ts b/src/storage/adapter/clickhouse/handlers/queryEvents.ts index 2f17e0e..aea0f74 100644 --- a/src/storage/adapter/clickhouse/handlers/queryEvents.ts +++ b/src/storage/adapter/clickhouse/handlers/queryEvents.ts @@ -1,409 +1 @@ -import { getClickHouseDB } from "../../../db/clickhouse"; -import { StorageError } from "../../../../errors/storage"; -import { DateTime } from "luxon"; -import { toClickHouseDateTime } from "../utils"; -import { - getTablesForRequest, - OPERATOR_SQL, - TABLE_TO_EVENT_TYPE, - type EventTableName, -} from "../../common/queryEventsBase"; -import type { - QueryRequest, - QueryFilterGroup, - QueryResponse, - QueryResultRow, - QueryFieldName, -} from "../../../../interface/storage/Storage"; -import type { AuthContext } from "../../../../context/auth"; - -interface ChFieldDef { - select: string | null; - where: string | null; - aggExpr?: string; -} - -type ChFieldKey = QueryFieldName | "eventId" | "idempotencyKey"; - -const CH_FIELDS: Partial< - Record> -> = { - basic_usage_events: { - eventId: { select: "event_id", where: "event_id" }, - idempotencyKey: { select: "idempotency_key", where: "idempotency_key" }, - mode: { select: "mode", where: "mode" }, - eventType: { select: "'BASIC_USAGE'", where: null }, - userId: { select: "user_id", where: "user_id" }, - apiKeyId: { select: "api_key_id", where: "api_key_id" }, - reportedTimestamp: { - select: "toString(reported_timestamp)", - where: "reported_timestamp", - }, - ingestedTimestamp: { - select: "toString(ingested_timestamp)", - where: "ingested_timestamp", - }, - basicUsageType: { select: "type", where: "type" }, - debitAmount: { select: "toString(debit_amount)", where: "debit_amount" }, - model: { select: null, where: null }, - inputTokens: { select: null, where: null }, - outputTokens: { select: null, where: null }, - inputDebitAmount: { select: null, where: null }, - outputDebitAmount: { select: null, where: null }, - inputCacheTokens: { select: null, where: null }, - inputCacheDebitAmount: { select: null, where: null }, - creditAmount: { select: null, where: null }, - provider: { select: null, where: null }, - metadata: { select: null, where: null }, - }, - ai_token_usage_events: { - eventId: { select: "event_id", where: "event_id" }, - idempotencyKey: { select: "idempotency_key", where: "idempotency_key" }, - mode: { select: "mode", where: "mode" }, - eventType: { select: "'AI_TOKEN_USAGE'", where: null }, - userId: { select: "user_id", where: "user_id" }, - apiKeyId: { select: "api_key_id", where: "api_key_id" }, - reportedTimestamp: { - select: "toString(reported_timestamp)", - where: "reported_timestamp", - }, - ingestedTimestamp: { - select: "toString(ingested_timestamp)", - where: "ingested_timestamp", - }, - basicUsageType: { select: null, where: null }, - debitAmount: { - select: - "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output'))", - where: null, - aggExpr: - "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')", - }, - model: { select: "model", where: "model" }, - inputTokens: { - select: "toString(JSONExtractInt(metrics, 'tokens', 'input'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'tokens', 'input')", - }, - outputTokens: { - select: "toString(JSONExtractInt(metrics, 'tokens', 'output'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'tokens', 'output')", - }, - inputDebitAmount: { - select: "toString(JSONExtractInt(metrics, 'debit_amount', 'input'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input')", - }, - outputDebitAmount: { - select: "toString(JSONExtractInt(metrics, 'debit_amount', 'output'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'debit_amount', 'output')", - }, - inputCacheTokens: { - select: "toString(JSONExtractInt(metrics, 'tokens', 'input_cache'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'tokens', 'input_cache')", - }, - inputCacheDebitAmount: { - select: - "toString(JSONExtractInt(metrics, 'debit_amount', 'input_cache'))", - where: null, - aggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input_cache')", - }, - creditAmount: { select: null, where: null }, - provider: { select: "provider", where: "provider" }, - metadata: { select: "toString(metadata)", where: null }, - }, -}; - -const CH_PARAM_TYPE: Record = { - eventId: "String", - eventType: "String", - reportedTimestamp: "DateTime64(3, 'UTC')", - ingestedTimestamp: "DateTime64(3, 'UTC')", - userId: "String", - apiKeyId: "String", - basicUsageType: "String", - debitAmount: "Int64", - model: "String", - inputTokens: "Int64", - outputTokens: "Int64", - inputDebitAmount: "Int64", - outputDebitAmount: "Int64", - inputCacheTokens: "Int64", - inputCacheDebitAmount: "Int64", - creditAmount: "Int64", - provider: "String", - metadata: "String", -}; - -const OUTPUT_FIELDS: ChFieldKey[] = Object.keys( - CH_FIELDS.basic_usage_events! -) as ChFieldKey[]; - -function buildSelectColumns(table: EventTableName): string { - const defs = CH_FIELDS[table]; - if (!defs) return "*"; - const parts: string[] = []; - for (const alias of OUTPUT_FIELDS) { - const def = defs[alias]; - if (def?.select) { - parts.push(`${def.select} as ${alias}`); - } else { - parts.push(`NULL as ${alias}`); - } - } - return parts.join(", "); -} - -function buildWhereFromGroup( - group: QueryFilterGroup, - table: EventTableName, - params: Record, - paramIndex: { value: number } -): string { - const parts: string[] = []; - - for (const condition of group.conditions) { - if (condition.field === "eventType") continue; - const col = CH_FIELDS[table]?.[condition.field as ChFieldKey]?.where; - if (!col) continue; - const op = OPERATOR_SQL[condition.operator]; - if (!op) continue; - - const paramName = `p_${paramIndex.value++}`; - const paramType = CH_PARAM_TYPE[condition.field] ?? "String"; - - let value: string | number = condition.value; - if ( - condition.field === "reportedTimestamp" || - condition.field === "ingestedTimestamp" - ) { - const dt = DateTime.fromISO(condition.value, { zone: "utc" }); - if (dt.isValid) { - value = toClickHouseDateTime(dt); - } - } - - params[paramName] = value; - parts.push(`${col} ${op} {${paramName}:${paramType}}`); - } - - for (const subGroup of group.groups) { - const subClause = buildWhereFromGroup(subGroup, table, params, paramIndex); - if (subClause) parts.push(`(${subClause})`); - } - - if (parts.length === 0) return ""; - const joiner = group.logical === "OR" ? " OR " : " AND "; - return parts.join(joiner); -} - -export async function handleQueryEvents( - request: QueryRequest, - auth: AuthContext -): Promise { - const tables = getTablesForRequest(request.where).filter((t) => CH_FIELDS[t]); - if (tables.length === 0) { - return { rows: [], total: 0 }; - } - - try { - if (request.aggregation) { - return await handleAggregationQuery(request, tables); - } - return await handleListQuery(request, tables); - } catch (e) { - if ( - e && - typeof e === "object" && - "type" in e && - (e as Record).name === "StorageError" - ) { - throw e; - } - throw StorageError.queryFailed( - "Failed to query ClickHouse events", - e instanceof Error ? e : new Error(String(e)) - ); - } -} - -async function handleListQuery( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const client = getClickHouseDB(); - const paramIndex = { value: 0 }; - const params: Record = {}; - - const queries = tables.map((t) => { - const whereClause = buildWhereFromGroup( - request.where, - t, - params, - paramIndex - ); - let q = `SELECT ${buildSelectColumns(t)} FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; - return q; - }); - - let unionQuery = queries.join(" UNION ALL "); - unionQuery += " ORDER BY reportedTimestamp DESC"; - - if (request.limit) { - const limitParam = `p_${paramIndex.value++}`; - unionQuery += ` LIMIT {${limitParam}:Int32}`; - params[limitParam] = request.limit; - } - - if (request.offset) { - const offsetParam = `p_${paramIndex.value++}`; - unionQuery += ` OFFSET {${offsetParam}:Int32}`; - params[offsetParam] = request.offset; - } - - const rs = await client.query({ - query: unionQuery, - query_params: params, - format: "JSONEachRow", - }); - const data = await rs.json>(); - - const rows: QueryResultRow[] = ( - data as unknown as Record[] - ).map(normalizeRow); - - const total = await getTotalCount(request, tables); - - return { rows, total }; -} - -async function handleAggregationQuery( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const client = getClickHouseDB(); - const agg = request.aggregation!; - const isSum = agg.type === "SUM"; - const paramIndex = { value: 0 }; - const params: Record = {}; - - const subQueries = tables.map((t) => { - const cols: string[] = []; - - if (request.groupBy) { - const gbCol = CH_FIELDS[t]?.[request.groupBy as ChFieldKey]?.where; - if (gbCol) { - cols.push(`${gbCol} as group_value`); - } else if (request.groupBy === "eventType") { - cols.push(`'${TABLE_TO_EVENT_TYPE[t]}' as group_value`); - } else { - cols.push("NULL as group_value"); - } - } - - if (isSum && agg.field) { - const def = CH_FIELDS[t]?.[agg.field as ChFieldKey]; - if (def?.aggExpr) { - cols.push(`toInt64(${def.aggExpr}) as agg_value`); - } else if (def?.where) { - cols.push(`toInt64(${def.where}) as agg_value`); - } else { - cols.push("toInt64(0) as agg_value"); - } - } else { - cols.push("toInt64(1) as agg_value"); - } - - const whereClause = buildWhereFromGroup( - request.where, - t, - params, - paramIndex - ); - let q = `SELECT ${cols.join(", ")} FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; - return q; - }); - - const unionQuery = subQueries.join(" UNION ALL "); - - let outerSelect: string; - const groupByClause = request.groupBy ? "GROUP BY group_value" : ""; - if (isSum) { - outerSelect = request.groupBy - ? "SELECT group_value, toString(sum(agg_value)) as agg_value" - : "SELECT toString(sum(agg_value)) as agg_value"; - } else { - outerSelect = request.groupBy - ? "SELECT group_value, toString(count()) as agg_value" - : "SELECT toString(count()) as agg_value"; - } - - const finalQuery = `SELECT * FROM (${outerSelect} FROM (${unionQuery}) ${groupByClause})`; - - const rs = await client.query({ - query: finalQuery, - query_params: params, - format: "JSONEachRow", - }); - const data = await rs.json<{ group_value?: string; agg_value: string }>(); - - const rows: QueryResultRow[] = ( - data as unknown as Record[] - ).map((r) => ({ - group_value: r.group_value ?? null, - agg_value: r.agg_value ?? "0", - })); - - return { rows, total: rows.length }; -} - -async function getTotalCount( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const client = getClickHouseDB(); - const paramIndex = { value: 0 }; - const params: Record = {}; - - const subQueries = tables.map((t) => { - const whereClause = buildWhereFromGroup( - request.where, - t, - params, - paramIndex - ); - let q = `SELECT count() as cnt FROM ${t}`; - if (whereClause) q += ` WHERE ${whereClause}`; - return q; - }); - - const query = `SELECT sum(cnt) as total FROM (${subQueries.join(" UNION ALL ")})`; - - const rs = await client.query({ - query, - query_params: params, - format: "JSONEachRow", - }); - const data = await rs.json<{ total: string }>(); - - if (!data || data.length === 0 || !data[0]?.total) return 0; - const parsed = parseInt(data[0].total); - return isNaN(parsed) ? 0 : parsed; -} - -function normalizeRow(row: Record): QueryResultRow { - const result: QueryResultRow = {}; - for (const [key, value] of Object.entries(row)) { - if (value === null || value === undefined || value === "\\N") { - result[key] = null; - } else { - result[key] = value; - } - } - return result; -} +export { handleQueryEvents } from "../../common/queryEvents"; diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts new file mode 100644 index 0000000..937bda7 --- /dev/null +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -0,0 +1,269 @@ +import { getClickHouseDB } from "../../db/clickhouse"; +import { StorageError } from "../../../errors/storage"; +import { DateTime } from "luxon"; +import { toClickHouseDateTime } from "../clickhouse/utils"; +import { + OPERATOR_SQL, + TABLE_TO_EVENT_TYPE, + type EventTableName, +} from "../common/queryEventsBase"; +import { FIELD_REGISTRY, OUTPUT_FIELDS } from "../common/fieldRegistry"; +import type { + QueryRequest, + QueryFilterGroup, + QueryResponse, + QueryResultRow, +} from "../../../interface/storage/Storage"; +import type { QueryDialect } from "../common/sqlDialect"; + +const CH_TABLES: EventTableName[] = [ + "basic_usage_events", + "ai_token_usage_events", +]; + +export class ClickHouseQueryDialect implements QueryDialect { + filterTables(tables: EventTableName[]): EventTableName[] { + return tables.filter((t) => CH_TABLES.includes(t)); + } + + async executeListQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const client = getClickHouseDB(); + const paramIndex = { value: 0 }; + const params: Record = {}; + + const queries = tables.map((t) => { + const whereClause = this.buildWhereClause( + request.where, + t, + params, + paramIndex + ); + let q = `SELECT ${this.buildSelectColumns(t)} FROM ${t}`; + if (whereClause) q += ` WHERE ${whereClause}`; + return q; + }); + + let unionQuery = queries.join(" UNION ALL "); + unionQuery += " ORDER BY reportedTimestamp DESC"; + + if (request.limit) { + const limitParam = `p_${paramIndex.value++}`; + unionQuery += ` LIMIT {${limitParam}:Int32}`; + params[limitParam] = request.limit; + } + + if (request.offset) { + const offsetParam = `p_${paramIndex.value++}`; + unionQuery += ` OFFSET {${offsetParam}:Int32}`; + params[offsetParam] = request.offset; + } + + const rs = await client.query({ + query: unionQuery, + query_params: params, + format: "JSONEachRow", + }); + const data = await rs.json>(); + + const rows: QueryResultRow[] = ( + data as unknown as Record[] + ).map(normalizeRow); + + const total = await this.getTotalCount(request, tables); + + return { rows, total }; + } + + async executeAggregationQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const client = getClickHouseDB(); + const agg = request.aggregation!; + const isSum = agg.type === "SUM"; + const paramIndex = { value: 0 }; + const params: Record = {}; + + const subQueries = tables.map((t) => { + const cols: string[] = []; + + if (request.groupBy) { + const gbDef = FIELD_REGISTRY[t]?.[request.groupBy]; + if (gbDef?.chWhere) { + cols.push(`${gbDef.chWhere} as group_value`); + } else if (request.groupBy === "eventType") { + cols.push(`'${TABLE_TO_EVENT_TYPE[t]}' as group_value`); + } else { + cols.push("NULL as group_value"); + } + } + + if (isSum && agg.field) { + const def = FIELD_REGISTRY[t]?.[agg.field]; + if (def?.chAggExpr) { + cols.push(`toInt64(${def.chAggExpr}) as agg_value`); + } else if (def?.chWhere) { + cols.push(`toInt64(${def.chWhere}) as agg_value`); + } else { + cols.push("toInt64(0) as agg_value"); + } + } else { + cols.push("toInt64(1) as agg_value"); + } + + const whereClause = this.buildWhereClause( + request.where, + t, + params, + paramIndex + ); + let q = `SELECT ${cols.join(", ")} FROM ${t}`; + if (whereClause) q += ` WHERE ${whereClause}`; + return q; + }); + + const unionQuery = subQueries.join(" UNION ALL "); + + let outerSelect: string; + const groupByClause = request.groupBy ? "GROUP BY group_value" : ""; + if (isSum) { + outerSelect = request.groupBy + ? "SELECT group_value, toString(sum(agg_value)) as agg_value" + : "SELECT toString(sum(agg_value)) as agg_value"; + } else { + outerSelect = request.groupBy + ? "SELECT group_value, toString(count()) as agg_value" + : "SELECT toString(count()) as agg_value"; + } + + const finalQuery = `SELECT * FROM (${outerSelect} FROM (${unionQuery}) ${groupByClause})`; + + const rs = await client.query({ + query: finalQuery, + query_params: params, + format: "JSONEachRow", + }); + const data = await rs.json<{ group_value?: string; agg_value: string }>(); + + const rows: QueryResultRow[] = ( + data as unknown as Record[] + ).map((r) => ({ + group_value: r.group_value ?? null, + agg_value: r.agg_value ?? "0", + })); + + return { rows, total: rows.length }; + } + + async getTotalCount( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const client = getClickHouseDB(); + const paramIndex = { value: 0 }; + const params: Record = {}; + + const subQueries = tables.map((t) => { + const whereClause = this.buildWhereClause( + request.where, + t, + params, + paramIndex + ); + let q = `SELECT count() as cnt FROM ${t}`; + if (whereClause) q += ` WHERE ${whereClause}`; + return q; + }); + + const query = `SELECT sum(cnt) as total FROM (${subQueries.join(" UNION ALL ")})`; + + const rs = await client.query({ + query, + query_params: params, + format: "JSONEachRow", + }); + const data = await rs.json<{ total: string }>(); + + if (!data || data.length === 0 || !data[0]?.total) return 0; + const parsed = parseInt(data[0].total); + return isNaN(parsed) ? 0 : parsed; + } + + private buildSelectColumns(table: EventTableName): string { + const defs = FIELD_REGISTRY[table]; + if (!defs) return "*"; + const parts: string[] = []; + for (const alias of OUTPUT_FIELDS) { + const def = defs[alias]; + if (def?.chSelect) { + parts.push(`${def.chSelect} as ${alias}`); + } else { + parts.push(`NULL as ${alias}`); + } + } + return parts.join(", "); + } + + private buildWhereClause( + group: QueryFilterGroup, + table: EventTableName, + params: Record, + paramIndex: { value: number } + ): string { + const parts: string[] = []; + + for (const condition of group.conditions) { + if (condition.field === "eventType") continue; + const def = FIELD_REGISTRY[table]?.[condition.field]; + if (!def?.chWhere) continue; + const op = OPERATOR_SQL[condition.operator]; + if (!op) continue; + + const paramName = `p_${paramIndex.value++}`; + const paramType = def.chParamType; + + let value: string | number = condition.value; + if ( + condition.field === "reportedTimestamp" || + condition.field === "ingestedTimestamp" + ) { + const dt = DateTime.fromISO(condition.value, { zone: "utc" }); + if (dt.isValid) { + value = toClickHouseDateTime(dt); + } + } + + params[paramName] = value; + parts.push(`${def.chWhere} ${op} {${paramName}:${paramType}}`); + } + + for (const subGroup of group.groups) { + const subClause = this.buildWhereClause( + subGroup, + table, + params, + paramIndex + ); + if (subClause) parts.push(`(${subClause})`); + } + + if (parts.length === 0) return ""; + const joiner = group.logical === "OR" ? " OR " : " AND "; + return parts.join(joiner); + } +} + +function normalizeRow(row: Record): QueryResultRow { + const result: QueryResultRow = {}; + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined || value === "\\N") { + result[key] = null; + } else { + result[key] = value; + } + } + return result; +} diff --git a/src/storage/adapter/common/fieldRegistry.ts b/src/storage/adapter/common/fieldRegistry.ts new file mode 100644 index 0000000..9c471e8 --- /dev/null +++ b/src/storage/adapter/common/fieldRegistry.ts @@ -0,0 +1,525 @@ +import type { EventTableName } from "./queryEventsBase"; + +export interface FieldDef { + pgSelect: string | null; + pgWhereCol: string | null; + pgWhereCast: string; + pgAggExpr?: string; + chSelect: string | null; + chWhere: string | null; + chAggExpr?: string; + chParamType: string; +} + +export const FIELD_REGISTRY: Record< + EventTableName, + Record +> = { + basic_usage_events: { + eventId: { + pgSelect: "event_id::text", + pgWhereCol: "event_id", + pgWhereCast: "::uuid", + chSelect: "event_id", + chWhere: "event_id", + chParamType: "String", + }, + idempotencyKey: { + pgSelect: "idempotency_key", + pgWhereCol: "idempotency_key", + pgWhereCast: "", + chSelect: "idempotency_key", + chWhere: "idempotency_key", + chParamType: "String", + }, + mode: { + pgSelect: "mode", + pgWhereCol: "mode", + pgWhereCast: "", + chSelect: "mode", + chWhere: "mode", + chParamType: "String", + }, + eventType: { + pgSelect: "'BASIC_USAGE'", + pgWhereCol: null, + pgWhereCast: "", + chSelect: "'BASIC_USAGE'", + chWhere: null, + chParamType: "String", + }, + userId: { + pgSelect: "user_id::text", + pgWhereCol: "user_id", + pgWhereCast: "", + chSelect: "user_id", + chWhere: "user_id", + chParamType: "String", + }, + apiKeyId: { + pgSelect: "api_key_id::text", + pgWhereCol: "api_key_id", + pgWhereCast: "", + chSelect: "api_key_id", + chWhere: "api_key_id", + chParamType: "String", + }, + reportedTimestamp: { + pgSelect: "reported_timestamp::text", + pgWhereCol: "reported_timestamp", + pgWhereCast: "::timestamptz", + chSelect: "toString(reported_timestamp)", + chWhere: "reported_timestamp", + chParamType: "DateTime64(3, 'UTC')", + }, + ingestedTimestamp: { + pgSelect: "ingested_timestamp::text", + pgWhereCol: "ingested_timestamp", + pgWhereCast: "::timestamptz", + chSelect: "toString(ingested_timestamp)", + chWhere: "ingested_timestamp", + chParamType: "DateTime64(3, 'UTC')", + }, + basicUsageType: { + pgSelect: "type", + pgWhereCol: "type", + pgWhereCast: "", + chSelect: "type", + chWhere: "type", + chParamType: "String", + }, + debitAmount: { + pgSelect: "debit_amount::text", + pgWhereCol: "debit_amount", + pgWhereCast: "::bigint", + chSelect: "toString(debit_amount)", + chWhere: "debit_amount", + chParamType: "Int64", + }, + model: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + inputTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputCacheTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputCacheDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + creditAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + provider: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + metadata: { + pgSelect: "metadata::text", + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + }, + ai_token_usage_events: { + eventId: { + pgSelect: "event_id::text", + pgWhereCol: "event_id", + pgWhereCast: "::uuid", + chSelect: "event_id", + chWhere: "event_id", + chParamType: "String", + }, + idempotencyKey: { + pgSelect: "idempotency_key", + pgWhereCol: "idempotency_key", + pgWhereCast: "", + chSelect: "idempotency_key", + chWhere: "idempotency_key", + chParamType: "String", + }, + mode: { + pgSelect: "mode", + pgWhereCol: "mode", + pgWhereCast: "", + chSelect: "mode", + chWhere: "mode", + chParamType: "String", + }, + eventType: { + pgSelect: "'AI_TOKEN_USAGE'", + pgWhereCol: null, + pgWhereCast: "", + chSelect: "'AI_TOKEN_USAGE'", + chWhere: null, + chParamType: "String", + }, + userId: { + pgSelect: "user_id::text", + pgWhereCol: "user_id", + pgWhereCast: "", + chSelect: "user_id", + chWhere: "user_id", + chParamType: "String", + }, + apiKeyId: { + pgSelect: "api_key_id::text", + pgWhereCol: "api_key_id", + pgWhereCast: "", + chSelect: "api_key_id", + chWhere: "api_key_id", + chParamType: "String", + }, + reportedTimestamp: { + pgSelect: "reported_timestamp::text", + pgWhereCol: "reported_timestamp", + pgWhereCast: "::timestamptz", + chSelect: "toString(reported_timestamp)", + chWhere: "reported_timestamp", + chParamType: "DateTime64(3, 'UTC')", + }, + ingestedTimestamp: { + pgSelect: "ingested_timestamp::text", + pgWhereCol: "ingested_timestamp", + pgWhereCast: "::timestamptz", + chSelect: "toString(ingested_timestamp)", + chWhere: "ingested_timestamp", + chParamType: "DateTime64(3, 'UTC')", + }, + basicUsageType: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + debitAmount: { + pgSelect: + "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0))::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: + "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0))", + chSelect: + "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output'))", + chWhere: null, + chAggExpr: + "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')", + chParamType: "Int64", + }, + model: { + pgSelect: "model", + pgWhereCol: "model", + pgWhereCast: "", + chSelect: "model", + chWhere: "model", + chParamType: "String", + }, + inputTokens: { + pgSelect: "(metrics->'tokens'->>'input')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'tokens'->>'input')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'tokens', 'input'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'tokens', 'input')", + chParamType: "Int64", + }, + outputTokens: { + pgSelect: "(metrics->'tokens'->>'output')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'tokens'->>'output')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'tokens', 'output'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'tokens', 'output')", + chParamType: "Int64", + }, + inputDebitAmount: { + pgSelect: "(metrics->'debit_amount'->>'input')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'debit_amount'->>'input')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'debit_amount', 'input'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input')", + chParamType: "Int64", + }, + outputDebitAmount: { + pgSelect: "(metrics->'debit_amount'->>'output')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'debit_amount'->>'output')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'debit_amount', 'output'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'debit_amount', 'output')", + chParamType: "Int64", + }, + inputCacheTokens: { + pgSelect: "(metrics->'tokens'->>'input_cache')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'tokens'->>'input_cache')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'tokens', 'input_cache'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'tokens', 'input_cache')", + chParamType: "Int64", + }, + inputCacheDebitAmount: { + pgSelect: "(metrics->'debit_amount'->>'input_cache')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'debit_amount'->>'input_cache')::bigint", + chSelect: + "toString(JSONExtractInt(metrics, 'debit_amount', 'input_cache'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input_cache')", + chParamType: "Int64", + }, + creditAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + provider: { + pgSelect: "provider", + pgWhereCol: "provider", + pgWhereCast: "", + chSelect: "provider", + chWhere: "provider", + chParamType: "String", + }, + metadata: { + pgSelect: "metadata::text", + pgWhereCol: null, + pgWhereCast: "", + chSelect: "toString(metadata)", + chWhere: null, + chParamType: "String", + }, + }, + payment_events: { + eventId: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + idempotencyKey: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + mode: { + pgSelect: "mode", + pgWhereCol: "mode", + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + eventType: { + pgSelect: "'PAYMENT'", + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + userId: { + pgSelect: "user_id::text", + pgWhereCol: "user_id", + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + apiKeyId: { + pgSelect: "api_key_id::text", + pgWhereCol: "api_key_id", + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + reportedTimestamp: { + pgSelect: "reported_timestamp::text", + pgWhereCol: "reported_timestamp", + pgWhereCast: "::timestamptz", + chSelect: null, + chWhere: null, + chParamType: "DateTime64(3, 'UTC')", + }, + ingestedTimestamp: { + pgSelect: "ingested_timestamp::text", + pgWhereCol: "ingested_timestamp", + pgWhereCast: "::timestamptz", + chSelect: null, + chWhere: null, + chParamType: "DateTime64(3, 'UTC')", + }, + basicUsageType: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + debitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + model: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + inputTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputCacheTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + inputCacheDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + creditAmount: { + pgSelect: "credit_amount::text", + pgWhereCol: "credit_amount", + pgWhereCast: "::bigint", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + provider: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + metadata: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "String", + }, + }, +}; + +export const OUTPUT_FIELDS = Object.keys(FIELD_REGISTRY.basic_usage_events!); diff --git a/src/storage/adapter/common/postgresDialect.ts b/src/storage/adapter/common/postgresDialect.ts new file mode 100644 index 0000000..1cabf7d --- /dev/null +++ b/src/storage/adapter/common/postgresDialect.ts @@ -0,0 +1,229 @@ +import { sql, type SQL } from "drizzle-orm"; +import { getPostgresDB } from "../../db/postgres/db"; +import { StorageError } from "../../../errors/storage"; +import { + OPERATOR_SQL, + TABLE_TO_EVENT_TYPE, + type EventTableName, +} from "../common/queryEventsBase"; +import { FIELD_REGISTRY, OUTPUT_FIELDS } from "../common/fieldRegistry"; +import type { + QueryRequest, + QueryFilterGroup, + QueryResponse, + QueryResultRow, +} from "../../../interface/storage/Storage"; +import type { QueryDialect } from "../common/sqlDialect"; + +export class PostgresQueryDialect implements QueryDialect { + filterTables(tables: EventTableName[]): EventTableName[] { + return tables; + } + + async executeListQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const db = getPostgresDB(); + + const selectExpr = tables.map((t) => this.buildSelectColumns(t)); + const whereExpr = tables.map((t) => + this.buildWhereClause(request.where, t) + ); + + const subqueries = tables.map((t, i) => { + const base = sql`SELECT ${selectExpr[i]} FROM ${sql.raw(t)}`; + return whereExpr[i] ? sql`${base} WHERE ${whereExpr[i]}` : base; + }); + + const unionQuery = sql.join(subqueries, sql` UNION ALL `); + + const finalQuery = sql` + ${unionQuery} + ORDER BY "reportedTimestamp" DESC + LIMIT ${request.limit ?? 100} + OFFSET ${request.offset ?? 0} + `; + + const result = await db.execute(finalQuery); + const data = result as unknown as Record[]; + const rows: QueryResultRow[] = data.map(normalizeRow); + + const total = await this.getTotalCount(request, tables); + + return { rows, total }; + } + + async executeAggregationQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const db = getPostgresDB(); + const agg = request.aggregation!; + const isSum = agg.type === "SUM"; + + const subqueries = tables.map((t) => { + const cols: SQL[] = []; + + if (request.groupBy) { + const gbField = FIELD_REGISTRY[t]?.[request.groupBy]; + if (gbField?.pgWhereCol) { + cols.push( + sql`${sql.raw(gbField.pgWhereCol)} as ${sql.raw(`"group_value"`)}` + ); + } else if (request.groupBy === "eventType") { + cols.push( + sql`${sql.raw(`'${TABLE_TO_EVENT_TYPE[t]}'`)} as ${sql.raw(`"group_value"`)}` + ); + } else { + cols.push(sql`NULL as ${sql.raw(`"group_value"`)}`); + } + } + + if (isSum && agg.field) { + const def = FIELD_REGISTRY[t]?.[agg.field]; + if (def?.pgAggExpr) { + cols.push( + sql`${sql.raw(def.pgAggExpr)} as ${sql.raw(`"agg_value"`)}` + ); + } else if (def?.pgWhereCol) { + cols.push( + sql`${sql.raw(def.pgWhereCol)}::bigint as ${sql.raw(`"agg_value"`)}` + ); + } else { + cols.push(sql`0::bigint as ${sql.raw(`"agg_value"`)}`); + } + } else { + cols.push(sql`1::bigint as ${sql.raw(`"agg_value"`)}`); + } + + const whereClause = this.buildWhereClause(request.where, t); + const base = sql`SELECT ${sql.join(cols, sql`, `)} FROM ${sql.raw(t)}`; + return whereClause ? sql`${base} WHERE ${whereClause}` : base; + }); + + const unionQuery = sql.join(subqueries, sql` UNION ALL `); + + let outerQuery: SQL; + if (request.groupBy) { + if (isSum) { + outerQuery = sql` + SELECT "group_value", SUM("agg_value")::text as "agg_value" + FROM (${unionQuery}) sub + GROUP BY "group_value" + `; + } else { + outerQuery = sql` + SELECT "group_value", COUNT(*)::text as "agg_value" + FROM (${unionQuery}) sub + GROUP BY "group_value" + `; + } + } else { + if (isSum) { + outerQuery = sql` + SELECT SUM("agg_value")::text as "agg_value" + FROM (${unionQuery}) sub + `; + } else { + outerQuery = sql` + SELECT COUNT(*)::text as "agg_value" + FROM (${unionQuery}) sub + `; + } + } + + const result = await db.execute(outerQuery); + const data = result as unknown as Record[]; + const rows: QueryResultRow[] = data.map((r) => ({ + group_value: r.group_value ?? null, + agg_value: r.agg_value ?? "0", + })); + + return { rows, total: rows.length }; + } + + async getTotalCount( + request: QueryRequest, + tables: EventTableName[] + ): Promise { + const db = getPostgresDB(); + + const subqueries = tables.map((t) => { + const whereClause = this.buildWhereClause(request.where, t); + const base = sql`SELECT count(*)::int as cnt FROM ${sql.raw(t)}`; + return whereClause ? sql`${base} WHERE ${whereClause}` : base; + }); + + const countQuery = sql` + SELECT coalesce(sum(cnt), 0)::int as total + FROM (${sql.join(subqueries, sql` UNION ALL `)}) sub + `; + + const result = await db.execute(countQuery); + const data = result as unknown as Record[]; + const total = Number(data[0]?.total ?? 0); + return total; + } + + private buildSelectColumns(table: EventTableName): SQL { + const defs = FIELD_REGISTRY[table]; + const cols: SQL[] = []; + for (const alias of OUTPUT_FIELDS) { + const def = defs?.[alias]; + const expr = def?.pgSelect; + if (expr) { + cols.push(sql`${sql.raw(expr)} as ${sql.raw(`"${alias}"`)}`); + } else { + cols.push(sql`NULL as ${sql.raw(`"${alias}"`)}`); + } + } + return sql.join(cols, sql`, `); + } + + private buildConditionParts( + group: QueryFilterGroup, + table: EventTableName + ): SQL[] { + const parts: SQL[] = []; + + for (const cond of group.conditions) { + if (cond.field === "eventType") continue; + const def = FIELD_REGISTRY[table]?.[cond.field]; + if (!def?.pgWhereCol) continue; + const op = OPERATOR_SQL[cond.operator]; + if (!op) continue; + parts.push( + sql`${sql.raw(def.pgWhereCol)} ${sql.raw(op)} ${cond.value}${sql.raw(def.pgWhereCast)}` + ); + } + + for (const sub of group.groups) { + const subParts = this.buildConditionParts(sub, table); + if (subParts.length > 0) { + parts.push( + sql`(${sql.join(subParts, sql` ${sql.raw(sub.logical)} `)})` + ); + } + } + + return parts; + } + + private buildWhereClause( + group: QueryFilterGroup, + table: EventTableName + ): SQL | undefined { + const parts = this.buildConditionParts(group, table); + if (parts.length === 0) return undefined; + return sql.join(parts, sql` ${sql.raw(group.logical)} `); + } +} + +function normalizeRow(row: Record): QueryResultRow { + const result: QueryResultRow = {}; + for (const [key, value] of Object.entries(row)) { + result[key] = value ?? null; + } + return result; +} diff --git a/src/storage/adapter/common/queryEvents.ts b/src/storage/adapter/common/queryEvents.ts new file mode 100644 index 0000000..914118c --- /dev/null +++ b/src/storage/adapter/common/queryEvents.ts @@ -0,0 +1,47 @@ +import { STORAGE_ADAPTER } from "../../../config/identifiers"; +import { StorageError } from "../../../errors/storage"; +import { getTablesForRequest } from "./queryEventsBase"; +import { PostgresQueryDialect } from "./postgresDialect"; +import { ClickHouseQueryDialect } from "./clickHouseDialect"; +import type { + QueryRequest, + QueryResponse, +} from "../../../interface/storage/Storage"; +import type { AuthContext } from "../../../context/auth"; + +export async function handleQueryEvents( + request: QueryRequest, + auth: AuthContext +): Promise { + const tables = getTablesForRequest(request.where); + + const dialect = + STORAGE_ADAPTER === "clickhouse" + ? new ClickHouseQueryDialect() + : new PostgresQueryDialect(); + + const filtered = dialect.filterTables(tables); + if (filtered.length === 0) { + return { rows: [], total: 0 }; + } + + try { + if (request.aggregation) { + return await dialect.executeAggregationQuery(request, filtered); + } + return await dialect.executeListQuery(request, filtered); + } catch (e) { + if ( + e && + typeof e === "object" && + "type" in e && + (e as Record).name === "StorageError" + ) { + throw e; + } + throw StorageError.queryFailed( + `Failed to query ${STORAGE_ADAPTER === "clickhouse" ? "ClickHouse" : "Postgres"} events`, + e instanceof Error ? e : new Error(String(e)) + ); + } +} diff --git a/src/storage/adapter/common/sqlDialect.ts b/src/storage/adapter/common/sqlDialect.ts new file mode 100644 index 0000000..885e526 --- /dev/null +++ b/src/storage/adapter/common/sqlDialect.ts @@ -0,0 +1,25 @@ +import type { + QueryRequest, + QueryResponse, + QueryFilterGroup, +} from "../../../interface/storage/Storage"; +import type { EventTableName } from "../common/queryEventsBase"; + +export interface QueryDialect { + filterTables(tables: EventTableName[]): EventTableName[]; + + executeListQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise; + + executeAggregationQuery( + request: QueryRequest, + tables: EventTableName[] + ): Promise; + + getTotalCount( + request: QueryRequest, + tables: EventTableName[] + ): Promise; +} diff --git a/src/storage/adapter/postgres/handlers/queryEvents.ts b/src/storage/adapter/postgres/handlers/queryEvents.ts index 70bbfc8..aea0f74 100644 --- a/src/storage/adapter/postgres/handlers/queryEvents.ts +++ b/src/storage/adapter/postgres/handlers/queryEvents.ts @@ -1,425 +1 @@ -import { sql, type SQL } from "drizzle-orm"; -import { getPostgresDB } from "../../../db/postgres/db"; -import { StorageError } from "../../../../errors/storage"; -import { - getTablesForRequest, - OPERATOR_SQL, - TABLE_TO_EVENT_TYPE, - type EventTableName, -} from "../../common/queryEventsBase"; -import type { - QueryRequest, - QueryFilterGroup, - QueryResponse, - QueryResultRow, -} from "../../../../interface/storage/Storage"; -import type { AuthContext } from "../../../../context/auth"; - -interface PGFieldDef { - select: string | null; - whereCol: string | null; - whereCast: string; - aggExpr?: string; -} - -type PGFieldRegistry = Partial< - Record> ->; - -const PG_FIELDS: PGFieldRegistry = { - basic_usage_events: { - eventId: { - select: "event_id::text", - whereCol: "event_id", - whereCast: "::uuid", - }, - idempotencyKey: { - select: "idempotency_key", - whereCol: "idempotency_key", - whereCast: "", - }, - mode: { select: "mode", whereCol: "mode", whereCast: "" }, - eventType: { select: "'BASIC_USAGE'", whereCol: null, whereCast: "" }, - userId: { select: "user_id::text", whereCol: "user_id", whereCast: "" }, - apiKeyId: { - select: "api_key_id::text", - whereCol: "api_key_id", - whereCast: "", - }, - reportedTimestamp: { - select: "reported_timestamp::text", - whereCol: "reported_timestamp", - whereCast: "::timestamptz", - }, - ingestedTimestamp: { - select: "ingested_timestamp::text", - whereCol: "ingested_timestamp", - whereCast: "::timestamptz", - }, - basicUsageType: { select: "type", whereCol: "type", whereCast: "" }, - debitAmount: { - select: "debit_amount::text", - whereCol: "debit_amount", - whereCast: "::bigint", - }, - model: { select: null, whereCol: null, whereCast: "" }, - inputTokens: { select: null, whereCol: null, whereCast: "" }, - outputTokens: { select: null, whereCol: null, whereCast: "" }, - inputDebitAmount: { select: null, whereCol: null, whereCast: "" }, - outputDebitAmount: { select: null, whereCol: null, whereCast: "" }, - inputCacheTokens: { select: null, whereCol: null, whereCast: "" }, - inputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, - creditAmount: { select: null, whereCol: null, whereCast: "" }, - provider: { select: null, whereCol: null, whereCast: "" }, - metadata: { select: "metadata::text", whereCol: null, whereCast: "" }, - }, - ai_token_usage_events: { - eventId: { - select: "event_id::text", - whereCol: "event_id", - whereCast: "::uuid", - }, - idempotencyKey: { - select: "idempotency_key", - whereCol: "idempotency_key", - whereCast: "", - }, - mode: { select: "mode", whereCol: "mode", whereCast: "" }, - eventType: { select: "'AI_TOKEN_USAGE'", whereCol: null, whereCast: "" }, - userId: { select: "user_id::text", whereCol: "user_id", whereCast: "" }, - apiKeyId: { - select: "api_key_id::text", - whereCol: "api_key_id", - whereCast: "", - }, - reportedTimestamp: { - select: "reported_timestamp::text", - whereCol: "reported_timestamp", - whereCast: "::timestamptz", - }, - ingestedTimestamp: { - select: "ingested_timestamp::text", - whereCol: "ingested_timestamp", - whereCast: "::timestamptz", - }, - basicUsageType: { select: null, whereCol: null, whereCast: "" }, - debitAmount: { - select: - "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0))::text", - whereCol: null, - whereCast: "", - aggExpr: - "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0))", - }, - model: { select: "model", whereCol: "model", whereCast: "" }, - inputTokens: { - select: "(metrics->'tokens'->>'input')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'tokens'->>'input')::bigint", - }, - outputTokens: { - select: "(metrics->'tokens'->>'output')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'tokens'->>'output')::bigint", - }, - inputDebitAmount: { - select: "(metrics->'debit_amount'->>'input')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'debit_amount'->>'input')::bigint", - }, - outputDebitAmount: { - select: "(metrics->'debit_amount'->>'output')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'debit_amount'->>'output')::bigint", - }, - inputCacheTokens: { - select: "(metrics->'tokens'->>'input_cache')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'tokens'->>'input_cache')::bigint", - }, - inputCacheDebitAmount: { - select: "(metrics->'debit_amount'->>'input_cache')::text", - whereCol: null, - whereCast: "", - aggExpr: "(metrics->'debit_amount'->>'input_cache')::bigint", - }, - creditAmount: { select: null, whereCol: null, whereCast: "" }, - provider: { select: "provider", whereCol: "provider", whereCast: "" }, - metadata: { select: "metadata::text", whereCol: null, whereCast: "" }, - }, - payment_events: { - eventId: { select: null, whereCol: null, whereCast: "" }, - idempotencyKey: { select: null, whereCol: null, whereCast: "" }, - mode: { select: "mode", whereCol: "mode", whereCast: "" }, - eventType: { select: "'PAYMENT'", whereCol: null, whereCast: "" }, - userId: { select: "user_id::text", whereCol: "user_id", whereCast: "" }, - apiKeyId: { - select: "api_key_id::text", - whereCol: "api_key_id", - whereCast: "", - }, - reportedTimestamp: { - select: "reported_timestamp::text", - whereCol: "reported_timestamp", - whereCast: "::timestamptz", - }, - ingestedTimestamp: { - select: "ingested_timestamp::text", - whereCol: "ingested_timestamp", - whereCast: "::timestamptz", - }, - basicUsageType: { select: null, whereCol: null, whereCast: "" }, - debitAmount: { select: null, whereCol: null, whereCast: "" }, - model: { select: null, whereCol: null, whereCast: "" }, - inputTokens: { select: null, whereCol: null, whereCast: "" }, - outputTokens: { select: null, whereCol: null, whereCast: "" }, - inputDebitAmount: { select: null, whereCol: null, whereCast: "" }, - outputDebitAmount: { select: null, whereCol: null, whereCast: "" }, - inputCacheTokens: { select: null, whereCol: null, whereCast: "" }, - inputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, - creditAmount: { - select: "credit_amount::text", - whereCol: "credit_amount", - whereCast: "::bigint", - }, - provider: { select: null, whereCol: null, whereCast: "" }, - metadata: { select: null, whereCol: null, whereCast: "" }, - }, -}; - -const OUTPUT_FIELDS = Object.keys(PG_FIELDS.basic_usage_events!); - -function buildConditionParts( - group: QueryFilterGroup, - table: EventTableName -): SQL[] { - const parts: SQL[] = []; - - for (const cond of group.conditions) { - if (cond.field === "eventType") continue; - const def = PG_FIELDS[table]?.[cond.field]; - if (!def?.whereCol) continue; - const op = OPERATOR_SQL[cond.operator]; - if (!op) continue; - parts.push( - sql`${sql.raw(def.whereCol)} ${sql.raw(op)} ${cond.value}${sql.raw(def.whereCast)}` - ); - } - - for (const sub of group.groups) { - const subParts = buildConditionParts(sub, table); - if (subParts.length > 0) { - parts.push(sql`(${sql.join(subParts, sql` ${sql.raw(sub.logical)} `)})`); - } - } - - return parts; -} - -function buildWhereClause( - group: QueryFilterGroup, - table: EventTableName -): SQL | undefined { - const parts = buildConditionParts(group, table); - if (parts.length === 0) return undefined; - return sql.join(parts, sql` ${sql.raw(group.logical)} `); -} - -function buildSelectColumns(table: EventTableName): SQL { - const cols: SQL[] = []; - for (const alias of OUTPUT_FIELDS) { - const def = PG_FIELDS[table]?.[alias]; - const expr = def?.select; - if (expr) { - cols.push(sql`${sql.raw(expr)} as ${sql.raw(`"${alias}"`)}`); - } else { - cols.push(sql`NULL as ${sql.raw(`"${alias}"`)}`); - } - } - return sql.join(cols, sql`, `); -} - -export async function handleQueryEvents( - request: QueryRequest, - auth: AuthContext -): Promise { - const tables = getTablesForRequest(request.where); - if (tables.length === 0) { - return { rows: [], total: 0 }; - } - - try { - if (request.aggregation) { - return await handleAggregationQuery(request, tables); - } - return await handleListQuery(request, tables); - } catch (e) { - if ( - e && - typeof e === "object" && - "type" in e && - (e as Record).name === "StorageError" - ) { - throw e; - } - throw StorageError.queryFailed( - "Failed to query Postgres events", - e instanceof Error ? e : new Error(String(e)) - ); - } -} - -async function handleListQuery( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const db = getPostgresDB(); - - const selectExpr = tables.map((t) => buildSelectColumns(t)); - const whereExpr = tables.map((t) => buildWhereClause(request.where, t)); - - const subqueries = tables.map((t, i) => { - const base = sql`SELECT ${selectExpr[i]} FROM ${sql.raw(t)}`; - return whereExpr[i] ? sql`${base} WHERE ${whereExpr[i]}` : base; - }); - - const unionQuery = sql.join(subqueries, sql` UNION ALL `); - - const finalQuery = sql` - ${unionQuery} - ORDER BY "reportedTimestamp" DESC - LIMIT ${request.limit ?? 100} - OFFSET ${request.offset ?? 0} - `; - - const result = await db.execute(finalQuery); - const data = result as unknown as Record[]; - const rows: QueryResultRow[] = data.map(normalizeRow); - - const total = await getTotalCount(request, tables); - - return { rows, total }; -} - -async function handleAggregationQuery( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const db = getPostgresDB(); - const agg = request.aggregation!; - const isSum = agg.type === "SUM"; - - const subqueries = tables.map((t) => { - const cols: SQL[] = []; - - if (request.groupBy) { - const gbField = PG_FIELDS[t]?.[request.groupBy]; - if (gbField?.whereCol) { - cols.push( - sql`${sql.raw(gbField.whereCol)} as ${sql.raw(`"group_value"`)}` - ); - } else if (request.groupBy === "eventType") { - cols.push( - sql`${sql.raw(`'${TABLE_TO_EVENT_TYPE[t]}'`)} as ${sql.raw(`"group_value"`)}` - ); - } else { - cols.push(sql`NULL as ${sql.raw(`"group_value"`)}`); - } - } - - if (isSum && agg.field) { - const def = PG_FIELDS[t]?.[agg.field]; - if (def?.aggExpr) { - cols.push(sql`${sql.raw(def.aggExpr)} as ${sql.raw(`"agg_value"`)}`); - } else if (def?.whereCol) { - cols.push( - sql`${sql.raw(def.whereCol)}::bigint as ${sql.raw(`"agg_value"`)}` - ); - } else { - cols.push(sql`0::bigint as ${sql.raw(`"agg_value"`)}`); - } - } else { - cols.push(sql`1::bigint as ${sql.raw(`"agg_value"`)}`); - } - - const whereClause = buildWhereClause(request.where, t); - const base = sql`SELECT ${sql.join(cols, sql`, `)} FROM ${sql.raw(t)}`; - return whereClause ? sql`${base} WHERE ${whereClause}` : base; - }); - - const unionQuery = sql.join(subqueries, sql` UNION ALL `); - - let outerQuery: SQL; - if (request.groupBy) { - if (isSum) { - outerQuery = sql` - SELECT "group_value", SUM("agg_value")::text as "agg_value" - FROM (${unionQuery}) sub - GROUP BY "group_value" - `; - } else { - outerQuery = sql` - SELECT "group_value", COUNT(*)::text as "agg_value" - FROM (${unionQuery}) sub - GROUP BY "group_value" - `; - } - } else { - if (isSum) { - outerQuery = sql` - SELECT SUM("agg_value")::text as "agg_value" - FROM (${unionQuery}) sub - `; - } else { - outerQuery = sql` - SELECT COUNT(*)::text as "agg_value" - FROM (${unionQuery}) sub - `; - } - } - - const result = await db.execute(outerQuery); - const data = result as unknown as Record[]; - const rows: QueryResultRow[] = data.map((r) => ({ - group_value: r.group_value ?? null, - agg_value: r.agg_value ?? "0", - })); - - return { rows, total: rows.length }; -} - -async function getTotalCount( - request: QueryRequest, - tables: EventTableName[] -): Promise { - const db = getPostgresDB(); - - const subqueries = tables.map((t) => { - const whereClause = buildWhereClause(request.where, t); - const base = sql`SELECT count(*)::int as cnt FROM ${sql.raw(t)}`; - return whereClause ? sql`${base} WHERE ${whereClause}` : base; - }); - - const countQuery = sql` - SELECT coalesce(sum(cnt), 0)::int as total - FROM (${sql.join(subqueries, sql` UNION ALL `)}) sub - `; - - const result = await db.execute(countQuery); - const data = result as unknown as Record[]; - const total = Number(data[0]?.total ?? 0); - return total; -} - -function normalizeRow(row: Record): QueryResultRow { - const result: QueryResultRow = {}; - for (const [key, value] of Object.entries(row)) { - result[key] = value ?? null; - } - return result; -} +export { handleQueryEvents } from "../../common/queryEvents"; diff --git a/src/storage/query/dataQuery.ts b/src/storage/query/dataQuery.ts new file mode 100644 index 0000000..a46cba9 --- /dev/null +++ b/src/storage/query/dataQuery.ts @@ -0,0 +1,266 @@ +import type { SQL } from "drizzle-orm"; +import { + eq, + gt, + gte, + lt, + lte, + ne, + like, + and, + or, + asc, + desc, + count, +} from "drizzle-orm"; +import type { AnyPgColumn, AnyPgTable } from "drizzle-orm/pg-core"; +import { getPostgresDB } from "../db/postgres/db"; +import { + usersTable, + sessionsTable, + tagsTable, + expressionsTable, + metadataTable, +} from "../db/postgres/schema"; +import type { DataQueryRequest } from "../../zod/data"; + +interface FieldDef { + col: AnyPgColumn; + cast: "text" | "integer" | "uuid" | "timestamptz" | "boolean"; +} + +interface TableDef { + tableName: string; + table: AnyPgTable; + fields: Record; +} + +function getCastForColumn(col: AnyPgColumn): FieldDef["cast"] { + const type = col.columnType; + switch (type) { + case "PgBigInt": + case "PgInteger": + return "integer"; + case "PgBoolean": + return "boolean"; + case "PgUUID": + return "uuid"; + case "PgTimestamp": + return "timestamptz"; + default: + return "text"; + } +} + +function fieldDef(col: AnyPgColumn): FieldDef { + return { col, cast: getCastForColumn(col) }; +} + +const TABLE_REGISTRY: Record = { + users: { + tableName: "users", + table: usersTable, + fields: { + id: fieldDef(usersTable.id), + lastBilledTimestamp: fieldDef(usersTable.last_billed_timestamp), + paymentProviderUserId: fieldDef(usersTable.payment_provider_user_id), + mode: fieldDef(usersTable.mode), + }, + }, + sessions: { + tableName: "sessions", + table: sessionsTable, + fields: { + proxy_link_id: fieldDef(sessionsTable.proxy_link_id), + sessionId: fieldDef(sessionsTable.sessionId), + processed: fieldDef(sessionsTable.processed), + userId: fieldDef(sessionsTable.userId), + billedUpto: fieldDef(sessionsTable.billed_upto), + createdAt: fieldDef(sessionsTable.createdAt), + mode: fieldDef(sessionsTable.mode), + }, + }, + tags: { + tableName: "tags", + table: tagsTable, + fields: { + id: fieldDef(tagsTable.id), + key: fieldDef(tagsTable.key), + amount: fieldDef(tagsTable.amount), + }, + }, + expressions: { + tableName: "expressions", + table: expressionsTable, + fields: { + id: fieldDef(expressionsTable.id), + key: fieldDef(expressionsTable.key), + expr: fieldDef(expressionsTable.expr), + }, + }, + metadata: { + tableName: "metadata", + table: metadataTable, + fields: { + id: fieldDef(metadataTable.id), + }, + }, +}; + +function castValue( + value: string, + fieldDef: FieldDef, + fieldName: string +): boolean | number | string { + if (fieldDef.cast === "boolean") { + if (value !== "true" && value !== "false") { + throw new Error( + `Invalid boolean value '${value}' for field '${fieldName}': must be "true" or "false"` + ); + } + return value === "true"; + } + if (fieldDef.cast === "integer") { + const n = Number(value); + if (!Number.isFinite(n) || !Number.isInteger(n)) { + throw new Error( + `Invalid integer value '${value}' for field '${fieldName}': must be a finite integer` + ); + } + return n; + } + return value; +} + +function applyOp( + col: AnyPgColumn, + op: string, + value: string, + fieldDef: FieldDef, + fieldName: string +): SQL { + const casted = castValue(value, fieldDef, fieldName); + switch (op) { + case "EQ": + return eq(col, casted); + case "GT": + return gt(col, casted); + case "GTE": + return gte(col, casted); + case "LT": + return lt(col, casted); + case "LTE": + return lte(col, casted); + case "NEQ": + return ne(col, casted); + case "CONTAINS": + return like(col, `%${value}%`); + default: + return eq(col, casted); + } +} + +function buildWhere( + group: DataQueryRequest["where"], + tableDef: TableDef +): SQL | undefined { + const parts: SQL[] = []; + + for (const condition of group.conditions) { + const fieldDef = tableDef.fields[condition.field]; + if (!fieldDef) { + throw new Error( + `Unknown field '${condition.field}' in table '${tableDef.tableName}'` + ); + } + const clause = applyOp( + fieldDef.col, + condition.operator, + condition.value, + fieldDef, + condition.field + ); + parts.push(clause); + } + + for (const subGroup of group.groups) { + const subWhere = buildWhere(subGroup, tableDef); + if (subWhere) parts.push(subWhere); + } + + if (parts.length === 0) return undefined; + return group.logical === "OR" ? or(...parts) : and(...parts); +} + +function buildSelect(tableDef: TableDef): Record { + const result: Record = {}; + for (const [name, def] of Object.entries(tableDef.fields)) { + result[name] = def.col; + } + return result; +} + +export interface DataQueryResult { + columns: string[]; + rows: Record[]; + total: number; + hasMore: boolean; +} + +export function getDataTable(table: string): TableDef | undefined { + return TABLE_REGISTRY[table]; +} + +export async function executeDataQuery( + config: DataQueryRequest, + tableDef: TableDef +): Promise { + const db = getPostgresDB(); + const whereClause = buildWhere(config.where, tableDef); + const selectCols = buildSelect(tableDef); + const columns = Object.keys(tableDef.fields); + + const orderClauses = config.orderBy.map((o) => { + const fieldDef = tableDef.fields[o.field]; + if (!fieldDef) { + throw new Error( + `Unknown field '${o.field}' for table '${tableDef.tableName}' in order_by` + ); + } + return o.descending ? desc(fieldDef.col) : asc(fieldDef.col); + }); + + const fetchLimit = config.limit + 1; + + const [countResult, rows] = await Promise.all([ + db + .select({ cnt: count() }) + .from(tableDef.table) + .where(whereClause) + .execute(), + db + .select(selectCols) + .from(tableDef.table) + .where(whereClause) + .orderBy(...orderClauses) + .limit(fetchLimit) + .offset(config.offset) + .execute(), + ]); + + const total = Number(countResult[0]?.cnt ?? 0); + const hasMore = rows.length > config.limit; + if (hasMore) { + rows.pop(); + } + + const result: Record[] = rows.map((row) => { + const r: Record = {}; + for (const c of columns) { + r[c] = row[c] ?? ""; + } + return r; + }); + + return { columns, rows: result, total, hasMore }; +} From 16bf9cb918acff17797ca3621af550722227939f Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Fri, 26 Jun 2026 03:43:09 +0530 Subject: [PATCH 16/55] chore: custom error types, derive Zod enum from registry, explicit OUTPUT_FIELDS --- src/storage/adapter/common/fieldRegistry.ts | 24 ++++++++++++++++++++- src/storage/query/dataQuery.ts | 14 ++++++++---- src/zod/data.ts | 9 +------- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/storage/adapter/common/fieldRegistry.ts b/src/storage/adapter/common/fieldRegistry.ts index 9c471e8..4af410d 100644 --- a/src/storage/adapter/common/fieldRegistry.ts +++ b/src/storage/adapter/common/fieldRegistry.ts @@ -358,6 +358,7 @@ export const FIELD_REGISTRY: Record< chParamType: "String", }, }, + // payment_events only exists in Postgres, not ClickHouse payment_events: { eventId: { pgSelect: null, @@ -522,4 +523,25 @@ export const FIELD_REGISTRY: Record< }, }; -export const OUTPUT_FIELDS = Object.keys(FIELD_REGISTRY.basic_usage_events!); +export const OUTPUT_FIELDS = [ + "eventId", + "idempotencyKey", + "mode", + "eventType", + "userId", + "apiKeyId", + "reportedTimestamp", + "ingestedTimestamp", + "basicUsageType", + "debitAmount", + "model", + "inputTokens", + "outputTokens", + "inputDebitAmount", + "outputDebitAmount", + "inputCacheTokens", + "inputCacheDebitAmount", + "creditAmount", + "provider", + "metadata", +]; diff --git a/src/storage/query/dataQuery.ts b/src/storage/query/dataQuery.ts index a46cba9..6a95ee5 100644 --- a/src/storage/query/dataQuery.ts +++ b/src/storage/query/dataQuery.ts @@ -23,6 +23,7 @@ import { metadataTable, } from "../db/postgres/schema"; import type { DataQueryRequest } from "../../zod/data"; +import { EventError } from "../../errors/event"; interface FieldDef { col: AnyPgColumn; @@ -107,6 +108,11 @@ const TABLE_REGISTRY: Record = { }, }; +export const DATA_TABLE_NAMES = Object.keys(TABLE_REGISTRY) as [ + string, + ...string[], +]; + function castValue( value: string, fieldDef: FieldDef, @@ -114,7 +120,7 @@ function castValue( ): boolean | number | string { if (fieldDef.cast === "boolean") { if (value !== "true" && value !== "false") { - throw new Error( + throw EventError.validationFailed( `Invalid boolean value '${value}' for field '${fieldName}': must be "true" or "false"` ); } @@ -123,7 +129,7 @@ function castValue( if (fieldDef.cast === "integer") { const n = Number(value); if (!Number.isFinite(n) || !Number.isInteger(n)) { - throw new Error( + throw EventError.validationFailed( `Invalid integer value '${value}' for field '${fieldName}': must be a finite integer` ); } @@ -169,7 +175,7 @@ function buildWhere( for (const condition of group.conditions) { const fieldDef = tableDef.fields[condition.field]; if (!fieldDef) { - throw new Error( + throw EventError.validationFailed( `Unknown field '${condition.field}' in table '${tableDef.tableName}'` ); } @@ -223,7 +229,7 @@ export async function executeDataQuery( const orderClauses = config.orderBy.map((o) => { const fieldDef = tableDef.fields[o.field]; if (!fieldDef) { - throw new Error( + throw EventError.validationFailed( `Unknown field '${o.field}' for table '${tableDef.tableName}' in order_by` ); } diff --git a/src/zod/data.ts b/src/zod/data.ts index 07c6bcb..22c4feb 100644 --- a/src/zod/data.ts +++ b/src/zod/data.ts @@ -1,14 +1,7 @@ import { z } from "zod"; import { Operator, LogicalOperator } from "../gen/data/v1/data"; import { createFilterGroupSchema } from "./internals"; - -const DATA_TABLE_NAMES = [ - "users", - "sessions", - "tags", - "expressions", - "metadata", -] as const; +import { DATA_TABLE_NAMES } from "../storage/query/dataQuery"; const OPERATOR_MAP = { [Operator.EQ]: "EQ", From 4788da898373ff377bfd8e4df0ae7cf4ebd4f4bb Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 26 Jun 2026 16:26:02 +0530 Subject: [PATCH 17/55] feat(createDashboardKey): made a new route to handle creation of dashboard keys --- src/routes/http/api/apiKeys.ts | 87 +++++++++++++++++++ src/routes/http/api/onboarding.ts | 32 +++++-- src/routes/http/api/registerApiRoutes.ts | 8 ++ .../postgres/handlers/addBasicUsage.ts | 2 +- src/storage/db/postgres/helpers/metadata.ts | 8 -- src/storage/db/postgres/schema.ts | 15 +++- 6 files changed, 133 insertions(+), 19 deletions(-) diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 2dc084d..385912c 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -19,12 +19,14 @@ import { getPostgresDB } from "../../../storage/db/postgres/db"; import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; import { apiKeysTable, + projectsTable, webhookEndpointsTable, } from "../../../storage/db/postgres/schema"; import { eq, and, isNull, ne, sql } from "drizzle-orm"; import type { ApiKeyRole } from "../../../utils/keyFormat"; import { invalidateWebhookEndpointCache } from "../../../interceptors/auth"; import { apiKeyCache } from "../../../utils/apiKeyCache"; +import { authenticateMasterApiKey } from "../../../utils/authenticateMasterApiKey.ts"; const createApiKeySchema = z.object({ name: z.string().min(1, "Name is required").max(255), @@ -289,3 +291,88 @@ export async function handleRevokeApiKey( logger.emit(builder.build()); } } + +export async function handleCreateDashboardKey( + request: FastifyRequest, + reply: FastifyReply +): Promise | { error: string }> { + const builder = createWideEventBuilder( + generateRequestId(), + request.method, + request.url + ); + + try { + const authHeader = request.headers.authorization; + authenticateMasterApiKey(authHeader); + + const body = await request.body; + const params = request.params as { project_id: string }; + + const appUrl = process.env.APP_URL; + if (!appUrl) { + builder.setError(500, { + type: "ConfigError", + message: "APP_URL environment variable is not set", + }); + reply.code(500); + return { error: "APP_URL environment variable is not set" }; + } + + const projectId = params.project_id; + + const existing = await getPostgresDB() + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(eq(projectsTable.id, projectId)) + .limit(1); + + if (existing.length === 0) { + builder.setError(404, { + type: "NotFound", + message: `Project with name '${projectId}' doesn't exist`, + }); + reply.code(404); + return { + error: `Project with name '${projectId}' doesn't exist`, + }; + } + + const dashboardKey = generateAPIKey("dashboard"); + const dashboardKeyHash = hashAPIKey(dashboardKey); + const expiresAt = DateTime.utc().plus({ years: 10 }).toISO(); + const db = getPostgresDB(); + + await db.insert(apiKeysTable).values({ + projectId, + name: "Default dashboard key", + key: dashboardKeyHash, + role: "dashboard", + expiresAt, + }); + + builder.setSuccess(201); + reply.code(201); + return { projectId, apiKey: dashboardKey }; + } catch (error) { + if (error instanceof AuthError) { + builder.setError(401, { type: error.type, message: error.message }); + reply.code(401); + return { error: error.message }; + } + + if (error instanceof ZodError) { + const issues = error.issues + .map((issue) => `${issue.path.join(".")}: ${issue.message}`) + .join("; "); + builder.setError(400, { type: "ValidationError", message: issues }); + reply.code(400); + return { error: issues }; + } + + const err = error instanceof Error ? error : new Error(String(error)); + builder.setError(500, { type: "InternalError", message: err.message }); + reply.code(500); + return { error: "Internal server error" }; + } +} diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index b347638..00e92db 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -10,6 +10,7 @@ import { } from "../../../context/requestContext.ts"; import { logger } from "../../../errors/logger.ts"; import { AuthError } from "../../../errors/auth"; +import { StorageError } from "../../../errors/storage"; import { authenticateMasterApiKey } from "../../../utils/authenticateMasterApiKey.ts"; import { authenticateHttpApiKey } from "../../../utils/authenticateHttpApiKey.ts"; import { generateAPIKey } from "../../../utils/generateAPIKey"; @@ -21,10 +22,7 @@ import { apiKeysTable, metadataTable, } from "../../../storage/db/postgres/schema"; -import { - getMetadata, - getAnyMetadata, -} from "../../../storage/db/postgres/helpers/metadata"; +import { getMetadata } from "../../../storage/db/postgres/helpers/metadata"; import { removeClient } from "../../gRPC/payment/paymentProvider.ts"; import { DateTime } from "luxon"; import { eq } from "drizzle-orm"; @@ -199,6 +197,18 @@ export async function handleOnboarding( extra: { context: "onboarding route handler" }, }); + if ( + error instanceof StorageError && + error.type === "CONSTRAINT_VIOLATION" + ) { + builder.setError(409, { + type: "ConflictError", + message: "A project with this name already exists", + }); + reply.code(409); + return {}; + } + if (error instanceof AuthError) { builder.setError(401, { type: error.type, @@ -268,9 +278,17 @@ export async function handleGetConfig( projectId = auth.projectId; } - const metadata = isMasterKey - ? await getAnyMetadata() - : await getMetadata(projectId!); + if (isMasterKey) { + const query = request.query as Record; + if (!query.projectId) { + throw AuthError.permissionDenied( + "projectId is required when using master key" + ); + } + projectId = query.projectId; + } + + const metadata = await getMetadata(projectId!); if (!metadata) { builder.setSuccess(200); diff --git a/src/routes/http/api/registerApiRoutes.ts b/src/routes/http/api/registerApiRoutes.ts index e17b649..557aa15 100644 --- a/src/routes/http/api/registerApiRoutes.ts +++ b/src/routes/http/api/registerApiRoutes.ts @@ -15,6 +15,7 @@ import { } from "./webhookEndpoints.ts"; import { handleCreateApiKey, + handleCreateDashboardKey, handleListApiKeys, handleRevokeApiKey, } from "./apiKeys.ts"; @@ -104,6 +105,13 @@ export async function registerApiRoutes( } ); + server.post( + "/api/v1/create-dashboard-key/:project-id", + async (request: FastifyRequest, reply: FastifyReply) => { + return handleCreateDashboardKey(request, reply); + } + ); + // Webhook endpoints server.post( "/api/v1/internals/webhook-endpoint", diff --git a/src/storage/adapter/postgres/handlers/addBasicUsage.ts b/src/storage/adapter/postgres/handlers/addBasicUsage.ts index 0de79d3..071d6c9 100644 --- a/src/storage/adapter/postgres/handlers/addBasicUsage.ts +++ b/src/storage/adapter/postgres/handlers/addBasicUsage.ts @@ -28,7 +28,7 @@ export async function handleAddBasicUsage( connectionObject, "storing BASIC_USAGE event", async (txn) => { - const ensurePromise = ensureUserExists( + const ensurePromise = await ensureUserExists( auth.projectId, event_data.userId, txn diff --git a/src/storage/db/postgres/helpers/metadata.ts b/src/storage/db/postgres/helpers/metadata.ts index d14389b..f696125 100644 --- a/src/storage/db/postgres/helpers/metadata.ts +++ b/src/storage/db/postgres/helpers/metadata.ts @@ -14,11 +14,3 @@ export async function getMetadata( .limit(1); return metadata; } - -export async function getAnyMetadata(): Promise< - typeof metadataTable.$inferSelect | undefined -> { - const db = getPostgresDB(); - const [metadata] = await db.select().from(metadataTable).limit(1); - return metadata; -} diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index 510c9ff..a6b4f21 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -8,6 +8,7 @@ import { text, boolean, jsonb, + unique, uniqueIndex, primaryKey, foreignKey, @@ -18,7 +19,7 @@ import { type Metrics } from "../../../zod/metrics"; export const projectsTable = pgTable("projects", { id: uuid("id").primaryKey().defaultRandom(), - name: text("name").notNull(), + name: text("name").notNull().unique(), createdAt: timestamp("created_at", { withTimezone: true, mode: "string", @@ -170,7 +171,7 @@ export const basicUsageEventsTable = pgTable( .references(() => projectsTable.id) .notNull(), eventId: uuid("event_id").notNull(), - idempotencyKey: text("idempotency_key").notNull().unique(), + idempotencyKey: text("idempotency_key").notNull(), reportedTimestamp: timestamp("reported_timestamp", { withTimezone: true, mode: "string", @@ -195,6 +196,10 @@ export const basicUsageEventsTable = pgTable( columns: [table.projectId, table.userId], foreignColumns: [usersTable.projectId, usersTable.id], }), + uniqueIdempotency: unique("basic_usage_events_idempotency_key_idx").on( + table.projectId, + table.idempotencyKey + ), }) ); @@ -281,7 +286,7 @@ export const aiTokenUsageEventsTable = pgTable( .references(() => projectsTable.id) .notNull(), eventId: uuid("event_id").notNull(), - idempotencyKey: text("idempotency_key").notNull().unique(), + idempotencyKey: text("idempotency_key").notNull(), reportedTimestamp: timestamp("reported_timestamp", { withTimezone: true, mode: "string", @@ -307,6 +312,10 @@ export const aiTokenUsageEventsTable = pgTable( columns: [table.projectId, table.userId], foreignColumns: [usersTable.projectId, usersTable.id], }), + uniqueIdempotency: unique("ai_token_usage_events_idempotency_key_idx").on( + table.projectId, + table.idempotencyKey + ), }) ); From 94184ac3d4fb85e544474dda8564844d6c54a57e Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 26 Jun 2026 16:48:22 +0530 Subject: [PATCH 18/55] fix(resgisterApiRoutes.ts): fixed params --- src/routes/http/api/apiKeys.ts | 2 +- src/routes/http/api/registerApiRoutes.ts | 2 +- src/utils/parseExpr.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 385912c..ce55a05 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -307,7 +307,7 @@ export async function handleCreateDashboardKey( authenticateMasterApiKey(authHeader); const body = await request.body; - const params = request.params as { project_id: string }; + const params = request.params as { projectId: string }; const appUrl = process.env.APP_URL; if (!appUrl) { diff --git a/src/routes/http/api/registerApiRoutes.ts b/src/routes/http/api/registerApiRoutes.ts index 557aa15..6626b93 100644 --- a/src/routes/http/api/registerApiRoutes.ts +++ b/src/routes/http/api/registerApiRoutes.ts @@ -106,7 +106,7 @@ export async function registerApiRoutes( ); server.post( - "/api/v1/create-dashboard-key/:project-id", + "/api/v1/create-dashboard-key/:projectId", async (request: FastifyRequest, reply: FastifyReply) => { return handleCreateDashboardKey(request, reply); } diff --git a/src/utils/parseExpr.ts b/src/utils/parseExpr.ts index 8a5559a..fe2841b 100644 --- a/src/utils/parseExpr.ts +++ b/src/utils/parseExpr.ts @@ -292,10 +292,10 @@ function resolveTokenPlaceholders( context: EvalTokenContext ): string { return exprString - .replace(/inputTokens\(\)/g, String(context.inputTokens ?? 0)) - .replace(/outputTokens\(\)/g, String(context.outputTokens ?? 0)) - .replace(/inputCacheTokens\(\)/g, String(context.inputCacheTokens ?? 0)) - .replace(/outputCacheTokens\(\)/g, String(context.outputCacheTokens ?? 0)); + .replace(/inputTokens\(\)/gi, String(context.inputTokens ?? 0)) + .replace(/outputTokens\(\)/gi, String(context.outputTokens ?? 0)) + .replace(/inputCacheTokens\(\)/gi, String(context.inputCacheTokens ?? 0)) + .replace(/outputCacheTokens\(\)/gi, String(context.outputCacheTokens ?? 0)); } /** From e9348f1b44517c690ad12ad7aae62cbe1725ca9b Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 26 Jun 2026 16:53:01 +0530 Subject: [PATCH 19/55] fix(apiKeys.ts): fixed linting errors --- src/routes/http/api/apiKeys.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index ce55a05..42abd4f 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -306,7 +306,6 @@ export async function handleCreateDashboardKey( const authHeader = request.headers.authorization; authenticateMasterApiKey(authHeader); - const body = await request.body; const params = request.params as { projectId: string }; const appUrl = process.env.APP_URL; @@ -319,22 +318,22 @@ export async function handleCreateDashboardKey( return { error: "APP_URL environment variable is not set" }; } - const projectId = params.project_id; + const project_id = params.projectId; const existing = await getPostgresDB() .select({ id: projectsTable.id }) .from(projectsTable) - .where(eq(projectsTable.id, projectId)) + .where(eq(projectsTable.id, project_id)) .limit(1); if (existing.length === 0) { builder.setError(404, { type: "NotFound", - message: `Project with name '${projectId}' doesn't exist`, + message: `Project with name '${project_id}' doesn't exist`, }); reply.code(404); return { - error: `Project with name '${projectId}' doesn't exist`, + error: `Project with name '${project_id}' doesn't exist`, }; } @@ -344,7 +343,7 @@ export async function handleCreateDashboardKey( const db = getPostgresDB(); await db.insert(apiKeysTable).values({ - projectId, + projectId: project_id, name: "Default dashboard key", key: dashboardKeyHash, role: "dashboard", @@ -353,7 +352,7 @@ export async function handleCreateDashboardKey( builder.setSuccess(201); reply.code(201); - return { projectId, apiKey: dashboardKey }; + return { project_id, apiKey: dashboardKey }; } catch (error) { if (error instanceof AuthError) { builder.setError(401, { type: error.type, message: error.message }); From 06e61d97c6e2c31caa750913ebefd4da27633199 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 27 Jun 2026 16:29:24 +0530 Subject: [PATCH 20/55] fix(apiKeys): fixed greptile issues --- src/interface/storage/Storage.ts | 2 + src/routes/http/api/apiKeys.ts | 51 ++++++++++++------- .../handlers/priceRequestAiTokenUsage.ts | 2 +- .../clickhouse/handlers/queryEvents.ts | 17 ++++++- .../adapter/postgres/handlers/queryEvents.ts | 16 +++++- 5 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/interface/storage/Storage.ts b/src/interface/storage/Storage.ts index a145180..c96f21e 100644 --- a/src/interface/storage/Storage.ts +++ b/src/interface/storage/Storage.ts @@ -20,6 +20,8 @@ export const QUERY_FIELD_NAMES = [ "outputDebitAmount", "inputCacheTokens", "inputCacheDebitAmount", + "outputCacheTokens", + "outputCacheDebitAmount", "creditAmount", "provider", "metadata", diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 42abd4f..91476da 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -308,16 +308,6 @@ export async function handleCreateDashboardKey( const params = request.params as { projectId: string }; - const appUrl = process.env.APP_URL; - if (!appUrl) { - builder.setError(500, { - type: "ConfigError", - message: "APP_URL environment variable is not set", - }); - reply.code(500); - return { error: "APP_URL environment variable is not set" }; - } - const project_id = params.projectId; const existing = await getPostgresDB() @@ -339,16 +329,41 @@ export async function handleCreateDashboardKey( const dashboardKey = generateAPIKey("dashboard"); const dashboardKeyHash = hashAPIKey(dashboardKey); - const expiresAt = DateTime.utc().plus({ years: 10 }).toISO(); + const expiresAt = DateTime.utc().plus({ years: 10 }).toISO()!; const db = getPostgresDB(); - await db.insert(apiKeysTable).values({ - projectId: project_id, - name: "Default dashboard key", - key: dashboardKeyHash, - role: "dashboard", - expiresAt, - }); + const existingKey = await db + .select({ id: apiKeysTable.id, key: apiKeysTable.key }) + .from(apiKeysTable) + .where( + and( + eq(apiKeysTable.projectId, project_id), + eq(apiKeysTable.name, "Default dashboard key"), + eq(apiKeysTable.revoked, false) + ) + ) + .limit(1); + + const existingDashboardKey = existingKey[0]; + if (existingDashboardKey) { + await db + .update(apiKeysTable) + .set({ + key: dashboardKeyHash, + expiresAt, + }) + .where(eq(apiKeysTable.id, existingDashboardKey.id)); + + apiKeyCache.delete(existingDashboardKey.key); + } else { + await db.insert(apiKeysTable).values({ + projectId: project_id, + name: "Default dashboard key", + key: dashboardKeyHash, + role: "dashboard", + expiresAt, + }); + } builder.setSuccess(201); reply.code(201); diff --git a/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts b/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts index 7cd4f13..d07795b 100644 --- a/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/priceRequestAiTokenUsage.ts @@ -4,7 +4,7 @@ import type { AuthContext } from "../../../../context/auth"; import { runClickHousePriceQuery } from "../utils"; const VALUE_EXPR = - "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')"; + "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')"; const BASE_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; const WINDOW_QUERY = `SELECT sum(${VALUE_EXPR}) as total FROM ai_token_usage_events WHERE project_id = {projectId:String} AND user_id = {userId:String} AND mode = {mode:String} AND reported_timestamp > {lastBilled:DateTime64(3, 'UTC')} AND reported_timestamp < {before:DateTime64(3, 'UTC')}`; diff --git a/src/storage/adapter/clickhouse/handlers/queryEvents.ts b/src/storage/adapter/clickhouse/handlers/queryEvents.ts index abf3fa9..55b19ee 100644 --- a/src/storage/adapter/clickhouse/handlers/queryEvents.ts +++ b/src/storage/adapter/clickhouse/handlers/queryEvents.ts @@ -74,10 +74,10 @@ const CH_FIELDS: Partial< basicUsageType: { select: null, where: null }, debitAmount: { select: - "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output'))", + "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output_cache') + JSONExtractInt(metrics, 'debit_amount', 'output'))", where: null, aggExpr: - "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')", + "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')", }, model: { select: "model", where: "model" }, inputTokens: { @@ -111,6 +111,17 @@ const CH_FIELDS: Partial< where: null, aggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input_cache')", }, + outputCacheTokens: { + select: "toString(JSONExtractInt(metrics, 'tokens', 'output_cache'))", + where: null, + aggExpr: "JSONExtractInt(metrics, 'tokens', 'output_cache')", + }, + outputCacheDebitAmount: { + select: + "toString(JSONExtractInt(metrics, 'debit_amount', 'output_cache'))", + where: null, + aggExpr: "JSONExtractInt(metrics, 'debit_amount', 'output_cache')", + }, creditAmount: { select: null, where: null }, provider: { select: "provider", where: "provider" }, metadata: { select: "toString(metadata)", where: null }, @@ -133,6 +144,8 @@ const CH_PARAM_TYPE: Record = { outputDebitAmount: "Int64", inputCacheTokens: "Int64", inputCacheDebitAmount: "Int64", + outputCacheTokens: "Int64", + outputCacheDebitAmount: "Int64", creditAmount: "Int64", provider: "String", metadata: "String", diff --git a/src/storage/adapter/postgres/handlers/queryEvents.ts b/src/storage/adapter/postgres/handlers/queryEvents.ts index 3bcf970..308effd 100644 --- a/src/storage/adapter/postgres/handlers/queryEvents.ts +++ b/src/storage/adapter/postgres/handlers/queryEvents.ts @@ -105,11 +105,11 @@ const PG_FIELDS: PGFieldRegistry = { basicUsageType: { select: null, whereCol: null, whereCast: "" }, debitAmount: { select: - "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0))::text", + "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0))::text", whereCol: null, whereCast: "", aggExpr: - "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0))", + "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0))", }, model: { select: "model", whereCol: "model", whereCast: "" }, inputTokens: { @@ -148,6 +148,18 @@ const PG_FIELDS: PGFieldRegistry = { whereCast: "", aggExpr: "(metrics->'debit_amount'->>'input_cache')::bigint", }, + outputCacheTokens: { + select: "(metrics->'tokens'->>'output_cache')::text", + whereCol: null, + whereCast: "", + aggExpr: "(metrics->'tokens'->>'output_cache')::bigint", + }, + outputCacheDebitAmount: { + select: "(metrics->'debit_amount'->>'output_cache')::text", + whereCol: null, + whereCast: "", + aggExpr: "(metrics->'debit_amount'->>'output_cache')::bigint", + }, creditAmount: { select: null, whereCol: null, whereCast: "" }, provider: { select: "provider", whereCol: "provider", whereCast: "" }, metadata: { select: "metadata::text", whereCol: null, whereCast: "" }, From 1f4553be52e6e2ce7f3387eafa493997ff629b3d Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 27 Jun 2026 16:31:40 +0530 Subject: [PATCH 21/55] fix(queryEvents): fixed greptile issues --- src/storage/adapter/clickhouse/handlers/queryEvents.ts | 2 ++ src/storage/adapter/postgres/handlers/queryEvents.ts | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/storage/adapter/clickhouse/handlers/queryEvents.ts b/src/storage/adapter/clickhouse/handlers/queryEvents.ts index 55b19ee..a7ba35d 100644 --- a/src/storage/adapter/clickhouse/handlers/queryEvents.ts +++ b/src/storage/adapter/clickhouse/handlers/queryEvents.ts @@ -52,6 +52,8 @@ const CH_FIELDS: Partial< outputDebitAmount: { select: null, where: null }, inputCacheTokens: { select: null, where: null }, inputCacheDebitAmount: { select: null, where: null }, + outputCacheTokens: { select: null, where: null }, + outputCacheDebitAmount: { select: null, where: null }, creditAmount: { select: null, where: null }, provider: { select: null, where: null }, metadata: { select: null, where: null }, diff --git a/src/storage/adapter/postgres/handlers/queryEvents.ts b/src/storage/adapter/postgres/handlers/queryEvents.ts index 308effd..b54a23a 100644 --- a/src/storage/adapter/postgres/handlers/queryEvents.ts +++ b/src/storage/adapter/postgres/handlers/queryEvents.ts @@ -69,6 +69,8 @@ const PG_FIELDS: PGFieldRegistry = { outputDebitAmount: { select: null, whereCol: null, whereCast: "" }, inputCacheTokens: { select: null, whereCol: null, whereCast: "" }, inputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, + outputCacheTokens: { select: null, whereCol: null, whereCast: "" }, + outputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, creditAmount: { select: null, whereCol: null, whereCast: "" }, provider: { select: null, whereCol: null, whereCast: "" }, metadata: { select: "metadata::text", whereCol: null, whereCast: "" }, @@ -194,6 +196,8 @@ const PG_FIELDS: PGFieldRegistry = { outputDebitAmount: { select: null, whereCol: null, whereCast: "" }, inputCacheTokens: { select: null, whereCol: null, whereCast: "" }, inputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, + outputCacheTokens: { select: null, whereCol: null, whereCast: "" }, + outputCacheDebitAmount: { select: null, whereCol: null, whereCast: "" }, creditAmount: { select: "credit_amount::text", whereCol: "credit_amount", From e3fee511f5b497586bc0652908f0aecc55a23d93 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 27 Jun 2026 17:02:52 +0530 Subject: [PATCH 22/55] fix(db): constrain config keys --- src/routes/http/api/webhookDeliveries.ts | 6 ++ .../clickhouse/handlers/addAiTokenUsage.ts | 12 ++-- .../postgres/handlers/addAiTokenUsage.ts | 8 ++- .../db/postgres/helpers/expressions.ts | 29 +++------ src/storage/db/postgres/helpers/tags.ts | 29 +++------ src/storage/db/postgres/schema.ts | 64 ++++++++++++------- 6 files changed, 74 insertions(+), 74 deletions(-) diff --git a/src/routes/http/api/webhookDeliveries.ts b/src/routes/http/api/webhookDeliveries.ts index 0bca7fb..e9a5963 100644 --- a/src/routes/http/api/webhookDeliveries.ts +++ b/src/routes/http/api/webhookDeliveries.ts @@ -40,6 +40,12 @@ export async function handleListDeliveries( try { const auth = await authenticateHttpApiKey(request.headers.authorization); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can read webhook deliveries" + ); + } + const query = listDeliveriesQuerySchema.parse(request.query); const db = getPostgresDB(); diff --git a/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts b/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts index 1061e1d..132b914 100644 --- a/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts +++ b/src/storage/adapter/clickhouse/handlers/addAiTokenUsage.ts @@ -165,13 +165,15 @@ export async function handleAddAiTokenUsage( validateAiTokenEvent(event_data); } - const firstEvent = events[0]; - if (firstEvent) { - await ensureUserExists(auth.projectId, firstEvent.userId); - } - const aggregatedEvents = aggregateAiTokenEvents(events); + const distinctUserIds = Array.from( + new Set(aggregatedEvents.map((e) => e.userId)) + ); + for (const userId of distinctUserIds) { + await ensureUserExists(auth.projectId, userId); + } + const firstId = crypto.randomUUID(); const now = toClickHouseDateTime(DateTime.utc()); const values = buildAiTokenInsertRows(aggregatedEvents, auth, firstId, now); diff --git a/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts b/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts index 1117fa2..bf40b51 100644 --- a/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts +++ b/src/storage/adapter/postgres/handlers/addAiTokenUsage.ts @@ -160,14 +160,16 @@ export async function handleAddAiTokenUsage( } const aggregatedEvents = await aggregateAiTokenEvents(events); - const firstEvent = events[0]; return await executeInTransaction( connectionObject, `storing ${events.length} AI_TOKEN_USAGE event(s)`, async (txn) => { - if (firstEvent) { - await ensureUserExists(auth.projectId, firstEvent.userId, txn); + const distinctUserIds = Array.from( + new Set(aggregatedEvents.map((e) => e.userId)) + ); + for (const userId of distinctUserIds) { + await ensureUserExists(auth.projectId, userId, txn); } try { diff --git a/src/storage/db/postgres/helpers/expressions.ts b/src/storage/db/postgres/helpers/expressions.ts index 41639d6..0d59c82 100644 --- a/src/storage/db/postgres/helpers/expressions.ts +++ b/src/storage/db/postgres/helpers/expressions.ts @@ -62,27 +62,14 @@ export async function createExpression( const db = getPostgresDB(); try { - const existing = await db - .select({ id: expressionsTable.id }) - .from(expressionsTable) - .where( - and( - eq(expressionsTable.projectId, projectId), - eq(expressionsTable.key, key), - isNull(expressionsTable.deletedAt) - ) - ) - .limit(1); - - if (existing[0]) { - await db - .update(expressionsTable) - .set({ expr }) - .where(eq(expressionsTable.id, existing[0].id)); - return; - } - - await db.insert(expressionsTable).values({ projectId, key, expr }); + await db + .insert(expressionsTable) + .values({ projectId, key, expr }) + .onConflictDoUpdate({ + target: [expressionsTable.projectId, expressionsTable.key], + targetWhere: isNull(expressionsTable.deletedAt), + set: { expr }, + }); } catch (e) { throw StorageError.insertFailed( `Failed to upsert expression '${key}'`, diff --git a/src/storage/db/postgres/helpers/tags.ts b/src/storage/db/postgres/helpers/tags.ts index e20d072..6d42143 100644 --- a/src/storage/db/postgres/helpers/tags.ts +++ b/src/storage/db/postgres/helpers/tags.ts @@ -34,28 +34,15 @@ export async function createTag( const db = getPostgresDB(); try { - const existing = await db - .select({ id: tagsTable.id }) - .from(tagsTable) - .where( - and( - eq(tagsTable.projectId, projectId), - eq(tagsTable.key, key), - isNull(tagsTable.deletedAt) - ) - ) - .limit(1); - - if (existing[0]) { - await db - .update(tagsTable) - .set({ amount }) - .where(eq(tagsTable.id, existing[0].id)); - tagCache.delete(`${projectId}:${key}`); - return; - } + await db + .insert(tagsTable) + .values({ projectId, key, amount }) + .onConflictDoUpdate({ + target: [tagsTable.projectId, tagsTable.key], + targetWhere: isNull(tagsTable.deletedAt), + set: { amount }, + }); - await db.insert(tagsTable).values({ projectId, key, amount }); tagCache.delete(`${projectId}:${key}`); } catch (e) { throw StorageError.insertFailed( diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index a6b4f21..b726540 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -340,18 +340,26 @@ export const aiTokenUsageEventsRelation = relations( }) ); -export const tagsTable = pgTable("tags", { - id: uuid("id").primaryKey().defaultRandom(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - key: text("key").notNull(), - amount: integer("amount").notNull(), - deletedAt: timestamp("deleted_at", { - withTimezone: true, - mode: "string", - }), -}); +export const tagsTable = pgTable( + "tags", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + key: text("key").notNull(), + amount: integer("amount").notNull(), + deletedAt: timestamp("deleted_at", { + withTimezone: true, + mode: "string", + }), + }, + (table) => ({ + uniqueActiveTag: uniqueIndex("unique_active_tag") + .on(table.projectId, table.key) + .where(sql`${table.deletedAt} IS NULL`), + }) +); export const tagsRelation = relations(tagsTable, ({ one }) => ({ project: one(projectsTable, { @@ -392,18 +400,26 @@ export const metadataRelation = relations(metadataTable, ({ one }) => ({ }), })); -export const expressionsTable = pgTable("expressions", { - id: uuid("id").primaryKey().defaultRandom(), - projectId: uuid("project_id") - .references(() => projectsTable.id) - .notNull(), - key: text("key").notNull(), - expr: text("expr").notNull(), - deletedAt: timestamp("deleted_at", { - withTimezone: true, - mode: "string", - }), -}); +export const expressionsTable = pgTable( + "expressions", + { + id: uuid("id").primaryKey().defaultRandom(), + projectId: uuid("project_id") + .references(() => projectsTable.id) + .notNull(), + key: text("key").notNull(), + expr: text("expr").notNull(), + deletedAt: timestamp("deleted_at", { + withTimezone: true, + mode: "string", + }), + }, + (table) => ({ + uniqueActiveExpr: uniqueIndex("unique_active_expr") + .on(table.projectId, table.key) + .where(sql`${table.deletedAt} IS NULL`), + }) +); export const expressionsRelation = relations(expressionsTable, ({ one }) => ({ project: one(projectsTable, { From 1550d8d97f6315f3eaa230e2e076bcbf7f808a21 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sun, 28 Jun 2026 11:39:53 +0530 Subject: [PATCH 23/55] feat(projects): added project create, update, delete routes and func --- src/routes/http/api/projects.ts | 227 +++++++++++++++++++++++ src/routes/http/api/registerApiRoutes.ts | 27 +++ 2 files changed, 254 insertions(+) create mode 100644 src/routes/http/api/projects.ts diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts new file mode 100644 index 0000000..7498c0a --- /dev/null +++ b/src/routes/http/api/projects.ts @@ -0,0 +1,227 @@ +import type { FastifyRequest, FastifyReply } from "fastify"; +import * as Sentry from "@sentry/bun"; +import { + createWideEventBuilder, + generateRequestId, +} from "../../../context/requestContext.ts"; +import { logger } from "../../../errors/logger.ts"; +import { AuthError } from "../../../errors/auth"; +import { authenticateMasterApiKey } from "../../../utils/authenticateMasterApiKey.ts"; +import { getPostgresDB } from "../../../storage/db/postgres/db"; +import { + projectsTable, + metadataTable, + apiKeysTable, + usersTable, + sessionsTable, + basicUsageEventsTable, + aiTokenUsageEventsTable, + paymentEventsTable, +} from "../../../storage/db/postgres/schema"; +import { eq, inArray } from "drizzle-orm"; +import { encrypt, decrypt } from "../../../utils/encryptMetadata.ts"; +import { executeInTransaction } from "../../../storage/adapter/postgres/handlers/addEventUtils"; + +function maskApiKey(key: string | null | undefined): string | null { + if (!key) return null; + if (key.length <= 16) return "****"; + return key.slice(0, 4) + "****" + key.slice(-4); +} + +export async function handleListProjects( + request: FastifyRequest, + reply: FastifyReply +): Promise> { + const builder = createWideEventBuilder( + generateRequestId(), + request.method, + request.url + ); + + try { + authenticateMasterApiKey(request.headers.authorization); + + const body = (await request.body) as { projectIds: string[] }; + if ( + !body || + !Array.isArray(body.projectIds) || + body.projectIds.length === 0 + ) { + builder.setSuccess(200); + reply.code(200); + return { projects: [] }; + } + + const db = getPostgresDB(); + const projects = await db + .select({ + id: projectsTable.id, + name: projectsTable.name, + dodo_live_api_key: metadataTable.dodo_live_api_key, + dodo_test_api_key: metadataTable.dodo_test_api_key, + dodo_live_product_id: metadataTable.dodo_live_product_id, + dodo_test_product_id: metadataTable.dodo_test_product_id, + currency: metadataTable.currency, + redirect_url: metadataTable.redirect_url, + }) + .from(projectsTable) + .innerJoin(metadataTable, eq(projectsTable.id, metadataTable.projectId)) + .where(inArray(projectsTable.id, body.projectIds)); + + builder.setSuccess(200); + reply.code(200); + + return { + projects: projects.map((p) => ({ + id: p.id, + name: p.name, + dodoLiveApiKey: maskApiKey(decrypt(p.dodo_live_api_key)), + dodoTestApiKey: maskApiKey(decrypt(p.dodo_test_api_key)), + dodoLiveProductId: p.dodo_live_product_id, + dodoTestProductId: p.dodo_test_product_id, + currency: p.currency, + redirectUrl: p.redirect_url, + })), + }; + } catch (error) { + Sentry.captureException(error, { + extra: { context: "handleListProjects" }, + }); + if (error instanceof AuthError) { + builder.setError(401, { type: error.type, message: error.message }); + reply.code(401); + return {}; + } + const err = error instanceof Error ? error : new Error(String(error)); + builder.setError(500, { type: "InternalError", message: err.message }); + reply.code(500); + return {}; + } finally { + logger.emit(builder.build()); + } +} + +export async function handleUpdateProject( + request: FastifyRequest, + reply: FastifyReply +): Promise> { + const builder = createWideEventBuilder( + generateRequestId(), + request.method, + request.url + ); + + try { + authenticateMasterApiKey(request.headers.authorization); + + const { projectId } = request.params as { projectId: string }; + const body = (await request.body) as { + name?: string; + dodoLiveProductId?: string; + dodoTestProductId?: string; + currency?: string; + redirectUrl?: string; + dodoLiveApiKey?: string; + dodoTestApiKey?: string; + }; + + const db = getPostgresDB(); + await executeInTransaction(db, "update project", async (txn) => { + if (body.name) { + await txn + .update(projectsTable) + .set({ name: body.name }) + .where(eq(projectsTable.id, projectId)); + } + + const metaUpdates: any = {}; + if (body.dodoLiveProductId) + metaUpdates.dodo_live_product_id = body.dodoLiveProductId; + if (body.dodoTestProductId) + metaUpdates.dodo_test_product_id = body.dodoTestProductId; + if (body.currency) metaUpdates.currency = body.currency; + if (body.redirectUrl) metaUpdates.redirect_url = body.redirectUrl; + + if (body.dodoLiveApiKey && !body.dodoLiveApiKey.includes("****")) { + metaUpdates.dodo_live_api_key = encrypt(body.dodoLiveApiKey); + } + if (body.dodoTestApiKey && !body.dodoTestApiKey.includes("****")) { + metaUpdates.dodo_test_api_key = encrypt(body.dodoTestApiKey); + } + + if (Object.keys(metaUpdates).length > 0) { + await txn + .update(metadataTable) + .set(metaUpdates) + .where(eq(metadataTable.projectId, projectId)); + } + }); + + builder.setSuccess(200); + reply.code(200); + return { success: true }; + } catch (error) { + Sentry.captureException(error, { + extra: { context: "handleUpdateProject" }, + }); + if (error instanceof AuthError) { + builder.setError(401, { type: error.type, message: error.message }); + reply.code(401); + return {}; + } + const err = error instanceof Error ? error : new Error(String(error)); + builder.setError(500, { type: "InternalError", message: err.message }); + reply.code(500); + return {}; + } finally { + logger.emit(builder.build()); + } +} + +export async function handleDeleteProject( + request: FastifyRequest, + reply: FastifyReply +): Promise> { + const builder = createWideEventBuilder( + generateRequestId(), + request.method, + request.url + ); + + try { + authenticateMasterApiKey(request.headers.authorization); + + const { projectId } = request.params as { projectId: string }; + const db = getPostgresDB(); + + // Manual cascading deletes + await executeInTransaction(db, "delete project", async (txn) => { + await txn + .delete(apiKeysTable) + .where(eq(apiKeysTable.projectId, projectId)); + await txn + .delete(metadataTable) + .where(eq(metadataTable.projectId, projectId)); + await txn.delete(projectsTable).where(eq(projectsTable.id, projectId)); + }); + + builder.setSuccess(200); + reply.code(200); + return { success: true }; + } catch (error) { + Sentry.captureException(error, { + extra: { context: "handleDeleteProject" }, + }); + if (error instanceof AuthError) { + builder.setError(401, { type: error.type, message: error.message }); + reply.code(401); + return {}; + } + const err = error instanceof Error ? error : new Error(String(error)); + builder.setError(500, { type: "InternalError", message: err.message }); + reply.code(500); + return {}; + } finally { + logger.emit(builder.build()); + } +} diff --git a/src/routes/http/api/registerApiRoutes.ts b/src/routes/http/api/registerApiRoutes.ts index 6626b93..18185f2 100644 --- a/src/routes/http/api/registerApiRoutes.ts +++ b/src/routes/http/api/registerApiRoutes.ts @@ -1,5 +1,10 @@ import type { FastifyRequest, FastifyReply } from "fastify"; import { handleOnboarding, handleGetConfig } from "./onboarding.ts"; +import { + handleListProjects, + handleUpdateProject, + handleDeleteProject, +} from "./projects.ts"; import { handleListTags, handleCreateTag, handleDeleteTag } from "./tags.ts"; import { handleListExpressions, @@ -39,6 +44,28 @@ export async function registerApiRoutes( } ); + // Projects + server.post( + "/api/v1/internals/projects/list", + async (request: FastifyRequest, reply: FastifyReply) => { + return handleListProjects(request, reply); + } + ); + + server.put( + "/api/v1/internals/projects/:projectId", + async (request: FastifyRequest, reply: FastifyReply) => { + return handleUpdateProject(request, reply); + } + ); + + server.delete( + "/api/v1/internals/projects/:projectId", + async (request: FastifyRequest, reply: FastifyReply) => { + return handleDeleteProject(request, reply); + } + ); + // Tags server.get( "/api/v1/tags", From 4ca65cb73f0f7bc1c0b053aece67e82658ca1c45 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sun, 28 Jun 2026 11:59:21 +0530 Subject: [PATCH 24/55] fix(projects): fixed projects checking --- src/routes/http/api/projects.ts | 119 ++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index 7498c0a..5c7bcc7 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -1,5 +1,7 @@ import type { FastifyRequest, FastifyReply } from "fastify"; import * as Sentry from "@sentry/bun"; +import DodoPayments from "dodopayments"; +import { removeClient } from "../../gRPC/payment/paymentProvider.ts"; import { createWideEventBuilder, generateRequestId, @@ -126,12 +128,19 @@ export async function handleUpdateProject( }; const db = getPostgresDB(); + + const appUrl = process.env.SCRAWN_HTTP_URL || "http://localhost:8070"; + await executeInTransaction(db, "update project", async (txn) => { + let rowsAffected = 0; + if (body.name) { - await txn + const updated = await txn .update(projectsTable) .set({ name: body.name }) - .where(eq(projectsTable.id, projectId)); + .where(eq(projectsTable.id, projectId)) + .returning({ id: projectsTable.id }); + rowsAffected += updated.length; } const metaUpdates: any = {}; @@ -144,19 +153,74 @@ export async function handleUpdateProject( if (body.dodoLiveApiKey && !body.dodoLiveApiKey.includes("****")) { metaUpdates.dodo_live_api_key = encrypt(body.dodoLiveApiKey); + + const liveClient = new DodoPayments({ + bearerToken: body.dodoLiveApiKey, + environment: "live_mode", + }); + + try { + const liveWebhook = await liveClient.webhooks.create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, + description: "Scrawn live payment webhook", + filter_types: ["payment.succeeded", "payment.failed"], + }); + const liveSecret = ( + await liveClient.webhooks.retrieveSecret(liveWebhook.id) + ).secret; + metaUpdates.dodo_live_webhook_secret = encrypt(liveSecret); + } catch (error) { + Sentry.captureException(error, { + extra: { context: "failed to register live webhook on update" }, + }); + } } + if (body.dodoTestApiKey && !body.dodoTestApiKey.includes("****")) { metaUpdates.dodo_test_api_key = encrypt(body.dodoTestApiKey); + + const testClient = new DodoPayments({ + bearerToken: body.dodoTestApiKey, + environment: "test_mode", + }); + + try { + const testWebhook = await testClient.webhooks.create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, + description: "Scrawn test payment webhook", + filter_types: ["payment.succeeded", "payment.failed"], + }); + const testSecret = ( + await testClient.webhooks.retrieveSecret(testWebhook.id) + ).secret; + metaUpdates.dodo_test_webhook_secret = encrypt(testSecret); + } catch (error) { + Sentry.captureException(error, { + extra: { context: "failed to register test webhook on update" }, + }); + } } if (Object.keys(metaUpdates).length > 0) { - await txn + const updated = await txn .update(metadataTable) .set(metaUpdates) - .where(eq(metadataTable.projectId, projectId)); + .where(eq(metadataTable.projectId, projectId)) + .returning({ projectId: metadataTable.projectId }); + rowsAffected += updated.length; + } + + if ( + rowsAffected === 0 && + (body.name || Object.keys(metaUpdates).length > 0) + ) { + throw new Error("PROJECT_NOT_FOUND"); } }); + // Invalidate cached clients + removeClient(projectId); + builder.setSuccess(200); reply.code(200); return { success: true }; @@ -170,6 +234,17 @@ export async function handleUpdateProject( return {}; } const err = error instanceof Error ? error : new Error(String(error)); + if ( + err.name === "StorageError" && + (err as any).originalError?.message === "PROJECT_NOT_FOUND" + ) { + builder.setError(404, { + type: "NotFoundError", + message: "Project not found", + }); + reply.code(404); + return {}; + } builder.setError(500, { type: "InternalError", message: err.message }); reply.code(500); return {}; @@ -194,17 +269,38 @@ export async function handleDeleteProject( const { projectId } = request.params as { projectId: string }; const db = getPostgresDB(); - // Manual cascading deletes await executeInTransaction(db, "delete project", async (txn) => { + await txn + .delete(basicUsageEventsTable) + .where(eq(basicUsageEventsTable.projectId, projectId)); + await txn + .delete(paymentEventsTable) + .where(eq(paymentEventsTable.projectId, projectId)); + await txn + .delete(aiTokenUsageEventsTable) + .where(eq(aiTokenUsageEventsTable.projectId, projectId)); + await txn + .delete(sessionsTable) + .where(eq(sessionsTable.projectId, projectId)); + await txn.delete(usersTable).where(eq(usersTable.projectId, projectId)); await txn .delete(apiKeysTable) .where(eq(apiKeysTable.projectId, projectId)); await txn .delete(metadataTable) .where(eq(metadataTable.projectId, projectId)); - await txn.delete(projectsTable).where(eq(projectsTable.id, projectId)); + const deletedProject = await txn + .delete(projectsTable) + .where(eq(projectsTable.id, projectId)) + .returning({ id: projectsTable.id }); + + if (deletedProject.length === 0) { + throw new Error("PROJECT_NOT_FOUND"); + } }); + removeClient(projectId); + builder.setSuccess(200); reply.code(200); return { success: true }; @@ -218,6 +314,17 @@ export async function handleDeleteProject( return {}; } const err = error instanceof Error ? error : new Error(String(error)); + if ( + err.name === "StorageError" && + (err as any).originalError?.message === "PROJECT_NOT_FOUND" + ) { + builder.setError(404, { + type: "NotFoundError", + message: "Project not found", + }); + reply.code(404); + return {}; + } builder.setError(500, { type: "InternalError", message: err.message }); reply.code(500); return {}; From 8b2fa2b6a69b3e734cd72cbe9b977919e12f6b08 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 01:20:08 +0530 Subject: [PATCH 25/55] fix(project.ts): fixed multiple grep issues --- src/routes/http/api/apiKeys.ts | 63 +++++++++++++----------- src/routes/http/api/expressions.ts | 12 +++++ src/routes/http/api/projects.ts | 58 +++++++++++++++------- src/routes/http/api/tags.ts | 8 +++ src/storage/db/postgres/helpers/users.ts | 2 +- 5 files changed, 95 insertions(+), 48 deletions(-) diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 91476da..12e4ac2 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -332,38 +332,43 @@ export async function handleCreateDashboardKey( const expiresAt = DateTime.utc().plus({ years: 10 }).toISO()!; const db = getPostgresDB(); - const existingKey = await db - .select({ id: apiKeysTable.id, key: apiKeysTable.key }) - .from(apiKeysTable) - .where( - and( - eq(apiKeysTable.projectId, project_id), - eq(apiKeysTable.name, "Default dashboard key"), - eq(apiKeysTable.revoked, false) + await executeInTransaction(db, "rotate dashboard key", async (txn) => { + const existingKey = await txn + .select({ id: apiKeysTable.id, key: apiKeysTable.key }) + .from(apiKeysTable) + .where( + and( + eq(apiKeysTable.projectId, project_id), + eq(apiKeysTable.name, "Default dashboard key"), + eq(apiKeysTable.role, "dashboard"), + eq(apiKeysTable.revoked, false) + ) ) - ) - .limit(1); - - const existingDashboardKey = existingKey[0]; - if (existingDashboardKey) { - await db - .update(apiKeysTable) - .set({ + .for("update") + .limit(1); + + const existingDashboardKey = existingKey[0]; + if (existingDashboardKey) { + await txn + .update(apiKeysTable) + .set({ + key: dashboardKeyHash, + role: "dashboard", + expiresAt, + }) + .where(eq(apiKeysTable.id, existingDashboardKey.id)); + + apiKeyCache.delete(existingDashboardKey.key); + } else { + await txn.insert(apiKeysTable).values({ + projectId: project_id, + name: "Default dashboard key", key: dashboardKeyHash, + role: "dashboard", expiresAt, - }) - .where(eq(apiKeysTable.id, existingDashboardKey.id)); - - apiKeyCache.delete(existingDashboardKey.key); - } else { - await db.insert(apiKeysTable).values({ - projectId: project_id, - name: "Default dashboard key", - key: dashboardKeyHash, - role: "dashboard", - expiresAt, - }); - } + }); + } + }); builder.setSuccess(201); reply.code(201); diff --git a/src/routes/http/api/expressions.ts b/src/routes/http/api/expressions.ts index 1b3cfeb..40c002b 100644 --- a/src/routes/http/api/expressions.ts +++ b/src/routes/http/api/expressions.ts @@ -86,6 +86,12 @@ export async function handleCreateExpression( const authHeader = request.headers.authorization; const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can manage expressions" + ); + } + const body = await request.body; const validated = createExpressionSchema.parse(body); @@ -149,6 +155,12 @@ export async function handleDeleteExpression( const authHeader = request.headers.authorization; const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied( + "Only dashboard keys can manage expressions" + ); + } + const params = request.params as { key: string }; const deleted = await deleteExpression(auth.projectId, params.key); diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index 5c7bcc7..112b5fd 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -19,6 +19,8 @@ import { basicUsageEventsTable, aiTokenUsageEventsTable, paymentEventsTable, + webhookDeliveriesTable, + webhookEndpointsTable, } from "../../../storage/db/postgres/schema"; import { eq, inArray } from "drizzle-orm"; import { encrypt, decrypt } from "../../../utils/encryptMetadata.ts"; @@ -44,11 +46,16 @@ export async function handleListProjects( authenticateMasterApiKey(request.headers.authorization); const body = (await request.body) as { projectIds: string[] }; - if ( - !body || - !Array.isArray(body.projectIds) || - body.projectIds.length === 0 - ) { + if (!body || !Array.isArray(body.projectIds)) { + builder.setError(400, { + type: "BadRequestError", + message: "Missing projectIds array", + }); + reply.code(400); + return {}; + } + + if (body.projectIds.length === 0) { builder.setSuccess(200); reply.code(200); return { projects: [] }; @@ -129,20 +136,17 @@ export async function handleUpdateProject( const db = getPostgresDB(); - const appUrl = process.env.SCRAWN_HTTP_URL || "http://localhost:8070"; + const appUrl = process.env.APP_URL; + if (!appUrl) { + builder.setError(500, { + type: "InternalError", + message: "APP_URL environment variable is not set", + }); + reply.code(500); + return {}; + } await executeInTransaction(db, "update project", async (txn) => { - let rowsAffected = 0; - - if (body.name) { - const updated = await txn - .update(projectsTable) - .set({ name: body.name }) - .where(eq(projectsTable.id, projectId)) - .returning({ id: projectsTable.id }); - rowsAffected += updated.length; - } - const metaUpdates: any = {}; if (body.dodoLiveProductId) metaUpdates.dodo_live_product_id = body.dodoLiveProductId; @@ -173,6 +177,7 @@ export async function handleUpdateProject( Sentry.captureException(error, { extra: { context: "failed to register live webhook on update" }, }); + throw new Error("FAILED_TO_REGISTER_LIVE_WEBHOOK"); } } @@ -198,9 +203,21 @@ export async function handleUpdateProject( Sentry.captureException(error, { extra: { context: "failed to register test webhook on update" }, }); + throw new Error("FAILED_TO_REGISTER_TEST_WEBHOOK"); } } + let rowsAffected = 0; + + if (body.name) { + const updated = await txn + .update(projectsTable) + .set({ name: body.name }) + .where(eq(projectsTable.id, projectId)) + .returning({ id: projectsTable.id }); + rowsAffected += updated.length; + } + if (Object.keys(metaUpdates).length > 0) { const updated = await txn .update(metadataTable) @@ -218,7 +235,6 @@ export async function handleUpdateProject( } }); - // Invalidate cached clients removeClient(projectId); builder.setSuccess(200); @@ -270,6 +286,12 @@ export async function handleDeleteProject( const db = getPostgresDB(); await executeInTransaction(db, "delete project", async (txn) => { + await txn + .delete(webhookDeliveriesTable) + .where(eq(webhookDeliveriesTable.projectId, projectId)); + await txn + .delete(webhookEndpointsTable) + .where(eq(webhookEndpointsTable.projectId, projectId)); await txn .delete(basicUsageEventsTable) .where(eq(basicUsageEventsTable.projectId, projectId)); diff --git a/src/routes/http/api/tags.ts b/src/routes/http/api/tags.ts index 444f875..284e1dc 100644 --- a/src/routes/http/api/tags.ts +++ b/src/routes/http/api/tags.ts @@ -88,6 +88,10 @@ export async function handleCreateTag( const authHeader = request.headers.authorization; const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied("Only dashboard keys can manage tags"); + } + const body = await request.body; const validated = createTagSchema.parse(body); @@ -139,6 +143,10 @@ export async function handleDeleteTag( const authHeader = request.headers.authorization; const auth = await authenticateHttpApiKey(authHeader); + if (auth.role !== "dashboard") { + throw AuthError.permissionDenied("Only dashboard keys can manage tags"); + } + const params = tagParamsSchema.parse(request.params); const deleted = await deleteTag(auth.projectId, params.key); diff --git a/src/storage/db/postgres/helpers/users.ts b/src/storage/db/postgres/helpers/users.ts index 624a5a5..63a2c0f 100644 --- a/src/storage/db/postgres/helpers/users.ts +++ b/src/storage/db/postgres/helpers/users.ts @@ -51,7 +51,7 @@ export async function ensureUserExists( await db .insert(usersTable) .values({ id: userId, projectId }) - .onConflictDoNothing(); + .onConflictDoNothing({ target: [usersTable.projectId, usersTable.id] }); } catch (e) { throw StorageError.queryFailed( "Failed to ensure user exists", From 84410404a953fd38fcffe01a96fac60b72428ddd Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 01:59:30 +0530 Subject: [PATCH 26/55] fix(projcets.ts): handling failure in updating dodo webhooks --- src/routes/http/api/projects.ts | 132 +++++++++++++++++++------------- 1 file changed, 78 insertions(+), 54 deletions(-) diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index 112b5fd..e9f6e66 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -120,6 +120,8 @@ export async function handleUpdateProject( request.url ); + const cleanupTasks: Array<() => Promise> = []; + try { authenticateMasterApiKey(request.headers.authorization); @@ -146,67 +148,85 @@ export async function handleUpdateProject( return {}; } - await executeInTransaction(db, "update project", async (txn) => { - const metaUpdates: any = {}; - if (body.dodoLiveProductId) - metaUpdates.dodo_live_product_id = body.dodoLiveProductId; - if (body.dodoTestProductId) - metaUpdates.dodo_test_product_id = body.dodoTestProductId; - if (body.currency) metaUpdates.currency = body.currency; - if (body.redirectUrl) metaUpdates.redirect_url = body.redirectUrl; - - if (body.dodoLiveApiKey && !body.dodoLiveApiKey.includes("****")) { - metaUpdates.dodo_live_api_key = encrypt(body.dodoLiveApiKey); - - const liveClient = new DodoPayments({ - bearerToken: body.dodoLiveApiKey, - environment: "live_mode", - }); + const metaUpdates: any = {}; + if (body.dodoLiveProductId) + metaUpdates.dodo_live_product_id = body.dodoLiveProductId; + if (body.dodoTestProductId) + metaUpdates.dodo_test_product_id = body.dodoTestProductId; + if (body.currency) metaUpdates.currency = body.currency; + if (body.redirectUrl) metaUpdates.redirect_url = body.redirectUrl; - try { - const liveWebhook = await liveClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, - description: "Scrawn live payment webhook", - filter_types: ["payment.succeeded", "payment.failed"], - }); - const liveSecret = ( - await liveClient.webhooks.retrieveSecret(liveWebhook.id) - ).secret; - metaUpdates.dodo_live_webhook_secret = encrypt(liveSecret); - } catch (error) { - Sentry.captureException(error, { - extra: { context: "failed to register live webhook on update" }, - }); - throw new Error("FAILED_TO_REGISTER_LIVE_WEBHOOK"); - } - } + if (body.dodoLiveApiKey && !body.dodoLiveApiKey.includes("****")) { + metaUpdates.dodo_live_api_key = encrypt(body.dodoLiveApiKey); - if (body.dodoTestApiKey && !body.dodoTestApiKey.includes("****")) { - metaUpdates.dodo_test_api_key = encrypt(body.dodoTestApiKey); + const liveClient = new DodoPayments({ + bearerToken: body.dodoLiveApiKey, + environment: "live_mode", + }); - const testClient = new DodoPayments({ - bearerToken: body.dodoTestApiKey, - environment: "test_mode", + try { + const liveWebhook = await liveClient.webhooks.create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, + description: "Scrawn live payment webhook", + filter_types: ["payment.succeeded", "payment.failed"], }); + cleanupTasks.push(async () => { + try { + await liveClient.webhooks.delete(liveWebhook.id); + } catch (e) { + Sentry.captureException(e, { + extra: { context: "failed to clean up live webhook on rollback" }, + }); + } + }); + const liveSecret = ( + await liveClient.webhooks.retrieveSecret(liveWebhook.id) + ).secret; + metaUpdates.dodo_live_webhook_secret = encrypt(liveSecret); + } catch (error) { + Sentry.captureException(error, { + extra: { context: "failed to register live webhook on update" }, + }); + throw new Error("FAILED_TO_REGISTER_LIVE_WEBHOOK"); + } + } + + if (body.dodoTestApiKey && !body.dodoTestApiKey.includes("****")) { + metaUpdates.dodo_test_api_key = encrypt(body.dodoTestApiKey); - try { - const testWebhook = await testClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, - description: "Scrawn test payment webhook", - filter_types: ["payment.succeeded", "payment.failed"], - }); - const testSecret = ( - await testClient.webhooks.retrieveSecret(testWebhook.id) - ).secret; - metaUpdates.dodo_test_webhook_secret = encrypt(testSecret); - } catch (error) { - Sentry.captureException(error, { - extra: { context: "failed to register test webhook on update" }, - }); - throw new Error("FAILED_TO_REGISTER_TEST_WEBHOOK"); - } + const testClient = new DodoPayments({ + bearerToken: body.dodoTestApiKey, + environment: "test_mode", + }); + + try { + const testWebhook = await testClient.webhooks.create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, + description: "Scrawn test payment webhook", + filter_types: ["payment.succeeded", "payment.failed"], + }); + cleanupTasks.push(async () => { + try { + await testClient.webhooks.delete(testWebhook.id); + } catch (e) { + Sentry.captureException(e, { + extra: { context: "failed to clean up test webhook on rollback" }, + }); + } + }); + const testSecret = ( + await testClient.webhooks.retrieveSecret(testWebhook.id) + ).secret; + metaUpdates.dodo_test_webhook_secret = encrypt(testSecret); + } catch (error) { + Sentry.captureException(error, { + extra: { context: "failed to register test webhook on update" }, + }); + throw new Error("FAILED_TO_REGISTER_TEST_WEBHOOK"); } + } + await executeInTransaction(db, "update project", async (txn) => { let rowsAffected = 0; if (body.name) { @@ -241,6 +261,10 @@ export async function handleUpdateProject( reply.code(200); return { success: true }; } catch (error) { + if (cleanupTasks.length > 0) { + await Promise.allSettled(cleanupTasks.map((task) => task())); + } + Sentry.captureException(error, { extra: { context: "handleUpdateProject" }, }); From 3a45993e6349f64174c3d83a19f59bb89bee631e Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 02:24:40 +0530 Subject: [PATCH 27/55] fix(projcets.ts): now accepting a dashboard-only target apiKeyId --- src/routes/http/api/projects.ts | 51 +++++++++++++++++++------ src/routes/http/api/webhookEndpoints.ts | 44 ++++++++++++++++++++- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index e9f6e66..804a5c3 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -9,6 +9,7 @@ import { import { logger } from "../../../errors/logger.ts"; import { AuthError } from "../../../errors/auth"; import { authenticateMasterApiKey } from "../../../utils/authenticateMasterApiKey.ts"; +import { apiKeyCache } from "../../../utils/apiKeyCache.ts"; import { getPostgresDB } from "../../../storage/db/postgres/db"; import { projectsTable, @@ -138,16 +139,6 @@ export async function handleUpdateProject( const db = getPostgresDB(); - const appUrl = process.env.APP_URL; - if (!appUrl) { - builder.setError(500, { - type: "InternalError", - message: "APP_URL environment variable is not set", - }); - reply.code(500); - return {}; - } - const metaUpdates: any = {}; if (body.dodoLiveProductId) metaUpdates.dodo_live_product_id = body.dodoLiveProductId; @@ -164,6 +155,16 @@ export async function handleUpdateProject( environment: "live_mode", }); + const appUrl = process.env.APP_URL; + if (!appUrl) { + builder.setError(500, { + type: "InternalError", + message: "APP_URL environment variable is not set", + }); + reply.code(500); + return {}; + } + try { const liveWebhook = await liveClient.webhooks.create({ url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, @@ -199,6 +200,16 @@ export async function handleUpdateProject( environment: "test_mode", }); + const appUrl = process.env.APP_URL; + if (!appUrl) { + builder.setError(500, { + type: "InternalError", + message: "APP_URL environment variable is not set", + }); + reply.code(500); + return {}; + } + try { const testWebhook = await testClient.webhooks.create({ url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, @@ -285,6 +296,17 @@ export async function handleUpdateProject( reply.code(404); return {}; } + if ( + err.name === "StorageError" && + (err as any).originalError?.code === "23505" + ) { + builder.setError(409, { + type: "ConflictError", + message: "A project with this name already exists", + }); + reply.code(409); + return {}; + } builder.setError(500, { type: "InternalError", message: err.message }); reply.code(500); return {}; @@ -329,9 +351,14 @@ export async function handleDeleteProject( .delete(sessionsTable) .where(eq(sessionsTable.projectId, projectId)); await txn.delete(usersTable).where(eq(usersTable.projectId, projectId)); - await txn + const deletedKeys = await txn .delete(apiKeysTable) - .where(eq(apiKeysTable.projectId, projectId)); + .where(eq(apiKeysTable.projectId, projectId)) + .returning({ key: apiKeysTable.key }); + + for (const k of deletedKeys) { + apiKeyCache.delete(k.key); + } await txn .delete(metadataTable) .where(eq(metadataTable.projectId, projectId)); diff --git a/src/routes/http/api/webhookEndpoints.ts b/src/routes/http/api/webhookEndpoints.ts index a3ef8f3..febd4be 100644 --- a/src/routes/http/api/webhookEndpoints.ts +++ b/src/routes/http/api/webhookEndpoints.ts @@ -194,9 +194,29 @@ export async function handleGetWebhookEndpoint( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); + const query = request.query as { apiKeyId?: string }; + const targetApiKeyId = + query.apiKeyId && auth.role === "dashboard" + ? query.apiKeyId + : auth.apiKeyId; + + if (targetApiKeyId !== auth.apiKeyId) { + const targetKey = await getApiKeyRoleById(targetApiKeyId); + if (!targetKey || targetKey.projectId !== auth.projectId) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key not found or belongs to another project", + }); + reply.code(403); + return { + error: "Target API key not found or belongs to another project", + }; + } + } + const endpoint = await getWebhookEndpointByApiKeyId( auth.projectId, - auth.apiKeyId + targetApiKeyId ); const endpoints: WebhookEndpointResponse[] = endpoint @@ -240,7 +260,27 @@ export async function handleDeleteWebhookEndpoint( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); - const deleted = await deleteWebhookEndpoint(auth.projectId, auth.apiKeyId); + const query = request.query as { apiKeyId?: string }; + const targetApiKeyId = + query.apiKeyId && auth.role === "dashboard" + ? query.apiKeyId + : auth.apiKeyId; + + if (targetApiKeyId !== auth.apiKeyId) { + const targetKey = await getApiKeyRoleById(targetApiKeyId); + if (!targetKey || targetKey.projectId !== auth.projectId) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key not found or belongs to another project", + }); + reply.code(403); + return { + error: "Target API key not found or belongs to another project", + }; + } + } + + const deleted = await deleteWebhookEndpoint(auth.projectId, targetApiKeyId); if (!deleted) { builder.setError(404, { From f076f9fa8bbef33bdf8ba201ed048d6d87d40a35 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 02:43:13 +0530 Subject: [PATCH 28/55] fix(projcets.ts): fixed greptile issues --- src/storage/adapter/clickhouse/handlers/queryEvents.ts | 6 +++--- src/storage/adapter/postgres/handlers/queryEvents.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storage/adapter/clickhouse/handlers/queryEvents.ts b/src/storage/adapter/clickhouse/handlers/queryEvents.ts index a7ba35d..d1c3db0 100644 --- a/src/storage/adapter/clickhouse/handlers/queryEvents.ts +++ b/src/storage/adapter/clickhouse/handlers/queryEvents.ts @@ -263,7 +263,7 @@ async function handleListQuery( ); let q = `SELECT ${buildSelectColumns(t)} FROM ${t}`; q += ` WHERE project_id = {projectId:String}`; - if (whereClause) q += ` AND ${whereClause}`; + if (whereClause) q += ` AND (${whereClause})`; return q; }); @@ -344,7 +344,7 @@ async function handleAggregationQuery( ); let q = `SELECT ${cols.join(", ")} FROM ${t}`; q += ` WHERE project_id = {projectId:String}`; - if (whereClause) q += ` AND ${whereClause}`; + if (whereClause) q += ` AND (${whereClause})`; return q; }); @@ -399,7 +399,7 @@ async function getTotalCount( ); let q = `SELECT count() as cnt FROM ${t}`; q += ` WHERE project_id = {projectId:String}`; - if (whereClause) q += ` AND ${whereClause}`; + if (whereClause) q += ` AND (${whereClause})`; return q; }); diff --git a/src/storage/adapter/postgres/handlers/queryEvents.ts b/src/storage/adapter/postgres/handlers/queryEvents.ts index b54a23a..e239305 100644 --- a/src/storage/adapter/postgres/handlers/queryEvents.ts +++ b/src/storage/adapter/postgres/handlers/queryEvents.ts @@ -304,7 +304,7 @@ async function handleListQuery( const subqueries = tables.map((t, i) => { const base = sql`SELECT ${selectExpr[i]} FROM ${sql.raw(t)}`; const fullWhere = whereExpr[i] - ? sql`${whereExpr[i]} AND ${projectFilter}` + ? sql`(${whereExpr[i]}) AND ${projectFilter}` : projectFilter; return sql`${base} WHERE ${fullWhere}`; }); @@ -373,7 +373,7 @@ async function handleAggregationQuery( const whereClause = buildWhereClause(request.where, t); const base = sql`SELECT ${sql.join(cols, sql`, `)} FROM ${sql.raw(t)}`; const fullWhere = whereClause - ? sql`${whereClause} AND ${projectFilter}` + ? sql`(${whereClause}) AND ${projectFilter}` : projectFilter; return sql`${base} WHERE ${fullWhere}`; }); @@ -431,7 +431,7 @@ async function getTotalCount( const whereClause = buildWhereClause(request.where, t); const base = sql`SELECT count(*)::int as cnt FROM ${sql.raw(t)}`; const fullWhere = whereClause - ? sql`${whereClause} AND ${projectFilter}` + ? sql`(${whereClause}) AND ${projectFilter}` : projectFilter; return sql`${base} WHERE ${fullWhere}`; }); From 0f6fae9db2eaec82c42d312299ae84cbec990140 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 08:05:21 +0530 Subject: [PATCH 29/55] fix(projcets.ts): using left join to insert data for projects --- src/routes/http/api/projects.ts | 10 +++++++--- src/routes/http/api/webhookEndpoints.ts | 24 ++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index 804a5c3..483d24c 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -75,7 +75,7 @@ export async function handleListProjects( redirect_url: metadataTable.redirect_url, }) .from(projectsTable) - .innerJoin(metadataTable, eq(projectsTable.id, metadataTable.projectId)) + .leftJoin(metadataTable, eq(projectsTable.id, metadataTable.projectId)) .where(inArray(projectsTable.id, body.projectIds)); builder.setSuccess(200); @@ -85,8 +85,12 @@ export async function handleListProjects( projects: projects.map((p) => ({ id: p.id, name: p.name, - dodoLiveApiKey: maskApiKey(decrypt(p.dodo_live_api_key)), - dodoTestApiKey: maskApiKey(decrypt(p.dodo_test_api_key)), + dodoLiveApiKey: p.dodo_live_api_key + ? maskApiKey(decrypt(p.dodo_live_api_key)) + : null, + dodoTestApiKey: p.dodo_test_api_key + ? maskApiKey(decrypt(p.dodo_test_api_key)) + : null, dodoLiveProductId: p.dodo_live_product_id, dodoTestProductId: p.dodo_test_product_id, currency: p.currency, diff --git a/src/routes/http/api/webhookEndpoints.ts b/src/routes/http/api/webhookEndpoints.ts index febd4be..494e40f 100644 --- a/src/routes/http/api/webhookEndpoints.ts +++ b/src/routes/http/api/webhookEndpoints.ts @@ -291,7 +291,7 @@ export async function handleDeleteWebhookEndpoint( return { error: "No webhook endpoint found for this API key" }; } - invalidateWebhookEndpointCache(auth.apiKeyId); + invalidateWebhookEndpointCache(targetApiKeyId); builder.setSuccess(200); reply.code(200); @@ -439,9 +439,29 @@ export async function handleGetPublicKey( const auth = await authenticateHttpApiKey(request.headers.authorization); builder.setApiKeyContext({ name: `webhook:${auth.apiKeyId}` }); + const query = request.query as { apiKeyId?: string }; + const targetApiKeyId = + query.apiKeyId && auth.role === "dashboard" + ? query.apiKeyId + : auth.apiKeyId; + + if (targetApiKeyId !== auth.apiKeyId) { + const targetKey = await getApiKeyRoleById(targetApiKeyId); + if (!targetKey || targetKey.projectId !== auth.projectId) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key not found or belongs to another project", + }); + reply.code(403); + return { + error: "Target API key not found or belongs to another project", + }; + } + } + const endpoint = await getWebhookEndpointByApiKeyId( auth.projectId, - auth.apiKeyId + targetApiKeyId ); if (!endpoint) { From a63ee9bb7b9ba5aa8517bb671661c06cf4e17d08 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 08:38:18 +0530 Subject: [PATCH 30/55] fix(projcets.ts): currency mismatch fixed --- src/routes/http/createdCheckout.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/http/createdCheckout.ts b/src/routes/http/createdCheckout.ts index 84700d3..e818b91 100644 --- a/src/routes/http/createdCheckout.ts +++ b/src/routes/http/createdCheckout.ts @@ -108,7 +108,7 @@ export async function handleDodoWebhook( return ignoredResponse(builder); } - const { payment_id, checkout_session_id } = webhookPayload.data; + const { payment_id, checkout_session_id, currency } = webhookPayload.data; builder.setWebhookContext({ webhookEvent: webhookPayload.type, @@ -238,7 +238,7 @@ export async function handleDodoWebhook( checkoutSessionId: checkout_session_id, userId, amount: creditAmount, - currency: "usd", + currency: currency, mode, billed_upto, createdAt: session.createdAt, From 609ab4ce2f9f24a570a48f1e15c5b39ea44dc3d1 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 08:52:59 +0530 Subject: [PATCH 31/55] fix(projcets.ts): added API key expiry handling to various files --- src/interceptors/auth.ts | 12 +++++++++--- src/routes/gRPC/auth/createAPIKey.ts | 12 +----------- src/routes/gRPC/data/query.ts | 14 ++++++++++---- src/routes/http/api/apiKeys.ts | 2 +- src/utils/authenticateHttpApiKey.ts | 7 +++++++ 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/interceptors/auth.ts b/src/interceptors/auth.ts index b271258..dcab5c4 100644 --- a/src/interceptors/auth.ts +++ b/src/interceptors/auth.ts @@ -122,8 +122,7 @@ export function authInterceptor( const wideEventBuilder = call[wideEventContextKey]; const authHeader = call.metadata.get("authorization")?.[0] as - | string - | undefined; + string | undefined; if (!authHeader) { return callback?.(AuthError.missingHeader()); @@ -156,6 +155,12 @@ export function authInterceptor( const cached = apiKeyCache.get(apiKeyHash); if (cached) { + if ( + cached.expiresAt && + DateTime.utc() > DateTime.fromISO(cached.expiresAt, { zone: "utc" }) + ) { + return callback?.(AuthError.expiredAPIKey()); + } if (cached.role !== role) { return callback?.( AuthError.roleMismatch( @@ -201,8 +206,9 @@ export function authInterceptor( } if ( + apiKeyRecord.expiresAt && DateTime.utc() > - DateTime.fromISO(apiKeyRecord.expiresAt, { zone: "utc" }) + DateTime.fromISO(apiKeyRecord.expiresAt, { zone: "utc" }) ) { return callback?.(AuthError.expiredAPIKey()); } diff --git a/src/routes/gRPC/auth/createAPIKey.ts b/src/routes/gRPC/auth/createAPIKey.ts index 535214c..17ebe69 100644 --- a/src/routes/gRPC/auth/createAPIKey.ts +++ b/src/routes/gRPC/auth/createAPIKey.ts @@ -3,7 +3,6 @@ import { CreateAPIKeyRequest, CreateAPIKeyResponse, } from "../../../gen/auth/v1/auth"; -import type { WideEventBuilder } from "../../../context/requestContext"; import { apiKeyContextKey } from "../../../context/auth"; import { createAPIKeySchema } from "../../../zod/apikey"; import { APIKeyError } from "../../../errors/apikey"; @@ -39,18 +38,9 @@ export async function createAPIKey( // Read role from gRPC metadata (not in proto message yet) const roleFromMeta = call.metadata.get("x-scrawn-role")?.[0] as - | string - | undefined; + string | undefined; const validatedData = validateRequest(req, roleFromMeta); - if (validatedData.role === "dashboard" && auth.role !== "dashboard") { - return callback?.( - AuthError.permissionDenied( - "Only dashboard keys can create dashboard keys" - ) - ); - } - wideEventBuilder?.setApiKeyContext({ name: validatedData.name }); const apiKey = generateAPIKey(validatedData.role as ApiKeyRole); diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index f52c852..0377e64 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -197,8 +197,7 @@ export async function queryData( callback?: sendUnaryData ): Promise { const wideEventBuilder = call[wideEventContextKey] as - | WideEventBuilder - | undefined; + WideEventBuilder | undefined; try { const auth = call[apiKeyContextKey]; @@ -225,9 +224,16 @@ export async function queryData( const db = getPostgresDB(); const userWhere = buildWhere(validated.where, tableDef); const projectFilter = eq(tableDef.table.projectId, auth.projectId) as SQL; - const whereClause = userWhere - ? and(projectFilter, userWhere) + + const modeFilter = tableDef.fields.mode + ? eq(tableDef.fields.mode.col, auth.mode) + : undefined; + + const baseFilter = modeFilter + ? and(projectFilter, modeFilter) : projectFilter; + + const whereClause = userWhere ? and(baseFilter, userWhere) : baseFilter; const selectCols = buildSelect(tableDef); const columns = Object.keys(tableDef.fields); diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 12e4ac2..7b9f258 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -372,7 +372,7 @@ export async function handleCreateDashboardKey( builder.setSuccess(201); reply.code(201); - return { project_id, apiKey: dashboardKey }; + return { projectId: project_id, apiKey: dashboardKey }; } catch (error) { if (error instanceof AuthError) { builder.setError(401, { type: error.type, message: error.message }); diff --git a/src/utils/authenticateHttpApiKey.ts b/src/utils/authenticateHttpApiKey.ts index 142853a..b236107 100644 --- a/src/utils/authenticateHttpApiKey.ts +++ b/src/utils/authenticateHttpApiKey.ts @@ -39,6 +39,12 @@ export async function authenticateHttpApiKey( const cached = apiKeyCache.get(apiKeyHash); if (cached) { + if ( + cached.expiresAt && + DateTime.utc() > DateTime.fromISO(cached.expiresAt, { zone: "utc" }) + ) { + throw AuthError.expiredAPIKey(); + } if (cached.role !== role) { throw AuthError.roleMismatch( `Key prefix ${role} doesn't match stored role ${cached.role}` @@ -63,6 +69,7 @@ export async function authenticateHttpApiKey( } if ( + apiKeyRecord.expiresAt && DateTime.utc() > DateTime.fromISO(apiKeyRecord.expiresAt, { zone: "utc" }) ) { throw AuthError.expiredAPIKey(); From ffef9a8aeae06e07d46dfacdb9a6a24bd0facac2 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 09:09:11 +0530 Subject: [PATCH 32/55] fix(projcets.ts): several auth issues are fixed --- src/routes/gRPC/auth/createAPIKey.ts | 8 ++++++++ src/routes/gRPC/data/query.ts | 13 ++++++++++--- src/routes/http/api/apiKeys.ts | 11 +++++++++++ src/routes/http/api/projects.ts | 29 ++++++++++++++++++---------- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/routes/gRPC/auth/createAPIKey.ts b/src/routes/gRPC/auth/createAPIKey.ts index 17ebe69..1dde9fc 100644 --- a/src/routes/gRPC/auth/createAPIKey.ts +++ b/src/routes/gRPC/auth/createAPIKey.ts @@ -41,6 +41,14 @@ export async function createAPIKey( string | undefined; const validatedData = validateRequest(req, roleFromMeta); + if (validatedData.role === "dashboard") { + return callback?.( + AuthError.permissionDenied( + "Dashboard role creation is reserved for master keys" + ) + ); + } + wideEventBuilder?.setApiKeyContext({ name: validatedData.name }); const apiKey = generateAPIKey(validatedData.role as ApiKeyRole); diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index 0377e64..d8e9c9a 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -205,6 +205,12 @@ export async function queryData( return callback?.(AuthError.invalidAPIKey("API key context not found")); } + if (auth.role !== "dashboard") { + return callback?.( + AuthError.permissionDenied("Only dashboard keys can query data") + ); + } + const req = { ...call.request } as Record; const validated = dataQuerySchema.parse(req); @@ -225,9 +231,10 @@ export async function queryData( const userWhere = buildWhere(validated.where, tableDef); const projectFilter = eq(tableDef.table.projectId, auth.projectId) as SQL; - const modeFilter = tableDef.fields.mode - ? eq(tableDef.fields.mode.col, auth.mode) - : undefined; + const modeFilter = + tableDef.fields.mode && auth.mode + ? eq(tableDef.fields.mode.col, auth.mode) + : undefined; const baseFilter = modeFilter ? and(projectFilter, modeFilter) diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index 7b9f258..ca2f4f8 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -59,6 +59,17 @@ export async function handleCreateApiKey( builder.setApiKeyContext({ name: `create-key:${auth.apiKeyId}` }); const body = await request.body; + + if ( + typeof body === "object" && + body !== null && + (body as Record).role === "dashboard" + ) { + throw AuthError.permissionDenied( + "Dashboard role creation is reserved for master keys" + ); + } + const validated = createApiKeySchema.parse(body); if ( diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index 483d24c..e9bc90c 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -144,12 +144,13 @@ export async function handleUpdateProject( const db = getPostgresDB(); const metaUpdates: any = {}; - if (body.dodoLiveProductId) + if (body.dodoLiveProductId !== undefined) metaUpdates.dodo_live_product_id = body.dodoLiveProductId; - if (body.dodoTestProductId) + if (body.dodoTestProductId !== undefined) metaUpdates.dodo_test_product_id = body.dodoTestProductId; - if (body.currency) metaUpdates.currency = body.currency; - if (body.redirectUrl) metaUpdates.redirect_url = body.redirectUrl; + if (body.currency !== undefined) metaUpdates.currency = body.currency; + if (body.redirectUrl !== undefined) + metaUpdates.redirect_url = body.redirectUrl; if (body.dodoLiveApiKey && !body.dodoLiveApiKey.includes("****")) { metaUpdates.dodo_live_api_key = encrypt(body.dodoLiveApiKey); @@ -244,7 +245,7 @@ export async function handleUpdateProject( await executeInTransaction(db, "update project", async (txn) => { let rowsAffected = 0; - if (body.name) { + if (body.name !== undefined) { const updated = await txn .update(projectsTable) .set({ name: body.name }) @@ -262,11 +263,19 @@ export async function handleUpdateProject( rowsAffected += updated.length; } - if ( - rowsAffected === 0 && - (body.name || Object.keys(metaUpdates).length > 0) - ) { - throw new Error("PROJECT_NOT_FOUND"); + if (rowsAffected === 0) { + if (body.name !== undefined || Object.keys(metaUpdates).length > 0) { + throw new Error("PROJECT_NOT_FOUND"); + } else { + const exists = await txn + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(eq(projectsTable.id, projectId)) + .limit(1); + if (exists.length === 0) { + throw new Error("PROJECT_NOT_FOUND"); + } + } } }); From 0b62aaf96a58c7f6567dd32ba69e6c39447246d8 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 09:41:20 +0530 Subject: [PATCH 33/55] fix(projects.ts): checking for existing checkout links THEN making the new link --- src/gen/auth/v1/auth.ts | 8 +- src/gen/data/v1/data.ts | 83 +------------------ src/gen/event/v1/event.ts | 8 +- src/gen/payment/v1/payment.ts | 8 +- src/gen/query/v1/query.ts | 52 ++++++++++-- src/routes/gRPC/data/query.ts | 11 +-- src/routes/gRPC/payment/createCheckoutLink.ts | 20 ++--- src/routes/gRPC/query/queryEvents.ts | 6 ++ src/routes/http/api/webhookEndpoints.ts | 42 ++++++++++ src/storage/adapter/clickhouse/schema.ts | 4 +- src/storage/db/postgres/helpers/apiKeys.ts | 11 ++- src/zod/data.ts | 2 +- 12 files changed, 123 insertions(+), 132 deletions(-) diff --git a/src/gen/auth/v1/auth.ts b/src/gen/auth/v1/auth.ts index f41d746..08f72c8 100644 --- a/src/gen/auth/v1/auth.ts +++ b/src/gen/auth/v1/auth.ts @@ -330,13 +330,7 @@ export const AuthServiceClient = makeGenericClientConstructor( }; type Builtin = - | Date - | Function - | Uint8Array - | string - | number - | boolean - | undefined; + Date | Function | Uint8Array | string | number | boolean | undefined; export type DeepPartial = T extends Builtin ? T diff --git a/src/gen/data/v1/data.ts b/src/gen/data/v1/data.ts index e796e63..05223da 100644 --- a/src/gen/data/v1/data.ts +++ b/src/gen/data/v1/data.ts @@ -249,81 +249,6 @@ export function sessionsFieldToJSON(object: SessionsField): string { } } -export enum PaymentEventsField { - PAYMENT_EVENTS_FIELD_UNSPECIFIED = 0, - PAYMENT_EVENTS_ID = 1, - PAYMENT_EVENTS_USER_ID = 2, - PAYMENT_EVENTS_API_KEY_ID = 3, - PAYMENT_EVENTS_MODE = 4, - PAYMENT_EVENTS_CREDIT_AMOUNT = 5, - PAYMENT_EVENTS_SESSION_ID = 6, - PAYMENT_EVENTS_REPORTED_TIMESTAMP = 7, - PAYMENT_EVENTS_INGESTED_TIMESTAMP = 8, - UNRECOGNIZED = -1, -} - -export function paymentEventsFieldFromJSON(object: any): PaymentEventsField { - switch (object) { - case 0: - case "PAYMENT_EVENTS_FIELD_UNSPECIFIED": - return PaymentEventsField.PAYMENT_EVENTS_FIELD_UNSPECIFIED; - case 1: - case "PAYMENT_EVENTS_ID": - return PaymentEventsField.PAYMENT_EVENTS_ID; - case 2: - case "PAYMENT_EVENTS_USER_ID": - return PaymentEventsField.PAYMENT_EVENTS_USER_ID; - case 3: - case "PAYMENT_EVENTS_API_KEY_ID": - return PaymentEventsField.PAYMENT_EVENTS_API_KEY_ID; - case 4: - case "PAYMENT_EVENTS_MODE": - return PaymentEventsField.PAYMENT_EVENTS_MODE; - case 5: - case "PAYMENT_EVENTS_CREDIT_AMOUNT": - return PaymentEventsField.PAYMENT_EVENTS_CREDIT_AMOUNT; - case 6: - case "PAYMENT_EVENTS_SESSION_ID": - return PaymentEventsField.PAYMENT_EVENTS_SESSION_ID; - case 7: - case "PAYMENT_EVENTS_REPORTED_TIMESTAMP": - return PaymentEventsField.PAYMENT_EVENTS_REPORTED_TIMESTAMP; - case 8: - case "PAYMENT_EVENTS_INGESTED_TIMESTAMP": - return PaymentEventsField.PAYMENT_EVENTS_INGESTED_TIMESTAMP; - case -1: - case "UNRECOGNIZED": - default: - return PaymentEventsField.UNRECOGNIZED; - } -} - -export function paymentEventsFieldToJSON(object: PaymentEventsField): string { - switch (object) { - case PaymentEventsField.PAYMENT_EVENTS_FIELD_UNSPECIFIED: - return "PAYMENT_EVENTS_FIELD_UNSPECIFIED"; - case PaymentEventsField.PAYMENT_EVENTS_ID: - return "PAYMENT_EVENTS_ID"; - case PaymentEventsField.PAYMENT_EVENTS_USER_ID: - return "PAYMENT_EVENTS_USER_ID"; - case PaymentEventsField.PAYMENT_EVENTS_API_KEY_ID: - return "PAYMENT_EVENTS_API_KEY_ID"; - case PaymentEventsField.PAYMENT_EVENTS_MODE: - return "PAYMENT_EVENTS_MODE"; - case PaymentEventsField.PAYMENT_EVENTS_CREDIT_AMOUNT: - return "PAYMENT_EVENTS_CREDIT_AMOUNT"; - case PaymentEventsField.PAYMENT_EVENTS_SESSION_ID: - return "PAYMENT_EVENTS_SESSION_ID"; - case PaymentEventsField.PAYMENT_EVENTS_REPORTED_TIMESTAMP: - return "PAYMENT_EVENTS_REPORTED_TIMESTAMP"; - case PaymentEventsField.PAYMENT_EVENTS_INGESTED_TIMESTAMP: - return "PAYMENT_EVENTS_INGESTED_TIMESTAMP"; - case PaymentEventsField.UNRECOGNIZED: - default: - return "UNRECOGNIZED"; - } -} - export enum TagsField { TAGS_FIELD_UNSPECIFIED = 0, TAGS_ID = 1, @@ -1148,13 +1073,7 @@ export const DataQueryServiceClient = makeGenericClientConstructor( }; type Builtin = - | Date - | Function - | Uint8Array - | string - | number - | boolean - | undefined; + Date | Function | Uint8Array | string | number | boolean | undefined; export type DeepPartial = T extends Builtin ? T diff --git a/src/gen/event/v1/event.ts b/src/gen/event/v1/event.ts index 456097b..a70b4fa 100644 --- a/src/gen/event/v1/event.ts +++ b/src/gen/event/v1/event.ts @@ -1616,13 +1616,7 @@ export const EventServiceClient = makeGenericClientConstructor( }; type Builtin = - | Date - | Function - | Uint8Array - | string - | number - | boolean - | undefined; + Date | Function | Uint8Array | string | number | boolean | undefined; export type DeepPartial = T extends Builtin ? T diff --git a/src/gen/payment/v1/payment.ts b/src/gen/payment/v1/payment.ts index 7390001..904a221 100644 --- a/src/gen/payment/v1/payment.ts +++ b/src/gen/payment/v1/payment.ts @@ -243,13 +243,7 @@ export const PaymentServiceClient = makeGenericClientConstructor( }; type Builtin = - | Date - | Function - | Uint8Array - | string - | number - | boolean - | undefined; + Date | Function | Uint8Array | string | number | boolean | undefined; export type DeepPartial = T extends Builtin ? T diff --git a/src/gen/query/v1/query.ts b/src/gen/query/v1/query.ts index 49fb0c1..212b222 100644 --- a/src/gen/query/v1/query.ts +++ b/src/gen/query/v1/query.ts @@ -457,6 +457,8 @@ export interface EventRow { inputCacheTokens?: number | undefined; inputCacheDebitAmount?: number | undefined; metadata?: string | undefined; + outputCacheTokens?: number | undefined; + outputCacheDebitAmount?: number | undefined; } export interface AggregationRow { @@ -1000,6 +1002,8 @@ function createBaseEventRow(): EventRow { inputCacheTokens: undefined, inputCacheDebitAmount: undefined, metadata: undefined, + outputCacheTokens: undefined, + outputCacheDebitAmount: undefined, }; } @@ -1056,6 +1060,12 @@ export const EventRow: MessageFns = { if (message.metadata !== undefined) { writer.uint32(138).string(message.metadata); } + if (message.outputCacheTokens !== undefined) { + writer.uint32(144).int32(message.outputCacheTokens); + } + if (message.outputCacheDebitAmount !== undefined) { + writer.uint32(152).int64(message.outputCacheDebitAmount); + } return writer; }, @@ -1195,6 +1205,22 @@ export const EventRow: MessageFns = { message.metadata = reader.string(); continue; } + case 18: { + if (tag !== 144) { + break; + } + + message.outputCacheTokens = reader.int32(); + continue; + } + case 19: { + if (tag !== 152) { + break; + } + + message.outputCacheDebitAmount = longToNumber(reader.int64()); + continue; + } } if ((tag & 7) === 4 || tag === 0) { break; @@ -1278,6 +1304,16 @@ export const EventRow: MessageFns = { metadata: isSet(object.metadata) ? globalThis.String(object.metadata) : undefined, + outputCacheTokens: isSet(object.outputCacheTokens) + ? globalThis.Number(object.outputCacheTokens) + : isSet(object.output_cache_tokens) + ? globalThis.Number(object.output_cache_tokens) + : undefined, + outputCacheDebitAmount: isSet(object.outputCacheDebitAmount) + ? globalThis.Number(object.outputCacheDebitAmount) + : isSet(object.output_cache_debit_amount) + ? globalThis.Number(object.output_cache_debit_amount) + : undefined, }; }, @@ -1331,6 +1367,12 @@ export const EventRow: MessageFns = { if (message.metadata !== undefined) { obj.metadata = message.metadata; } + if (message.outputCacheTokens !== undefined) { + obj.outputCacheTokens = Math.round(message.outputCacheTokens); + } + if (message.outputCacheDebitAmount !== undefined) { + obj.outputCacheDebitAmount = Math.round(message.outputCacheDebitAmount); + } return obj; }, @@ -1355,6 +1397,8 @@ export const EventRow: MessageFns = { message.inputCacheTokens = object.inputCacheTokens ?? undefined; message.inputCacheDebitAmount = object.inputCacheDebitAmount ?? undefined; message.metadata = object.metadata ?? undefined; + message.outputCacheTokens = object.outputCacheTokens ?? undefined; + message.outputCacheDebitAmount = object.outputCacheDebitAmount ?? undefined; return message; }, }; @@ -1623,13 +1667,7 @@ export const QueryServiceClient = makeGenericClientConstructor( }; type Builtin = - | Date - | Function - | Uint8Array - | string - | number - | boolean - | undefined; + Date | Function | Uint8Array | string | number | boolean | undefined; export type DeepPartial = T extends Builtin ? T diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index d8e9c9a..62e4f35 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -100,14 +100,15 @@ const TABLE_REGISTRY: Record = { }; function castValue( - value: string, + value: string | number | boolean, fieldDef: FieldDef, fieldName: string ): boolean | number | string { if (fieldDef.cast === "boolean") { + if (typeof value === "boolean") return value; if (value !== "true" && value !== "false") { throw EventError.validationFailed( - `Invalid boolean value '${value}' for field '${fieldName}': must be "true" or "false"` + `Invalid boolean value '${String(value)}' for field '${fieldName}': must be "true" or "false"` ); } return value === "true"; @@ -116,18 +117,18 @@ function castValue( const n = Number(value); if (!Number.isFinite(n) || !Number.isInteger(n)) { throw EventError.validationFailed( - `Invalid integer value '${value}' for field '${fieldName}': must be a finite integer` + `Invalid integer value '${String(value)}' for field '${fieldName}': must be a finite integer` ); } return n; } - return value; + return typeof value === "string" ? value : String(value); } function applyOp( col: AnyPgColumn, op: string, - value: string, + value: string | number | boolean, fieldDef: FieldDef, fieldName: string ): SQL { diff --git a/src/routes/gRPC/payment/createCheckoutLink.ts b/src/routes/gRPC/payment/createCheckoutLink.ts index 6973f22..2b2d2e1 100644 --- a/src/routes/gRPC/payment/createCheckoutLink.ts +++ b/src/routes/gRPC/payment/createCheckoutLink.ts @@ -79,16 +79,6 @@ export async function createCheckoutLink( ); wideEventBuilder?.setPaymentContext({ priceAmount: custom_price }); - const checkoutResult = await createCheckoutSession( - auth.projectId, - config, - custom_price, - validatedData.userId, - auth.apiKeyId, - beforeTimestamp, - mode - ); - const checkoutLink = await executeInTransaction( db, "create checkout link", @@ -118,6 +108,16 @@ export async function createCheckoutLink( return proxyUrl; } + const checkoutResult = await createCheckoutSession( + auth.projectId, + config, + custom_price, + validatedData.userId, + auth.apiKeyId, + beforeTimestamp, + mode + ); + const sessionResult = await handleAddSession( auth.projectId, validatedData.userId, diff --git a/src/routes/gRPC/query/queryEvents.ts b/src/routes/gRPC/query/queryEvents.ts index 7378884..e2a3a4a 100644 --- a/src/routes/gRPC/query/queryEvents.ts +++ b/src/routes/gRPC/query/queryEvents.ts @@ -101,6 +101,12 @@ function buildEventRow(row: QueryResponse["rows"][number]): EventRow { if (row.inputCacheDebitAmount != null) { eventRow.inputCacheDebitAmount = Number(row.inputCacheDebitAmount); } + if (row.outputCacheTokens != null) { + eventRow.outputCacheTokens = Number(row.outputCacheTokens); + } + if (row.outputCacheDebitAmount != null) { + eventRow.outputCacheDebitAmount = Number(row.outputCacheDebitAmount); + } if (row.metadata != null) { eventRow.metadata = JSON.stringify(row.metadata); } diff --git a/src/routes/http/api/webhookEndpoints.ts b/src/routes/http/api/webhookEndpoints.ts index 494e40f..e30bd4f 100644 --- a/src/routes/http/api/webhookEndpoints.ts +++ b/src/routes/http/api/webhookEndpoints.ts @@ -106,6 +106,15 @@ export async function handleCreateWebhookEndpoint( return { error: "Target API key not found" }; } + if (targetKey.revoked) { + builder.setError(400, { + type: "ValidationError", + message: "Cannot set webhook for a revoked API key", + }); + reply.code(400); + return { error: "Cannot set webhook for a revoked API key" }; + } + if (targetKey.projectId !== auth.projectId) { builder.setError(403, { type: "PermissionDenied", @@ -212,6 +221,14 @@ export async function handleGetWebhookEndpoint( error: "Target API key not found or belongs to another project", }; } + if (targetKey.revoked) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key is revoked", + }); + reply.code(403); + return { error: "Target API key is revoked" }; + } } const endpoint = await getWebhookEndpointByApiKeyId( @@ -278,6 +295,14 @@ export async function handleDeleteWebhookEndpoint( error: "Target API key not found or belongs to another project", }; } + if (targetKey.revoked) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key is revoked", + }); + reply.code(403); + return { error: "Target API key is revoked" }; + } } const deleted = await deleteWebhookEndpoint(auth.projectId, targetApiKeyId); @@ -352,6 +377,15 @@ export async function handleSendTestWebhook( return { error: "API key not found" }; } + if (targetKey.revoked) { + builder.setError(400, { + type: "ValidationError", + message: "Cannot send test webhook to a revoked API key", + }); + reply.code(400); + return { error: "Cannot send test webhook to a revoked API key" }; + } + if (targetKey.projectId !== auth.projectId) { builder.setError(403, { type: "PermissionDenied", @@ -457,6 +491,14 @@ export async function handleGetPublicKey( error: "Target API key not found or belongs to another project", }; } + if (targetKey.revoked) { + builder.setError(403, { + type: "PermissionDenied", + message: "Target API key is revoked", + }); + reply.code(403); + return { error: "Target API key is revoked" }; + } } const endpoint = await getWebhookEndpointByApiKeyId( diff --git a/src/storage/adapter/clickhouse/schema.ts b/src/storage/adapter/clickhouse/schema.ts index 653c662..675774d 100644 --- a/src/storage/adapter/clickhouse/schema.ts +++ b/src/storage/adapter/clickhouse/schema.ts @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS basic_usage_events ( debit_amount Int64, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (project_id, idempotency_key, user_id) +ORDER BY (project_id, idempotency_key, user_id, event_id) `; const AI_TOKEN_USAGE_EVENTS_TABLE = ` @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS ai_token_usage_events ( metrics String, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (project_id, idempotency_key, user_id) +ORDER BY (project_id, idempotency_key, user_id, event_id) `; export async function runClickHouseMigrations(): Promise { diff --git a/src/storage/db/postgres/helpers/apiKeys.ts b/src/storage/db/postgres/helpers/apiKeys.ts index 950ae8b..d58fa42 100644 --- a/src/storage/db/postgres/helpers/apiKeys.ts +++ b/src/storage/db/postgres/helpers/apiKeys.ts @@ -79,17 +79,20 @@ type ApiKeyRecord = { projectId: string; }; -export async function getApiKeyRoleById( - id: string -): Promise<{ +export async function getApiKeyRoleById(id: string): Promise<{ role: "dashboard" | "production" | "test"; projectId: string; + revoked: boolean; } | null> { const db = getPostgresDB(); try { const [record] = await db - .select({ role: apiKeysTable.role, projectId: apiKeysTable.projectId }) + .select({ + role: apiKeysTable.role, + projectId: apiKeysTable.projectId, + revoked: apiKeysTable.revoked, + }) .from(apiKeysTable) .where(eq(apiKeysTable.id, id)) .limit(1); diff --git a/src/zod/data.ts b/src/zod/data.ts index 07c6bcb..9f16c13 100644 --- a/src/zod/data.ts +++ b/src/zod/data.ts @@ -33,7 +33,7 @@ const filterConditionSchema = z.object({ .min(1) .max(7) .transform((v) => OPERATOR_MAP[v as keyof typeof OPERATOR_MAP]), - value: z.string(), + value: z.union([z.string(), z.number(), z.boolean()]), }); const filterGroupSchema = createFilterGroupSchema( From 6f309e7290794e4c59df984fe6cc22306a3c89b6 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 09:44:16 +0530 Subject: [PATCH 34/55] fix(projects.ts): clickhouse startup cleanup --- src/storage/adapter/clickhouse/schema.ts | 1 - src/zod/data.ts | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/storage/adapter/clickhouse/schema.ts b/src/storage/adapter/clickhouse/schema.ts index 675774d..94eff8e 100644 --- a/src/storage/adapter/clickhouse/schema.ts +++ b/src/storage/adapter/clickhouse/schema.ts @@ -1,6 +1,5 @@ import { getClickHouseDB } from "../../db/clickhouse"; import { logger } from "../../../errors/logger"; -import { innerProduct } from "drizzle-orm"; const BASIC_USAGE_EVENTS_TABLE = ` CREATE TABLE IF NOT EXISTS basic_usage_events ( diff --git a/src/zod/data.ts b/src/zod/data.ts index 9f16c13..ace275d 100644 --- a/src/zod/data.ts +++ b/src/zod/data.ts @@ -2,13 +2,7 @@ import { z } from "zod"; import { Operator, LogicalOperator } from "../gen/data/v1/data"; import { createFilterGroupSchema } from "./internals"; -const DATA_TABLE_NAMES = [ - "users", - "sessions", - "tags", - "expressions", - "metadata", -] as const; +const DATA_TABLE_NAMES = ["users", "sessions", "tags", "expressions"] as const; const OPERATOR_MAP = { [Operator.EQ]: "EQ", From aa3ea4486cec3a3c38b603308a3b338319daf771 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 10:06:41 +0530 Subject: [PATCH 35/55] fix(projects): handling partial metadata insertion failure --- src/gen/data/v1/data.ts | 45 ------------------------ src/routes/http/api/apiKeys.ts | 13 +++++++ src/routes/http/api/onboarding.ts | 28 ++++++++------- src/routes/http/api/projects.ts | 34 ++++++++---------- src/storage/adapter/clickhouse/schema.ts | 4 +-- src/storage/db/postgres/schema.ts | 14 ++++---- 6 files changed, 53 insertions(+), 85 deletions(-) diff --git a/src/gen/data/v1/data.ts b/src/gen/data/v1/data.ts index 05223da..7f7ec34 100644 --- a/src/gen/data/v1/data.ts +++ b/src/gen/data/v1/data.ts @@ -339,51 +339,6 @@ export function expressionsFieldToJSON(object: ExpressionsField): string { } } -export enum MetadataField { - METADATA_FIELD_UNSPECIFIED = 0, - METADATA_ID = 1, - METADATA_PAYMENT_CRON = 2, - METADATA_PAYMENT_WEBHOOK = 3, - UNRECOGNIZED = -1, -} - -export function metadataFieldFromJSON(object: any): MetadataField { - switch (object) { - case 0: - case "METADATA_FIELD_UNSPECIFIED": - return MetadataField.METADATA_FIELD_UNSPECIFIED; - case 1: - case "METADATA_ID": - return MetadataField.METADATA_ID; - case 2: - case "METADATA_PAYMENT_CRON": - return MetadataField.METADATA_PAYMENT_CRON; - case 3: - case "METADATA_PAYMENT_WEBHOOK": - return MetadataField.METADATA_PAYMENT_WEBHOOK; - case -1: - case "UNRECOGNIZED": - default: - return MetadataField.UNRECOGNIZED; - } -} - -export function metadataFieldToJSON(object: MetadataField): string { - switch (object) { - case MetadataField.METADATA_FIELD_UNSPECIFIED: - return "METADATA_FIELD_UNSPECIFIED"; - case MetadataField.METADATA_ID: - return "METADATA_ID"; - case MetadataField.METADATA_PAYMENT_CRON: - return "METADATA_PAYMENT_CRON"; - case MetadataField.METADATA_PAYMENT_WEBHOOK: - return "METADATA_PAYMENT_WEBHOOK"; - case MetadataField.UNRECOGNIZED: - default: - return "UNRECOGNIZED"; - } -} - export interface FilterCondition { field: string; operator: Operator; diff --git a/src/routes/http/api/apiKeys.ts b/src/routes/http/api/apiKeys.ts index ca2f4f8..96ac975 100644 --- a/src/routes/http/api/apiKeys.ts +++ b/src/routes/http/api/apiKeys.ts @@ -8,6 +8,7 @@ import { } from "../../../context/requestContext.ts"; import { logger } from "../../../errors/logger.ts"; import { AuthError } from "../../../errors/auth.ts"; +import { StorageError } from "../../../errors/storage.ts"; import { authenticateHttpApiKey } from "../../../utils/authenticateHttpApiKey.ts"; import { generateAPIKey } from "../../../utils/generateAPIKey"; import { hashAPIKey } from "../../../utils/hashAPIKey"; @@ -139,6 +140,18 @@ export async function handleCreateApiKey( extra: { context: "create API key handler" }, }); + if ( + error instanceof StorageError && + error.type === "CONSTRAINT_VIOLATION" + ) { + builder.setError(409, { + type: "ConflictError", + message: "An API key with this name already exists", + }); + reply.code(409); + return { error: "An API key with this name already exists" }; + } + if (error instanceof AuthError) { builder.setError(401, { type: error.type, message: error.message }); reply.code(401); diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 00e92db..8213d45 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -105,14 +105,14 @@ export async function handleOnboarding( .secret; } catch (error) { if (liveWebhookId) { - liveClient.webhooks.delete(liveWebhookId).catch((e) => + await liveClient.webhooks.delete(liveWebhookId).catch((e) => Sentry.captureException(e, { extra: { context: "rollback: failed to delete live webhook" }, }) ); } if (testWebhookId) { - testClient.webhooks.delete(testWebhookId).catch((e) => + await testClient.webhooks.delete(testWebhookId).catch((e) => Sentry.captureException(e, { extra: { context: "rollback: failed to delete test webhook" }, }) @@ -164,7 +164,7 @@ export async function handleOnboarding( }); } catch (txnError) { if (liveWebhookId) { - liveClient.webhooks.delete(liveWebhookId).catch((e) => + await liveClient.webhooks.delete(liveWebhookId).catch((e) => Sentry.captureException(e, { extra: { context: @@ -174,7 +174,7 @@ export async function handleOnboarding( ); } if (testWebhookId) { - testClient.webhooks.delete(testWebhookId).catch((e) => + await testClient.webhooks.delete(testWebhookId).catch((e) => Sentry.captureException(e, { extra: { context: @@ -300,16 +300,20 @@ export async function handleGetConfig( reply.code(200); return { configured: true, - dodo_live_api_key: maskApiKey(decrypt(metadata.dodo_live_api_key)), - dodo_test_api_key: maskApiKey(decrypt(metadata.dodo_test_api_key)), + dodo_live_api_key: metadata.dodo_live_api_key + ? maskApiKey(decrypt(metadata.dodo_live_api_key)) + : null, + dodo_test_api_key: metadata.dodo_test_api_key + ? maskApiKey(decrypt(metadata.dodo_test_api_key)) + : null, dodo_live_product_id: metadata.dodo_live_product_id, dodo_test_product_id: metadata.dodo_test_product_id, - dodo_live_webhook_secret: maskApiKey( - decrypt(metadata.dodo_live_webhook_secret) - ), - dodo_test_webhook_secret: maskApiKey( - decrypt(metadata.dodo_test_webhook_secret) - ), + dodo_live_webhook_secret: metadata.dodo_live_webhook_secret + ? maskApiKey(decrypt(metadata.dodo_live_webhook_secret)) + : null, + dodo_test_webhook_secret: metadata.dodo_test_webhook_secret + ? maskApiKey(decrypt(metadata.dodo_test_webhook_secret)) + : null, currency: metadata.currency, redirect_url: metadata.redirect_url, }; diff --git a/src/routes/http/api/projects.ts b/src/routes/http/api/projects.ts index e9bc90c..88112ae 100644 --- a/src/routes/http/api/projects.ts +++ b/src/routes/http/api/projects.ts @@ -243,15 +243,20 @@ export async function handleUpdateProject( } await executeInTransaction(db, "update project", async (txn) => { - let rowsAffected = 0; + const exists = await txn + .select({ id: projectsTable.id }) + .from(projectsTable) + .where(eq(projectsTable.id, projectId)) + .limit(1); + if (exists.length === 0) { + throw new Error("PROJECT_NOT_FOUND"); + } if (body.name !== undefined) { - const updated = await txn + await txn .update(projectsTable) .set({ name: body.name }) - .where(eq(projectsTable.id, projectId)) - .returning({ id: projectsTable.id }); - rowsAffected += updated.length; + .where(eq(projectsTable.id, projectId)); } if (Object.keys(metaUpdates).length > 0) { @@ -260,21 +265,12 @@ export async function handleUpdateProject( .set(metaUpdates) .where(eq(metadataTable.projectId, projectId)) .returning({ projectId: metadataTable.projectId }); - rowsAffected += updated.length; - } - if (rowsAffected === 0) { - if (body.name !== undefined || Object.keys(metaUpdates).length > 0) { - throw new Error("PROJECT_NOT_FOUND"); - } else { - const exists = await txn - .select({ id: projectsTable.id }) - .from(projectsTable) - .where(eq(projectsTable.id, projectId)) - .limit(1); - if (exists.length === 0) { - throw new Error("PROJECT_NOT_FOUND"); - } + if (updated.length === 0) { + await txn.insert(metadataTable).values({ + projectId, + ...metaUpdates, + }); } } }); diff --git a/src/storage/adapter/clickhouse/schema.ts b/src/storage/adapter/clickhouse/schema.ts index 94eff8e..691ebce 100644 --- a/src/storage/adapter/clickhouse/schema.ts +++ b/src/storage/adapter/clickhouse/schema.ts @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS basic_usage_events ( debit_amount Int64, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (project_id, idempotency_key, user_id, event_id) +ORDER BY (project_id, idempotency_key, user_id) `; const AI_TOKEN_USAGE_EVENTS_TABLE = ` @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS ai_token_usage_events ( metrics String, metadata JSON ) ENGINE = ReplacingMergeTree() -ORDER BY (project_id, idempotency_key, user_id, event_id) +ORDER BY (project_id, idempotency_key, user_id) `; export async function runClickHouseMigrations(): Promise { diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index b726540..33bd7f0 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -379,14 +379,14 @@ export const metadataTable = pgTable( withTimezone: true, mode: "string", }), - dodo_live_api_key: text("dodo_live_api_key").notNull(), - dodo_test_api_key: text("dodo_test_api_key").notNull(), - dodo_live_product_id: text("dodo_live_product_id").notNull(), - dodo_test_product_id: text("dodo_test_product_id").notNull(), - dodo_live_webhook_secret: text("dodo_live_webhook_secret").notNull(), - dodo_test_webhook_secret: text("dodo_test_webhook_secret").notNull(), + dodo_live_api_key: text("dodo_live_api_key"), + dodo_test_api_key: text("dodo_test_api_key"), + dodo_live_product_id: text("dodo_live_product_id"), + dodo_test_product_id: text("dodo_test_product_id"), + dodo_live_webhook_secret: text("dodo_live_webhook_secret"), + dodo_test_webhook_secret: text("dodo_test_webhook_secret"), currency: text("currency").notNull().default("usd"), - redirect_url: text("redirect_url").notNull(), + redirect_url: text("redirect_url"), }, (table) => ({ uniqueProjectId: uniqueIndex("unique_project_id").on(table.projectId), From 4951286b7dc91167e9022f348d75284741c69047 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 29 Jun 2026 10:23:13 +0530 Subject: [PATCH 36/55] fix(projects): fixed multiple greptile issues --- src/routes/http/createdCheckout.ts | 26 ++++++++++----------- src/storage/db/postgres/helpers/sessions.ts | 10 +++++++- src/storage/db/postgres/schema.ts | 7 ++++-- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/routes/http/createdCheckout.ts b/src/routes/http/createdCheckout.ts index e818b91..afa81f0 100644 --- a/src/routes/http/createdCheckout.ts +++ b/src/routes/http/createdCheckout.ts @@ -124,7 +124,10 @@ export async function handleDodoWebhook( ); } - const session = await getSessionByCheckoutId(checkout_session_id); + const session = await getSessionByCheckoutId( + projectId, + checkout_session_id + ); if (!session) { return errorResponse( @@ -135,15 +138,6 @@ export async function handleDodoWebhook( ); } - if (session.projectId !== projectId) { - return errorResponse( - 404, - "NotFoundError", - `Session not found for checkout_session_id: ${checkout_session_id}`, - builder - ); - } - if (session.processed !== "pending") { Sentry.captureMessage( `Webhook received for session ${checkout_session_id} with non-pending status: ${session.processed}`, @@ -157,7 +151,12 @@ export async function handleDodoWebhook( if (webhookPayload.type === "payment.failed") { let claimed: boolean = false; await executeInTransaction(db, "process failed", async (txn) => { - claimed = await updateSessionStatus(checkout_session_id, "failed", txn); + claimed = await updateSessionStatus( + projectId, + checkout_session_id, + "failed", + txn + ); if (!claimed) return; }); if (!claimed) { @@ -169,7 +168,7 @@ export async function handleDodoWebhook( } builder.setSuccess(200); - forwardWebhook(session.projectId, session.apiKeyId, { + await forwardWebhook(session.projectId, session.apiKeyId, { eventType: "payment.failed", resource: "payment", action: "failed", @@ -196,6 +195,7 @@ export async function handleDodoWebhook( await executeInTransaction(db, "process checkout", async (txn) => { claimed = await updateSessionStatus( + projectId, checkout_session_id, "succeeded", txn @@ -229,7 +229,7 @@ export async function handleDodoWebhook( builder.setPaymentContext({ creditAmount }); builder.setSuccess(200); - forwardWebhook(session.projectId, apiKeyId, { + await forwardWebhook(session.projectId, apiKeyId, { eventType: "payment.succeeded", resource: "payment", action: "succeeded", diff --git a/src/storage/db/postgres/helpers/sessions.ts b/src/storage/db/postgres/helpers/sessions.ts index 8598390..3c6cec9 100644 --- a/src/storage/db/postgres/helpers/sessions.ts +++ b/src/storage/db/postgres/helpers/sessions.ts @@ -7,6 +7,7 @@ import type { UserId } from "../../../../config/identifiers"; import type { PgTransaction } from "drizzle-orm/pg-core"; export async function updateSessionStatus( + projectId: string, checkoutSessionId: string, status: "failed" | "succeeded", txn: PgTransaction @@ -17,6 +18,7 @@ export async function updateSessionStatus( .set({ processed: status }) .where( and( + eq(sessionsTable.projectId, projectId), eq(sessionsTable.sessionId, checkoutSessionId), eq(sessionsTable.processed, "pending") ) @@ -139,6 +141,7 @@ export async function handleAddSession( } export async function getSessionByCheckoutId( + projectId: string, checkoutSessionId: string ): Promise { const db = getPostgresDB(); @@ -147,7 +150,12 @@ export async function getSessionByCheckoutId( const [session] = await db .select() .from(sessionsTable) - .where(eq(sessionsTable.sessionId, checkoutSessionId)) + .where( + and( + eq(sessionsTable.projectId, projectId), + eq(sessionsTable.sessionId, checkoutSessionId) + ) + ) .limit(1); return session ?? undefined; diff --git a/src/storage/db/postgres/schema.ts b/src/storage/db/postgres/schema.ts index 33bd7f0..ecf1111 100644 --- a/src/storage/db/postgres/schema.ts +++ b/src/storage/db/postgres/schema.ts @@ -69,7 +69,7 @@ export const sessionsTable = pgTable( projectId: uuid("project_id") .references(() => projectsTable.id) .notNull(), - sessionId: text("session_id").notNull().unique(), + sessionId: text("session_id").notNull(), processed: text("processed", { enum: ["pending", "failed", "succeeded"] }) .default("pending") .notNull(), @@ -93,7 +93,10 @@ export const sessionsTable = pgTable( .default("production"), }, (table) => ({ - uniqueSessionId: uniqueIndex("unique_session_id").on(table.sessionId), + uniqueSessionId: uniqueIndex("unique_session_id").on( + table.projectId, + table.sessionId + ), userFk: foreignKey({ columns: [table.projectId, table.userId], foreignColumns: [usersTable.projectId, usersTable.id], From 518433277a2e601bf3182ec280df05da3553b747 Mon Sep 17 00:00:00 2001 From: Jayadeep Bejoy Date: Wed, 1 Jul 2026 20:05:34 +0530 Subject: [PATCH 37/55] chore: rename SCRAWN_KEY to MASTER_API_KEY, add MASTER_API_KEY_HASH, update proto submodule --- docker-compose.yml | 3 ++- proto | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8a13f6d..51af698 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,7 @@ services: NODE_ENV: production HMAC_SECRET: ${HMAC_SECRET} APP_URL: ${APP_URL} + MASTER_API_KEY_HASH: ${MASTER_API_KEY_HASH} dashboard: profiles: @@ -77,6 +78,6 @@ services: BETTER_AUTH_URL: http://localhost:3000 SCRAWN_BASE_URL: http://server:8060 SCRAWN_HTTP_URL: http://server:8060 - SCRAWN_KEY: ${SCRAWN_KEY} + MASTER_API_KEY: ${MASTER_API_KEY} diff --git a/proto b/proto index d9ceee6..e4b7237 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit d9ceee648e170df091eff3fbd04dbc29ebd056bd +Subproject commit e4b7237de066160224b9a71d72d5c3824897d8a3 From c71e76055cc6604f25ffbaf935fbf5ceefd1ca34 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 07:26:30 +0530 Subject: [PATCH 38/55] fix(analytics): computed filters now have the condition --- src/storage/adapter/common/clickHouseDialect.ts | 7 ++++--- src/storage/adapter/common/postgresDialect.ts | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts index 937bda7..4e8fac5 100644 --- a/src/storage/adapter/common/clickHouseDialect.ts +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -218,12 +218,13 @@ export class ClickHouseQueryDialect implements QueryDialect { for (const condition of group.conditions) { if (condition.field === "eventType") continue; const def = FIELD_REGISTRY[table]?.[condition.field]; - if (!def?.chWhere) continue; + + const colExpr = def?.chWhere || def?.chAggExpr || def?.chSelect || "NULL"; const op = OPERATOR_SQL[condition.operator]; if (!op) continue; const paramName = `p_${paramIndex.value++}`; - const paramType = def.chParamType; + const paramType = def?.chParamType || "String"; let value: string | number = condition.value; if ( @@ -237,7 +238,7 @@ export class ClickHouseQueryDialect implements QueryDialect { } params[paramName] = value; - parts.push(`${def.chWhere} ${op} {${paramName}:${paramType}}`); + parts.push(`${colExpr} ${op} {${paramName}:${paramType}}`); } for (const subGroup of group.groups) { diff --git a/src/storage/adapter/common/postgresDialect.ts b/src/storage/adapter/common/postgresDialect.ts index 1cabf7d..5b952a7 100644 --- a/src/storage/adapter/common/postgresDialect.ts +++ b/src/storage/adapter/common/postgresDialect.ts @@ -190,11 +190,14 @@ export class PostgresQueryDialect implements QueryDialect { for (const cond of group.conditions) { if (cond.field === "eventType") continue; const def = FIELD_REGISTRY[table]?.[cond.field]; - if (!def?.pgWhereCol) continue; + + const colExpr = + def?.pgWhereCol || def?.pgAggExpr || def?.pgSelect || "NULL"; const op = OPERATOR_SQL[cond.operator]; if (!op) continue; + parts.push( - sql`${sql.raw(def.pgWhereCol)} ${sql.raw(op)} ${cond.value}${sql.raw(def.pgWhereCast)}` + sql`${sql.raw(colExpr)} ${sql.raw(op)} ${cond.value}${sql.raw(def?.pgWhereCast || "")}` ); } From c37c05cdba9b1916307fe151741b2452f6a0493b Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 12:22:55 +0530 Subject: [PATCH 39/55] fix(analytics): implemented auth checks in query --- src/storage/query/dataQuery.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/storage/query/dataQuery.ts b/src/storage/query/dataQuery.ts index a2778f6..2eefbf5 100644 --- a/src/storage/query/dataQuery.ts +++ b/src/storage/query/dataQuery.ts @@ -24,6 +24,7 @@ import { } from "../db/postgres/schema"; import type { DataQueryRequest } from "../../zod/data"; import { EventError } from "../../errors/event"; +import type { AuthContext } from "../../context/auth"; interface FieldDef { col: AnyPgColumn; @@ -62,6 +63,7 @@ const TABLE_REGISTRY: Record = { tableName: "users", table: usersTable, fields: { + projectId: fieldDef(usersTable.projectId), id: fieldDef(usersTable.id), lastBilledTimestamp: fieldDef(usersTable.last_billed_timestamp), paymentProviderUserId: fieldDef(usersTable.payment_provider_user_id), @@ -72,6 +74,7 @@ const TABLE_REGISTRY: Record = { tableName: "sessions", table: sessionsTable, fields: { + projectId: fieldDef(sessionsTable.projectId), proxy_link_id: fieldDef(sessionsTable.proxy_link_id), sessionId: fieldDef(sessionsTable.sessionId), processed: fieldDef(sessionsTable.processed), @@ -85,6 +88,7 @@ const TABLE_REGISTRY: Record = { tableName: "tags", table: tagsTable, fields: { + projectId: fieldDef(tagsTable.projectId), id: fieldDef(tagsTable.id), key: fieldDef(tagsTable.key), amount: fieldDef(tagsTable.amount), @@ -94,6 +98,7 @@ const TABLE_REGISTRY: Record = { tableName: "expressions", table: expressionsTable, fields: { + projectId: fieldDef(expressionsTable.projectId), id: fieldDef(expressionsTable.id), key: fieldDef(expressionsTable.key), expr: fieldDef(expressionsTable.expr), @@ -103,6 +108,7 @@ const TABLE_REGISTRY: Record = { tableName: "metadata", table: metadataTable, fields: { + projectId: fieldDef(metadataTable.projectId), id: fieldDef(metadataTable.id), }, }, @@ -227,11 +233,22 @@ export function getDataTable(table: string): TableDef | undefined { } export async function executeDataQuery( + auth: AuthContext, config: DataQueryRequest, tableDef: TableDef ): Promise { const db = getPostgresDB(); - const whereClause = buildWhere(config.where, tableDef); + const userWhere = buildWhere(config.where, tableDef); + + const authConditions: SQL[] = []; + if (tableDef.fields.projectId) { + authConditions.push(eq(tableDef.fields.projectId.col, auth.projectId)); + } + if (tableDef.fields.mode) { + authConditions.push(eq(tableDef.fields.mode.col, auth.mode)); + } + const finalWhere = and(userWhere, ...authConditions); + const selectCols = buildSelect(tableDef); const columns = Object.keys(tableDef.fields); @@ -251,12 +268,12 @@ export async function executeDataQuery( db .select({ cnt: count() }) .from(tableDef.table) - .where(whereClause) + .where(finalWhere) .execute(), db .select(selectCols) .from(tableDef.table) - .where(whereClause) + .where(finalWhere) .orderBy(...orderClauses) .limit(fetchLimit) .offset(config.offset) From b47f89f494662c8cb72bdcd63db27cc1c029ecee Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 12:25:01 +0530 Subject: [PATCH 40/55] fix(analytics): fixed linting errors --- src/routes/gRPC/data/query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/gRPC/data/query.ts b/src/routes/gRPC/data/query.ts index eec482b..4ef99a0 100644 --- a/src/routes/gRPC/data/query.ts +++ b/src/routes/gRPC/data/query.ts @@ -47,7 +47,7 @@ export async function queryData( ); } - const result = await executeDataQuery(validated, tableDef); + const result = await executeDataQuery(auth, validated, tableDef); const response = QueryResponse.create(); response.columns = result.columns; From e527bc4a97ed10a72bc1a7544c381b54c4c64eaf Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 19:10:27 +0530 Subject: [PATCH 41/55] fix(analytics): taking orderBy in query now --- src/interface/storage/Storage.ts | 2 + .../adapter/common/clickHouseDialect.ts | 5 +- src/storage/adapter/common/postgresDialect.ts | 4 +- src/storage/adapter/common/queryEventsBase.ts | 63 +++++++------------ src/zod/data.ts | 3 +- 5 files changed, 33 insertions(+), 44 deletions(-) diff --git a/src/interface/storage/Storage.ts b/src/interface/storage/Storage.ts index c96f21e..baac20d 100644 --- a/src/interface/storage/Storage.ts +++ b/src/interface/storage/Storage.ts @@ -2,6 +2,7 @@ import type { SerializedEvent, EventKind } from "../event/Event"; import { type UserId } from "../../config/identifiers"; import type { DateTime } from "luxon"; import type { AuthContext } from "../../context/auth"; +import type { OrderByRequest } from "../../zod/data"; export type QueryOperator = "EQ" | "GT" | "GTE" | "LT" | "LTE" | "NEQ"; @@ -54,6 +55,7 @@ export interface QueryRequest { groupBy?: string; limit?: number; offset?: number; + orderBy?: OrderByRequest; } export type QueryResultRow = Record; diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts index 4e8fac5..472cc67 100644 --- a/src/storage/adapter/common/clickHouseDialect.ts +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -47,7 +47,10 @@ export class ClickHouseQueryDialect implements QueryDialect { }); let unionQuery = queries.join(" UNION ALL "); - unionQuery += " ORDER BY reportedTimestamp DESC"; + const orderByField = request.orderBy?.field ?? "reportedTimestamp"; + const orderByDir = request.orderBy?.descending ? "DESC" : "ASC"; + + unionQuery += ` ORDER BY ${orderByField} ${orderByDir}`; if (request.limit) { const limitParam = `p_${paramIndex.value++}`; diff --git a/src/storage/adapter/common/postgresDialect.ts b/src/storage/adapter/common/postgresDialect.ts index 5b952a7..79ecdd5 100644 --- a/src/storage/adapter/common/postgresDialect.ts +++ b/src/storage/adapter/common/postgresDialect.ts @@ -37,10 +37,12 @@ export class PostgresQueryDialect implements QueryDialect { }); const unionQuery = sql.join(subqueries, sql` UNION ALL `); + const orderByField = request.orderBy?.field ?? "reportedTimestamp"; + const orderByDir = request.orderBy?.descending ? "DESC" : "ASC"; const finalQuery = sql` ${unionQuery} - ORDER BY "reportedTimestamp" DESC + ORDER BY ${sql.raw(`"${orderByField}" ${orderByDir}`)} LIMIT ${request.limit ?? 100} OFFSET ${request.offset ?? 0} `; diff --git a/src/storage/adapter/common/queryEventsBase.ts b/src/storage/adapter/common/queryEventsBase.ts index 5728ccb..8d801dc 100644 --- a/src/storage/adapter/common/queryEventsBase.ts +++ b/src/storage/adapter/common/queryEventsBase.ts @@ -6,9 +6,7 @@ import type { export type EventTypeLabel = "BASIC_USAGE" | "AI_TOKEN_USAGE" | "PAYMENT"; export type EventTableName = - | "basic_usage_events" - | "ai_token_usage_events" - | "payment_events"; + "basic_usage_events" | "ai_token_usage_events" | "payment_events"; const EVENT_TYPE_TO_TABLE: Record = { BASIC_USAGE: "basic_usage_events", @@ -43,50 +41,33 @@ export const OPERATOR_SQL: Record = { NEQ: "!=", }; -interface EventTypeFilter { - operator: string; - value: string; -} +function canTableMatch( + group: QueryFilterGroup, + table: EventTableName +): boolean { + const tableEventType = TABLE_TO_EVENT_TYPE[table]; -function collectEventTypeFilters(group: QueryFilterGroup): EventTypeFilter[] { - const filters: EventTypeFilter[] = []; - for (const c of group.conditions) { + const conditionResults = group.conditions.map((c) => { if (c.field === "eventType") { - filters.push({ operator: c.operator, value: c.value }); + if (c.operator === "EQ") return c.value === tableEventType; + if (c.operator === "NEQ") return c.value !== tableEventType; + return true; } - } - for (const sub of group.groups) { - filters.push(...collectEventTypeFilters(sub)); - } - return filters; -} + return true; + }); -export function getTablesForRequest(where: QueryFilterGroup): EventTableName[] { - const filters = collectEventTypeFilters(where); - if (filters.length === 0) { - return [...ALL_TABLES]; - } + const groupResults = group.groups.map((g) => canTableMatch(g, table)); + const allResults = [...conditionResults, ...groupResults]; - const included = new Set(); - const excluded = new Set(); + if (allResults.length === 0) return true; - for (const { operator, value } of filters) { - if (!(value in EVENT_TYPE_TO_TABLE)) continue; - const table = EVENT_TYPE_TO_TABLE[value as EventTypeLabel]; - if (operator === "EQ") { - included.add(table); - } else if (operator === "NEQ") { - excluded.add(table); - } - } - - if (included.size > 0) { - return [...included]; - } - - if (excluded.size > 0) { - return ALL_TABLES.filter((t) => !excluded.has(t)); + if (group.logical === "AND") { + return allResults.every((res) => res); + } else { + return allResults.some((res) => res); } +} - return []; +export function getTablesForRequest(where: QueryFilterGroup): EventTableName[] { + return ALL_TABLES.filter((table) => canTableMatch(where, table)); } diff --git a/src/zod/data.ts b/src/zod/data.ts index a594e44..1276009 100644 --- a/src/zod/data.ts +++ b/src/zod/data.ts @@ -34,10 +34,11 @@ const filterGroupSchema = createFilterGroupSchema( LOGICAL_MAP ); -const orderBySchema = z.object({ +export const orderBySchema = z.object({ field: z.string(), descending: z.boolean().default(false), }); +export type OrderByRequest = z.output; export const dataQuerySchema = z .object({ From 99407f52e57ac0ce8e54fd51bbd1fb7b135ff399 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 21:05:14 +0530 Subject: [PATCH 42/55] fix(analytics): applied pagination globally --- src/storage/adapter/common/clickHouseDialect.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts index 472cc67..0ab7464 100644 --- a/src/storage/adapter/common/clickHouseDialect.ts +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -46,26 +46,28 @@ export class ClickHouseQueryDialect implements QueryDialect { return q; }); - let unionQuery = queries.join(" UNION ALL "); + const unionQuery = queries.join(" UNION ALL "); + let finalQuery = `SELECT * FROM (${unionQuery})`; + const orderByField = request.orderBy?.field ?? "reportedTimestamp"; const orderByDir = request.orderBy?.descending ? "DESC" : "ASC"; - unionQuery += ` ORDER BY ${orderByField} ${orderByDir}`; + finalQuery += ` ORDER BY ${orderByField} ${orderByDir}`; - if (request.limit) { + if (request.limit !== undefined) { const limitParam = `p_${paramIndex.value++}`; - unionQuery += ` LIMIT {${limitParam}:Int32}`; + finalQuery += ` LIMIT {${limitParam}:Int32}`; params[limitParam] = request.limit; } - if (request.offset) { + if (request.offset !== undefined) { const offsetParam = `p_${paramIndex.value++}`; - unionQuery += ` OFFSET {${offsetParam}:Int32}`; + finalQuery += ` OFFSET {${offsetParam}:Int32}`; params[offsetParam] = request.offset; } const rs = await client.query({ - query: unionQuery, + query: finalQuery, query_params: params, format: "JSONEachRow", }); From 6eea7cfec0407e17b54f66606ae462629b248679 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Fri, 3 Jul 2026 21:12:23 +0530 Subject: [PATCH 43/55] fix(analytics): matched protobuf orderBy field with zod --- src/zod/data.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zod/data.ts b/src/zod/data.ts index 1276009..c37cb33 100644 --- a/src/zod/data.ts +++ b/src/zod/data.ts @@ -44,7 +44,7 @@ export const dataQuerySchema = z .object({ table: z.enum(DATA_TABLE_NAMES), where: filterGroupSchema.optional(), - orderByList: z.array(orderBySchema).default([]), + orderBy: z.array(orderBySchema).default([]), limit: z.number().int().min(0).max(1000).default(100), offset: z.number().int().min(0).default(0), }) @@ -55,7 +55,7 @@ export const dataQuerySchema = z conditions: [], groups: [], }, - orderBy: v.orderByList, + orderBy: v.orderBy, limit: v.limit, offset: v.offset, })); From ea6d3ac467cbaace2dfc12a597c097f18941dd9a Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 4 Jul 2026 01:36:06 +0530 Subject: [PATCH 44/55] fix(analytics): restored output-cache fields --- src/storage/adapter/common/fieldRegistry.ts | 63 +++++++++++++++++++-- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/src/storage/adapter/common/fieldRegistry.ts b/src/storage/adapter/common/fieldRegistry.ts index 4af410d..a3e03a8 100644 --- a/src/storage/adapter/common/fieldRegistry.ts +++ b/src/storage/adapter/common/fieldRegistry.ts @@ -152,6 +152,22 @@ export const FIELD_REGISTRY: Record< chWhere: null, chParamType: "Int64", }, + outputCacheTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputCacheDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, creditAmount: { pgSelect: null, pgWhereCol: null, @@ -252,16 +268,16 @@ export const FIELD_REGISTRY: Record< }, debitAmount: { pgSelect: - "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0))::text", + "(COALESCE((metrics->'debit_amount'->>'input')::integer,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::integer,0) + COALESCE((metrics->'debit_amount'->>'output')::integer,0) + COALESCE((metrics->'debit_amount'->>'output_cache')::integer,0))::text", pgWhereCol: null, pgWhereCast: "", pgAggExpr: - "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0))", + "(COALESCE((metrics->'debit_amount'->>'input')::bigint,0) + COALESCE((metrics->'debit_amount'->>'input_cache')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output')::bigint,0) + COALESCE((metrics->'debit_amount'->>'output_cache')::bigint,0))", chSelect: - "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output'))", + "toString(JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output') + JSONExtractInt(metrics, 'debit_amount', 'output_cache'))", chWhere: null, chAggExpr: - "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output')", + "JSONExtractInt(metrics, 'debit_amount', 'input') + JSONExtractInt(metrics, 'debit_amount', 'input_cache') + JSONExtractInt(metrics, 'debit_amount', 'output') + JSONExtractInt(metrics, 'debit_amount', 'output_cache')", chParamType: "Int64", }, model: { @@ -333,6 +349,27 @@ export const FIELD_REGISTRY: Record< chAggExpr: "JSONExtractInt(metrics, 'debit_amount', 'input_cache')", chParamType: "Int64", }, + outputCacheTokens: { + pgSelect: "(metrics->'tokens'->>'output_cache')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'tokens'->>'output_cache')::bigint", + chSelect: "toString(JSONExtractInt(metrics, 'tokens', 'output_cache'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'tokens', 'output_cache')", + chParamType: "Int64", + }, + outputCacheDebitAmount: { + pgSelect: "(metrics->'debit_amount'->>'output_cache')::text", + pgWhereCol: null, + pgWhereCast: "", + pgAggExpr: "(metrics->'debit_amount'->>'output_cache')::bigint", + chSelect: + "toString(JSONExtractInt(metrics, 'debit_amount', 'output_cache'))", + chWhere: null, + chAggExpr: "JSONExtractInt(metrics, 'debit_amount', 'output_cache')", + chParamType: "Int64", + }, creditAmount: { pgSelect: null, pgWhereCol: null, @@ -496,6 +533,22 @@ export const FIELD_REGISTRY: Record< chWhere: null, chParamType: "Int64", }, + outputCacheTokens: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, + outputCacheDebitAmount: { + pgSelect: null, + pgWhereCol: null, + pgWhereCast: "", + chSelect: null, + chWhere: null, + chParamType: "Int64", + }, creditAmount: { pgSelect: "credit_amount::text", pgWhereCol: "credit_amount", @@ -541,6 +594,8 @@ export const OUTPUT_FIELDS = [ "outputDebitAmount", "inputCacheTokens", "inputCacheDebitAmount", + "outputCacheTokens", + "outputCacheDebitAmount", "creditAmount", "provider", "metadata", From 37c086220a2469f489898718c6d66db3c4603c1f Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 4 Jul 2026 01:45:44 +0530 Subject: [PATCH 45/55] fix(analytics): preserve eventType predicates --- src/storage/adapter/common/clickHouseDialect.ts | 12 +++++++++++- src/storage/adapter/common/postgresDialect.ts | 11 ++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts index 0ab7464..5a84090 100644 --- a/src/storage/adapter/common/clickHouseDialect.ts +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -221,7 +221,17 @@ export class ClickHouseQueryDialect implements QueryDialect { const parts: string[] = []; for (const condition of group.conditions) { - if (condition.field === "eventType") continue; + if (condition.field === "eventType") { + const tableEventType = TABLE_TO_EVENT_TYPE[table]; + let isTrue = false; + if (condition.operator === "EQ") + isTrue = condition.value === tableEventType; + else if (condition.operator === "NEQ") + isTrue = condition.value !== tableEventType; + else isTrue = true; + parts.push(isTrue ? "1=1" : "1=0"); + continue; + } const def = FIELD_REGISTRY[table]?.[condition.field]; const colExpr = def?.chWhere || def?.chAggExpr || def?.chSelect || "NULL"; diff --git a/src/storage/adapter/common/postgresDialect.ts b/src/storage/adapter/common/postgresDialect.ts index 79ecdd5..539a04e 100644 --- a/src/storage/adapter/common/postgresDialect.ts +++ b/src/storage/adapter/common/postgresDialect.ts @@ -190,7 +190,16 @@ export class PostgresQueryDialect implements QueryDialect { const parts: SQL[] = []; for (const cond of group.conditions) { - if (cond.field === "eventType") continue; + if (cond.field === "eventType") { + const tableEventType = TABLE_TO_EVENT_TYPE[table]; + let isTrue = false; + if (cond.operator === "EQ") isTrue = cond.value === tableEventType; + else if (cond.operator === "NEQ") + isTrue = cond.value !== tableEventType; + else isTrue = true; + parts.push(isTrue ? sql`1=1` : sql`1=0`); + continue; + } const def = FIELD_REGISTRY[table]?.[cond.field]; const colExpr = From b0fed5c2d9188f23b95b6636b34038bb9839e34e Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 4 Jul 2026 01:52:57 +0530 Subject: [PATCH 46/55] fix(analytics): fixed values in fieldRegistery --- src/routes/gRPC/query/queryEvents.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/routes/gRPC/query/queryEvents.ts b/src/routes/gRPC/query/queryEvents.ts index e2a3a4a..8dbcf4c 100644 --- a/src/routes/gRPC/query/queryEvents.ts +++ b/src/routes/gRPC/query/queryEvents.ts @@ -107,8 +107,11 @@ function buildEventRow(row: QueryResponse["rows"][number]): EventRow { if (row.outputCacheDebitAmount != null) { eventRow.outputCacheDebitAmount = Number(row.outputCacheDebitAmount); } + if (row.provider != null) { + eventRow.provider = String(row.provider); + } if (row.metadata != null) { - eventRow.metadata = JSON.stringify(row.metadata); + eventRow.metadata = String(row.metadata); } return eventRow; From 043c29f9a0da67a2af6b451295fb8df0d4bd578c Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 4 Jul 2026 02:10:47 +0530 Subject: [PATCH 47/55] fix(analytics): fixed AI debit assertions --- src/__tests__/queryEvents.test.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/__tests__/queryEvents.test.ts b/src/__tests__/queryEvents.test.ts index 6f7ac7b..f76ebc3 100644 --- a/src/__tests__/queryEvents.test.ts +++ b/src/__tests__/queryEvents.test.ts @@ -480,10 +480,9 @@ describe("QueryEvents", () => { grpcMetadata(`Bearer ${rawKey}`) ); - // NOTE: debitAmount filter is silently dropped for AI_TOKEN_USAGE events - // (whereCol is null in PG_FIELDS), so all AI events pass through const basicMatch = seededBasic.filter((e) => e.debitAmount > 60); - expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + const aiMatch = seededAi.filter((e) => e.debitAmount > 60); + expect(res.rows.length).toBe(basicMatch.length + aiMatch.length); }); it("filters by debitAmount GTE", async () => { @@ -499,7 +498,8 @@ describe("QueryEvents", () => { ); const basicMatch = seededBasic.filter((e) => e.debitAmount >= 50); - expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + const aiMatch = seededAi.filter((e) => e.debitAmount >= 50); + expect(res.rows.length).toBe(basicMatch.length + aiMatch.length); }); it("filters by debitAmount LT", async () => { @@ -515,7 +515,8 @@ describe("QueryEvents", () => { ); const basicMatch = seededBasic.filter((e) => e.debitAmount < 100); - expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + const aiMatch = seededAi.filter((e) => e.debitAmount < 100); + expect(res.rows.length).toBe(basicMatch.length + aiMatch.length); }); it("filters by debitAmount LTE", async () => { @@ -531,7 +532,8 @@ describe("QueryEvents", () => { ); const basicMatch = seededBasic.filter((e) => e.debitAmount <= 100); - expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + const aiMatch = seededAi.filter((e) => e.debitAmount <= 100); + expect(res.rows.length).toBe(basicMatch.length + aiMatch.length); }); it("filters by debitAmount NEQ", async () => { @@ -547,7 +549,8 @@ describe("QueryEvents", () => { ); const basicMatch = seededBasic.filter((e) => e.debitAmount !== 100); - expect(res.rows.length).toBe(basicMatch.length + seededAi.length); + const aiMatch = seededAi.filter((e) => e.debitAmount !== 100); + expect(res.rows.length).toBe(basicMatch.length + aiMatch.length); }); it("combines multiple conditions with AND", async () => { From feddc3aa523cbfd3a872475155bcb85e81375995 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Sat, 4 Jul 2026 02:31:17 +0530 Subject: [PATCH 48/55] fix(analytics): reject unknown aggregations --- src/storage/adapter/common/clickHouseDialect.ts | 4 ++++ src/storage/adapter/common/postgresDialect.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/storage/adapter/common/clickHouseDialect.ts b/src/storage/adapter/common/clickHouseDialect.ts index 5a84090..ae785f6 100644 --- a/src/storage/adapter/common/clickHouseDialect.ts +++ b/src/storage/adapter/common/clickHouseDialect.ts @@ -89,6 +89,10 @@ export class ClickHouseQueryDialect implements QueryDialect { const client = getClickHouseDB(); const agg = request.aggregation!; const isSum = agg.type === "SUM"; + + if (isSum && agg.field && !OUTPUT_FIELDS.includes(agg.field)) { + throw StorageError.invalidData(`Unknown aggregation field: ${agg.field}`); + } const paramIndex = { value: 0 }; const params: Record = {}; diff --git a/src/storage/adapter/common/postgresDialect.ts b/src/storage/adapter/common/postgresDialect.ts index 539a04e..441f6c4 100644 --- a/src/storage/adapter/common/postgresDialect.ts +++ b/src/storage/adapter/common/postgresDialect.ts @@ -64,6 +64,10 @@ export class PostgresQueryDialect implements QueryDialect { const agg = request.aggregation!; const isSum = agg.type === "SUM"; + if (isSum && agg.field && !OUTPUT_FIELDS.includes(agg.field)) { + throw StorageError.invalidData(`Unknown aggregation field: ${agg.field}`); + } + const subqueries = tables.map((t) => { const cols: SQL[] = []; From 7610b1dd4a845f58266c6b086abadc507a407c71 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 05:45:54 +0530 Subject: [PATCH 49/55] fix(products): products are now created automatically --- src/routes/http/api/onboarding.ts | 113 +++++++++++++++++++++++++----- src/zod/internals.ts | 16 ++++- 2 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 8213d45..ca18854 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -2,7 +2,11 @@ import { randomUUID } from "crypto"; import type { FastifyRequest, FastifyReply } from "fastify"; import * as Sentry from "@sentry/bun"; import { ZodError } from "zod"; -import DodoPayments from "dodopayments"; +import DodoPayments, { + AuthenticationError, + BadRequestError, +} from "dodopayments"; +import type { Currency } from "dodopayments/resources/misc"; import { onboardingSchema } from "../../../zod/internals.ts"; import { createWideEventBuilder, @@ -85,24 +89,72 @@ export async function handleOnboarding( let testSecret: string; let liveWebhookId: string | undefined; let testWebhookId: string | undefined; + let liveProductId: string; + let testProductId: string; try { - const liveWebhook = await liveClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, - description: "Scrawn live payment webhook", - filter_types: ["payment.succeeded", "payment.failed"], - }); - liveWebhookId = liveWebhook.id; + const [liveWebhook, testWebhook, liveProduct, testProduct] = + await Promise.all([ + liveClient.webhooks + .create({ + url: `${appUrl}/webhooks/payment/createCheckout?mode=production`, + description: "Scrawn live payment webhook", + filter_types: [ + "payment.succeeded", + "payment.processing", + "payment.failed", + ], + }) + .then((w) => { + liveWebhookId = w.id; + return w; + }), + testClient.webhooks + .create({ + url: `${appUrl}/webhooks/payment/createCheckout?mode=test`, + description: "Scrawn test payment webhook", + filter_types: [ + "payment.succeeded", + "payment.processing", + "payment.failed", + ], + }) + .then((w) => { + testWebhookId = w.id; + return w; + }), + liveClient.products.create({ + name: "Scrawn Billing", + price: { + type: "one_time_price", + currency: validated.currency as Currency, + price: 0, + pay_what_you_want: true, + purchasing_power_parity: false, + discount: 0, + }, + tax_category: "saas", + }), + testClient.products.create({ + name: "Scrawn Billing", + price: { + type: "one_time_price", + currency: validated.currency as Currency, + price: 0, + pay_what_you_want: true, + purchasing_power_parity: false, + discount: 0, + }, + tax_category: "saas", + }), + ]); + liveSecret = (await liveClient.webhooks.retrieveSecret(liveWebhook.id)) .secret; - - const testWebhook = await testClient.webhooks.create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, - description: "Scrawn test payment webhook", - filter_types: ["payment.succeeded", "payment.failed"], - }); - testWebhookId = testWebhook.id; testSecret = (await testClient.webhooks.retrieveSecret(testWebhook.id)) .secret; + + liveProductId = liveProduct.product_id; + testProductId = testProduct.product_id; } catch (error) { if (liveWebhookId) { await liveClient.webhooks.delete(liveWebhookId).catch((e) => @@ -118,13 +170,36 @@ export async function handleOnboarding( }) ); } - const errMsg = error instanceof Error ? error.message : String(error); + Sentry.captureException(error, { - extra: { context: "dodo webhook registration during onboarding" }, + extra: { + context: "dodo webhook/product registration during onboarding", + }, }); + + if (error instanceof AuthenticationError) { + builder.setError(400, { + type: "DodoAuthError", + message: + "Invalid Dodo Payments API key(s) provided. Please ensure both live and test keys are correct.", + }); + reply.code(400); + return {}; + } + + if (error instanceof BadRequestError) { + builder.setError(400, { + type: "DodoConfigError", + message: `Invalid configuration for Dodo Payments: ${error.message}`, + }); + reply.code(400); + return {}; + } + + const errMsg = error instanceof Error ? error.message : String(error); builder.setError(400, { type: "DodoApiError", - message: `Failed to register webhook with Dodo: ${errMsg}`, + message: `Failed to register with Dodo: ${errMsg}`, }); reply.code(400); return {}; @@ -146,8 +221,8 @@ export async function handleOnboarding( projectId, dodo_live_api_key: encrypt(validated.dodoLiveApiKey), dodo_test_api_key: encrypt(validated.dodoTestApiKey), - dodo_live_product_id: validated.dodoLiveProductId, - dodo_test_product_id: validated.dodoTestProductId, + dodo_live_product_id: validated.liveProductId, + dodo_test_product_id: validated.testProductId, dodo_live_webhook_secret: encrypt(liveSecret), dodo_test_webhook_secret: encrypt(testSecret), currency: validated.currency, diff --git a/src/zod/internals.ts b/src/zod/internals.ts index 525b71d..82ea408 100644 --- a/src/zod/internals.ts +++ b/src/zod/internals.ts @@ -1,5 +1,13 @@ import { z } from "zod"; +const currencyMap = { + usd: "USD", + eur: "EUR", + gbp: "GBP", + inr: "INR", + jpy: "JPY", +} as const; + export interface FilterGroupOutput { logical: "AND" | "OR"; conditions: C[]; @@ -36,8 +44,10 @@ export const onboardingSchema = z.object({ name: z.string().min(1, "Project name is required").max(255), dodoLiveApiKey: z.string().min(1, "Dodo live API key is required"), dodoTestApiKey: z.string().min(1, "Dodo test API key is required"), - dodoLiveProductId: z.string().min(1, "Dodo live product ID is required"), - dodoTestProductId: z.string().min(1, "Dodo test product ID is required"), - currency: z.string().min(1, "Currency is required"), + liveProductId: z.string().min(1, "Dodo live product ID is required"), + testProductId: z.string().min(1, "Dodo test product ID is required"), + currency: z + .enum(["usd", "eur", "gbp", "inr", "jpy"]) + .transform((c) => currencyMap[c]), redirectUrl: z.url("Redirect URL must be a valid URL"), }); From 9bde46f0691b5fa01869bc30c9346382aa4bc399 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 05:51:48 +0530 Subject: [PATCH 50/55] fix(products): removed product_id from zod --- src/routes/http/api/onboarding.ts | 4 ++-- src/zod/internals.ts | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index ca18854..e9ce0e7 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -221,8 +221,8 @@ export async function handleOnboarding( projectId, dodo_live_api_key: encrypt(validated.dodoLiveApiKey), dodo_test_api_key: encrypt(validated.dodoTestApiKey), - dodo_live_product_id: validated.liveProductId, - dodo_test_product_id: validated.testProductId, + dodo_live_product_id: liveProductId, + dodo_test_product_id: testProductId, dodo_live_webhook_secret: encrypt(liveSecret), dodo_test_webhook_secret: encrypt(testSecret), currency: validated.currency, diff --git a/src/zod/internals.ts b/src/zod/internals.ts index 82ea408..5fca269 100644 --- a/src/zod/internals.ts +++ b/src/zod/internals.ts @@ -44,8 +44,6 @@ export const onboardingSchema = z.object({ name: z.string().min(1, "Project name is required").max(255), dodoLiveApiKey: z.string().min(1, "Dodo live API key is required"), dodoTestApiKey: z.string().min(1, "Dodo test API key is required"), - liveProductId: z.string().min(1, "Dodo live product ID is required"), - testProductId: z.string().min(1, "Dodo test product ID is required"), currency: z .enum(["usd", "eur", "gbp", "inr", "jpy"]) .transform((c) => currencyMap[c]), From 6f55b52be6a05599cf7b5102d8138d0391a5a3bb Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 06:15:27 +0530 Subject: [PATCH 51/55] fix(products): webhook url now contains product id --- src/routes/http/api/onboarding.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index e9ce0e7..3a5fa7d 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -96,7 +96,7 @@ export async function handleOnboarding( await Promise.all([ liveClient.webhooks .create({ - url: `${appUrl}/webhooks/payment/createCheckout?mode=production`, + url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, description: "Scrawn live payment webhook", filter_types: [ "payment.succeeded", @@ -110,7 +110,7 @@ export async function handleOnboarding( }), testClient.webhooks .create({ - url: `${appUrl}/webhooks/payment/createCheckout?mode=test`, + url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, description: "Scrawn test payment webhook", filter_types: [ "payment.succeeded", From ed81030b63ec37304fbd8523d03999770af625e6 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 06:32:38 +0530 Subject: [PATCH 52/55] fix(products): handling rejected promise better now --- src/routes/http/api/onboarding.ts | 125 +++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 35 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index 3a5fa7d..f1c8554 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -89,40 +89,40 @@ export async function handleOnboarding( let testSecret: string; let liveWebhookId: string | undefined; let testWebhookId: string | undefined; - let liveProductId: string; - let testProductId: string; + let liveProductId: string | undefined; + let testProductId: string | undefined; try { - const [liveWebhook, testWebhook, liveProduct, testProduct] = - await Promise.all([ - liveClient.webhooks - .create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, - description: "Scrawn live payment webhook", - filter_types: [ - "payment.succeeded", - "payment.processing", - "payment.failed", - ], - }) - .then((w) => { - liveWebhookId = w.id; - return w; - }), - testClient.webhooks - .create({ - url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, - description: "Scrawn test payment webhook", - filter_types: [ - "payment.succeeded", - "payment.processing", - "payment.failed", - ], - }) - .then((w) => { - testWebhookId = w.id; - return w; - }), - liveClient.products.create({ + const results = await Promise.allSettled([ + liveClient.webhooks + .create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=production&projectId=${projectId}`, + description: "Scrawn live payment webhook", + filter_types: [ + "payment.succeeded", + "payment.processing", + "payment.failed", + ], + }) + .then((w) => { + liveWebhookId = w.id; + return w; + }), + testClient.webhooks + .create({ + url: `${appUrl}/webhooks/payment/createdCheckout?mode=test&projectId=${projectId}`, + description: "Scrawn test payment webhook", + filter_types: [ + "payment.succeeded", + "payment.processing", + "payment.failed", + ], + }) + .then((w) => { + testWebhookId = w.id; + return w; + }), + liveClient.products + .create({ name: "Scrawn Billing", price: { type: "one_time_price", @@ -133,8 +133,13 @@ export async function handleOnboarding( discount: 0, }, tax_category: "saas", + }) + .then((p) => { + liveProductId = p.product_id; + return p; }), - testClient.products.create({ + testClient.products + .create({ name: "Scrawn Billing", price: { type: "one_time_price", @@ -145,8 +150,24 @@ export async function handleOnboarding( discount: 0, }, tax_category: "saas", + }) + .then((p) => { + testProductId = p.product_id; + return p; }), - ]); + ]); + + const rejections = results.filter( + (r) => r.status === "rejected" + ) as PromiseRejectedResult[]; + if (rejections.length > 0) { + throw rejections[0]!.reason; + } + + const liveWebhook = (results[0] as PromiseFulfilledResult).value; + const testWebhook = (results[1] as PromiseFulfilledResult).value; + const liveProduct = (results[2] as PromiseFulfilledResult).value; + const testProduct = (results[3] as PromiseFulfilledResult).value; liveSecret = (await liveClient.webhooks.retrieveSecret(liveWebhook.id)) .secret; @@ -170,6 +191,20 @@ export async function handleOnboarding( }) ); } + if (liveProductId) { + await liveClient.products.archive(liveProductId).catch((e: unknown) => + Sentry.captureException(e, { + extra: { context: "rollback: failed to archive live product" }, + }) + ); + } + if (testProductId) { + await testClient.products.archive(testProductId).catch((e: unknown) => + Sentry.captureException(e, { + extra: { context: "rollback: failed to archive test product" }, + }) + ); + } Sentry.captureException(error, { extra: { @@ -258,6 +293,26 @@ export async function handleOnboarding( }) ); } + if (liveProductId) { + await liveClient.products.archive(liveProductId).catch((e: unknown) => + Sentry.captureException(e, { + extra: { + context: + "rollback: failed to archive live product after DB failure", + }, + }) + ); + } + if (testProductId) { + await testClient.products.archive(testProductId).catch((e: unknown) => + Sentry.captureException(e, { + extra: { + context: + "rollback: failed to archive test product after DB failure", + }, + }) + ); + } throw txnError; } From 2595e009837ec866faa20c7fbdb4f05555c9a805 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 13:23:15 +0530 Subject: [PATCH 53/55] fix(products): removed typecasting from promise results --- src/routes/http/api/onboarding.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/routes/http/api/onboarding.ts b/src/routes/http/api/onboarding.ts index f1c8554..5f285c9 100644 --- a/src/routes/http/api/onboarding.ts +++ b/src/routes/http/api/onboarding.ts @@ -157,17 +157,20 @@ export async function handleOnboarding( }), ]); - const rejections = results.filter( - (r) => r.status === "rejected" - ) as PromiseRejectedResult[]; - if (rejections.length > 0) { - throw rejections[0]!.reason; + if ( + results[0].status === "rejected" || + results[1].status === "rejected" || + results[2].status === "rejected" || + results[3].status === "rejected" + ) { + const rejected = results.find((r) => r.status === "rejected"); + throw rejected!.reason; } - const liveWebhook = (results[0] as PromiseFulfilledResult).value; - const testWebhook = (results[1] as PromiseFulfilledResult).value; - const liveProduct = (results[2] as PromiseFulfilledResult).value; - const testProduct = (results[3] as PromiseFulfilledResult).value; + const liveWebhook = results[0].value; + const testWebhook = results[1].value; + const liveProduct = results[2].value; + const testProduct = results[3].value; liveSecret = (await liveClient.webhooks.retrieveSecret(liveWebhook.id)) .secret; From f9be0c57329a7c88ffbf3ed02a755f5398ef1c73 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 13:52:48 +0530 Subject: [PATCH 54/55] fix(products): changed to Currency type from dodo --- src/zod/internals.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/zod/internals.ts b/src/zod/internals.ts index 5fca269..22c2cae 100644 --- a/src/zod/internals.ts +++ b/src/zod/internals.ts @@ -1,12 +1,5 @@ import { z } from "zod"; - -const currencyMap = { - usd: "USD", - eur: "EUR", - gbp: "GBP", - inr: "INR", - jpy: "JPY", -} as const; +import type { Currency } from "dodopayments/resources"; export interface FilterGroupOutput { logical: "AND" | "OR"; @@ -46,6 +39,6 @@ export const onboardingSchema = z.object({ dodoTestApiKey: z.string().min(1, "Dodo test API key is required"), currency: z .enum(["usd", "eur", "gbp", "inr", "jpy"]) - .transform((c) => currencyMap[c]), + .transform((c) => c.toUpperCase() as Currency), redirectUrl: z.url("Redirect URL must be a valid URL"), }); From e2cb64abb1de33c9825468aa9f71c795f01c6122 Mon Sep 17 00:00:00 2001 From: 0xanshu Date: Mon, 6 Jul 2026 14:43:13 +0530 Subject: [PATCH 55/55] fix(products): correct path to import Currency type now --- src/zod/internals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zod/internals.ts b/src/zod/internals.ts index 22c2cae..3c97688 100644 --- a/src/zod/internals.ts +++ b/src/zod/internals.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import type { Currency } from "dodopayments/resources"; +import type { Currency } from "dodopayments/resources/misc"; export interface FilterGroupOutput { logical: "AND" | "OR";