diff --git a/apps/cruncher-server/src/engineV2/engine.ts b/apps/cruncher-server/src/engineV2/engine.ts index c82cba2..46749f3 100644 --- a/apps/cruncher-server/src/engineV2/engine.ts +++ b/apps/cruncher-server/src/engineV2/engine.ts @@ -510,7 +510,7 @@ export class Engine { const onBatchDone = async (data: ProcessedData[]): Promise => { if (queryTaskState.abortController.signal.aborted) return; for (const item of data) { - if (!item.id) item.id = crypto.randomUUID(); + if (!item.id) item.id = uuidv7(); item.object["_source"] = { type: "string", value: instanceRef }; } const bytes = serializeBatch(data); @@ -621,7 +621,7 @@ export class Engine { cancelToken: queryTaskState.abortController.signal, onBatchDone: async (batch) => { for (const item of batch) { - if (!item.id) item.id = crypto.randomUUID(); + if (!item.id) item.id = uuidv7(); item.object["_source"] = { type: "string", value: instanceRef, @@ -779,7 +779,7 @@ export class Engine { searchProfile: taskState.task.input.searchProfileRef, createdAt: taskState.task.createdAt, completedAt, - rowCount: taskState.lastBatchStatus?.views.events.total ?? (totalRowCount > 0 ? totalRowCount : null), + rowCount: taskState.lastBatchStatus?.views.events.total ?? null, status: finalStatus, error: taskState.task.error, subtaskIds: taskState.subTasks.map((s) => s.subtaskId), @@ -937,18 +937,18 @@ export class Engine { toTime: number, ): Promise { const queryTaskState = this.taskStore.get(taskId)!; - const [ - { displayResults: pipelineData, rawEventCount, autoCompleteKeys }, - backendBuckets, - ] = await Promise.all([ - runPipelineAndSave(this.ctx, taskId, parsedTree, { ...queryTaskState.task.input.queryOptions, fromTime, toTime }), - this.ctx.backend.computeHistogram(queryTaskState.chunkPaths, fromTime, toTime, 100), - ]); + const { displayResults: pipelineData, autoCompleteKeys, deferredWrites } = await runPipelineAndSave( + this.ctx, taskId, parsedTree, { ...queryTaskState.task.input.queryOptions, fromTime, toTime }, + ); + await deferredWrites; + const backendBuckets = queryTaskState.eventsResultPath + ? await this.ctx.backend.computeHistogram([queryTaskState.eventsResultPath], fromTime, toTime, 100) + : []; await queryTaskState.mutex.runExclusive(async () => { queryTaskState.lastBatchStatus = this._buildBatchStatus(queryTaskState, { scale: { from: fromTime, to: toTime }, - total: rawEventCount, + total: pipelineData.events.data.length, buckets: backendBuckets, autoCompleteKeys, tableDataPoints: pipelineData.table?.dataPoints, diff --git a/apps/cruncher-server/src/engineV2/localstate/LocalStateDB.ts b/apps/cruncher-server/src/engineV2/localstate/LocalStateDB.ts index 801adeb..757e8f3 100644 --- a/apps/cruncher-server/src/engineV2/localstate/LocalStateDB.ts +++ b/apps/cruncher-server/src/engineV2/localstate/LocalStateDB.ts @@ -285,7 +285,7 @@ export class LocalStateDB { } deleteEntry(id: string): void { - this.db.delete(queryHistory).where(sql`${queryHistory.id} = ${id}`).run(); + this.db.delete(queryHistory).where(eq(queryHistory.id, id)).run(); } clearHistory(): void { @@ -713,7 +713,7 @@ export class LocalStateDB { searchProfile: r.searchProfile, createdAt: r.createdAt, completedAt: r.completedAt ?? null, - rowCount: rowCounts ? (rowCounts.get(r.id) ?? r.rowCount ?? null) : (r.rowCount ?? null), + rowCount: r.rowCount ?? (rowCounts ? (rowCounts.get(r.id) ?? null) : null), status: r.status, error: r.error ?? null, subtaskIds, diff --git a/apps/cruncher-server/src/engineV2/pipelineExecution.ts b/apps/cruncher-server/src/engineV2/pipelineExecution.ts index a0ce0f7..0e05f6f 100644 --- a/apps/cruncher-server/src/engineV2/pipelineExecution.ts +++ b/apps/cruncher-server/src/engineV2/pipelineExecution.ts @@ -6,19 +6,19 @@ import { asDisplayString, ProcessedData, } from "@cruncher/adapter-utils/logTypes"; -import { DisplayResults } from "@cruncher/server-shared"; -import { processEval } from "@cruncher/server-shared"; -import { processRegex } from "@cruncher/server-shared"; import { + DisplayResults, PipelineItemProcessor, + processEval, processPipelineV2, + processRegex, + processSort, + processStats, + processTable, + processTimeChart, + processUnpack, + processWhere, } from "@cruncher/server-shared"; -import { processSort } from "@cruncher/server-shared"; -import { processStats } from "@cruncher/server-shared"; -import { processTable } from "@cruncher/server-shared"; -import { processTimeChart } from "@cruncher/server-shared"; -import { processWhere } from "@cruncher/server-shared"; -import { processUnpack } from "@cruncher/server-shared"; import { ParsedQuery } from "@cruncher/qql"; import { createSignal } from "@cruncher/utils"; import { deserializeRows, serializeBatch } from "./duckdb/serialization"; @@ -117,11 +117,11 @@ export async function runPipelineAndSave( taskId: TaskRef, parsedTree: ParsedQuery, queryOptions: SerializableParams, -): Promise<{ displayResults: DisplayResults; rawEventCount: number; autoCompleteKeys: string[] }> { +): Promise<{ displayResults: DisplayResults; rawEventCount: number; autoCompleteKeys: string[]; deferredWrites: Promise }> { const taskState: QueryTaskState | undefined = ctx.taskStore.get(taskId); if (!taskState) { const empty: DisplayResults = { events: { type: "events", data: [] }, table: undefined, view: undefined }; - return { displayResults: empty, rawEventCount: 0, autoCompleteKeys: [] }; + return { displayResults: empty, rawEventCount: 0, autoCompleteKeys: [], deferredWrites: Promise.resolve() }; } const context = { @@ -243,5 +243,5 @@ export async function runPipelineAndSave( ]), ]; - return { displayResults, rawEventCount, autoCompleteKeys }; + return { displayResults, rawEventCount, autoCompleteKeys, deferredWrites }; } diff --git a/apps/cruncher-server/src/engineV2/sessionManager.ts b/apps/cruncher-server/src/engineV2/sessionManager.ts index 2100c6d..461baa8 100644 --- a/apps/cruncher-server/src/engineV2/sessionManager.ts +++ b/apps/cruncher-server/src/engineV2/sessionManager.ts @@ -113,19 +113,20 @@ export async function restoreSession(ctx: EngineCtx, taskId: TaskRef): Promise { queryTaskState.lastBatchStatus = ctx.buildBatchStatus(queryTaskState, { scale: { from: queryOptions.fromTime, to: queryOptions.toTime }, - total: rawEventCount, + total: pipelineData.events.data.length, buckets: [], - autoCompleteKeys: [], + autoCompleteKeys, tableDataPoints: pipelineData.table?.dataPoints, }); });