Lazy cache Knowhere chunk pages

next.config.ts

@@ -14,6 +14,9 @@ const nextConfig: NextConfig = {
     "notebook.127.0.0.1.nip.io",
     "dashboard.127.0.0.1.nip.io",
   ],
+  turbopack: {
+    root: process.cwd(),
+  },
 };
 
 export default nextConfig;

package.json

@@ -24,7 +24,7 @@
     "@antv/chart-visualization-skills": "0.1.3",
     "@effect/platform": "^0.96.1",
     "@neondatabase/serverless": "^1.1.0",
-    "@ontos-ai/knowhere-sdk": "^0.6.0",
+    "@ontos-ai/knowhere-sdk": "^0.10.0",
     "@radix-ui/react-alert-dialog": "^1.1.15",
     "@radix-ui/react-checkbox": "^1.3.3",
     "@radix-ui/react-dialog": "^1.1.15",

src/app/api/sources/[sourceId]/chunks/route.test.ts

@@ -2,14 +2,19 @@ import { NextRequest } from "next/server"
 import { beforeEach, describe, expect, it, vi } from "vitest"
 
 const mocks = vi.hoisted(() => ({
+  blobGet: vi.fn(),
+  blobPut: vi.fn(),
+  deleteBlob: vi.fn(),
   ensureApiKeyForWorkspace: vi.fn(),
   ensureWorkspace: vi.fn(),
   fetchDemoChunkPage: vi.fn(),
   findSourceInWorkspace: vi.fn(),
   getCurrentUser: vi.fn(),
   getSourceParseAssetUrls: vi.fn(),
+  localizeRemoteDocument: vi.fn(),
   makeKnowhereClient: vi.fn(),
   requireUser: vi.fn(),
+  updateSourceRevisionKey: vi.fn(),
 }))
 
 vi.mock("next/headers", () => ({
@@ -36,10 +41,18 @@ vi.mock("@/integrations/knowhere", () => ({
   makeKnowhereClient: mocks.makeKnowhereClient,
 }))
 
+vi.mock("@vercel/blob", () => ({
+  del: mocks.deleteBlob,
+  get: mocks.blobGet,
+  put: mocks.blobPut,
+}))
+
 vi.mock("@/domains/sources/service", () => ({
   sourceService: {
     findInWorkspace: mocks.findSourceInWorkspace,
     getParseAssetUrls: mocks.getSourceParseAssetUrls,
+    localizeRemoteDocument: mocks.localizeRemoteDocument,
+    updateSourceRevisionKey: mocks.updateSourceRevisionKey,
   },
 }))
 
@@ -54,6 +67,11 @@ import { GET } from "./route"
 describe("GET /api/sources/[sourceId]/chunks", () => {
   beforeEach(() => {
     vi.clearAllMocks()
+    mocks.blobGet.mockResolvedValue(null)
+    mocks.blobPut.mockImplementation(async (pathname: string) => ({
+      url: `https://blob.example/${pathname}`,
+    }))
+    mocks.updateSourceRevisionKey.mockResolvedValue(null)
   })
 
   it("serves API-owned demo chunks for anonymous canonical demo sources", async () => {
@@ -502,7 +520,48 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
     })
   })
 
-  it("does not load chunks from unlocalized remote source ids", async () => {
+  it("materializes a remote source id on open before loading chunks", async () => {
+    const knowhereClient = {
+      documents: {
+        list: vi.fn(async () => ({
+          documents: [
+            {
+              documentId: "doc_remote",
+              namespace: "default",
+              status: "active",
+              currentJobResultId: "job_result_1",
+              sourceFileName: "remote.pdf",
+              documentMetadata: {
+                mimeType: "application/pdf",
+              },
+            },
+          ],
+        })),
+        listChunks: vi.fn(async () => ({
+          documentId: "doc_remote",
+          jobResultId: "job_result_1",
+          chunks: [
+            {
+              id: "dchk_remote",
+              chunkId: "parser_remote",
+              chunkType: "text",
+              content: "Remote chunk",
+              sectionPath: "Summary",
+              sourceChunkPath: "Default_Root/remote.pdf/Summary",
+              filePath: null,
+              metadata: {},
+              sortOrder: 0,
+            },
+          ],
+          pagination: {
+            page: 1,
+            pageSize: 1,
+            total: 1,
+            totalPages: 1,
+          },
+        })),
+      },
+    }
     mocks.getCurrentUser.mockResolvedValue({
       id: "user_1",
       email: null,
@@ -515,6 +574,27 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
       createdAt: new Date("2026-05-10T00:00:00.000Z"),
     })
     mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo"))
+    mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123")
+    mocks.makeKnowhereClient.mockReturnValue(knowhereClient)
+    mocks.localizeRemoteDocument.mockResolvedValue({
+      id: "00000000-0000-0000-0000-000000000009",
+      workspaceId: "workspace_1",
+      title: "remote.pdf",
+      mimeType: "application/pdf",
+      sizeBytes: 0,
+      status: "ready",
+      failureReason: null,
+      knowhereJobId: "job_result_1",
+      knowhereDocumentId: "doc_remote",
+      stagedBlobPathname: null,
+      stagedBlobUrl: null,
+      originalBlobPathname: null,
+      originalBlobUrl: null,
+      demoKey: null,
+      createdAt: new Date("2026-05-10T00:00:00.000Z"),
+      updatedAt: new Date("2026-05-10T00:00:00.000Z"),
+      deletedAt: null,
+    })
 
     const response = await GET(
       new NextRequest(
@@ -527,12 +607,46 @@ describe("GET /api/sources/[sourceId]/chunks", () => {
       },
     )
 
-    await expect(response.json()).resolves.toEqual({
-      message: "Source not found.",
+    await expect(response.json()).resolves.toMatchObject({
+      chunks: [
+        {
+          chunkId: "dchk_remote",
+          parserChunkId: "parser_remote",
+          documentId: "doc_remote",
+          sourceTitle: "remote.pdf",
+        },
+      ],
+      pagination: {
+        page: 1,
+        pageSize: 1,
+        total: 1,
+      },
     })
-    expect(response.status).toBe(404)
+    expect(response.status).toBe(200)
     expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled()
-    expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled()
-    expect(mocks.makeKnowhereClient).not.toHaveBeenCalled()
+    expect(mocks.ensureApiKeyForWorkspace).toHaveBeenCalledWith(
+      "workspace_1",
+      "session=abc",
+    )
+    expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith(
+      "workspace_1",
+      {
+        documentId: "doc_remote",
+        namespace: "default",
+        status: "ready",
+        title: "remote.pdf",
+        mimeType: "application/pdf",
+        sizeBytes: undefined,
+        revisionKey: "job_result_1",
+      },
+    )
+    expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith(
+      "doc_remote",
+      {
+        page: 1,
+        pageSize: 1,
+        includeAssetUrls: true,
+      },
+    )
   })
 })

src/app/api/sources/reconcile/route.ts

@@ -1,50 +1,27 @@
 import { serve } from "@upstash/workflow/nextjs"
 
-import { reconcileSourcesForWorkspace } from "@/domains/sources/reconcile"
-import { makeKnowhereClient } from "@/integrations/knowhere"
-import { logger } from "@/lib/logger"
-
-type ReconcilePayload = {
-  readonly workspaceId: string
-  readonly sourceId: string
-  readonly apiKey: string
-}
-
-const MAX_POLL_ATTEMPTS = 60
-const INITIAL_DELAY_S = 3
-const MAX_DELAY_S = 30
-
-export const { POST } = serve<ReconcilePayload>(async (context) => {
-  const { workspaceId, sourceId, apiKey } = context.requestPayload
-  const workspace = { id: workspaceId }
-  const client = makeKnowhereClient(apiKey)
-  let delay = INITIAL_DELAY_S
-
-  for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt++) {
-    const resolved = await context.run(`poll-${attempt}`, async () => {
-      const sources = await reconcileSourcesForWorkspace(workspace, client)
-      const source = sources.find((s) => s.id === sourceId)
-      if (!source || source.status !== "parsing") {
-        return { done: true, status: source?.status ?? "gone" } as const
-      }
-      return { done: false } as const
+import { sourceReconcileRouteWorkflow } from "@/domains/sources/source-reconcile-route-workflow"
+
+type ReconcilePayload = Parameters<
+  typeof sourceReconcileRouteWorkflow.normalizeReconcilePayload
+>[0]
+
+export const { POST } = serve<ReconcilePayload>(
+  async (context) => {
+    const payload = sourceReconcileRouteWorkflow.normalizeReconcilePayload(
+      context.requestPayload,
+    )
+    await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({
+      context,
+      payload,
     })
-
-    if (resolved.done) {
-      logger.info("workflow: source resolved", {
-        sourceId,
-        status: resolved.status,
-        attempts: attempt + 1,
-      })
-      return
-    }
-
-    await context.sleep(`wait-${attempt}`, delay)
-    delay = Math.min(Math.round(delay * 1.5), MAX_DELAY_S)
-  }
-
-  logger.error("workflow: exhausted poll attempts", {
-    sourceId,
-    maxAttempts: MAX_POLL_ATTEMPTS,
-  })
-})
+  },
+  {
+    failureFunction: async ({ context, failResponse }) => {
+      await sourceReconcileRouteWorkflow.markSourceFailedAfterWorkflowFailure(
+        context.requestPayload,
+        failResponse,
+      )
+    },
+  },
+)

src/components/source-row.tsx

@@ -37,6 +37,7 @@ export function SourceRow({
   const isBusy = source.status === "uploading" || source.status === "parsing";
   const isFailed = source.status === "failed";
   const isLibrarySource = source.officialLibrary !== undefined;
+  const isRemoteSource = source.kind === "remote";
 
   const iconBg = fileIconTint(source.title);
 
@@ -57,7 +58,7 @@ export function SourceRow({
       >
         <Checkbox
           checked={!source.excludedFromQuery}
-          disabled={!isReady || !onToggleIncluded || isAdding}
+          disabled={!isReady || !onToggleIncluded || isAdding || isRemoteSource}
           onCheckedChange={(checked) =>
             onToggleIncluded?.(source.id, checked === true)
           }
@@ -96,9 +97,7 @@ export function SourceRow({
             }`}
           >
             {isReady
-              ? `${isLibrarySource ? "Official Library" : "Processed"} · ${
-                  source.chunkCount ?? 0
-                } chunks`
+              ? `${getReadySourceLabel(source)} · ${source.chunkCount ?? 0} chunks`
               : source.status === "parsing"
                 ? "Preparing"
                 : source.status === "uploading"
@@ -162,6 +161,12 @@ export function SourceRow({
   );
 }
 
+function getReadySourceLabel(source: SourceView): string {
+  if (source.officialLibrary !== undefined) return "Official Library";
+  if (source.kind === "remote") return "Remote";
+  return "Processed";
+}
+
 function fileIconTint(title: string): { bg: string; fg: string } {
   const ext = title.split(".").pop()?.toLowerCase();
   switch (ext) {

src/components/workspace-chat-workflow.ts

@@ -250,7 +250,8 @@ export function useWorkspaceChatWorkflow({
   async function handleChatSend(text: string): Promise<void> {
     const sendStart = Date.now()
     const selectedSourcesCount = sources.filter(
-      (source) => source.status === "ready" && !source.excludedFromQuery,
+      (source) =>
+        isQueryableReadySource(source) && !source.excludedFromQuery,
     ).length
     const demoSourceIds = getMaterializableDemoSourceIds(sources)
     if (demoSourceIds.length > 0) {
@@ -396,10 +397,14 @@ function getMaterializableDemoSourceIds(
 }
 
 function hasQueryableReadySource(sources: readonly SourceView[]): boolean {
-  return sources.some(
-    (source) =>
-      source.status === "ready" &&
-      !isUnmaterializedOfficialLibrarySource(source),
+  return sources.some(isQueryableReadySource)
+}
+
+function isQueryableReadySource(source: SourceView): boolean {
+  return (
+    source.status === "ready" &&
+    !isUnmaterializedOfficialLibrarySource(source) &&
+    source.kind !== "remote"
   )
 }
 

src/components/workspace-source-workflow.ts

@@ -221,13 +221,17 @@ export function useWorkspaceSourceWorkflow({
 function isQueryableReadySource(source: SourceView): boolean {
   if (source.status !== "ready") return false
 
-  return !isUnmaterializedOfficialLibrarySource(source)
+  return !isUnmaterializedOfficialLibrarySource(source) && !isRemoteSource(source)
 }
 
 function isUnmaterializedOfficialLibrarySource(source: SourceView): boolean {
   return source.kind === "demo" && source.officialLibrary !== undefined
 }
 
+function isRemoteSource(source: SourceView): boolean {
+  return source.kind === "remote"
+}
+
 function archiveSourceMutation(
   _key: string,
   { arg: sourceId }: { readonly arg: string },