feat: update index

.env.example

@@ -45,5 +45,29 @@ TEMPORAL_TASK_QUEUE=brain-core
 # Port for the temporal-worker's local health/status endpoint.
 TEMPORAL_WORKER_HEALTH_PORT=4100
 
+# ── Embeddings (knowledge pipeline) ─────────────────────────────
+# Provider: local (offline, dev default) | openai | gemini | voyage
+EMBEDDINGS_PROVIDER=local
+# EMBEDDINGS_MODEL=text-embedding-3-small
+# EMBEDDINGS_DIMENSION=384
+# OPENAI_API_KEY=
+# GEMINI_API_KEY=
+# VOYAGE_API_KEY=
+
+# ── Chunking ────────────────────────────────────────────────────
+CHUNK_SIZE=400
+CHUNK_OVERLAP=60
+
+# ── Connectors (Google Workspace OAuth) ─────────────────────────
+# Create OAuth client (Web application) at console.cloud.google.com,
+# authorized redirect URI = GOOGLE_REDIRECT_URI below.
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+GOOGLE_REDIRECT_URI=http://localhost:4000/api/v1/connectors/google/callback
+# Refresh-token encryption at rest. Generate: openssl rand -hex 32
+TOKEN_ENCRYPTION_KEY=change-me-generate-with-openssl-rand-hex-32
+CONNECTOR_TASK_QUEUE=brain-connectors
+CONNECTOR_WORKER_HEALTH_PORT=4101
+
 # ── Frontend ────────────────────────────────────────────────────
 NEXT_PUBLIC_API_URL=http://localhost:4000

README.md

@@ -28,12 +28,18 @@ apps/
   web/            Next.js dashboard (login, register, dashboard, profile, settings)
 packages/
   activities/     Temporal activity implementations (all side effects)
+  auth/           OAuth2 helpers + AES-256-GCM token encryption
   config/         Shared ESLint + tsconfig presets
+  connectors/
+    core/         Connector SDK (provider-agnostic framework)
+    google/       Google Workspace connector (Drive/Docs/Sheets/Slides/Gmail/Calendar)
+  events/         Redis-backed platform event bus
   types/          Shared TypeScript contracts (API envelope, auth, roles)
   ui/             Shared shadcn-style React primitives
   utils/          Small dependency-free helpers
   workflows/      Temporal workflow definitions (deterministic orchestration)
 services/
+  connector-worker/ Temporal worker for connector sync (queue brain-connectors, health :4101)
   temporal-worker/ Temporal worker (workflow + activity host, health :4100)
   worker/         BullMQ worker (generic "system" queue)
 infrastructure/
@@ -126,5 +132,6 @@ Every endpoint returns the standard envelope:
 ```
 
 See [docs/architecture.md](docs/architecture.md) for design decisions and
-[docs/temporal.md](docs/temporal.md) for the workflow orchestration guide
-(why Temporal, how workflows/activities work, how to add new workflows).
+[docs/temporal.md](docs/temporal.md) for the workflow orchestration guide and
+[docs/connectors.md](docs/connectors.md) for the Knowledge Connector Platform
+(Connector SDK, OAuth flow, sync architecture, extension guide).

apps/api/package.json

@@ -19,13 +19,19 @@
   },
   "dependencies": {
     "@company-brain/activities": "workspace:*",
+    "@company-brain/auth": "workspace:*",
+    "@company-brain/connector-core": "workspace:*",
+    "@company-brain/connector-google": "workspace:*",
+    "@company-brain/events": "workspace:*",
+    "@company-brain/knowledge": "workspace:*",
     "@company-brain/types": "workspace:*",
     "@company-brain/utils": "workspace:*",
     "@company-brain/workflows": "workspace:*",
     "@fastify/compress": "^8.0.1",
     "@fastify/cookie": "^11.0.2",
     "@fastify/cors": "^11.0.1",
     "@fastify/helmet": "^13.0.1",
+    "@fastify/multipart": "^10.1.0",
     "@fastify/rate-limit": "^10.2.2",
     "@fastify/swagger": "^9.5.0",
     "@fastify/swagger-ui": "^5.2.2",

apps/api/prisma/migrations/20260714075643_knowledge_brain/migration.sql

@@ -0,0 +1,296 @@
+-- CreateEnum
+CREATE TYPE "DocumentStatus" AS ENUM ('UPLOADED', 'PROCESSING', 'READY', 'FAILED', 'ARCHIVED');
+
+-- CreateEnum
+CREATE TYPE "ProcessingStatus" AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED');
+
+-- CreateEnum
+CREATE TYPE "ProcessingStage" AS ENUM ('VALIDATE', 'PARSE', 'CLEAN', 'CHUNK', 'METADATA', 'EMBED', 'INDEX', 'PERSIST', 'COMPLETE');
+
+-- CreateEnum
+CREATE TYPE "KnowledgeSourceType" AS ENUM ('UPLOAD', 'API', 'INTEGRATION');
+
+-- CreateTable
+CREATE TABLE "folders" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "parentId" UUID,
+    "organizationId" UUID NOT NULL,
+    "projectId" UUID,
+    "ownerId" UUID,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "folders_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "tags" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "slug" TEXT NOT NULL,
+    "color" TEXT,
+    "organizationId" UUID NOT NULL,
+    "ownerId" UUID,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "tags_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "knowledge_sources" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "type" "KnowledgeSourceType" NOT NULL DEFAULT 'UPLOAD',
+    "config" JSONB,
+    "organizationId" UUID NOT NULL,
+    "projectId" UUID,
+    "ownerId" UUID,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "knowledge_sources_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "documents" (
+    "id" UUID NOT NULL,
+    "title" TEXT NOT NULL,
+    "description" TEXT,
+    "fileName" TEXT NOT NULL,
+    "mimeType" TEXT NOT NULL,
+    "fileSizeBytes" INTEGER NOT NULL,
+    "storageBucket" TEXT NOT NULL,
+    "storageKey" TEXT NOT NULL,
+    "checksum" TEXT,
+    "status" "DocumentStatus" NOT NULL DEFAULT 'UPLOADED',
+    "language" TEXT,
+    "metadata" JSONB,
+    "currentVersion" INTEGER NOT NULL DEFAULT 1,
+    "folderId" UUID,
+    "sourceId" UUID,
+    "organizationId" UUID NOT NULL,
+    "projectId" UUID,
+    "ownerId" UUID,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "documents_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "document_versions" (
+    "id" UUID NOT NULL,
+    "version" INTEGER NOT NULL,
+    "documentId" UUID NOT NULL,
+    "storageKey" TEXT NOT NULL,
+    "fileSizeBytes" INTEGER NOT NULL,
+    "checksum" TEXT,
+    "metadata" JSONB,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "document_versions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "document_tags" (
+    "documentId" UUID NOT NULL,
+    "tagId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "document_tags_pkey" PRIMARY KEY ("documentId","tagId")
+);
+
+-- CreateTable
+CREATE TABLE "chunks" (
+    "id" UUID NOT NULL,
+    "index" INTEGER NOT NULL,
+    "content" TEXT NOT NULL,
+    "tokenCount" INTEGER NOT NULL,
+    "heading" TEXT,
+    "section" TEXT,
+    "startOffset" INTEGER NOT NULL DEFAULT 0,
+    "endOffset" INTEGER NOT NULL DEFAULT 0,
+    "metadata" JSONB,
+    "documentId" UUID NOT NULL,
+    "versionId" UUID NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "chunks_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "embedding_references" (
+    "id" UUID NOT NULL,
+    "chunkId" UUID NOT NULL,
+    "documentId" UUID NOT NULL,
+    "collection" TEXT NOT NULL,
+    "pointId" UUID NOT NULL,
+    "provider" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "dimension" INTEGER NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "embedding_references_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "processing_jobs" (
+    "id" UUID NOT NULL,
+    "documentId" UUID NOT NULL,
+    "documentVersion" INTEGER NOT NULL DEFAULT 1,
+    "workflowId" TEXT NOT NULL,
+    "runId" TEXT,
+    "stage" "ProcessingStage" NOT NULL DEFAULT 'VALIDATE',
+    "status" "ProcessingStatus" NOT NULL DEFAULT 'PENDING',
+    "attempt" INTEGER NOT NULL DEFAULT 1,
+    "error" TEXT,
+    "logs" JSONB[],
+    "chunkCount" INTEGER,
+    "embeddingCount" INTEGER,
+    "organizationId" UUID NOT NULL,
+    "startedAt" TIMESTAMP(3),
+    "completedAt" TIMESTAMP(3),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "processing_jobs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "folders_organizationId_idx" ON "folders"("organizationId");
+
+-- CreateIndex
+CREATE INDEX "folders_parentId_idx" ON "folders"("parentId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "tags_organizationId_slug_key" ON "tags"("organizationId", "slug");
+
+-- CreateIndex
+CREATE INDEX "knowledge_sources_organizationId_idx" ON "knowledge_sources"("organizationId");
+
+-- CreateIndex
+CREATE INDEX "documents_organizationId_idx" ON "documents"("organizationId");
+
+-- CreateIndex
+CREATE INDEX "documents_organizationId_status_idx" ON "documents"("organizationId", "status");
+
+-- CreateIndex
+CREATE INDEX "documents_projectId_idx" ON "documents"("projectId");
+
+-- CreateIndex
+CREATE INDEX "documents_folderId_idx" ON "documents"("folderId");
+
+-- CreateIndex
+CREATE INDEX "document_versions_organizationId_idx" ON "document_versions"("organizationId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "document_versions_documentId_version_key" ON "document_versions"("documentId", "version");
+
+-- CreateIndex
+CREATE INDEX "chunks_documentId_idx" ON "chunks"("documentId");
+
+-- CreateIndex
+CREATE INDEX "chunks_organizationId_idx" ON "chunks"("organizationId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "chunks_versionId_index_key" ON "chunks"("versionId", "index");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "embedding_references_chunkId_key" ON "embedding_references"("chunkId");
+
+-- CreateIndex
+CREATE INDEX "embedding_references_documentId_idx" ON "embedding_references"("documentId");
+
+-- CreateIndex
+CREATE INDEX "embedding_references_organizationId_idx" ON "embedding_references"("organizationId");
+
+-- CreateIndex
+CREATE INDEX "processing_jobs_documentId_idx" ON "processing_jobs"("documentId");
+
+-- CreateIndex
+CREATE INDEX "processing_jobs_organizationId_status_idx" ON "processing_jobs"("organizationId", "status");
+
+-- CreateIndex
+CREATE INDEX "processing_jobs_workflowId_idx" ON "processing_jobs"("workflowId");
+
+-- AddForeignKey
+ALTER TABLE "folders" ADD CONSTRAINT "folders_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES "folders"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "folders" ADD CONSTRAINT "folders_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "folders" ADD CONSTRAINT "folders_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "folders" ADD CONSTRAINT "folders_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tags" ADD CONSTRAINT "tags_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tags" ADD CONSTRAINT "tags_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "documents" ADD CONSTRAINT "documents_folderId_fkey" FOREIGN KEY ("folderId") REFERENCES "folders"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "documents" ADD CONSTRAINT "documents_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "knowledge_sources"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "documents" ADD CONSTRAINT "documents_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "documents" ADD CONSTRAINT "documents_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "documents" ADD CONSTRAINT "documents_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_versions" ADD CONSTRAINT "document_versions_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_tags" ADD CONSTRAINT "document_tags_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_tags" ADD CONSTRAINT "document_tags_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "tags"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "chunks" ADD CONSTRAINT "chunks_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "chunks" ADD CONSTRAINT "chunks_versionId_fkey" FOREIGN KEY ("versionId") REFERENCES "document_versions"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "embedding_references" ADD CONSTRAINT "embedding_references_chunkId_fkey" FOREIGN KEY ("chunkId") REFERENCES "chunks"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "embedding_references" ADD CONSTRAINT "embedding_references_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "processing_jobs" ADD CONSTRAINT "processing_jobs_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;

apps/api/prisma/migrations/20260714080100_chunk_fulltext_index/migration.sql

@@ -0,0 +1,3 @@
+-- Full-text search over chunk content for the hybrid (keyword) search arm.
+CREATE INDEX IF NOT EXISTS "chunks_content_fts_idx"
+  ON "chunks" USING GIN (to_tsvector('english', "content"));