diff --git a/.env.example b/.env.example index ed9a56b..6fe7c24 100644 --- a/.env.example +++ b/.env.example @@ -45,5 +45,29 @@ TEMPORAL_TASK_QUEUE=brain-core # Port for the temporal-worker's local health/status endpoint. TEMPORAL_WORKER_HEALTH_PORT=4100 +# ── Embeddings (knowledge pipeline) ───────────────────────────── +# Provider: local (offline, dev default) | openai | gemini | voyage +EMBEDDINGS_PROVIDER=local +# EMBEDDINGS_MODEL=text-embedding-3-small +# EMBEDDINGS_DIMENSION=384 +# OPENAI_API_KEY= +# GEMINI_API_KEY= +# VOYAGE_API_KEY= + +# ── Chunking ──────────────────────────────────────────────────── +CHUNK_SIZE=400 +CHUNK_OVERLAP=60 + +# ── Connectors (Google Workspace OAuth) ───────────────────────── +# Create OAuth client (Web application) at console.cloud.google.com, +# authorized redirect URI = GOOGLE_REDIRECT_URI below. +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= +GOOGLE_REDIRECT_URI=http://localhost:4000/api/v1/connectors/google/callback +# Refresh-token encryption at rest. Generate: openssl rand -hex 32 +TOKEN_ENCRYPTION_KEY=change-me-generate-with-openssl-rand-hex-32 +CONNECTOR_TASK_QUEUE=brain-connectors +CONNECTOR_WORKER_HEALTH_PORT=4101 + # ── Frontend ──────────────────────────────────────────────────── NEXT_PUBLIC_API_URL=http://localhost:4000 diff --git a/README.md b/README.md index f67d84c..e4eae74 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,18 @@ apps/ web/ Next.js dashboard (login, register, dashboard, profile, settings) packages/ activities/ Temporal activity implementations (all side effects) + auth/ OAuth2 helpers + AES-256-GCM token encryption config/ Shared ESLint + tsconfig presets + connectors/ + core/ Connector SDK (provider-agnostic framework) + google/ Google Workspace connector (Drive/Docs/Sheets/Slides/Gmail/Calendar) + events/ Redis-backed platform event bus types/ Shared TypeScript contracts (API envelope, auth, roles) ui/ Shared shadcn-style React primitives utils/ Small dependency-free helpers workflows/ Temporal workflow definitions (deterministic orchestration) services/ + connector-worker/ Temporal worker for connector sync (queue brain-connectors, health :4101) temporal-worker/ Temporal worker (workflow + activity host, health :4100) worker/ BullMQ worker (generic "system" queue) infrastructure/ @@ -126,5 +132,6 @@ Every endpoint returns the standard envelope: ``` See [docs/architecture.md](docs/architecture.md) for design decisions and -[docs/temporal.md](docs/temporal.md) for the workflow orchestration guide -(why Temporal, how workflows/activities work, how to add new workflows). +[docs/temporal.md](docs/temporal.md) for the workflow orchestration guide and +[docs/connectors.md](docs/connectors.md) for the Knowledge Connector Platform +(Connector SDK, OAuth flow, sync architecture, extension guide). diff --git a/apps/api/package.json b/apps/api/package.json index 457cc85..f819639 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,6 +19,11 @@ }, "dependencies": { "@company-brain/activities": "workspace:*", + "@company-brain/auth": "workspace:*", + "@company-brain/connector-core": "workspace:*", + "@company-brain/connector-google": "workspace:*", + "@company-brain/events": "workspace:*", + "@company-brain/knowledge": "workspace:*", "@company-brain/types": "workspace:*", "@company-brain/utils": "workspace:*", "@company-brain/workflows": "workspace:*", @@ -26,6 +31,7 @@ "@fastify/cookie": "^11.0.2", "@fastify/cors": "^11.0.1", "@fastify/helmet": "^13.0.1", + "@fastify/multipart": "^10.1.0", "@fastify/rate-limit": "^10.2.2", "@fastify/swagger": "^9.5.0", "@fastify/swagger-ui": "^5.2.2", diff --git a/apps/api/prisma/migrations/20260714075643_knowledge_brain/migration.sql b/apps/api/prisma/migrations/20260714075643_knowledge_brain/migration.sql new file mode 100644 index 0000000..522ba94 --- /dev/null +++ b/apps/api/prisma/migrations/20260714075643_knowledge_brain/migration.sql @@ -0,0 +1,296 @@ +-- CreateEnum +CREATE TYPE "DocumentStatus" AS ENUM ('UPLOADED', 'PROCESSING', 'READY', 'FAILED', 'ARCHIVED'); + +-- CreateEnum +CREATE TYPE "ProcessingStatus" AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED'); + +-- CreateEnum +CREATE TYPE "ProcessingStage" AS ENUM ('VALIDATE', 'PARSE', 'CLEAN', 'CHUNK', 'METADATA', 'EMBED', 'INDEX', 'PERSIST', 'COMPLETE'); + +-- CreateEnum +CREATE TYPE "KnowledgeSourceType" AS ENUM ('UPLOAD', 'API', 'INTEGRATION'); + +-- CreateTable +CREATE TABLE "folders" ( + "id" UUID NOT NULL, + "name" TEXT NOT NULL, + "parentId" UUID, + "organizationId" UUID NOT NULL, + "projectId" UUID, + "ownerId" UUID, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "folders_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "tags" ( + "id" UUID NOT NULL, + "name" TEXT NOT NULL, + "slug" TEXT NOT NULL, + "color" TEXT, + "organizationId" UUID NOT NULL, + "ownerId" UUID, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "tags_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "knowledge_sources" ( + "id" UUID NOT NULL, + "name" TEXT NOT NULL, + "type" "KnowledgeSourceType" NOT NULL DEFAULT 'UPLOAD', + "config" JSONB, + "organizationId" UUID NOT NULL, + "projectId" UUID, + "ownerId" UUID, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "knowledge_sources_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "documents" ( + "id" UUID NOT NULL, + "title" TEXT NOT NULL, + "description" TEXT, + "fileName" TEXT NOT NULL, + "mimeType" TEXT NOT NULL, + "fileSizeBytes" INTEGER NOT NULL, + "storageBucket" TEXT NOT NULL, + "storageKey" TEXT NOT NULL, + "checksum" TEXT, + "status" "DocumentStatus" NOT NULL DEFAULT 'UPLOADED', + "language" TEXT, + "metadata" JSONB, + "currentVersion" INTEGER NOT NULL DEFAULT 1, + "folderId" UUID, + "sourceId" UUID, + "organizationId" UUID NOT NULL, + "projectId" UUID, + "ownerId" UUID, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "documents_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "document_versions" ( + "id" UUID NOT NULL, + "version" INTEGER NOT NULL, + "documentId" UUID NOT NULL, + "storageKey" TEXT NOT NULL, + "fileSizeBytes" INTEGER NOT NULL, + "checksum" TEXT, + "metadata" JSONB, + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "document_versions_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "document_tags" ( + "documentId" UUID NOT NULL, + "tagId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "document_tags_pkey" PRIMARY KEY ("documentId","tagId") +); + +-- CreateTable +CREATE TABLE "chunks" ( + "id" UUID NOT NULL, + "index" INTEGER NOT NULL, + "content" TEXT NOT NULL, + "tokenCount" INTEGER NOT NULL, + "heading" TEXT, + "section" TEXT, + "startOffset" INTEGER NOT NULL DEFAULT 0, + "endOffset" INTEGER NOT NULL DEFAULT 0, + "metadata" JSONB, + "documentId" UUID NOT NULL, + "versionId" UUID NOT NULL, + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "chunks_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "embedding_references" ( + "id" UUID NOT NULL, + "chunkId" UUID NOT NULL, + "documentId" UUID NOT NULL, + "collection" TEXT NOT NULL, + "pointId" UUID NOT NULL, + "provider" TEXT NOT NULL, + "model" TEXT NOT NULL, + "dimension" INTEGER NOT NULL, + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "embedding_references_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "processing_jobs" ( + "id" UUID NOT NULL, + "documentId" UUID NOT NULL, + "documentVersion" INTEGER NOT NULL DEFAULT 1, + "workflowId" TEXT NOT NULL, + "runId" TEXT, + "stage" "ProcessingStage" NOT NULL DEFAULT 'VALIDATE', + "status" "ProcessingStatus" NOT NULL DEFAULT 'PENDING', + "attempt" INTEGER NOT NULL DEFAULT 1, + "error" TEXT, + "logs" JSONB[], + "chunkCount" INTEGER, + "embeddingCount" INTEGER, + "organizationId" UUID NOT NULL, + "startedAt" TIMESTAMP(3), + "completedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "processing_jobs_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "folders_organizationId_idx" ON "folders"("organizationId"); + +-- CreateIndex +CREATE INDEX "folders_parentId_idx" ON "folders"("parentId"); + +-- CreateIndex +CREATE UNIQUE INDEX "tags_organizationId_slug_key" ON "tags"("organizationId", "slug"); + +-- CreateIndex +CREATE INDEX "knowledge_sources_organizationId_idx" ON "knowledge_sources"("organizationId"); + +-- CreateIndex +CREATE INDEX "documents_organizationId_idx" ON "documents"("organizationId"); + +-- CreateIndex +CREATE INDEX "documents_organizationId_status_idx" ON "documents"("organizationId", "status"); + +-- CreateIndex +CREATE INDEX "documents_projectId_idx" ON "documents"("projectId"); + +-- CreateIndex +CREATE INDEX "documents_folderId_idx" ON "documents"("folderId"); + +-- CreateIndex +CREATE INDEX "document_versions_organizationId_idx" ON "document_versions"("organizationId"); + +-- CreateIndex +CREATE UNIQUE INDEX "document_versions_documentId_version_key" ON "document_versions"("documentId", "version"); + +-- CreateIndex +CREATE INDEX "chunks_documentId_idx" ON "chunks"("documentId"); + +-- CreateIndex +CREATE INDEX "chunks_organizationId_idx" ON "chunks"("organizationId"); + +-- CreateIndex +CREATE UNIQUE INDEX "chunks_versionId_index_key" ON "chunks"("versionId", "index"); + +-- CreateIndex +CREATE UNIQUE INDEX "embedding_references_chunkId_key" ON "embedding_references"("chunkId"); + +-- CreateIndex +CREATE INDEX "embedding_references_documentId_idx" ON "embedding_references"("documentId"); + +-- CreateIndex +CREATE INDEX "embedding_references_organizationId_idx" ON "embedding_references"("organizationId"); + +-- CreateIndex +CREATE INDEX "processing_jobs_documentId_idx" ON "processing_jobs"("documentId"); + +-- CreateIndex +CREATE INDEX "processing_jobs_organizationId_status_idx" ON "processing_jobs"("organizationId", "status"); + +-- CreateIndex +CREATE INDEX "processing_jobs_workflowId_idx" ON "processing_jobs"("workflowId"); + +-- AddForeignKey +ALTER TABLE "folders" ADD CONSTRAINT "folders_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES "folders"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "folders" ADD CONSTRAINT "folders_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "folders" ADD CONSTRAINT "folders_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "folders" ADD CONSTRAINT "folders_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "tags" ADD CONSTRAINT "tags_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "tags" ADD CONSTRAINT "tags_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "knowledge_sources" ADD CONSTRAINT "knowledge_sources_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "documents" ADD CONSTRAINT "documents_folderId_fkey" FOREIGN KEY ("folderId") REFERENCES "folders"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "documents" ADD CONSTRAINT "documents_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "knowledge_sources"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "documents" ADD CONSTRAINT "documents_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "documents" ADD CONSTRAINT "documents_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "documents" ADD CONSTRAINT "documents_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "document_versions" ADD CONSTRAINT "document_versions_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "document_tags" ADD CONSTRAINT "document_tags_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "document_tags" ADD CONSTRAINT "document_tags_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "tags"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "chunks" ADD CONSTRAINT "chunks_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "chunks" ADD CONSTRAINT "chunks_versionId_fkey" FOREIGN KEY ("versionId") REFERENCES "document_versions"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "embedding_references" ADD CONSTRAINT "embedding_references_chunkId_fkey" FOREIGN KEY ("chunkId") REFERENCES "chunks"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "embedding_references" ADD CONSTRAINT "embedding_references_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "processing_jobs" ADD CONSTRAINT "processing_jobs_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20260714080100_chunk_fulltext_index/migration.sql b/apps/api/prisma/migrations/20260714080100_chunk_fulltext_index/migration.sql new file mode 100644 index 0000000..6d4f135 --- /dev/null +++ b/apps/api/prisma/migrations/20260714080100_chunk_fulltext_index/migration.sql @@ -0,0 +1,3 @@ +-- Full-text search over chunk content for the hybrid (keyword) search arm. +CREATE INDEX IF NOT EXISTS "chunks_content_fts_idx" + ON "chunks" USING GIN (to_tsvector('english', "content")); diff --git a/apps/api/prisma/migrations/20260714103641_connector_platform/migration.sql b/apps/api/prisma/migrations/20260714103641_connector_platform/migration.sql new file mode 100644 index 0000000..77b9916 --- /dev/null +++ b/apps/api/prisma/migrations/20260714103641_connector_platform/migration.sql @@ -0,0 +1,348 @@ +-- CreateEnum +CREATE TYPE "ConnectorProvider" AS ENUM ('GOOGLE_WORKSPACE', 'SLACK', 'GITHUB', 'NOTION', 'CONFLUENCE', 'MICROSOFT_365', 'DROPBOX', 'JIRA', 'LINEAR', 'SALESFORCE'); + +-- CreateEnum +CREATE TYPE "ConnectorStatus" AS ENUM ('PENDING', 'CONNECTED', 'SYNCING', 'ERROR', 'DISCONNECTED', 'REVOKED'); + +-- CreateEnum +CREATE TYPE "CredentialStatus" AS ENUM ('ACTIVE', 'EXPIRED', 'REVOKED'); + +-- CreateEnum +CREATE TYPE "SyncJobType" AS ENUM ('INITIAL', 'INCREMENTAL', 'DISCOVERY', 'PERMISSION', 'MANUAL'); + +-- CreateEnum +CREATE TYPE "SyncJobStatus" AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'PARTIAL', 'FAILED', 'CANCELLED'); + +-- CreateEnum +CREATE TYPE "ExternalResourceType" AS ENUM ('GOOGLE_DOC', 'GOOGLE_SHEET', 'GOOGLE_SLIDES', 'PDF', 'FOLDER', 'DRIVE_FILE', 'SHARED_DRIVE', 'EMAIL', 'EMAIL_THREAD', 'CALENDAR', 'CALENDAR_EVENT', 'ATTACHMENT', 'OTHER'); + +-- CreateEnum +CREATE TYPE "ExternalResourceStatus" AS ENUM ('ACTIVE', 'TRASHED', 'DELETED'); + +-- CreateEnum +CREATE TYPE "ChangeType" AS ENUM ('CREATED', 'UPDATED', 'DELETED', 'PERMISSION_CHANGED'); + +-- CreateEnum +CREATE TYPE "PermissionRole" AS ENUM ('OWNER', 'EDITOR', 'COMMENTER', 'VIEWER'); + +-- CreateEnum +CREATE TYPE "PrincipalType" AS ENUM ('USER', 'GROUP', 'DOMAIN', 'ANYONE'); + +-- CreateEnum +CREATE TYPE "ConnectorLogLevel" AS ENUM ('DEBUG', 'INFO', 'WARN', 'ERROR'); + +-- CreateTable +CREATE TABLE "connectors" ( + "id" UUID NOT NULL, + "provider" "ConnectorProvider" NOT NULL, + "name" TEXT NOT NULL, + "status" "ConnectorStatus" NOT NULL DEFAULT 'PENDING', + "config" JSONB, + "error" TEXT, + "lastSyncAt" TIMESTAMP(3), + "nextSyncAt" TIMESTAMP(3), + "organizationId" UUID NOT NULL, + "ownerId" UUID, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "connectors_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "organization_connectors" ( + "id" UUID NOT NULL, + "organizationId" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "isEnabled" BOOLEAN NOT NULL DEFAULT true, + "settings" JSONB, + "status" "ConnectorStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "organization_connectors_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "workspaces" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "externalId" TEXT, + "domain" TEXT, + "name" TEXT, + "adminEmail" TEXT, + "metadata" JSONB, + "status" "ConnectorStatus" NOT NULL DEFAULT 'CONNECTED', + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "workspaces_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "oauth_credentials" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "userEmail" TEXT, + "scopes" TEXT[], + "encryptedRefreshToken" TEXT NOT NULL, + "accessTokenExpiresAt" TIMESTAMP(3), + "tokenType" TEXT NOT NULL DEFAULT 'Bearer', + "status" "CredentialStatus" NOT NULL DEFAULT 'ACTIVE', + "lastRefreshedAt" TIMESTAMP(3), + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "oauth_credentials_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "sync_jobs" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "type" "SyncJobType" NOT NULL, + "status" "SyncJobStatus" NOT NULL DEFAULT 'PENDING', + "service" TEXT, + "workflowId" TEXT NOT NULL, + "runId" TEXT, + "stats" JSONB, + "error" TEXT, + "organizationId" UUID NOT NULL, + "startedAt" TIMESTAMP(3), + "completedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "sync_jobs_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "sync_cursors" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "service" TEXT NOT NULL, + "resourceScope" TEXT NOT NULL DEFAULT '', + "cursor" TEXT NOT NULL, + "status" "ConnectorStatus" NOT NULL DEFAULT 'CONNECTED', + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "sync_cursors_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "external_resources" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "externalId" TEXT NOT NULL, + "type" "ExternalResourceType" NOT NULL, + "status" "ExternalResourceStatus" NOT NULL DEFAULT 'ACTIVE', + "title" TEXT, + "mimeType" TEXT, + "url" TEXT, + "ownerEmail" TEXT, + "parentExternalId" TEXT, + "driveId" TEXT, + "sizeBytes" BIGINT, + "checksum" TEXT, + "version" TEXT, + "externalCreatedAt" TIMESTAMP(3), + "externalUpdatedAt" TIMESTAMP(3), + "metadata" JSONB, + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "external_resources_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "external_changes" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "resourceId" UUID, + "externalId" TEXT NOT NULL, + "service" TEXT NOT NULL, + "changeType" "ChangeType" NOT NULL, + "payload" JSONB, + "occurredAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "processedAt" TIMESTAMP(3), + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "external_changes_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "resource_permissions" ( + "id" UUID NOT NULL, + "resourceId" UUID NOT NULL, + "externalPermissionId" TEXT, + "principalType" "PrincipalType" NOT NULL, + "principalEmail" TEXT, + "domain" TEXT, + "role" "PermissionRole" NOT NULL, + "status" "ExternalResourceStatus" NOT NULL DEFAULT 'ACTIVE', + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "resource_permissions_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "resource_versions" ( + "id" UUID NOT NULL, + "resourceId" UUID NOT NULL, + "version" TEXT NOT NULL, + "checksum" TEXT, + "sizeBytes" BIGINT, + "modifiedByEmail" TEXT, + "externalModifiedAt" TIMESTAMP(3), + "metadata" JSONB, + "status" "ExternalResourceStatus" NOT NULL DEFAULT 'ACTIVE', + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "resource_versions_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "connector_logs" ( + "id" UUID NOT NULL, + "connectorId" UUID NOT NULL, + "level" "ConnectorLogLevel" NOT NULL DEFAULT 'INFO', + "event" TEXT NOT NULL, + "message" TEXT NOT NULL, + "context" JSONB, + "status" "ConnectorStatus", + "organizationId" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "deletedAt" TIMESTAMP(3), + + CONSTRAINT "connector_logs_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "connectors_organizationId_provider_idx" ON "connectors"("organizationId", "provider"); + +-- CreateIndex +CREATE INDEX "connectors_organizationId_status_idx" ON "connectors"("organizationId", "status"); + +-- CreateIndex +CREATE UNIQUE INDEX "organization_connectors_organizationId_connectorId_key" ON "organization_connectors"("organizationId", "connectorId"); + +-- CreateIndex +CREATE UNIQUE INDEX "workspaces_connectorId_key" ON "workspaces"("connectorId"); + +-- CreateIndex +CREATE INDEX "workspaces_organizationId_idx" ON "workspaces"("organizationId"); + +-- CreateIndex +CREATE INDEX "workspaces_domain_idx" ON "workspaces"("domain"); + +-- CreateIndex +CREATE INDEX "oauth_credentials_connectorId_idx" ON "oauth_credentials"("connectorId"); + +-- CreateIndex +CREATE INDEX "oauth_credentials_organizationId_idx" ON "oauth_credentials"("organizationId"); + +-- CreateIndex +CREATE INDEX "sync_jobs_connectorId_status_idx" ON "sync_jobs"("connectorId", "status"); + +-- CreateIndex +CREATE INDEX "sync_jobs_organizationId_createdAt_idx" ON "sync_jobs"("organizationId", "createdAt"); + +-- CreateIndex +CREATE INDEX "sync_jobs_workflowId_idx" ON "sync_jobs"("workflowId"); + +-- CreateIndex +CREATE UNIQUE INDEX "sync_cursors_connectorId_service_resourceScope_key" ON "sync_cursors"("connectorId", "service", "resourceScope"); + +-- CreateIndex +CREATE INDEX "external_resources_organizationId_type_idx" ON "external_resources"("organizationId", "type"); + +-- CreateIndex +CREATE INDEX "external_resources_connectorId_type_idx" ON "external_resources"("connectorId", "type"); + +-- CreateIndex +CREATE INDEX "external_resources_parentExternalId_idx" ON "external_resources"("parentExternalId"); + +-- CreateIndex +CREATE UNIQUE INDEX "external_resources_connectorId_externalId_key" ON "external_resources"("connectorId", "externalId"); + +-- CreateIndex +CREATE INDEX "external_changes_connectorId_occurredAt_idx" ON "external_changes"("connectorId", "occurredAt"); + +-- CreateIndex +CREATE INDEX "external_changes_organizationId_changeType_idx" ON "external_changes"("organizationId", "changeType"); + +-- CreateIndex +CREATE INDEX "resource_permissions_resourceId_idx" ON "resource_permissions"("resourceId"); + +-- CreateIndex +CREATE INDEX "resource_permissions_organizationId_principalEmail_idx" ON "resource_permissions"("organizationId", "principalEmail"); + +-- CreateIndex +CREATE UNIQUE INDEX "resource_versions_resourceId_version_key" ON "resource_versions"("resourceId", "version"); + +-- CreateIndex +CREATE INDEX "connector_logs_connectorId_createdAt_idx" ON "connector_logs"("connectorId", "createdAt"); + +-- CreateIndex +CREATE INDEX "connector_logs_organizationId_level_idx" ON "connector_logs"("organizationId", "level"); + +-- AddForeignKey +ALTER TABLE "connectors" ADD CONSTRAINT "connectors_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "organization_connectors" ADD CONSTRAINT "organization_connectors_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "organization_connectors" ADD CONSTRAINT "organization_connectors_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "workspaces" ADD CONSTRAINT "workspaces_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "oauth_credentials" ADD CONSTRAINT "oauth_credentials_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "sync_jobs" ADD CONSTRAINT "sync_jobs_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "sync_cursors" ADD CONSTRAINT "sync_cursors_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "external_resources" ADD CONSTRAINT "external_resources_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "external_changes" ADD CONSTRAINT "external_changes_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "external_changes" ADD CONSTRAINT "external_changes_resourceId_fkey" FOREIGN KEY ("resourceId") REFERENCES "external_resources"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "resource_permissions" ADD CONSTRAINT "resource_permissions_resourceId_fkey" FOREIGN KEY ("resourceId") REFERENCES "external_resources"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "resource_versions" ADD CONSTRAINT "resource_versions_resourceId_fkey" FOREIGN KEY ("resourceId") REFERENCES "external_resources"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "connector_logs" ADD CONSTRAINT "connector_logs_connectorId_fkey" FOREIGN KEY ("connectorId") REFERENCES "connectors"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 41aab1c..f2bb902 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -33,6 +33,11 @@ model User { auditLogs AuditLog[] apiKeys APIKey[] + folders Folder[] + tags Tag[] + knowledgeSources KnowledgeSource[] + documents Document[] + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt deletedAt DateTime? @@ -51,6 +56,14 @@ model Organization { apiKeys APIKey[] auditLogs AuditLog[] + folders Folder[] + tags Tag[] + knowledgeSources KnowledgeSource[] + documents Document[] + + connectors Connector[] + connectorLinks OrganizationConnector[] + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt deletedAt DateTime? @@ -66,6 +79,10 @@ model Project { organizationId String @db.Uuid organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + folders Folder[] + knowledgeSources KnowledgeSource[] + documents Document[] + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt deletedAt DateTime? @@ -180,3 +197,710 @@ model AuditLog { @@index([createdAt]) @@map("audit_logs") } + +// ──────────────────────────────────────────────────────────────── +// Phase 1 — Knowledge Brain +// Documents, versions, chunks, embeddings, folders, tags, sources +// and processing jobs. Every model carries organization scoping, +// optional project scoping, ownership and soft-delete + audit +// timestamps, matching the Phase 0 conventions. +// ──────────────────────────────────────────────────────────────── + +enum DocumentStatus { + UPLOADED + PROCESSING + READY + FAILED + ARCHIVED +} + +enum ProcessingStatus { + PENDING + RUNNING + COMPLETED + FAILED + CANCELLED +} + +enum ProcessingStage { + VALIDATE + PARSE + CLEAN + CHUNK + METADATA + EMBED + INDEX + PERSIST + COMPLETE +} + +enum KnowledgeSourceType { + UPLOAD + API + INTEGRATION +} + +model Folder { + id String @id @default(uuid()) @db.Uuid + name String + + parentId String? @db.Uuid + parent Folder? @relation("FolderTree", fields: [parentId], references: [id], onDelete: SetNull) + children Folder[] @relation("FolderTree") + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + projectId String? @db.Uuid + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + ownerId String? @db.Uuid + owner User? @relation(fields: [ownerId], references: [id], onDelete: SetNull) + + documents Document[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([organizationId]) + @@index([parentId]) + @@map("folders") +} + +model Tag { + id String @id @default(uuid()) @db.Uuid + name String + slug String + color String? + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + ownerId String? @db.Uuid + owner User? @relation(fields: [ownerId], references: [id], onDelete: SetNull) + + documents DocumentTag[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([organizationId, slug]) + @@map("tags") +} + +model KnowledgeSource { + id String @id @default(uuid()) @db.Uuid + name String + type KnowledgeSourceType @default(UPLOAD) + // Source-specific configuration (e.g. integration credentials pointer). + config Json? + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + projectId String? @db.Uuid + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + ownerId String? @db.Uuid + owner User? @relation(fields: [ownerId], references: [id], onDelete: SetNull) + + documents Document[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([organizationId]) + @@map("knowledge_sources") +} + +model Document { + id String @id @default(uuid()) @db.Uuid + title String + description String? + + fileName String + mimeType String + fileSizeBytes Int + storageBucket String + storageKey String + checksum String? + status DocumentStatus @default(UPLOADED) + language String? + // Extracted metadata: author, keywords, headings, sections, tables, + // creationDate — schema-free so parsers can evolve. + metadata Json? + + currentVersion Int @default(1) + + folderId String? @db.Uuid + folder Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull) + sourceId String? @db.Uuid + source KnowledgeSource? @relation(fields: [sourceId], references: [id], onDelete: SetNull) + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + projectId String? @db.Uuid + project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull) + ownerId String? @db.Uuid + owner User? @relation(fields: [ownerId], references: [id], onDelete: SetNull) + + versions DocumentVersion[] + chunks Chunk[] + embeddings EmbeddingReference[] + tags DocumentTag[] + processingJobs ProcessingJob[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([organizationId]) + @@index([organizationId, status]) + @@index([projectId]) + @@index([folderId]) + @@map("documents") +} + +model DocumentVersion { + id String @id @default(uuid()) @db.Uuid + version Int + + documentId String @db.Uuid + document Document @relation(fields: [documentId], references: [id], onDelete: Cascade) + + storageKey String + fileSizeBytes Int + checksum String? + // Parser output snapshot for this version (headings, sections, stats). + metadata Json? + + organizationId String @db.Uuid + + chunks Chunk[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([documentId, version]) + @@index([organizationId]) + @@map("document_versions") +} + +model DocumentTag { + documentId String @db.Uuid + document Document @relation(fields: [documentId], references: [id], onDelete: Cascade) + tagId String @db.Uuid + tag Tag @relation(fields: [tagId], references: [id], onDelete: Cascade) + + createdAt DateTime @default(now()) + + @@id([documentId, tagId]) + @@map("document_tags") +} + +model Chunk { + id String @id @default(uuid()) @db.Uuid + index Int + + content String @db.Text + tokenCount Int + heading String? + section String? + startOffset Int @default(0) + endOffset Int @default(0) + metadata Json? + + documentId String @db.Uuid + document Document @relation(fields: [documentId], references: [id], onDelete: Cascade) + versionId String @db.Uuid + version DocumentVersion @relation(fields: [versionId], references: [id], onDelete: Cascade) + + organizationId String @db.Uuid + + embedding EmbeddingReference? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([versionId, index]) + @@index([documentId]) + @@index([organizationId]) + @@map("chunks") +} + +model EmbeddingReference { + id String @id @default(uuid()) @db.Uuid + + chunkId String @unique @db.Uuid + chunk Chunk @relation(fields: [chunkId], references: [id], onDelete: Cascade) + documentId String @db.Uuid + document Document @relation(fields: [documentId], references: [id], onDelete: Cascade) + + // Qdrant location + provenance of the vector. + collection String + pointId String @db.Uuid + provider String + model String + dimension Int + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([documentId]) + @@index([organizationId]) + @@map("embedding_references") +} + +model ProcessingJob { + id String @id @default(uuid()) @db.Uuid + + documentId String @db.Uuid + document Document @relation(fields: [documentId], references: [id], onDelete: Cascade) + documentVersion Int @default(1) + + // Temporal linkage for observability. + workflowId String + runId String? + + stage ProcessingStage @default(VALIDATE) + status ProcessingStatus @default(PENDING) + attempt Int @default(1) + error String? + // Append-only stage log: [{stage, message, at}] + logs Json[] + + chunkCount Int? + embeddingCount Int? + + organizationId String @db.Uuid + + startedAt DateTime? + completedAt DateTime? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([documentId]) + @@index([organizationId, status]) + @@index([workflowId]) + @@map("processing_jobs") +} + +// ──────────────────────────────────────────────────────────────── +// Phase 1 — Knowledge Connector Platform +// Generic connector framework tables: one Connector row per linked +// external system (Google Workspace first; Slack/GitHub/Notion/… later), +// with credentials, sync jobs, cursors, discovered resources, changes, +// permissions, versions and an audit log. Organization scoping, +// soft delete and audit timestamps everywhere. Child tables carry +// organizationId as a plain scalar for isolation queries. +// ──────────────────────────────────────────────────────────────── + +enum ConnectorProvider { + GOOGLE_WORKSPACE + SLACK + GITHUB + NOTION + CONFLUENCE + MICROSOFT_365 + DROPBOX + JIRA + LINEAR + SALESFORCE +} + +enum ConnectorStatus { + PENDING + CONNECTED + SYNCING + ERROR + DISCONNECTED + REVOKED +} + +enum CredentialStatus { + ACTIVE + EXPIRED + REVOKED +} + +enum SyncJobType { + INITIAL + INCREMENTAL + DISCOVERY + PERMISSION + MANUAL +} + +enum SyncJobStatus { + PENDING + RUNNING + COMPLETED + PARTIAL + FAILED + CANCELLED +} + +enum ExternalResourceType { + GOOGLE_DOC + GOOGLE_SHEET + GOOGLE_SLIDES + PDF + FOLDER + DRIVE_FILE + SHARED_DRIVE + EMAIL + EMAIL_THREAD + CALENDAR + CALENDAR_EVENT + ATTACHMENT + OTHER +} + +enum ExternalResourceStatus { + ACTIVE + TRASHED + DELETED +} + +enum ChangeType { + CREATED + UPDATED + DELETED + PERMISSION_CHANGED +} + +enum PermissionRole { + OWNER + EDITOR + COMMENTER + VIEWER +} + +enum PrincipalType { + USER + GROUP + DOMAIN + ANYONE +} + +enum ConnectorLogLevel { + DEBUG + INFO + WARN + ERROR +} + +model Connector { + id String @id @default(uuid()) @db.Uuid + provider ConnectorProvider + name String + status ConnectorStatus @default(PENDING) + // Provider-specific, non-secret configuration (sync intervals, filters). + config Json? + error String? + + lastSyncAt DateTime? + nextSyncAt DateTime? + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + ownerId String? @db.Uuid + + workspace Workspace? + credentials OAuthCredential[] + syncJobs SyncJob[] + syncCursors SyncCursor[] + resources ExternalResource[] + changes ExternalChange[] + logs ConnectorLog[] + organizations OrganizationConnector[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([organizationId, provider]) + @@index([organizationId, status]) + @@map("connectors") +} + +model OrganizationConnector { + id String @id @default(uuid()) @db.Uuid + + organizationId String @db.Uuid + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + isEnabled Boolean @default(true) + // Per-organization overrides (which services to sync, schedules…). + settings Json? + status ConnectorStatus @default(PENDING) + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([organizationId, connectorId]) + @@map("organization_connectors") +} + +model Workspace { + id String @id @default(uuid()) @db.Uuid + + connectorId String @unique @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + // Provider-side identity: Google customer id, Slack team id, … + externalId String? + domain String? + name String? + adminEmail String? + metadata Json? + status ConnectorStatus @default(CONNECTED) + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([organizationId]) + @@index([domain]) + @@map("workspaces") +} + +model OAuthCredential { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + // Account the grant belongs to (the connecting admin). + userEmail String? + scopes String[] + // AES-256-GCM encrypted refresh token — never stored in plaintext, + // never returned by any API. + encryptedRefreshToken String + accessTokenExpiresAt DateTime? + tokenType String @default("Bearer") + status CredentialStatus @default(ACTIVE) + lastRefreshedAt DateTime? + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([connectorId]) + @@index([organizationId]) + @@map("oauth_credentials") +} + +model SyncJob { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + type SyncJobType + status SyncJobStatus @default(PENDING) + // Which provider service this job covers: drive, docs, gmail, calendar… + service String? + + // Temporal linkage for observability. + workflowId String + runId String? + + // Counters: {discovered, created, updated, deleted, permissions, errors} + stats Json? + error String? + + organizationId String @db.Uuid + + startedAt DateTime? + completedAt DateTime? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([connectorId, status]) + @@index([organizationId, createdAt]) + @@index([workflowId]) + @@map("sync_jobs") +} + +model SyncCursor { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + // drive | gmail | calendar | permissions … + service String + // Narrower scope inside a service (a shared drive id, a calendar id). + resourceScope String @default("") + // Opaque provider cursor: Drive changes pageToken, Gmail historyId, + // Calendar syncToken. + cursor String + status ConnectorStatus @default(CONNECTED) + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([connectorId, service, resourceScope]) + @@map("sync_cursors") +} + +model ExternalResource { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + externalId String + type ExternalResourceType + status ExternalResourceStatus @default(ACTIVE) + + title String? + mimeType String? + url String? + ownerEmail String? + // Provider-side hierarchy (Drive parent folder id, Gmail thread id…). + parentExternalId String? + driveId String? + sizeBytes BigInt? + checksum String? + version String? + externalCreatedAt DateTime? + externalUpdatedAt DateTime? + // Service-specific extras (labels, attendees, worksheet names, …). + metadata Json? + + organizationId String @db.Uuid + + permissions ResourcePermission[] + versions ResourceVersion[] + changeEvents ExternalChange[] + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([connectorId, externalId]) + @@index([organizationId, type]) + @@index([connectorId, type]) + @@index([parentExternalId]) + @@map("external_resources") +} + +model ExternalChange { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + resourceId String? @db.Uuid + resource ExternalResource? @relation(fields: [resourceId], references: [id], onDelete: SetNull) + + externalId String + service String + changeType ChangeType + // Raw provider change payload for replay/debugging. + payload Json? + + occurredAt DateTime @default(now()) + processedAt DateTime? + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([connectorId, occurredAt]) + @@index([organizationId, changeType]) + @@map("external_changes") +} + +model ResourcePermission { + id String @id @default(uuid()) @db.Uuid + + resourceId String @db.Uuid + resource ExternalResource @relation(fields: [resourceId], references: [id], onDelete: Cascade) + + externalPermissionId String? + principalType PrincipalType + // Email of the user/group; null for DOMAIN/ANYONE grants. + principalEmail String? + domain String? + role PermissionRole + status ExternalResourceStatus @default(ACTIVE) + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([resourceId]) + @@index([organizationId, principalEmail]) + @@map("resource_permissions") +} + +model ResourceVersion { + id String @id @default(uuid()) @db.Uuid + + resourceId String @db.Uuid + resource ExternalResource @relation(fields: [resourceId], references: [id], onDelete: Cascade) + + version String + checksum String? + sizeBytes BigInt? + modifiedByEmail String? + externalModifiedAt DateTime? + metadata Json? + status ExternalResourceStatus @default(ACTIVE) + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@unique([resourceId, version]) + @@map("resource_versions") +} + +model ConnectorLog { + id String @id @default(uuid()) @db.Uuid + + connectorId String @db.Uuid + connector Connector @relation(fields: [connectorId], references: [id], onDelete: Cascade) + + level ConnectorLogLevel @default(INFO) + // Namespaced event name: oauth.connected, sync.drive.completed, … + event String + message String + context Json? + status ConnectorStatus? + + organizationId String @db.Uuid + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + + @@index([connectorId, createdAt]) + @@index([organizationId, level]) + @@map("connector_logs") +} diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index c8781a1..0125ac4 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -13,6 +13,8 @@ import authRoutes from './modules/auth/auth.routes.js'; import healthRoutes from './modules/health/health.routes.js'; import userRoutes from './modules/users/user.routes.js'; import workflowRoutes from './modules/workflows/workflow.routes.js'; +import knowledgeRoutes from './modules/knowledge/knowledge.routes.js'; +import connectorRoutes from './modules/connectors/connector.routes.js'; /** * Builds a fully configured Fastify instance. Kept separate from the @@ -54,6 +56,8 @@ export async function buildApp(): Promise { await app.register(authRoutes, { prefix: '/api/v1/auth' }); await app.register(userRoutes, { prefix: '/api/v1/users' }); await app.register(workflowRoutes, { prefix: '/api/v1/workflows' }); + await app.register(knowledgeRoutes, { prefix: '/api/v1/knowledge' }); + await app.register(connectorRoutes, { prefix: '/api/v1/connectors' }); return app; } diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index fe804b7..c02c5dc 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -51,6 +51,31 @@ const envSchema = z.object({ TEMPORAL_NAMESPACE: z.string().default('company-brain'), TEMPORAL_TASK_QUEUE: z.string().default('brain-core'), TEMPORAL_WORKER_HEALTH_URL: z.string().url().default('http://localhost:4100/health'), + + // Must match the worker's provider so query vectors live in the same + // space as the indexed chunk vectors. + EMBEDDINGS_PROVIDER: z.enum(['local', 'openai', 'gemini', 'voyage']).default('local'), + EMBEDDINGS_MODEL: z.string().optional(), + EMBEDDINGS_DIMENSION: z.coerce.number().int().positive().optional(), + OPENAI_API_KEY: z.string().optional(), + GEMINI_API_KEY: z.string().optional(), + VOYAGE_API_KEY: z.string().optional(), + + UPLOAD_MAX_FILE_SIZE_MB: z.coerce.number().int().positive().default(50), + + GOOGLE_CLIENT_ID: z.string().optional().default(''), + GOOGLE_CLIENT_SECRET: z.string().optional().default(''), + GOOGLE_REDIRECT_URI: z + .string() + .url() + .default('http://localhost:4000/api/v1/connectors/google/callback'), + TOKEN_ENCRYPTION_KEY: z + .string() + .regex(/^[0-9a-fA-F]{64}$/, 'TOKEN_ENCRYPTION_KEY must be 64 hex chars') + .optional(), + CONNECTOR_TASK_QUEUE: z.string().default('brain-connectors'), + CONNECTOR_WORKER_HEALTH_URL: z.string().url().default('http://localhost:4101/health'), + WEB_APP_URL: z.string().url().default('http://localhost:3000'), }); const parsed = envSchema.safeParse(process.env); diff --git a/apps/api/src/config/index.ts b/apps/api/src/config/index.ts index f6f5005..c353b5e 100644 --- a/apps/api/src/config/index.ts +++ b/apps/api/src/config/index.ts @@ -49,6 +49,33 @@ export const config = { taskQueue: env.TEMPORAL_TASK_QUEUE, workerHealthUrl: env.TEMPORAL_WORKER_HEALTH_URL, }, + embeddings: { + provider: env.EMBEDDINGS_PROVIDER, + model: env.EMBEDDINGS_MODEL, + dimension: env.EMBEDDINGS_DIMENSION, + apiKey: { + local: undefined, + openai: env.OPENAI_API_KEY, + gemini: env.GEMINI_API_KEY, + voyage: env.VOYAGE_API_KEY, + }[env.EMBEDDINGS_PROVIDER], + }, + uploads: { + maxFileSizeBytes: env.UPLOAD_MAX_FILE_SIZE_MB * 1024 * 1024, + }, + connectors: { + taskQueue: env.CONNECTOR_TASK_QUEUE, + workerHealthUrl: env.CONNECTOR_WORKER_HEALTH_URL, + webAppUrl: env.WEB_APP_URL, + // Refresh-token encryption; required as soon as a connector is used. + tokenEncryptionKey: env.TOKEN_ENCRYPTION_KEY, + stateSecret: env.COOKIE_SECRET, + google: { + clientId: env.GOOGLE_CLIENT_ID, + clientSecret: env.GOOGLE_CLIENT_SECRET, + redirectUri: env.GOOGLE_REDIRECT_URI, + }, + }, } as const; export type AppConfig = typeof config; diff --git a/apps/api/src/modules/connectors/connector.routes.ts b/apps/api/src/modules/connectors/connector.routes.ts new file mode 100644 index 0000000..ff9eed6 --- /dev/null +++ b/apps/api/src/modules/connectors/connector.routes.ts @@ -0,0 +1,205 @@ +import type { FastifyInstance } from 'fastify'; +import type { ZodTypeProvider } from 'fastify-type-provider-zod'; +import { authenticate } from '../../middleware/authenticate.js'; +import { ok } from '../../utils/response.js'; +import { config } from '../../config/index.js'; +import { ConnectorApiService } from './connector.service.js'; +import { + connectorIdParamsSchema, + disconnectBodySchema, + listLogsQuerySchema, + listResourcesQuerySchema, + oauthCallbackQuerySchema, +} from './connector.schemas.js'; + +/** + * Knowledge Connector Platform API. Google Workspace is the first + * provider; the endpoints are provider-scoped only where OAuth demands it + * (connect/callback) — everything else is generic. + */ +export default async function connectorRoutes(fastify: FastifyInstance): Promise { + const app = fastify.withTypeProvider(); + const service = new ConnectorApiService({ + prisma: app.prisma, + temporal: app.temporal, + redis: app.redis, + }); + + // ── OAuth flow ──────────────────────────────────────────────── + + app.post( + '/google/connect', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Begin the Google Workspace OAuth flow (returns the consent URL)', + security: [{ bearerAuth: [] }], + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(service.buildGoogleConnectUrl(request.user!.id, organizationId))); + }, + ); + + // Google redirects here — authenticated by the signed state, not a JWT. + app.get( + '/google/callback', + { + schema: { + tags: ['connectors'], + summary: 'OAuth redirect endpoint (browser lands here from Google)', + querystring: oauthCallbackQuerySchema, + }, + }, + async (request, reply) => { + const { code, state, error } = request.query; + const webUrl = `${config.connectors.webAppUrl}/connectors`; + if (error || !code || !state) { + return reply.redirect(`${webUrl}?error=${encodeURIComponent(error ?? 'missing_code')}`); + } + try { + const { connectorId } = await service.handleGoogleCallback(code, state); + return reply.redirect(`${webUrl}?connected=${connectorId}`); + } catch (callbackError) { + request.log.error({ err: callbackError }, 'google oauth callback failed'); + const message = + callbackError instanceof Error ? callbackError.message : 'connection_failed'; + return reply.redirect(`${webUrl}?error=${encodeURIComponent(message)}`); + } + }, + ); + + app.post( + '/google/disconnect', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Disconnect a Google Workspace connector and revoke its tokens', + security: [{ bearerAuth: [] }], + body: disconnectBodySchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok(await service.disconnect(organizationId, request.body.connectorId), 'Disconnected'), + ); + }, + ); + + // ── Generic connector APIs ──────────────────────────────────── + + app.get( + '/', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'List connectors for the organization', + security: [{ bearerAuth: [] }], + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.list(organizationId))); + }, + ); + + app.get( + '/:connectorId', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Connector detail: workspace, cursors, resource counts', + security: [{ bearerAuth: [] }], + params: connectorIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.get(organizationId, request.params.connectorId))); + }, + ); + + app.post( + '/:connectorId/sync', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Trigger a manual full synchronization', + security: [{ bearerAuth: [] }], + params: connectorIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply + .status(202) + .send( + ok(await service.triggerSync(organizationId, request.params.connectorId), 'Sync started'), + ); + }, + ); + + app.get( + '/:connectorId/status', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Connection + sync status: running jobs, recent jobs, worker health', + security: [{ bearerAuth: [] }], + params: connectorIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.status(organizationId, request.params.connectorId))); + }, + ); + + app.get( + '/:connectorId/resources', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Browse synchronized resource metadata', + security: [{ bearerAuth: [] }], + params: connectorIdParamsSchema, + querystring: listResourcesQuerySchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok(await service.resources(organizationId, request.params.connectorId, request.query)), + ); + }, + ); + + app.get( + '/:connectorId/logs', + { + preHandler: [authenticate], + schema: { + tags: ['connectors'], + summary: 'Synchronization / audit log entries', + security: [{ bearerAuth: [] }], + params: connectorIdParamsSchema, + querystring: listLogsQuerySchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok(await service.logs(organizationId, request.params.connectorId, request.query)), + ); + }, + ); +} diff --git a/apps/api/src/modules/connectors/connector.schemas.ts b/apps/api/src/modules/connectors/connector.schemas.ts new file mode 100644 index 0000000..be45b8c --- /dev/null +++ b/apps/api/src/modules/connectors/connector.schemas.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; + +export const connectorIdParamsSchema = z.object({ + connectorId: z.string().uuid(), +}); + +export const oauthCallbackQuerySchema = z.object({ + code: z.string().optional(), + state: z.string().optional(), + error: z.string().optional(), +}); + +export const disconnectBodySchema = z.object({ + connectorId: z.string().uuid(), +}); + +export const listResourcesQuerySchema = z.object({ + page: z.coerce.number().int().positive().default(1), + limit: z.coerce.number().int().positive().max(100).default(25), + type: z.string().optional(), + search: z.string().optional(), +}); + +export const listLogsQuerySchema = z.object({ + page: z.coerce.number().int().positive().default(1), + limit: z.coerce.number().int().positive().max(200).default(50), + level: z.enum(['DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), +}); diff --git a/apps/api/src/modules/connectors/connector.service.ts b/apps/api/src/modules/connectors/connector.service.ts new file mode 100644 index 0000000..90c1acc --- /dev/null +++ b/apps/api/src/modules/connectors/connector.service.ts @@ -0,0 +1,426 @@ +import type { Prisma, PrismaClient } from '@prisma/client'; +import type { Redis } from 'ioredis'; +import { + buildAuthorizationUrl, + encryptSecret, + decryptSecret, + exchangeAuthorizationCode, + parseEncryptionKey, + revokeToken, + signState, + verifyState, + type OAuthClientConfig, +} from '@company-brain/auth'; +import { + GOOGLE_AUTH_PARAMS, + GOOGLE_OAUTH_ENDPOINTS, + GOOGLE_PROVIDER, + GOOGLE_SCOPES, +} from '@company-brain/connector-google'; +import { EventBus } from '@company-brain/events'; +import { TASK_QUEUES, WORKFLOW_TYPES } from '@company-brain/workflows'; +import type { TemporalService } from '../../services/temporal.service.js'; +import { BadRequestError, ForbiddenError, NotFoundError } from '../../utils/errors.js'; +import { config } from '../../config/index.js'; + +const INCREMENTAL_CRON = '*/15 * * * *'; + +interface Deps { + prisma: PrismaClient; + temporal: TemporalService; + redis: Redis; +} + +/** JSON-safe view of an ExternalResource (BigInt → number). */ +function serializeResource(row: T) { + return { ...row, sizeBytes: row.sizeBytes === null ? null : Number(row.sizeBytes) }; +} + +export class ConnectorApiService { + private readonly events: EventBus; + + constructor(private readonly deps: Deps) { + this.events = new EventBus(deps.redis); + } + + private googleOAuth(): OAuthClientConfig { + const { clientId, clientSecret, redirectUri } = config.connectors.google; + if (!clientId || !clientSecret) { + throw new BadRequestError( + 'Google OAuth is not configured — set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET', + ); + } + return { clientId, clientSecret, redirectUri, endpoints: GOOGLE_OAUTH_ENDPOINTS }; + } + + private encryptionKey(): Buffer { + if (!config.connectors.tokenEncryptionKey) { + throw new BadRequestError('TOKEN_ENCRYPTION_KEY is not configured'); + } + return parseEncryptionKey(config.connectors.tokenEncryptionKey); + } + + async resolveOrganization(userId: string): Promise { + const membership = await this.deps.prisma.membership.findFirst({ + where: { userId, deletedAt: null }, + orderBy: { createdAt: 'asc' }, + }); + if (!membership) { + throw new ForbiddenError('You must belong to an organization to manage connectors'); + } + return membership.organizationId; + } + + private async requireConnector(organizationId: string, connectorId: string) { + const connector = await this.deps.prisma.connector.findFirst({ + where: { id: connectorId, organizationId, deletedAt: null }, + }); + if (!connector) throw new NotFoundError('Connector not found'); + return connector; + } + + // ── OAuth connect / callback / disconnect ───────────────────── + + buildGoogleConnectUrl(userId: string, organizationId: string): { authUrl: string } { + const oauth = this.googleOAuth(); + this.encryptionKey(); // fail early if encryption is not configured + const state = signState( + { organizationId, userId, provider: GOOGLE_PROVIDER }, + config.connectors.stateSecret, + ); + const authUrl = buildAuthorizationUrl(oauth, { + scopes: [...GOOGLE_SCOPES], + state, + extraParams: { ...GOOGLE_AUTH_PARAMS }, + }); + return { authUrl }; + } + + async handleGoogleCallback(code: string, state: string): Promise<{ connectorId: string }> { + const payload = verifyState(state, config.connectors.stateSecret); + const oauth = this.googleOAuth(); + const tokens = await exchangeAuthorizationCode(oauth, code); + if (!tokens.refreshToken) { + throw new BadRequestError( + 'Google did not return a refresh token — remove prior access at myaccount.google.com/permissions and retry', + ); + } + + // Identify the connecting account. + const userinfo = (await ( + await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + headers: { authorization: `Bearer ${tokens.accessToken}` }, + }) + ).json()) as { email?: string; hd?: string; name?: string }; + + const { prisma } = this.deps; + const domain = userinfo.hd ?? userinfo.email?.split('@')[1] ?? null; + + // Reconnect flow: reuse the org's existing Google connector row. + let connector = await prisma.connector.findFirst({ + where: { + organizationId: payload.organizationId, + provider: 'GOOGLE_WORKSPACE', + deletedAt: null, + }, + }); + if (connector) { + connector = await prisma.connector.update({ + where: { id: connector.id }, + data: { status: 'PENDING', error: null }, + }); + await prisma.oAuthCredential.updateMany({ + where: { connectorId: connector.id, status: 'ACTIVE' }, + data: { status: 'REVOKED' }, + }); + } else { + connector = await prisma.connector.create({ + data: { + provider: 'GOOGLE_WORKSPACE', + name: domain ? `Google Workspace (${domain})` : 'Google Workspace', + status: 'PENDING', + organizationId: payload.organizationId, + ownerId: payload.userId, + }, + }); + await prisma.organizationConnector.create({ + data: { + organizationId: payload.organizationId, + connectorId: connector.id, + status: 'PENDING', + }, + }); + } + + await prisma.oAuthCredential.create({ + data: { + connectorId: connector.id, + organizationId: payload.organizationId, + userEmail: userinfo.email ?? null, + scopes: tokens.scope?.split(' ') ?? [...GOOGLE_SCOPES], + encryptedRefreshToken: encryptSecret(tokens.refreshToken, this.encryptionKey()), + accessTokenExpiresAt: new Date(Date.now() + tokens.expiresInSeconds * 1000), + tokenType: tokens.tokenType, + status: 'ACTIVE', + }, + }); + + await prisma.connectorLog.create({ + data: { + connectorId: connector.id, + organizationId: payload.organizationId, + level: 'INFO', + event: 'oauth.connected', + message: `Google Workspace connected by ${userinfo.email ?? 'unknown'}`, + context: { domain, scopes: tokens.scope } as Prisma.InputJsonValue, + }, + }); + await this.events.publish({ + type: 'connector.connected', + organizationId: payload.organizationId, + connectorId: connector.id, + provider: GOOGLE_PROVIDER, + payload: { domain, adminEmail: userinfo.email }, + }); + + // Kick off the initial full sync + the continuous incremental cron. + await this.deps.temporal.start(WORKFLOW_TYPES.workspaceInitialSync, { + workflowId: `workspace-initial-${connector.id}-${Date.now()}`, + taskQueue: TASK_QUEUES.connectors, + args: [{ connectorId: connector.id }], + }); + try { + await this.deps.temporal.start(WORKFLOW_TYPES.incrementalSync, { + workflowId: `connector-incremental-${connector.id}`, + taskQueue: TASK_QUEUES.connectors, + args: [{ connectorId: connector.id }], + cronSchedule: INCREMENTAL_CRON, + }); + } catch { + // Already scheduled from a previous connect — that's fine. + } + + return { connectorId: connector.id }; + } + + async disconnect(organizationId: string, connectorId: string) { + const connector = await this.requireConnector(organizationId, connectorId); + const { prisma } = this.deps; + + const credential = await prisma.oAuthCredential.findFirst({ + where: { connectorId, status: 'ACTIVE', deletedAt: null }, + orderBy: { createdAt: 'desc' }, + }); + if (credential) { + try { + const refreshToken = decryptSecret(credential.encryptedRefreshToken, this.encryptionKey()); + await revokeToken(this.googleOAuth(), refreshToken); + } catch { + // Provider-side revocation is best effort; local state is truth. + } + await prisma.oAuthCredential.update({ + where: { id: credential.id }, + data: { status: 'REVOKED' }, + }); + } + + await prisma.connector.update({ + where: { id: connectorId }, + data: { status: 'DISCONNECTED', error: null }, + }); + try { + await this.deps.temporal.terminate( + `connector-incremental-${connectorId}`, + 'connector disconnected', + ); + } catch { + // Cron workflow may not exist anymore. + } + + await prisma.connectorLog.create({ + data: { + connectorId, + organizationId, + level: 'INFO', + event: 'oauth.disconnected', + message: 'Connector disconnected and tokens revoked', + }, + }); + await this.events.publish({ + type: 'connector.disconnected', + organizationId, + connectorId, + provider: GOOGLE_PROVIDER, + }); + return { disconnected: true, connectorId: connector.id }; + } + + // ── Read APIs ───────────────────────────────────────────────── + + async list(organizationId: string) { + const connectors = await this.deps.prisma.connector.findMany({ + where: { organizationId, deletedAt: null }, + orderBy: { createdAt: 'desc' }, + include: { + workspace: true, + _count: { select: { resources: { where: { deletedAt: null } } } }, + }, + }); + return connectors; + } + + async get(organizationId: string, connectorId: string) { + const connector = await this.deps.prisma.connector.findFirst({ + where: { id: connectorId, organizationId, deletedAt: null }, + include: { + workspace: true, + syncCursors: { where: { deletedAt: null } }, + credentials: { + where: { deletedAt: null }, + orderBy: { createdAt: 'desc' }, + take: 1, + // Never expose token material — selected fields only. + select: { + id: true, + userEmail: true, + scopes: true, + status: true, + lastRefreshedAt: true, + createdAt: true, + }, + }, + }, + }); + if (!connector) throw new NotFoundError('Connector not found'); + + const resourceCounts = await this.deps.prisma.externalResource.groupBy({ + by: ['type'], + where: { connectorId, deletedAt: null }, + _count: { _all: true }, + }); + return { + ...connector, + resourceCounts: Object.fromEntries(resourceCounts.map((r) => [r.type, r._count._all])), + }; + } + + async triggerSync(organizationId: string, connectorId: string) { + const connector = await this.requireConnector(organizationId, connectorId); + if (connector.status === 'REVOKED' || connector.status === 'DISCONNECTED') { + throw new BadRequestError('Connector is not connected — reconnect first'); + } + const workflowId = `workspace-manual-${connectorId}-${Date.now()}`; + await this.deps.temporal.start(WORKFLOW_TYPES.workspaceInitialSync, { + workflowId, + taskQueue: TASK_QUEUES.connectors, + args: [{ connectorId }], + }); + return { workflowId }; + } + + async status(organizationId: string, connectorId: string) { + const connector = await this.requireConnector(organizationId, connectorId); + const [runningJobs, recentJobs] = await Promise.all([ + this.deps.prisma.syncJob.findMany({ + where: { connectorId, status: 'RUNNING', deletedAt: null }, + orderBy: { createdAt: 'desc' }, + }), + this.deps.prisma.syncJob.findMany({ + where: { connectorId, deletedAt: null }, + orderBy: { createdAt: 'desc' }, + take: 15, + }), + ]); + + let worker: unknown = { reachable: false }; + try { + const response = await fetch(config.connectors.workerHealthUrl, { + signal: AbortSignal.timeout(2000), + }); + worker = { reachable: true, ...((await response.json()) as Record) }; + } catch { + // worker offline + } + + return { + connector: { + id: connector.id, + status: connector.status, + error: connector.error, + lastSyncAt: connector.lastSyncAt, + nextSyncAt: connector.nextSyncAt, + }, + runningJobs, + recentJobs, + worker, + }; + } + + async resources( + organizationId: string, + connectorId: string, + query: { page: number; limit: number; type?: string; search?: string }, + ) { + await this.requireConnector(organizationId, connectorId); + const where: Prisma.ExternalResourceWhereInput = { + connectorId, + organizationId, + deletedAt: null, + ...(query.type ? { type: query.type as never } : {}), + ...(query.search ? { title: { contains: query.search, mode: 'insensitive' } } : {}), + }; + const [items, total] = await this.deps.prisma.$transaction([ + this.deps.prisma.externalResource.findMany({ + where, + orderBy: { externalUpdatedAt: 'desc' }, + skip: (query.page - 1) * query.limit, + take: query.limit, + include: { _count: { select: { permissions: { where: { deletedAt: null } } } } }, + }), + this.deps.prisma.externalResource.count({ where }), + ]); + const counts = await this.deps.prisma.externalResource.groupBy({ + by: ['type'], + where: { connectorId, organizationId, deletedAt: null }, + _count: { _all: true }, + }); + return { + items: items.map(serializeResource), + total, + page: query.page, + limit: query.limit, + totalPages: Math.ceil(total / query.limit) || 1, + typeCounts: Object.fromEntries(counts.map((c) => [c.type, c._count._all])), + }; + } + + async logs( + organizationId: string, + connectorId: string, + query: { page: number; limit: number; level?: string }, + ) { + await this.requireConnector(organizationId, connectorId); + const where: Prisma.ConnectorLogWhereInput = { + connectorId, + organizationId, + deletedAt: null, + ...(query.level ? { level: query.level as never } : {}), + }; + const [items, total] = await this.deps.prisma.$transaction([ + this.deps.prisma.connectorLog.findMany({ + where, + orderBy: { createdAt: 'desc' }, + skip: (query.page - 1) * query.limit, + take: query.limit, + }), + this.deps.prisma.connectorLog.count({ where }), + ]); + return { + items, + total, + page: query.page, + limit: query.limit, + totalPages: Math.ceil(total / query.limit) || 1, + }; + } +} diff --git a/apps/api/src/modules/knowledge/fusion.ts b/apps/api/src/modules/knowledge/fusion.ts new file mode 100644 index 0000000..7ca198d --- /dev/null +++ b/apps/api/src/modules/knowledge/fusion.ts @@ -0,0 +1,48 @@ +/** + * Reciprocal Rank Fusion: merges ranked result lists from the vector and + * keyword arms into one ranking. score(d) = Σ_lists 1 / (k + rank_d). + * k=60 is the standard damping constant — high enough that a single list's + * top rank cannot dominate items that appear in both lists. + */ +export interface RankedItem { + id: string; + /** Arm-specific score, kept for transparency in responses. */ + score: number; +} + +export interface FusedResult { + id: string; + fusedScore: number; + vectorScore: number | null; + keywordScore: number | null; +} + +export function reciprocalRankFusion( + vectorResults: RankedItem[], + keywordResults: RankedItem[], + k = 60, +): FusedResult[] { + const fused = new Map(); + + const ensure = (id: string): FusedResult => { + let entry = fused.get(id); + if (!entry) { + entry = { id, fusedScore: 0, vectorScore: null, keywordScore: null }; + fused.set(id, entry); + } + return entry; + }; + + vectorResults.forEach((item, rank) => { + const entry = ensure(item.id); + entry.fusedScore += 1 / (k + rank + 1); + entry.vectorScore = item.score; + }); + keywordResults.forEach((item, rank) => { + const entry = ensure(item.id); + entry.fusedScore += 1 / (k + rank + 1); + entry.keywordScore = item.score; + }); + + return [...fused.values()].sort((a, b) => b.fusedScore - a.fusedScore); +} diff --git a/apps/api/src/modules/knowledge/knowledge.routes.ts b/apps/api/src/modules/knowledge/knowledge.routes.ts new file mode 100644 index 0000000..6b844fe --- /dev/null +++ b/apps/api/src/modules/knowledge/knowledge.routes.ts @@ -0,0 +1,264 @@ +import type { FastifyInstance } from 'fastify'; +import type { ZodTypeProvider } from 'fastify-type-provider-zod'; +import multipart from '@fastify/multipart'; +import { SUPPORTED_MIME_TYPES, SUPPORTED_EXTENSIONS } from '@company-brain/knowledge'; +import { authenticate } from '../../middleware/authenticate.js'; +import { BadRequestError } from '../../utils/errors.js'; +import { ok } from '../../utils/response.js'; +import { config } from '../../config/index.js'; +import { KnowledgeService } from './knowledge.service.js'; +import { + documentIdParamsSchema, + listDocumentsQuerySchema, + searchBodySchema, +} from './knowledge.schemas.js'; + +function multipartField(fields: Record, name: string): string | undefined { + const field = fields[name] as { value?: string } | Array<{ value?: string }> | undefined; + if (!field) return undefined; + const single = Array.isArray(field) ? field[0] : field; + const value = single?.value?.trim(); + return value || undefined; +} + +/** + * Knowledge Brain API: document lifecycle + hybrid search. + * All routes are organization-isolated via the caller's membership and + * guarded by knowledge:* permissions. + */ +export default async function knowledgeRoutes(fastify: FastifyInstance): Promise { + const app = fastify.withTypeProvider(); + await app.register(multipart, { + limits: { fileSize: config.uploads.maxFileSizeBytes, files: 1 }, + }); + + const service = new KnowledgeService({ + prisma: app.prisma, + storage: app.storage, + vector: app.vector, + temporal: app.temporal, + embeddings: app.embeddings, + }); + + // ── Upload ──────────────────────────────────────────────────── + + app.post( + '/documents', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: + 'Upload a document (multipart/form-data: file, title?, description?, projectId?, folderId?, tags?)', + security: [{ bearerAuth: [] }], + consumes: ['multipart/form-data'], + }, + }, + async (request, reply) => { + const data = await request.file(); + if (!data) throw new BadRequestError('Missing file field in multipart form'); + const buffer = await data.toBuffer(); + if (buffer.length === 0) throw new BadRequestError('Uploaded file is empty'); + + const fields = data.fields as Record; + const organizationId = await service.resolveOrganization(request.user!.id); + const tagsRaw = multipartField(fields, 'tags'); + + const result = await service.uploadDocument(request.user!.id, organizationId, { + fileName: data.filename, + mimeType: data.mimetype, + buffer, + title: multipartField(fields, 'title'), + description: multipartField(fields, 'description'), + projectId: multipartField(fields, 'projectId'), + folderId: multipartField(fields, 'folderId'), + tags: tagsRaw + ? tagsRaw + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + : undefined, + }); + return reply.status(202).send(ok(result, 'Document uploaded — ingestion workflow started')); + }, + ); + + // ── Listing / details ───────────────────────────────────────── + + app.get( + '/documents', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'List documents (filters: status, project, folder, tag, search)', + security: [{ bearerAuth: [] }], + querystring: listDocumentsQuerySchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.listDocuments(organizationId, request.query))); + }, + ); + + app.get( + '/supported-types', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Supported MIME types and file extensions', + security: [{ bearerAuth: [] }], + }, + }, + async (_request, reply) => + reply.send(ok({ mimeTypes: SUPPORTED_MIME_TYPES, extensions: SUPPORTED_EXTENSIONS })), + ); + + app.get( + '/documents/:documentId', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Get a document with versions, tags and chunk count', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.getDocument(organizationId, request.params.documentId))); + }, + ); + + app.get( + '/documents/:documentId/chunks', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'List the chunks of a document (viewer)', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok(await service.getDocumentChunks(organizationId, request.params.documentId)), + ); + }, + ); + + app.get( + '/documents/:documentId/status', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Processing status: latest job, stage log and workflow state', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok(await service.getProcessingStatus(organizationId, request.params.documentId)), + ); + }, + ); + + // ── Mutations ───────────────────────────────────────────────── + + app.delete( + '/documents/:documentId', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Delete a document (soft delete + vector removal)', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send( + ok( + await service.deleteDocument(organizationId, request.params.documentId), + 'Document deleted', + ), + ); + }, + ); + + app.post( + '/documents/:documentId/reindex', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Re-run the full ingestion pipeline for a document', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply + .status(202) + .send( + ok( + await service.reindexDocument(organizationId, request.params.documentId), + 'Reindex started', + ), + ); + }, + ); + + app.post( + '/documents/:documentId/retry', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Retry processing of a failed document', + security: [{ bearerAuth: [] }], + params: documentIdParamsSchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply + .status(202) + .send( + ok( + await service.retryProcessing(organizationId, request.params.documentId), + 'Retry started', + ), + ); + }, + ); + + // ── Search ──────────────────────────────────────────────────── + + app.post( + '/search', + { + preHandler: [authenticate], + schema: { + tags: ['knowledge'], + summary: 'Hybrid search (vector + keyword + metadata filters, RRF-ranked)', + security: [{ bearerAuth: [] }], + body: searchBodySchema, + }, + }, + async (request, reply) => { + const organizationId = await service.resolveOrganization(request.user!.id); + return reply.send(ok(await service.search(organizationId, request.body))); + }, + ); +} diff --git a/apps/api/src/modules/knowledge/knowledge.schemas.ts b/apps/api/src/modules/knowledge/knowledge.schemas.ts new file mode 100644 index 0000000..34ed89b --- /dev/null +++ b/apps/api/src/modules/knowledge/knowledge.schemas.ts @@ -0,0 +1,29 @@ +import { z } from 'zod'; + +export const listDocumentsQuerySchema = z.object({ + page: z.coerce.number().int().positive().default(1), + limit: z.coerce.number().int().positive().max(100).default(20), + status: z.enum(['UPLOADED', 'PROCESSING', 'READY', 'FAILED', 'ARCHIVED']).optional(), + projectId: z.string().uuid().optional(), + folderId: z.string().uuid().optional(), + tag: z.string().optional(), + search: z.string().optional(), +}); + +export const documentIdParamsSchema = z.object({ + documentId: z.string().uuid(), +}); + +export const searchBodySchema = z.object({ + query: z.string().min(1).max(2000), + limit: z.coerce.number().int().positive().max(50).default(10), + mode: z.enum(['hybrid', 'vector', 'keyword']).default('hybrid'), + projectId: z.string().uuid().optional(), + folderId: z.string().uuid().optional(), + documentIds: z.array(z.string().uuid()).optional(), + tags: z.array(z.string()).optional(), + mimeTypes: z.array(z.string()).optional(), +}); + +export type ListDocumentsQuery = z.infer; +export type SearchBody = z.infer; diff --git a/apps/api/src/modules/knowledge/knowledge.service.ts b/apps/api/src/modules/knowledge/knowledge.service.ts new file mode 100644 index 0000000..b462ed0 --- /dev/null +++ b/apps/api/src/modules/knowledge/knowledge.service.ts @@ -0,0 +1,428 @@ +import { createHash, randomUUID } from 'node:crypto'; +import type { Prisma, PrismaClient } from '@prisma/client'; +import { isSupported, type EmbeddingProvider } from '@company-brain/knowledge'; +import { WORKFLOW_TYPES } from '@company-brain/workflows'; +import { collectionForOrganization } from '@company-brain/activities'; +import type { StorageService } from '../../services/storage.service.js'; +import type { VectorService } from '../../services/vector.service.js'; +import type { TemporalService } from '../../services/temporal.service.js'; +import { BadRequestError, ForbiddenError, NotFoundError } from '../../utils/errors.js'; +import { reciprocalRankFusion } from './fusion.js'; +import type { ListDocumentsQuery, SearchBody } from './knowledge.schemas.js'; + +export interface UploadInput { + fileName: string; + mimeType: string; + buffer: Buffer; + title?: string; + description?: string; + projectId?: string; + folderId?: string; + tags?: string[]; +} + +interface Deps { + prisma: PrismaClient; + storage: StorageService; + vector: VectorService; + temporal: TemporalService; + embeddings: EmbeddingProvider; +} + +const slugify = (value: string): string => + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + +export class KnowledgeService { + constructor(private readonly deps: Deps) {} + + /** + * Organization isolation: every operation is scoped to the caller's + * organization, resolved through their membership. Users without a + * membership cannot touch the knowledge base at all. + */ + async resolveOrganization(userId: string): Promise { + const membership = await this.deps.prisma.membership.findFirst({ + where: { userId, deletedAt: null }, + orderBy: { createdAt: 'asc' }, + }); + if (!membership) { + throw new ForbiddenError('You must belong to an organization to use the knowledge base'); + } + return membership.organizationId; + } + + /** Project isolation: a supplied projectId must belong to the same org. */ + private async assertProject(organizationId: string, projectId?: string): Promise { + if (!projectId) return; + const project = await this.deps.prisma.project.findFirst({ + where: { id: projectId, organizationId, deletedAt: null }, + }); + if (!project) throw new BadRequestError('Unknown project for this organization'); + } + + // ── Upload ──────────────────────────────────────────────────── + + async uploadDocument(userId: string, organizationId: string, input: UploadInput) { + if (!isSupported(input.mimeType, input.fileName)) { + throw new BadRequestError( + `Unsupported file type: ${input.mimeType}. Supported: PDF, DOCX, TXT, Markdown, CSV, JSON, HTML`, + ); + } + await this.assertProject(organizationId, input.projectId); + + const { prisma, storage } = this.deps; + const documentId = randomUUID(); + const version = 1; + const storageKey = `documents/${organizationId}/${documentId}/v${version}/${input.fileName}`; + const checksum = createHash('sha256').update(input.buffer).digest('hex'); + + await storage.upload(storageKey, input.buffer, { contentType: input.mimeType }); + + const document = await prisma.document.create({ + data: { + id: documentId, + title: input.title?.trim() || input.fileName.replace(/\.[^.]+$/, ''), + description: input.description, + fileName: input.fileName, + mimeType: input.mimeType, + fileSizeBytes: input.buffer.length, + storageBucket: 'company-brain', + storageKey, + checksum, + status: 'UPLOADED', + currentVersion: version, + organizationId, + projectId: input.projectId, + folderId: input.folderId, + ownerId: userId, + versions: { + create: { + version, + storageKey, + fileSizeBytes: input.buffer.length, + checksum, + organizationId, + }, + }, + }, + }); + + if (input.tags?.length) { + for (const rawTag of input.tags) { + const slug = slugify(rawTag); + if (!slug) continue; + const tag = await prisma.tag.upsert({ + where: { organizationId_slug: { organizationId, slug } }, + create: { name: rawTag.trim(), slug, organizationId, ownerId: userId }, + update: {}, + }); + await prisma.documentTag.upsert({ + where: { documentId_tagId: { documentId, tagId: tag.id } }, + create: { documentId, tagId: tag.id }, + update: {}, + }); + } + } + + const { workflowId } = await this.startIngestion(documentId, organizationId, 1); + return { document, workflowId }; + } + + /** Creates the ProcessingJob and starts the Temporal workflow. */ + private async startIngestion( + documentId: string, + organizationId: string, + attempt: number, + ): Promise<{ workflowId: string }> { + const workflowId = `ingest-${documentId}-${Date.now()}`; + await this.deps.prisma.processingJob.create({ + data: { + documentId, + workflowId, + organizationId, + attempt, + status: 'PENDING', + stage: 'VALIDATE', + logs: [{ stage: 'VALIDATE', message: 'workflow queued', at: new Date().toISOString() }], + }, + }); + const run = await this.deps.temporal.start(WORKFLOW_TYPES.documentIngestion, { + workflowId, + args: [{ documentId }], + }); + await this.deps.prisma.processingJob.updateMany({ + where: { workflowId }, + data: { runId: run.runId }, + }); + return { workflowId }; + } + + // ── Read ────────────────────────────────────────────────────── + + async listDocuments(organizationId: string, query: ListDocumentsQuery) { + const where: Prisma.DocumentWhereInput = { + organizationId, + deletedAt: null, + ...(query.status ? { status: query.status } : {}), + ...(query.projectId ? { projectId: query.projectId } : {}), + ...(query.folderId ? { folderId: query.folderId } : {}), + ...(query.tag ? { tags: { some: { tag: { slug: query.tag } } } } : {}), + ...(query.search + ? { + OR: [ + { title: { contains: query.search, mode: 'insensitive' } }, + { fileName: { contains: query.search, mode: 'insensitive' } }, + ], + } + : {}), + }; + const [items, total] = await this.deps.prisma.$transaction([ + this.deps.prisma.document.findMany({ + where, + orderBy: { createdAt: 'desc' }, + skip: (query.page - 1) * query.limit, + take: query.limit, + include: { + tags: { include: { tag: true } }, + _count: { select: { chunks: { where: { deletedAt: null } } } }, + }, + }), + this.deps.prisma.document.count({ where }), + ]); + return { + items: items.map((d) => this.serializeDocument(d)), + total, + page: query.page, + limit: query.limit, + totalPages: Math.ceil(total / query.limit) || 1, + }; + } + + async getDocument(organizationId: string, documentId: string) { + const document = await this.deps.prisma.document.findFirst({ + where: { id: documentId, organizationId, deletedAt: null }, + include: { + tags: { include: { tag: true } }, + folder: true, + project: true, + owner: { select: { id: true, name: true, email: true } }, + versions: { orderBy: { version: 'desc' } }, + _count: { select: { chunks: { where: { deletedAt: null } } } }, + }, + }); + if (!document) throw new NotFoundError('Document not found'); + return this.serializeDocument(document); + } + + async getDocumentChunks(organizationId: string, documentId: string, limit = 200) { + await this.getDocument(organizationId, documentId); + return this.deps.prisma.chunk.findMany({ + where: { documentId, organizationId, deletedAt: null }, + orderBy: { index: 'asc' }, + take: limit, + select: { + id: true, + index: true, + content: true, + tokenCount: true, + heading: true, + section: true, + }, + }); + } + + // ── Delete / Reindex / Retry ────────────────────────────────── + + async deleteDocument(organizationId: string, documentId: string) { + const document = await this.deps.prisma.document.findFirst({ + where: { id: documentId, organizationId, deletedAt: null }, + }); + if (!document) throw new NotFoundError('Document not found'); + + // Vectors are removed immediately; rows are soft-deleted for audit. + await this.deps.vector.deleteByFilter(collectionForOrganization(organizationId), { + must: [{ key: 'documentId', match: { value: documentId } }], + }); + + const now = new Date(); + await this.deps.prisma.$transaction([ + this.deps.prisma.embeddingReference.updateMany({ + where: { documentId }, + data: { deletedAt: now }, + }), + this.deps.prisma.chunk.updateMany({ where: { documentId }, data: { deletedAt: now } }), + this.deps.prisma.document.update({ + where: { id: documentId }, + data: { deletedAt: now, status: 'ARCHIVED' }, + }), + ]); + return { deleted: true }; + } + + async reindexDocument(organizationId: string, documentId: string) { + const document = await this.deps.prisma.document.findFirst({ + where: { id: documentId, organizationId, deletedAt: null }, + }); + if (!document) throw new NotFoundError('Document not found'); + const previous = await this.deps.prisma.processingJob.count({ where: { documentId } }); + return this.startIngestion(documentId, organizationId, previous + 1); + } + + async retryProcessing(organizationId: string, documentId: string) { + const document = await this.deps.prisma.document.findFirst({ + where: { id: documentId, organizationId, deletedAt: null }, + }); + if (!document) throw new NotFoundError('Document not found'); + if (document.status !== 'FAILED') { + throw new BadRequestError('Only failed documents can be retried; use reindex otherwise'); + } + const previous = await this.deps.prisma.processingJob.count({ where: { documentId } }); + return this.startIngestion(documentId, organizationId, previous + 1); + } + + async getProcessingStatus(organizationId: string, documentId: string) { + const document = await this.deps.prisma.document.findFirst({ + where: { id: documentId, organizationId, deletedAt: null }, + select: { id: true, status: true, title: true }, + }); + if (!document) throw new NotFoundError('Document not found'); + + const jobs = await this.deps.prisma.processingJob.findMany({ + where: { documentId, deletedAt: null }, + orderBy: { createdAt: 'desc' }, + take: 10, + }); + + const latest = jobs[0] ?? null; + let workflow: unknown = null; + if (latest) { + try { + workflow = await this.deps.temporal.describe(latest.workflowId); + } catch { + workflow = null; // Temporal may prune old histories — job row remains. + } + } + return { document, latestJob: latest, workflow, history: jobs }; + } + + // ── Hybrid search ───────────────────────────────────────────── + + async search(organizationId: string, body: SearchBody) { + const collection = collectionForOrganization(organizationId); + + const [vectorArm, keywordArm] = await Promise.all([ + body.mode !== 'keyword' + ? this.vectorSearch(collection, organizationId, body) + : Promise.resolve([]), + body.mode !== 'vector' ? this.keywordSearch(organizationId, body) : Promise.resolve([]), + ]); + + const fused = reciprocalRankFusion(vectorArm, keywordArm).slice(0, body.limit); + if (fused.length === 0) return { query: body.query, mode: body.mode, results: [] }; + + const chunks = await this.deps.prisma.chunk.findMany({ + where: { id: { in: fused.map((f) => f.id) }, organizationId, deletedAt: null }, + include: { + document: { + select: { id: true, title: true, fileName: true, mimeType: true, status: true }, + }, + }, + }); + const byId = new Map(chunks.map((c) => [c.id, c])); + + return { + query: body.query, + mode: body.mode, + results: fused.flatMap((f) => { + const chunk = byId.get(f.id); + if (!chunk || chunk.document.status === 'ARCHIVED') return []; + return [ + { + chunkId: chunk.id, + documentId: chunk.document.id, + documentTitle: chunk.document.title, + fileName: chunk.document.fileName, + mimeType: chunk.document.mimeType, + heading: chunk.heading, + index: chunk.index, + content: chunk.content, + score: Number(f.fusedScore.toFixed(6)), + vectorScore: f.vectorScore, + keywordScore: f.keywordScore, + }, + ]; + }), + }; + } + + private async vectorSearch(collection: string, organizationId: string, body: SearchBody) { + if (!(await this.deps.vector.collectionExists(collection))) return []; + const [queryVector] = await this.deps.embeddings.embed([body.query]); + + const must: Record[] = [ + { key: 'organizationId', match: { value: organizationId } }, + ]; + if (body.projectId) must.push({ key: 'projectId', match: { value: body.projectId } }); + if (body.folderId) must.push({ key: 'folderId', match: { value: body.folderId } }); + if (body.documentIds?.length) + must.push({ key: 'documentId', match: { any: body.documentIds } }); + if (body.tags?.length) must.push({ key: 'tags', match: { any: body.tags } }); + if (body.mimeTypes?.length) must.push({ key: 'mimeType', match: { any: body.mimeTypes } }); + + const results = await this.deps.vector.search(collection, queryVector!, body.limit * 2, { + must, + }); + return results.map((r) => ({ id: String(r.id), score: r.score })); + } + + /** Postgres full-text arm, backed by the GIN index on chunks.content. */ + private async keywordSearch(organizationId: string, body: SearchBody) { + const rows = await this.deps.prisma.$queryRaw>` + SELECT c.id::text AS id, + ts_rank(to_tsvector('english', c.content), + plainto_tsquery('english', ${body.query})) AS rank + FROM chunks c + JOIN documents d ON d.id = c."documentId" + WHERE c."organizationId" = ${organizationId}::uuid + AND c."deletedAt" IS NULL + AND d."deletedAt" IS NULL + AND (${body.projectId ?? null}::uuid IS NULL OR d."projectId" = ${body.projectId ?? null}::uuid) + AND (${body.folderId ?? null}::uuid IS NULL OR d."folderId" = ${body.folderId ?? null}::uuid) + AND to_tsvector('english', c.content) @@ plainto_tsquery('english', ${body.query}) + ORDER BY rank DESC + LIMIT ${body.limit * 2} + `; + let filtered = rows; + if (body.documentIds?.length || body.tags?.length || body.mimeTypes?.length) { + const allowed = await this.deps.prisma.chunk.findMany({ + where: { + id: { in: rows.map((r) => r.id) }, + ...(body.documentIds?.length ? { documentId: { in: body.documentIds } } : {}), + ...(body.mimeTypes?.length ? { document: { mimeType: { in: body.mimeTypes } } } : {}), + ...(body.tags?.length + ? { document: { tags: { some: { tag: { slug: { in: body.tags } } } } } } + : {}), + }, + select: { id: true }, + }); + const allowedIds = new Set(allowed.map((a) => a.id)); + filtered = rows.filter((r) => allowedIds.has(r.id)); + } + return filtered.map((r) => ({ id: r.id, score: Number(r.rank) })); + } + + // ── Serialization ───────────────────────────────────────────── + + private serializeDocument(document: Record & { tags?: unknown }) { + const tags = Array.isArray(document.tags) + ? (document.tags as Array<{ tag: { slug: string; name: string } }>).map((t) => ({ + slug: t.tag.slug, + name: t.tag.name, + })) + : undefined; + return { ...document, tags }; + } +} diff --git a/apps/api/src/plugins/services.ts b/apps/api/src/plugins/services.ts index fbd2036..fdb9c28 100644 --- a/apps/api/src/plugins/services.ts +++ b/apps/api/src/plugins/services.ts @@ -1,9 +1,11 @@ import fp from 'fastify-plugin'; import type { FastifyInstance } from 'fastify'; +import { createEmbeddingProvider, type EmbeddingProvider } from '@company-brain/knowledge'; import { StorageService } from '../services/storage.service.js'; import { VectorService } from '../services/vector.service.js'; import { QueueService } from '../services/queue.service.js'; import { TemporalService } from '../services/temporal.service.js'; +import { config } from '../config/index.js'; import { createRedisConnection } from './redis.js'; declare module 'fastify' { @@ -12,6 +14,7 @@ declare module 'fastify' { vector: VectorService; queues: QueueService; temporal: TemporalService; + embeddings: EmbeddingProvider; } } @@ -28,6 +31,9 @@ export default fp( const queues = new QueueService(queueConnection); // Lazy client: no connection is opened until the first workflow call. const temporal = new TemporalService(); + // Same provider config as the worker: query vectors must share the + // embedding space of the indexed chunks. + const embeddings = createEmbeddingProvider(config.embeddings); // Non-fatal in dev: infra containers may still be starting. try { @@ -40,6 +46,7 @@ export default fp( app.decorate('vector', vector); app.decorate('queues', queues); app.decorate('temporal', temporal); + app.decorate('embeddings', embeddings); app.addHook('onClose', async () => { await queues.close(); diff --git a/apps/api/src/services/temporal.service.ts b/apps/api/src/services/temporal.service.ts index bd1d26f..73c9e7d 100644 --- a/apps/api/src/services/temporal.service.ts +++ b/apps/api/src/services/temporal.service.ts @@ -55,17 +55,29 @@ export class TemporalService { /** Start a workflow without waiting for its result. */ async start( workflowType: string, - options: { workflowId: string; args?: unknown[]; taskQueue?: string }, + options: { + workflowId: string; + args?: unknown[]; + taskQueue?: string; + cronSchedule?: string; + }, ): Promise<{ workflowId: string; runId: string }> { const client = await this.getClient(); const handle = await client.workflow.start(workflowType, { taskQueue: options.taskQueue ?? config.temporal.taskQueue, workflowId: options.workflowId, args: (options.args ?? []) as never[], + ...(options.cronSchedule ? { cronSchedule: options.cronSchedule } : {}), }); return { workflowId: handle.workflowId, runId: handle.firstExecutionRunId }; } + /** Terminate a workflow (e.g. a connector's cron sync) if it exists. */ + async terminate(workflowId: string, reason: string): Promise { + const handle = await this.getHandle(workflowId); + await handle.terminate(reason); + } + /** Start a workflow and await its result (only for short-lived workflows). */ async execute( workflowType: string, diff --git a/apps/api/src/services/vector.service.ts b/apps/api/src/services/vector.service.ts index 7241005..c29d71e 100644 --- a/apps/api/src/services/vector.service.ts +++ b/apps/api/src/services/vector.service.ts @@ -65,6 +65,18 @@ export class VectorService { return results.map((r) => ({ id: r.id, score: r.score, payload: r.payload })); } + /** Remove all points matching a payload filter (e.g. one document). */ + async deleteByFilter(collection: string, filter: Record): Promise { + const { exists } = await this.client.collectionExists(collection); + if (!exists) return; + await this.client.delete(collection, { wait: true, filter }); + } + + async collectionExists(name: string): Promise { + const { exists } = await this.client.collectionExists(name); + return exists; + } + async health(): Promise { try { await this.client.getCollections(); diff --git a/apps/api/tests/connector.schemas.test.ts b/apps/api/tests/connector.schemas.test.ts new file mode 100644 index 0000000..c0db929 --- /dev/null +++ b/apps/api/tests/connector.schemas.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { + connectorIdParamsSchema, + listLogsQuerySchema, + listResourcesQuerySchema, + oauthCallbackQuerySchema, +} from '../src/modules/connectors/connector.schemas.js'; + +describe('connector API schemas', () => { + it('validates connector id params', () => { + expect(() => connectorIdParamsSchema.parse({ connectorId: 'not-a-uuid' })).toThrow(); + expect( + connectorIdParamsSchema.parse({ connectorId: '0b0e8bde-8b3f-4f26-9d0e-111111111111' }), + ).toBeTruthy(); + }); + + it('accepts partial OAuth callback queries (error-only redirects)', () => { + expect(oauthCallbackQuerySchema.parse({ error: 'access_denied' })).toMatchObject({ + error: 'access_denied', + }); + expect(oauthCallbackQuerySchema.parse({ code: 'abc', state: 'xyz' })).toMatchObject({ + code: 'abc', + }); + }); + + it('applies resource list defaults and caps', () => { + expect(listResourcesQuerySchema.parse({})).toMatchObject({ page: 1, limit: 25 }); + expect(() => listResourcesQuerySchema.parse({ limit: 1000 })).toThrow(); + }); + + it('validates log level filters', () => { + expect(listLogsQuerySchema.parse({ level: 'ERROR' }).level).toBe('ERROR'); + expect(() => listLogsQuerySchema.parse({ level: 'TRACE' })).toThrow(); + }); +}); diff --git a/apps/api/tests/fusion.test.ts b/apps/api/tests/fusion.test.ts new file mode 100644 index 0000000..50ed2e6 --- /dev/null +++ b/apps/api/tests/fusion.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { reciprocalRankFusion } from '../src/modules/knowledge/fusion.js'; + +describe('reciprocal rank fusion', () => { + it('ranks items found by both arms above single-arm items', () => { + const vector = [ + { id: 'a', score: 0.9 }, + { id: 'b', score: 0.8 }, + ]; + const keyword = [ + { id: 'c', score: 3.2 }, + { id: 'b', score: 1.1 }, + ]; + const fused = reciprocalRankFusion(vector, keyword); + expect(fused[0]!.id).toBe('b'); // appears in both lists + expect(fused[0]!.vectorScore).toBe(0.8); + expect(fused[0]!.keywordScore).toBe(1.1); + }); + + it('preserves order within a single arm', () => { + const fused = reciprocalRankFusion( + [ + { id: 'x', score: 0.9 }, + { id: 'y', score: 0.5 }, + ], + [], + ); + expect(fused.map((f) => f.id)).toEqual(['x', 'y']); + expect(fused[0]!.keywordScore).toBeNull(); + }); + + it('returns empty for empty inputs', () => { + expect(reciprocalRankFusion([], [])).toEqual([]); + }); + + it('fused score equals the sum of reciprocal ranks', () => { + const fused = reciprocalRankFusion([{ id: 'a', score: 1 }], [{ id: 'a', score: 1 }], 60); + expect(fused[0]!.fusedScore).toBeCloseTo(2 / 61, 10); + }); +}); diff --git a/apps/api/tests/knowledge.schemas.test.ts b/apps/api/tests/knowledge.schemas.test.ts new file mode 100644 index 0000000..403371d --- /dev/null +++ b/apps/api/tests/knowledge.schemas.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { + listDocumentsQuerySchema, + searchBodySchema, +} from '../src/modules/knowledge/knowledge.schemas.js'; + +describe('knowledge API schemas', () => { + it('applies list defaults and coerces pagination', () => { + const parsed = listDocumentsQuerySchema.parse({ page: '2', limit: '10' }); + expect(parsed).toMatchObject({ page: 2, limit: 10 }); + expect(listDocumentsQuerySchema.parse({})).toMatchObject({ page: 1, limit: 20 }); + }); + + it('caps the list page size', () => { + expect(() => listDocumentsQuerySchema.parse({ limit: 1000 })).toThrow(); + }); + + it('rejects unknown status filters', () => { + expect(() => listDocumentsQuerySchema.parse({ status: 'BOGUS' })).toThrow(); + }); + + it('validates search bodies with defaults', () => { + const parsed = searchBodySchema.parse({ query: 'vacation policy' }); + expect(parsed.mode).toBe('hybrid'); + expect(parsed.limit).toBe(10); + }); + + it('rejects empty queries and bad modes', () => { + expect(() => searchBodySchema.parse({ query: '' })).toThrow(); + expect(() => searchBodySchema.parse({ query: 'x', mode: 'psychic' })).toThrow(); + }); + + it('accepts metadata filters', () => { + const parsed = searchBodySchema.parse({ + query: 'roadmap', + tags: ['planning'], + mimeTypes: ['application/pdf'], + documentIds: ['0b0e8bde-8b3f-4f26-9d0e-111111111111'], + }); + expect(parsed.tags).toEqual(['planning']); + }); +}); diff --git a/apps/web/src/app/(app)/connectors/[id]/page.tsx b/apps/web/src/app/(app)/connectors/[id]/page.tsx new file mode 100644 index 0000000..76bf1b5 --- /dev/null +++ b/apps/web/src/app/(app)/connectors/[id]/page.tsx @@ -0,0 +1,468 @@ +'use client'; + +import Link from 'next/link'; +import { useCallback, useEffect, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { ArrowLeft, ExternalLink, Loader2, RefreshCw, Unplug } from 'lucide-react'; +import { + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, + Input, +} from '@company-brain/ui'; +import { + api, + type ConnectorDetail, + type ConnectorLogList, + type ConnectorResourceList, + type ConnectorStatusReport, +} from '@/lib/api'; +import { StatusBadge, formatBytes, formatDate } from '@/components/knowledge/status-badge'; + +const TABS = ['Status', 'Resources', 'Logs', 'Settings'] as const; +type Tab = (typeof TABS)[number]; + +const TYPE_LABELS: Record = { + GOOGLE_DOC: 'Docs', + GOOGLE_SHEET: 'Sheets', + GOOGLE_SLIDES: 'Slides', + PDF: 'PDFs', + FOLDER: 'Folders', + DRIVE_FILE: 'Files', + SHARED_DRIVE: 'Shared drives', + EMAIL: 'Emails', + EMAIL_THREAD: 'Threads', + CALENDAR: 'Calendars', + CALENDAR_EVENT: 'Events', + ATTACHMENT: 'Attachments', + OTHER: 'Other', +}; + +export default function ConnectorDetailPage() { + const params = useParams<{ id: string }>(); + const router = useRouter(); + const connectorId = params.id; + + const [tab, setTab] = useState('Status'); + const [detail, setDetail] = useState(null); + const [status, setStatus] = useState(null); + const [resources, setResources] = useState(null); + const [logs, setLogs] = useState(null); + const [resourceType, setResourceType] = useState(''); + const [resourceSearch, setResourceSearch] = useState(''); + const [resourcePage, setResourcePage] = useState(1); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + + const load = useCallback(async () => { + try { + const [d, s] = await Promise.all([ + api.getConnector(connectorId), + api.getConnectorStatus(connectorId), + ]); + setDetail(d); + setStatus(s); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load connector'); + } + }, [connectorId]); + + useEffect(() => { + void load(); + const timer = setInterval(() => void load(), 10_000); + return () => clearInterval(timer); + }, [load]); + + useEffect(() => { + if (tab !== 'Resources') return; + void api + .listConnectorResources(connectorId, { + page: resourcePage, + limit: 25, + type: resourceType || undefined, + search: resourceSearch || undefined, + }) + .then(setResources) + .catch(() => {}); + }, [tab, connectorId, resourcePage, resourceType, resourceSearch]); + + useEffect(() => { + if (tab !== 'Logs') return; + void api + .listConnectorLogs(connectorId, { limit: 100 }) + .then(setLogs) + .catch(() => {}); + }, [tab, connectorId]); + + async function triggerSync() { + setBusy(true); + try { + await api.triggerConnectorSync(connectorId); + await load(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Sync trigger failed'); + } finally { + setBusy(false); + } + } + + async function disconnect() { + if (!window.confirm('Disconnect this workspace? Tokens will be revoked.')) return; + setBusy(true); + try { + await api.disconnectGoogle(connectorId); + router.push('/connectors'); + } catch (err) { + setError(err instanceof Error ? err.message : 'Disconnect failed'); + setBusy(false); + } + } + + if (!detail) { + return ( +
+ {error ?? } +
+ ); + } + + return ( +
+
+
+ + + Connectors + +

{detail.name}

+

+ {detail.workspace?.adminEmail ?? '—'} · connected {formatDate(detail.createdAt)} +

+
+
+ + +
+
+ + {error &&

{error}

} + +
+ {TABS.map((t) => ( + + ))} +
+ + {tab === 'Status' && status && ( +
+ + + Connection + + +
+ Status + +
+
+ Last sync + + {status.connector.lastSyncAt ? formatDate(status.connector.lastSyncAt) : 'never'} + +
+
+ Next sync + + {status.connector.nextSyncAt ? formatDate(status.connector.nextSyncAt) : '—'} + +
+
+ Sync worker + + {status.worker.reachable ? `up (${status.worker.taskQueue})` : 'offline'} + +
+
+ Running workflows + {status.runningJobs.length} +
+ {status.connector.error && ( +

+ {status.connector.error} +

+ )} +
+

Resources

+
+ {Object.entries(detail.resourceCounts).map(([type, count]) => ( + + {TYPE_LABELS[type] ?? type}: {count} + + ))} + {Object.keys(detail.resourceCounts).length === 0 && ( + nothing synchronized yet + )} +
+
+ + + + + + Sync jobs + Latest workflow executions + + +
    + {status.recentJobs.map((job) => ( +
  • +
    +

    + {job.service ?? 'workspace'} · {job.type.toLowerCase()} +

    +

    + {job.stats + ? `${job.stats.discovered ?? 0} seen · ${job.stats.created ?? 0} new · ${job.stats.updated ?? 0} changed` + : formatDate(job.createdAt)} + {job.error ? ` · ${job.error}` : ''} +

    +
    + +
    • + ))} + {status.recentJobs.length === 0 && ( +

    No sync jobs yet

    + )} +
+ + +
+ )} + + {tab === 'Resources' && ( + + +
+ { + setResourcePage(1); + setResourceSearch(e.target.value); + }} + placeholder="Search titles…" + className="w-56" + /> + + + {resources ? `${resources.total} resources` : ''} + +
+ + +
+ + + + + + + + + + + + + {resources?.items.map((resource) => ( + + + + + + + + + ))} + {resources && resources.items.length === 0 && ( + + + + )} + +
TitleTypeOwnerSizePermsUpdated
+ + {resource.title ?? resource.externalId} + {resource.url && ( + + + + )} + + + {TYPE_LABELS[resource.type] ?? resource.type} + + {resource.ownerEmail ?? '—'} + + {resource.sizeBytes ? formatBytes(resource.sizeBytes) : '—'} + + {resource._count?.permissions ?? 0} + + {resource.externalUpdatedAt ? formatDate(resource.externalUpdatedAt) : '—'} +
+ No resources synchronized yet. +
+
+ {resources && resources.totalPages > 1 && ( +
+ + + Page {resources.page} of {resources.totalPages} + + +
+ )} + + + )} + + {tab === 'Logs' && ( + + +
    + {logs?.items.map((log) => ( +
  • + + {log.level} + +
    +

    {log.event}

    +

    {log.message}

    +
    + + {formatDate(log.createdAt)} + +
    • + ))} + {logs && logs.items.length === 0 && ( +

    No log entries yet.

    + )} +
+ + + )} + + {tab === 'Settings' && ( +
+ + + Credential + Tokens are encrypted at rest and never displayed + + + {detail.credentials[0] ? ( + <> +
+ Account + {detail.credentials[0].userEmail ?? '—'} +
+
+ Status + +
+
+ Last token refresh + + {detail.credentials[0].lastRefreshedAt + ? formatDate(detail.credentials[0].lastRefreshedAt) + : 'not yet'} + +
+
+

Granted scopes

+
    + {detail.credentials[0].scopes.map((scope) => ( +
  • + {scope} +
    • + ))} +
+
+ + ) : ( +

No credential on file.

+ )} + + + + + + Danger zone + + +

+ Disconnecting revokes the OAuth grant at Google, stops the incremental sync schedule + and keeps already-synchronized metadata (soft-deleted on request). +

+ + + +
+ )} +
+ ); +} diff --git a/apps/web/src/app/(app)/connectors/page.tsx b/apps/web/src/app/(app)/connectors/page.tsx new file mode 100644 index 0000000..1919b65 --- /dev/null +++ b/apps/web/src/app/(app)/connectors/page.tsx @@ -0,0 +1,153 @@ +'use client'; + +import Link from 'next/link'; +import { Suspense, useCallback, useEffect, useState } from 'react'; +import { useSearchParams } from 'next/navigation'; +import { Cable, Loader2, Plus, RefreshCw } from 'lucide-react'; +import { + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@company-brain/ui'; +import { api, ApiRequestError, type ConnectorSummary } from '@/lib/api'; +import { StatusBadge, formatDate } from '@/components/knowledge/status-badge'; + +function ConnectorsContent() { + const searchParams = useSearchParams(); + const [connectors, setConnectors] = useState(null); + const [error, setError] = useState(searchParams.get('error')); + const [connecting, setConnecting] = useState(false); + const justConnected = searchParams.get('connected'); + + const load = useCallback(async () => { + try { + setConnectors(await api.listConnectors()); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load connectors'); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + async function connect() { + setConnecting(true); + setError(null); + try { + const { authUrl } = await api.connectGoogle(); + window.location.href = authUrl; + } catch (err) { + setError(err instanceof ApiRequestError ? err.message : 'Could not start the connect flow'); + setConnecting(false); + } + } + + return ( +
+
+
+

Connectors

+

+ Connect company data sources — they synchronize continuously, no manual uploads. +

+
+ +
+ + {justConnected && ( +

+ Workspace connected — initial synchronization is running in the background. +

+ )} + {error && ( +

+ {decodeURIComponent(error)} +

+ )} + + {connectors && connectors.length === 0 && ( + + + +

No connectors yet

+

+ Connect your Google Workspace to automatically discover and synchronize Drive, Docs, + Sheets, Slides, Gmail and Calendar metadata. +

+ + + )} + +
+ {connectors?.map((connector) => ( + + +
+ + + {connector.name} + + + + {connector.workspace?.domain ?? connector.provider} + {connector.workspace?.adminEmail && ` · ${connector.workspace.adminEmail}`} + +
+ + + +
+ Resources + + {connector._count?.resources ?? 0} + +
+
+ Last sync + {connector.lastSyncAt ? formatDate(connector.lastSyncAt) : 'never'} +
+
+ Next sync + {connector.nextSyncAt ? formatDate(connector.nextSyncAt) : '—'} +
+ {connector.error &&

{connector.error}

} +
+ + Status, resources & logs + +
+ + + ))} +
+
+ ); +} + +export default function ConnectorsPage() { + return ( + + + + } + > + + + ); +} diff --git a/apps/web/src/app/(app)/knowledge/documents/[id]/page.tsx b/apps/web/src/app/(app)/knowledge/documents/[id]/page.tsx new file mode 100644 index 0000000..7823074 --- /dev/null +++ b/apps/web/src/app/(app)/knowledge/documents/[id]/page.tsx @@ -0,0 +1,259 @@ +'use client'; + +import Link from 'next/link'; +import { useCallback, useEffect, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { ArrowLeft, Loader2, RefreshCw, Trash2 } from 'lucide-react'; +import { + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@company-brain/ui'; +import { api, type DocumentChunk, type KnowledgeDocument, type ProcessingStatus } from '@/lib/api'; +import { StatusBadge, formatBytes, formatDate } from '@/components/knowledge/status-badge'; + +const ACTIVE_STATUSES = new Set(['UPLOADED', 'PROCESSING', 'PENDING', 'RUNNING']); + +export default function DocumentViewerPage() { + const params = useParams<{ id: string }>(); + const router = useRouter(); + const documentId = params.id; + + const [doc, setDoc] = useState(null); + const [status, setStatus] = useState(null); + const [chunks, setChunks] = useState(null); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + + const load = useCallback(async () => { + try { + const [document, processing] = await Promise.all([ + api.getDocument(documentId), + api.getProcessingStatus(documentId), + ]); + setDoc(document); + setStatus(processing); + if (document.status === 'READY') { + setChunks(await api.getDocumentChunks(documentId)); + } + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load document'); + } + }, [documentId]); + + useEffect(() => { + void load(); + }, [load]); + + // Poll while the ingestion pipeline is running. + useEffect(() => { + if (!doc || !ACTIVE_STATUSES.has(doc.status)) return; + const timer = setInterval(() => void load(), 3000); + return () => clearInterval(timer); + }, [doc, load]); + + async function act(action: 'reindex' | 'retry' | 'delete') { + setBusy(true); + try { + if (action === 'delete') { + if (!window.confirm('Delete this document and its embeddings?')) return; + await api.deleteDocument(documentId); + router.push('/knowledge/library'); + return; + } + await (action === 'reindex' + ? api.reindexDocument(documentId) + : api.retryProcessing(documentId)); + await load(); + } catch (err) { + setError(err instanceof Error ? err.message : `${action} failed`); + } finally { + setBusy(false); + } + } + + if (error) { + return ( +
+ + + Back to library + +

{error}

+
+ ); + } + if (!doc) { + return ( +
+ +
+ ); + } + + const metadata = (doc.metadata ?? {}) as { + keywords?: string[]; + headings?: string[]; + author?: string; + pageCount?: number; + tableCount?: number; + }; + + return ( +
+
+
+ + + Back to library + +

{doc.title}

+

+ {doc.fileName} · {formatBytes(doc.fileSizeBytes)} · v{doc.currentVersion} · uploaded{' '} + {formatDate(doc.createdAt)} +

+ {doc.tags && doc.tags.length > 0 && ( +

+ {doc.tags.map((t) => `#${t.slug}`).join(' ')} +

+ )} +
+
+ + + {doc.status === 'FAILED' && ( + + )} + +
+
+ +
+ + + Processing + + {status?.latestJob + ? `Attempt ${status.latestJob.attempt} — ${status.latestJob.status}` + : 'No processing job yet'} + + + + {status?.latestJob?.error && ( +

+ {status.latestJob.error} +

+ )} +
    + {status?.latestJob?.logs.map((log, i) => ( +
  1. + +
    +

    {log.stage}

    +

    + {log.message} · {formatDate(log.at)} +

    +
    +
    2. + ))} +
+ {status?.latestJob && ( +

+ workflow {status.latestJob.workflowId} + {status.latestJob.chunkCount !== null && ` · ${status.latestJob.chunkCount} chunks`} + {status.latestJob.embeddingCount !== null && + ` · ${status.latestJob.embeddingCount} vectors`} +

+ )} + + + + + + Metadata + + +
+

Language

+

{doc.language ?? '—'}

+
+
+

Author

+

{metadata.author ?? '—'}

+
+
+

Pages / tables

+

+ {metadata.pageCount ?? '—'} / {metadata.tableCount ?? 0} +

+
+
+

Chunks

+

{doc._count?.chunks ?? 0}

+
+ {metadata.keywords && metadata.keywords.length > 0 && ( +
+

Keywords

+

{metadata.keywords.join(', ')}

+
+ )} + {metadata.headings && metadata.headings.length > 0 && ( +
+

Outline

+
    + {metadata.headings.slice(0, 8).map((h, i) => ( +
  • + {h} +
    • + ))} +
+
+ )} + + +
+ + + + Content + + {doc.status === 'READY' + ? `${chunks?.length ?? 0} chunk${(chunks?.length ?? 0) === 1 ? '' : 's'} — as indexed for search` + : 'Content appears here once processing completes'} + + + + {chunks?.map((chunk) => ( +
+

+ + #{chunk.index} + {chunk.heading ? ` · ${chunk.heading}` : ''} + + {chunk.tokenCount} tokens +

+

{chunk.content}

+
+ ))} + {doc.status !== 'READY' && ( +
+ {ACTIVE_STATUSES.has(doc.status) && } + {ACTIVE_STATUSES.has(doc.status) + ? 'Processing — this page refreshes automatically' + : 'Processing has not completed for this document'} +
+ )} + + +
+ ); +} diff --git a/apps/web/src/app/(app)/knowledge/library/page.tsx b/apps/web/src/app/(app)/knowledge/library/page.tsx new file mode 100644 index 0000000..b941f0e --- /dev/null +++ b/apps/web/src/app/(app)/knowledge/library/page.tsx @@ -0,0 +1,212 @@ +'use client'; + +import Link from 'next/link'; +import { useCallback, useEffect, useState } from 'react'; +import { FileText, RefreshCw, Trash2 } from 'lucide-react'; +import { Button, Card, CardContent, Input } from '@company-brain/ui'; +import { api, type DocumentList } from '@/lib/api'; +import { StatusBadge, formatBytes, formatDate } from '@/components/knowledge/status-badge'; + +const STATUS_FILTERS = ['ALL', 'READY', 'PROCESSING', 'FAILED', 'UPLOADED'] as const; + +export default function LibraryPage() { + const [list, setList] = useState(null); + const [page, setPage] = useState(1); + const [status, setStatus] = useState<(typeof STATUS_FILTERS)[number]>('ALL'); + const [search, setSearch] = useState(''); + const [error, setError] = useState(null); + const [busyId, setBusyId] = useState(null); + + const load = useCallback(async () => { + try { + setError(null); + setList( + await api.listDocuments({ + page, + limit: 15, + status: status === 'ALL' ? undefined : status, + search: search || undefined, + }), + ); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load documents'); + } + }, [page, status, search]); + + useEffect(() => { + void load(); + }, [load]); + + async function remove(id: string) { + if (!window.confirm('Delete this document and its embeddings?')) return; + setBusyId(id); + try { + await api.deleteDocument(id); + await load(); + } finally { + setBusyId(null); + } + } + + async function reindex(id: string) { + setBusyId(id); + try { + await api.reindexDocument(id); + await load(); + } finally { + setBusyId(null); + } + } + + return ( +
+
+
+

Document library

+

+ {list ? `${list.total} document${list.total === 1 ? '' : 's'}` : 'Loading…'} +

+
+
+ { + setPage(1); + setSearch(e.target.value); + }} + placeholder="Filter by title or file name…" + className="w-56" + /> +
+ {STATUS_FILTERS.map((s) => ( + + ))} +
+
+
+ + {error &&

{error}

} + + + +
+ + + + + + + + + + + + + + {list?.items.map((doc) => ( + + + + + + + + + + ))} + {list && list.items.length === 0 && ( + + + + )} + +
DocumentTypeSizeChunksUploadedStatusActions
+ + + {doc.title} + + {doc.tags && doc.tags.length > 0 && ( +

+ {doc.tags.map((t) => `#${t.slug}`).join(' ')} +

+ )} + 		+ {doc.fileName.split('.').pop()?.toUpperCase()} + + {formatBytes(doc.fileSizeBytes)} + + {doc._count?.chunks ?? '—'} + + {formatDate(doc.createdAt)} + + + +
+ + +
+
+ No documents match this filter. +
+
+ + + + {list && list.totalPages > 1 && ( +
+ + + Page {list.page} of {list.totalPages} + + +
+ )} +
+ ); +} diff --git a/apps/web/src/app/(app)/knowledge/page.tsx b/apps/web/src/app/(app)/knowledge/page.tsx new file mode 100644 index 0000000..a1dba72 --- /dev/null +++ b/apps/web/src/app/(app)/knowledge/page.tsx @@ -0,0 +1,130 @@ +'use client'; + +import Link from 'next/link'; +import { useEffect, useState } from 'react'; +import { FileText, Layers, Search, Upload } from 'lucide-react'; +import { + buttonVariants, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@company-brain/ui'; +import { api, type DocumentList } from '@/lib/api'; +import { StatusBadge, formatBytes, formatDate } from '@/components/knowledge/status-badge'; + +interface Stats { + total: number; + ready: number; + processing: number; + failed: number; + chunks: number; +} + +export default function KnowledgeDashboardPage() { + const [recent, setRecent] = useState(null); + const [stats, setStats] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + async function load() { + try { + const [all, ready, processing, failed] = await Promise.all([ + api.listDocuments({ limit: 8 }), + api.listDocuments({ limit: 1, status: 'READY' }), + api.listDocuments({ limit: 1, status: 'PROCESSING' }), + api.listDocuments({ limit: 1, status: 'FAILED' }), + ]); + setRecent(all); + setStats({ + total: all.total, + ready: ready.total, + processing: processing.total, + failed: failed.total, + chunks: all.items.reduce((sum, d) => sum + (d._count?.chunks ?? 0), 0), + }); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load knowledge base'); + } + } + void load(); + }, []); + + return ( +
+
+
+

Knowledge Brain

+

+ Your organization's searchable knowledge base +

+
+
+ + Search + + + Upload + +
+
+ + {error &&

{error}

} + +
+ {[ + { label: 'Documents', value: stats?.total, icon: FileText }, + { label: 'Ready', value: stats?.ready, icon: Layers }, + { label: 'Processing', value: stats?.processing, icon: Layers }, + { label: 'Failed', value: stats?.failed, icon: Layers }, + ].map(({ label, value, icon: Icon }) => ( + + + {label} + + + +

{value ?? '—'}

+ + + ))} +
+ + + + Recent documents + Latest uploads across your organization + + + {recent && recent.items.length === 0 && ( +

+ No documents yet.{' '} + + Upload your first document + + . +

+ )} +
    + {recent?.items.map((doc) => ( +
  • + + + {doc.title} + + + {formatBytes(doc.fileSizeBytes)} · {formatDate(doc.createdAt)} + + +
    • + ))} +
+ + +
+ ); +} diff --git a/apps/web/src/app/(app)/knowledge/search/page.tsx b/apps/web/src/app/(app)/knowledge/search/page.tsx new file mode 100644 index 0000000..398c75e --- /dev/null +++ b/apps/web/src/app/(app)/knowledge/search/page.tsx @@ -0,0 +1,139 @@ +'use client'; + +import Link from 'next/link'; +import { useState } from 'react'; +import { FileText, Loader2, Search as SearchIcon } from 'lucide-react'; +import { Button, Card, CardContent, Input } from '@company-brain/ui'; +import { api, type SearchResponse } from '@/lib/api'; + +const MODES = [ + { id: 'hybrid', label: 'Hybrid' }, + { id: 'vector', label: 'Semantic' }, + { id: 'keyword', label: 'Keyword' }, +] as const; + +function highlight(content: string, query: string): React.ReactNode { + const terms = query + .toLowerCase() + .split(/\W+/) + .filter((t) => t.length > 2); + if (terms.length === 0) return content; + const parts = content.split(new RegExp(`(${terms.join('|')})`, 'gi')); + return parts.map((part, i) => + terms.includes(part.toLowerCase()) ? ( + + {part} + + ) : ( + part + ), + ); +} + +export default function SearchPage() { + const [query, setQuery] = useState(''); + const [mode, setMode] = useState<(typeof MODES)[number]['id']>('hybrid'); + const [response, setResponse] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + async function submit(event: React.FormEvent) { + event.preventDefault(); + if (!query.trim() || busy) return; + setBusy(true); + setError(null); + try { + setResponse(await api.searchKnowledge({ query: query.trim(), mode, limit: 15 })); + } catch (err) { + setError(err instanceof Error ? err.message : 'Search failed'); + } finally { + setBusy(false); + } + } + + return ( +
+
+

Search knowledge

+

+ Hybrid retrieval: semantic vectors + keyword full-text, fused with reciprocal rank fusion. +

+
+ +
+
+ + setQuery(e.target.value)} + placeholder="Ask about anything in your documents…" + className="pl-9" + /> +
+
+ {MODES.map((m) => ( + + ))} +
+ +
+ + {error &&

{error}

} + + {response && ( +
+

+ {response.results.length} result{response.results.length === 1 ? '' : 's'} for{' '} + “{response.query}” +

+ {response.results.map((result) => ( + + +
+ + + {result.documentTitle} + {result.heading && ( + › {result.heading} + )} + + + score {result.score.toFixed(4)} + {result.vectorScore !== null && ' · semantic'} + {result.keywordScore !== null && ' · keyword'} + +
+

+ {highlight(result.content, response.query)} +

+ + + ))} + {response.results.length === 0 && ( + + + Nothing found. Try different wording or upload more documents. + + + )} +
+ )} +
+ ); +} diff --git a/apps/web/src/app/(app)/knowledge/upload/page.tsx b/apps/web/src/app/(app)/knowledge/upload/page.tsx new file mode 100644 index 0000000..241c1a3 --- /dev/null +++ b/apps/web/src/app/(app)/knowledge/upload/page.tsx @@ -0,0 +1,159 @@ +'use client'; + +import { useCallback, useRef, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { FileUp, Loader2, UploadCloud } from 'lucide-react'; +import { + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, + Input, + Label, + cn, +} from '@company-brain/ui'; +import { api, ApiRequestError } from '@/lib/api'; +import { formatBytes } from '@/components/knowledge/status-badge'; + +const ACCEPT = '.pdf,.docx,.txt,.md,.markdown,.csv,.tsv,.json,.html,.htm'; + +export default function UploadPage() { + const router = useRouter(); + const inputRef = useRef(null); + const [file, setFile] = useState(null); + const [title, setTitle] = useState(''); + const [description, setDescription] = useState(''); + const [tags, setTags] = useState(''); + const [dragging, setDragging] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + const onDrop = useCallback((event: React.DragEvent) => { + event.preventDefault(); + setDragging(false); + const dropped = event.dataTransfer.files[0]; + if (dropped) setFile(dropped); + }, []); + + async function submit(event: React.FormEvent) { + event.preventDefault(); + if (!file || busy) return; + setBusy(true); + setError(null); + try { + const { document } = await api.uploadDocument({ + file, + title: title || undefined, + description: description || undefined, + tags: tags || undefined, + }); + router.push(`/knowledge/documents/${document.id}`); + } catch (err) { + setError(err instanceof ApiRequestError ? err.message : 'Upload failed — please try again'); + setBusy(false); + } + } + + return ( +
+
+

Upload document

+

+ PDF, DOCX, TXT, Markdown, CSV, JSON or HTML — parsed, chunked, embedded and indexed + automatically. +

+
+ +
+ + + File + Drag & drop or browse. Max 50 MB. + + + + setFile(e.target.files?.[0] ?? null)} + /> + +
+
+ + setTitle(e.target.value)} + placeholder="Defaults to the file name" + /> +
+
+ + setTags(e.target.value)} + placeholder="handbook, onboarding" + /> +
+
+
+ + setDescription(e.target.value)} + placeholder="What is this document about?" + /> +
+ + {error &&

{error}

} + + + + +
+
+ ); +} diff --git a/apps/web/src/app/(app)/layout.tsx b/apps/web/src/app/(app)/layout.tsx index 5fa5be0..417611b 100644 --- a/apps/web/src/app/(app)/layout.tsx +++ b/apps/web/src/app/(app)/layout.tsx @@ -2,13 +2,29 @@ import Link from 'next/link'; import { usePathname } from 'next/navigation'; -import { Brain, LayoutDashboard, LogOut, Settings, User } from 'lucide-react'; +import { + Brain, + Cable, + FileText, + LayoutDashboard, + Library, + LogOut, + Search, + Settings, + Upload, + User, +} from 'lucide-react'; import { Button, cn } from '@company-brain/ui'; import { AuthProvider, useAuth } from '@/components/auth-provider'; import { ThemeToggle } from '@/components/theme-toggle'; const NAV_ITEMS = [ { href: '/dashboard', label: 'Dashboard', icon: LayoutDashboard }, + { href: '/knowledge', label: 'Knowledge', icon: FileText }, + { href: '/knowledge/upload', label: 'Upload', icon: Upload }, + { href: '/knowledge/library', label: 'Library', icon: Library }, + { href: '/knowledge/search', label: 'Search', icon: Search }, + { href: '/connectors', label: 'Connectors', icon: Cable }, { href: '/profile', label: 'Profile', icon: User }, { href: '/settings', label: 'Settings', icon: Settings }, ]; diff --git a/apps/web/src/components/knowledge/status-badge.tsx b/apps/web/src/components/knowledge/status-badge.tsx new file mode 100644 index 0000000..5ac1b6d --- /dev/null +++ b/apps/web/src/components/knowledge/status-badge.tsx @@ -0,0 +1,35 @@ +import { cn } from '@company-brain/ui'; + +const STYLES: Record = { + READY: 'bg-emerald-500/15 text-emerald-600 dark:text-emerald-400', + PROCESSING: 'bg-blue-500/15 text-blue-600 dark:text-blue-400 animate-pulse', + UPLOADED: 'bg-amber-500/15 text-amber-600 dark:text-amber-400', + FAILED: 'bg-red-500/15 text-red-600 dark:text-red-400', + ARCHIVED: 'bg-muted text-muted-foreground', + COMPLETED: 'bg-emerald-500/15 text-emerald-600 dark:text-emerald-400', + RUNNING: 'bg-blue-500/15 text-blue-600 dark:text-blue-400 animate-pulse', + PENDING: 'bg-amber-500/15 text-amber-600 dark:text-amber-400', +}; + +export function StatusBadge({ status }: { status: string }) { + return ( + + {status} + + ); +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export function formatDate(value: string): string { + return new Date(value).toLocaleString(); +} diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index e8d5e44..91ff4a7 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -33,7 +33,8 @@ async function request(path: string, init: RequestInit = {}, retryOn401 = tru // Send the httpOnly refresh cookie along with every auth request. credentials: 'include', headers: { - 'Content-Type': 'application/json', + // FormData bodies set their own multipart boundary header. + ...(init.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(token ? { Authorization: `Bearer ${token}` } : {}), ...init.headers, }, @@ -115,4 +116,309 @@ export const api = { getHealth(): Promise { return request('/health'); }, + + // ── Knowledge Brain ───────────────────────────────────────────── + + uploadDocument(input: { + file: File; + title?: string; + description?: string; + tags?: string; + }): Promise<{ document: KnowledgeDocument; workflowId: string }> { + const form = new FormData(); + form.append('file', input.file); + if (input.title) form.append('title', input.title); + if (input.description) form.append('description', input.description); + if (input.tags) form.append('tags', input.tags); + return request('/api/v1/knowledge/documents', { method: 'POST', body: form }); + }, + + listDocuments( + params: { + page?: number; + limit?: number; + status?: string; + search?: string; + tag?: string; + } = {}, + ): Promise { + const query = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== '') query.set(key, String(value)); + } + const suffix = query.toString() ? `?${query.toString()}` : ''; + return request(`/api/v1/knowledge/documents${suffix}`); + }, + + getDocument(documentId: string): Promise { + return request(`/api/v1/knowledge/documents/${documentId}`); + }, + + getDocumentChunks(documentId: string): Promise { + return request(`/api/v1/knowledge/documents/${documentId}/chunks`); + }, + + getProcessingStatus(documentId: string): Promise { + return request(`/api/v1/knowledge/documents/${documentId}/status`); + }, + + deleteDocument(documentId: string): Promise<{ deleted: boolean }> { + return request(`/api/v1/knowledge/documents/${documentId}`, { method: 'DELETE' }); + }, + + reindexDocument(documentId: string): Promise<{ workflowId: string }> { + return request(`/api/v1/knowledge/documents/${documentId}/reindex`, { + method: 'POST', + body: JSON.stringify({}), + }); + }, + + retryProcessing(documentId: string): Promise<{ workflowId: string }> { + return request(`/api/v1/knowledge/documents/${documentId}/retry`, { + method: 'POST', + body: JSON.stringify({}), + }); + }, + + searchKnowledge(input: { + query: string; + limit?: number; + mode?: 'hybrid' | 'vector' | 'keyword'; + tags?: string[]; + mimeTypes?: string[]; + }): Promise { + return request('/api/v1/knowledge/search', { method: 'POST', body: JSON.stringify(input) }); + }, + + // ── Connectors ────────────────────────────────────────────────── + + connectGoogle(): Promise<{ authUrl: string }> { + return request('/api/v1/connectors/google/connect', { + method: 'POST', + body: JSON.stringify({}), + }); + }, + + disconnectGoogle(connectorId: string): Promise<{ disconnected: boolean }> { + return request('/api/v1/connectors/google/disconnect', { + method: 'POST', + body: JSON.stringify({ connectorId }), + }); + }, + + listConnectors(): Promise { + return request('/api/v1/connectors'); + }, + + getConnector(connectorId: string): Promise { + return request(`/api/v1/connectors/${connectorId}`); + }, + + triggerConnectorSync(connectorId: string): Promise<{ workflowId: string }> { + return request(`/api/v1/connectors/${connectorId}/sync`, { + method: 'POST', + body: JSON.stringify({}), + }); + }, + + getConnectorStatus(connectorId: string): Promise { + return request(`/api/v1/connectors/${connectorId}/status`); + }, + + listConnectorResources( + connectorId: string, + params: { page?: number; limit?: number; type?: string; search?: string } = {}, + ): Promise { + const query = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== '') query.set(key, String(value)); + } + const suffix = query.toString() ? `?${query.toString()}` : ''; + return request(`/api/v1/connectors/${connectorId}/resources${suffix}`); + }, + + listConnectorLogs( + connectorId: string, + params: { page?: number; limit?: number; level?: string } = {}, + ): Promise { + const query = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== '') query.set(key, String(value)); + } + const suffix = query.toString() ? `?${query.toString()}` : ''; + return request(`/api/v1/connectors/${connectorId}/logs${suffix}`); + }, }; + +// ── Connector types ─────────────────────────────────────────────── + +export interface ConnectorSummary { + id: string; + provider: string; + name: string; + status: string; + error: string | null; + lastSyncAt: string | null; + nextSyncAt: string | null; + createdAt: string; + workspace: { domain: string | null; adminEmail: string | null; name: string | null } | null; + _count?: { resources: number }; +} + +export interface ConnectorDetail extends ConnectorSummary { + syncCursors: Array<{ service: string; cursor: string; updatedAt: string }>; + credentials: Array<{ + userEmail: string | null; + scopes: string[]; + status: string; + lastRefreshedAt: string | null; + }>; + resourceCounts: Record; +} + +export interface ConnectorSyncJob { + id: string; + type: string; + status: string; + service: string | null; + workflowId: string; + stats: Record | null; + error: string | null; + startedAt: string | null; + completedAt: string | null; + createdAt: string; +} + +export interface ConnectorStatusReport { + connector: { + id: string; + status: string; + error: string | null; + lastSyncAt: string | null; + nextSyncAt: string | null; + }; + runningJobs: ConnectorSyncJob[]; + recentJobs: ConnectorSyncJob[]; + worker: { reachable: boolean; status?: string; taskQueue?: string }; +} + +export interface ConnectorResource { + id: string; + externalId: string; + type: string; + status: string; + title: string | null; + mimeType: string | null; + url: string | null; + ownerEmail: string | null; + sizeBytes: number | null; + version: string | null; + externalUpdatedAt: string | null; + _count?: { permissions: number }; +} + +export interface ConnectorResourceList { + items: ConnectorResource[]; + total: number; + page: number; + limit: number; + totalPages: number; + typeCounts: Record; +} + +export interface ConnectorLogEntry { + id: string; + level: string; + event: string; + message: string; + context: Record | null; + createdAt: string; +} + +export interface ConnectorLogList { + items: ConnectorLogEntry[]; + total: number; + page: number; + limit: number; + totalPages: number; +} + +// ── Knowledge types (client-side view models) ───────────────────── + +export interface KnowledgeDocument { + id: string; + title: string; + description: string | null; + fileName: string; + mimeType: string; + fileSizeBytes: number; + status: 'UPLOADED' | 'PROCESSING' | 'READY' | 'FAILED' | 'ARCHIVED'; + language: string | null; + metadata: Record | null; + currentVersion: number; + createdAt: string; + updatedAt: string; + tags?: Array<{ slug: string; name: string }>; + owner?: { id: string; name: string; email: string }; + _count?: { chunks: number }; +} + +export interface DocumentList { + items: KnowledgeDocument[]; + total: number; + page: number; + limit: number; + totalPages: number; +} + +export interface DocumentChunk { + id: string; + index: number; + content: string; + tokenCount: number; + heading: string | null; + section: string | null; +} + +export interface ProcessingJobInfo { + id: string; + workflowId: string; + runId: string | null; + stage: string; + status: string; + attempt: number; + error: string | null; + logs: Array<{ stage: string; message: string; at: string }>; + chunkCount: number | null; + embeddingCount: number | null; + createdAt: string; + startedAt: string | null; + completedAt: string | null; +} + +export interface ProcessingStatus { + document: { id: string; status: string; title: string }; + latestJob: ProcessingJobInfo | null; + workflow: { status: string; startTime: string; closeTime: string | null } | null; + history: ProcessingJobInfo[]; +} + +export interface SearchResult { + chunkId: string; + documentId: string; + documentTitle: string; + fileName: string; + mimeType: string; + heading: string | null; + index: number; + content: string; + score: number; + vectorScore: number | null; + keywordScore: number | null; +} + +export interface SearchResponse { + query: string; + mode: string; + results: SearchResult[]; +} diff --git a/docs/connectors.md b/docs/connectors.md new file mode 100644 index 0000000..63420ce --- /dev/null +++ b/docs/connectors.md @@ -0,0 +1,237 @@ +# Knowledge Connector Platform + +Continuously synchronizes external company data sources into the platform. +This phase ships the **connector framework** plus the first implementation, +**Google Workspace** (Drive, Docs, Sheets, Slides, Gmail, Calendar) — +**metadata only**: no parsing, no embeddings, no AI. Future phases consume +the synchronized resources and events. + +## Architecture + +``` +apps/web ── /connectors UI + │ +apps/api ── /api/v1/connectors/* (OAuth flow, status, resources, logs) + │ │ + │ ├── packages/auth (AES-256-GCM token vault, OAuth2, signed state) + │ ├── packages/events (Redis stream + pub/sub event bus) + │ └── Temporal client ──────────────┐ + │ ▼ + │ Temporal Server (namespace company-brain) + │ task queue: brain-connectors + │ │ +services/connector-worker ────────────────────────┘ + │ activities: discover, syncServicePage, runIncrementalSync, jobs + │ + ├── packages/connectors/core (Connector SDK — provider-agnostic) + └── packages/connectors/google (GoogleWorkspaceConnector) + │ + Google APIs (Drive/Docs/Sheets/Slides/Gmail/Calendar, read-only) + +PostgreSQL: Connector, Workspace, OAuthCredential, SyncJob, SyncCursor, + ExternalResource, ExternalChange, ResourcePermission, + ResourceVersion, ConnectorLog, OrganizationConnector +``` + +## Connector SDK (`packages/connectors/core`) + +Every provider implements one interface: + +```ts +interface Connector { + descriptor: ConnectorDescriptor; // provider id, scopes, services + connect(ctx): Promise; + disconnect(ctx): Promise; + validate(ctx): Promise; // credentials still usable? + health(ctx): Promise; // per-service reachability + discover(ctx): Promise; + refresh(ctx): Promise; + sync(ctx, service, pageCursor?): Promise; // one page + incrementalSync(ctx, service, cursor): Promise; +} +``` + +Key design points: + +- **Page-based full sync** — `sync()` returns one page + an opaque + `nextPageCursor`; each page is one retryable Temporal activity, so a + 100k-file drive never blows an activity timeout and progress survives + worker restarts. The last page returns `incrementalCursor`, anchoring + change detection. +- **`ConnectorContext`** — connectors never see credentials; they call + `ctx.getAccessToken()` and the platform's TokenManager refreshes/rotates + behind the scenes. +- **Typed errors** — `TokenExpiredError`, `RateLimitError(retryAfterMs)`, + `QuotaExceededError`, `PermissionDeniedError`, `CursorExpiredError`, + `ProviderApiError`. `retryable` maps directly onto Temporal retry + policies (revoked grants fail fast; rate limits back off). +- **`ConnectorRegistry`** — the only place providers are wired in. + +## OAuth 2.0 flow (Google) + +``` +Admin clicks "Connect Google Workspace" + │ POST /api/v1/connectors/google/connect (JWT) + │ → signed state (HMAC, org+user+nonce, 15 min TTL) + │ → consent URL: access_type=offline, prompt=consent, + │ include_granted_scopes=true, 10 read-only scopes + ▼ +Google consent screen ──► GET /api/v1/connectors/google/callback?code&state + │ 1. verifyState (CSRF) 4. OAuthCredential created: + │ 2. exchange code → tokens refresh token AES-256-GCM + │ 3. userinfo → workspace identity encrypted at rest + ▼ +workspaceInitialSyncWorkflow started (task queue brain-connectors) +incrementalSyncWorkflow started with cron */15 * * * * + ▼ +browser redirected to /connectors?connected= +``` + +Reconnect uses the same flow — the existing connector row is reused, old +credentials are marked REVOKED, a new one becomes ACTIVE. Disconnect +revokes at Google, marks the credential REVOKED, terminates the cron +workflow and emits `connector.disconnected`. + +## Synchronization + +### Temporal workflows (all on `brain-connectors`) + +| Workflow | Purpose | +| ------------------------------ | ------------------------------------------------------------------------ | +| `workspaceInitialSyncWorkflow` | discovery, then fans out the 7 service workflows as independent children | +| `driveSyncWorkflow` | shared drives + all files incl. permissions | +| `docsSyncWorkflow` | Google Docs metadata (mime-filtered Drive view) | +| `sheetsSyncWorkflow` | Sheets metadata + worksheet structure | +| `slidesSyncWorkflow` | Slides metadata | +| `emailSyncWorkflow` | Gmail message metadata (headers, labels, attachments) | +| `calendarSyncWorkflow` | calendars + events (attendees, recurrence, links) | +| `permissionSyncWorkflow` | permission-bearing file pages | +| `incrementalSyncWorkflow` | cron `*/15m`: consumes change feeds via cursors | + +Every service workflow: `startSyncJob → syncServicePage×N → completeSyncJob`. +A failing service never blocks the others (`Promise.allSettled` fan-out). +Live progress is queryable (`getSyncProgress`). + +### Incremental sync cursors + +| Service | Provider mechanism | Cursor stored in SyncCursor | +| -------- | ------------------------ | ---------------------------------- | +| drive | Drive Changes API | `startPageToken` → next page token | +| gmail | Gmail History API | mailbox `historyId` | +| calendar | per-calendar `syncToken` | JSON map calendarId → syncToken | + +The Drive change feed also covers docs/sheets/slides/permissions. Expired +cursors (HTTP 410) surface as `CursorExpiredError`; the cursor row is +dropped and the job reports PARTIAL so a full resync can be scheduled. +Nothing is ever re-downloaded wholesale during incremental runs. + +## Event model (`packages/events`) + +Every change publishes a `PlatformEvent` to Redis — durable +(`XADD brain:events:stream`, capped ~100k) + realtime +(`PUBLISH brain:events`): + +```json +{ + "id": "uuid", + "type": "resource.document.updated", + "occurredAt": "ISO", + "organizationId": "…", + "connectorId": "…", + "provider": "google-workspace", + "resource": { "externalId": "…", "type": "GOOGLE_DOC", "title": "…" } +} +``` + +Types: `connector.connected|disconnected|error`, +`sync.started|completed|failed`, +`resource.document.created|updated|deleted`, `resource.sheet.updated`, +`resource.slides.updated`, `resource.file.created|updated|deleted`, +`resource.email.received`, `resource.calendar.updated`, +`resource.permission.changed`. Future phases consume with `XREADGROUP`. + +## Security decisions + +- **Refresh tokens** encrypted with AES-256-GCM (`TOKEN_ENCRYPTION_KEY`, + 32-byte hex; format `v1:iv:tag:ciphertext` allows key/scheme rotation). +- **Access tokens** live only in worker memory; no API ever returns token + material (credential endpoints select non-secret columns only). +- **Token rotation** — new refresh tokens from providers replace the stored + ciphertext automatically. +- **Revocation** — `invalid_grant` marks credential + connector REVOKED; + the UI offers reconnect. Disconnect revokes provider-side too. +- **CSRF-safe OAuth** — HMAC-signed state with nonce and 15-minute expiry. +- **Organization isolation** — every row carries `organizationId`; every + API call resolves the caller's organization from their membership. All + members of a company share visibility (per product decision — no + role-based gating); other organizations see nothing. +- **Audit** — ConnectorLog rows for oauth/sync/discovery events; SyncJob + rows for every workflow run. +- **Least privilege** — 10 read-only Google scopes; incremental + authorization enabled for future scope additions. + +## Error handling + +| Failure | Behavior | +| -------------------- | -------------------------------------------------------------------- | +| Expired access token | auto-refresh via TokenManager | +| Revoked grant | credential+connector → REVOKED, non-retryable, UI reconnect | +| Rate limit (429) | `RateLimitError` → Temporal backoff retry | +| Quota exceeded | `QuotaExceededError` → retry with long backoff | +| 5xx / network | `ProviderApiError` → retried up to 6 attempts | +| Expired cursor (410) | cursor dropped, job PARTIAL, full resync path | +| Partial sync failure | per-service children isolate failures; job FAILED with error message | + +## API + +| Method | Path | Description | +| ------ | -------------------------------------- | -------------------------------- | +| POST | `/api/v1/connectors/google/connect` | returns Google consent URL | +| GET | `/api/v1/connectors/google/callback` | OAuth redirect target | +| POST | `/api/v1/connectors/google/disconnect` | revoke + disconnect | +| GET | `/api/v1/connectors` | list org connectors | +| GET | `/api/v1/connectors/:id` | detail, cursors, resource counts | +| POST | `/api/v1/connectors/:id/sync` | trigger manual full sync | +| GET | `/api/v1/connectors/:id/status` | connection, jobs, worker health | +| GET | `/api/v1/connectors/:id/resources` | browse synced metadata | +| GET | `/api/v1/connectors/:id/logs` | audit/sync log | + +## Running + +```bash +# Configure OAuth (console.cloud.google.com → OAuth client, Web application, +# redirect URI = http://localhost:4000/api/v1/connectors/google/callback) +GOOGLE_CLIENT_ID=… # .env +GOOGLE_CLIENT_SECRET=… +TOKEN_ENCRYPTION_KEY=$(openssl rand -hex 32) + +pnpm infra:up # Temporal, Postgres, Redis, … +pnpm dev # api + web + workers (incl. connector-worker :4101) +# or: pnpm dev:connector-worker +``` + +UI: http://localhost:3000/connectors · Temporal UI: http://localhost:8233 + +## Adding a new connector (Slack, GitHub, Notion, Microsoft 365, Jira…) + +1. **Package** — `packages/connectors//` exporting a class that + extends `BaseConnector` and implements `descriptor`, `validate`, + `discover`, `sync`, `incrementalSync` (map provider objects to + `SyncedResource`; throw the SDK's typed errors). +2. **OAuth config** — endpoints + scopes in the package (`packages/auth` + handles the wire protocol). Add client id/secret env vars. +3. **Register** — in `services/connector-worker/src/context.ts` + (`registry.register('slack', () => new SlackConnector())`) and map the + Prisma enum in `PROVIDER_IDS`. Add the enum value to `ConnectorProvider`. +4. **API** — add `POST /connectors//connect` + callback wiring in + the connectors module (a ~30-line copy of the Google pair using the same + `ConnectorApiService` helpers). +5. **Cursors** — pick the provider's change mechanism (Slack: event cursor, + GitHub: webhooks + `since` params, Notion: `last_edited_time` filters) + and return it as `incrementalCursor` from the last sync page. +6. Everything else — workflows, jobs, cursors, events, resources UI, + logs, status — works unchanged: it is all keyed on the Connector row. + +``` + +``` diff --git a/package.json b/package.json index 5ead6d8..80ece23 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,13 @@ "name": "company-brain", "version": "0.1.0", "private": true, - "description": "AI-powered Company Brain — Phase 0 platform foundation", + "description": "AI-powered Company Brain \u2014 Phase 0 platform foundation", "packageManager": "pnpm@10.33.2", "engines": { "node": ">=22" }, "scripts": { - "dev": "pnpm --parallel --filter @company-brain/api --filter @company-brain/web --filter @company-brain/worker --filter @company-brain/temporal-worker dev", + "dev": "pnpm --parallel --filter @company-brain/api --filter @company-brain/web --filter @company-brain/worker --filter @company-brain/temporal-worker --filter @company-brain/connector-worker dev", "dev:api": "pnpm --filter @company-brain/api dev", "dev:web": "pnpm --filter @company-brain/web dev", "dev:worker": "pnpm --filter @company-brain/worker dev", @@ -26,7 +26,8 @@ "db:migrate": "pnpm --filter @company-brain/api db:migrate", "db:seed": "pnpm --filter @company-brain/api db:seed", "db:studio": "pnpm --filter @company-brain/api db:studio", - "prepare": "husky" + "prepare": "husky", + "dev:connector-worker": "pnpm --filter @company-brain/connector-worker dev" }, "devDependencies": { "@commitlint/cli": "^19.8.0", diff --git a/packages/activities/package.json b/packages/activities/package.json index 6420a65..0f9a4b0 100644 --- a/packages/activities/package.json +++ b/packages/activities/package.json @@ -15,6 +15,9 @@ "test": "echo 'no tests' && exit 0" }, "dependencies": { + "@company-brain/knowledge": "workspace:*", + "@prisma/client": "^6.19.3", + "@qdrant/js-client-rest": "^1.18.0", "@temporalio/activity": "^1.20.2", "ioredis": "^5.6.1", "minio": "^8.0.5" diff --git a/packages/activities/src/index.ts b/packages/activities/src/index.ts index f4e9540..c80cbc3 100644 --- a/packages/activities/src/index.ts +++ b/packages/activities/src/index.ts @@ -8,3 +8,15 @@ export type { UploadFileInput, UploadFileResult, } from './activities.js'; +export { createKnowledgeActivityContext } from './knowledge.context.js'; +export type { KnowledgeActivityContext, KnowledgeConfig } from './knowledge.context.js'; +export { createKnowledgeActivities, collectionForOrganization } from './knowledge.activities.js'; +export type { + KnowledgeActivities, + IngestionInput, + ValidateResult, + ExtractResult, + ChunkResult, + EmbedResult, + FinalizeInput, +} from './knowledge.activities.js'; diff --git a/packages/activities/src/knowledge.activities.ts b/packages/activities/src/knowledge.activities.ts new file mode 100644 index 0000000..38485bf --- /dev/null +++ b/packages/activities/src/knowledge.activities.ts @@ -0,0 +1,383 @@ +import type { Readable } from 'node:stream'; +import { ApplicationFailure, log } from '@temporalio/activity'; +import type { Prisma } from '@prisma/client'; +import { + buildDocumentMetadata, + chunkDocument, + cleanText, + embedAll, + findParser, + type DocumentSection, +} from '@company-brain/knowledge'; +import type { KnowledgeActivityContext } from './knowledge.context.js'; + +// ── Activity IO contracts (shared with the workflow via type imports) ── + +export interface IngestionInput { + documentId: string; + workflowId: string; +} + +export interface ValidateResult { + versionId: string; + version: number; + organizationId: string; + storageBucket: string; + storageKey: string; + fileName: string; + mimeType: string; + fileSizeBytes: number; +} + +export interface ExtractResult { + textKey: string; + sectionsKey: string; + characterCount: number; + sectionCount: number; + title: string; +} + +export interface ChunkResult { + chunkCount: number; +} + +export interface EmbedResult { + embeddingCount: number; + collection: string; +} + +export interface FinalizeInput extends IngestionInput { + success: boolean; + error?: string; + chunkCount?: number; + embeddingCount?: number; +} + +/** Qdrant collection per organization. */ +export function collectionForOrganization(organizationId: string): string { + return `org_${organizationId.replace(/-/g, '')}`; +} + +function streamToBuffer(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const parts: Buffer[] = []; + stream.on('data', (part: Buffer) => parts.push(part)); + stream.on('end', () => resolve(Buffer.concat(parts))); + stream.on('error', reject); + }); +} + +function derivedKey(storageKey: string, name: string): string { + const base = storageKey.includes('/') + ? storageKey.slice(0, storageKey.lastIndexOf('/')) + : storageKey; + return `${base}/derived/${name}`; +} + +export function createKnowledgeActivities(ctx: KnowledgeActivityContext) { + const { prisma, qdrant, storage, embeddings } = ctx; + const bucket = ctx.config.storage.defaultBucket; + + /** Advance the ProcessingJob and append to its observable stage log. */ + async function trackStage( + workflowId: string, + stage: Prisma.ProcessingJobUpdateInput['stage'], + message: string, + extra: Partial = {}, + ): Promise { + await prisma.processingJob.updateMany({ + where: { workflowId }, + data: { + stage: stage as never, + status: 'RUNNING', + logs: { push: { stage, message, at: new Date().toISOString() } }, + ...extra, + }, + }); + } + + async function getDocument(documentId: string) { + const document = await prisma.document.findFirst({ + where: { id: documentId, deletedAt: null }, + include: { versions: { orderBy: { version: 'desc' }, take: 1 } }, + }); + if (!document) { + throw ApplicationFailure.nonRetryable(`Document ${documentId} not found`, 'NotFound'); + } + const version = document.versions[0]; + if (!version) { + throw ApplicationFailure.nonRetryable(`Document ${documentId} has no version`, 'NoVersion'); + } + return { document, version }; + } + + return { + /** VALIDATE — document exists, format is supported, blob is readable. */ + async validateDocument(input: IngestionInput): Promise { + const { document, version } = await getDocument(input.documentId); + + await prisma.processingJob.updateMany({ + where: { workflowId: input.workflowId }, + data: { status: 'RUNNING', startedAt: new Date() }, + }); + await trackStage(input.workflowId, 'VALIDATE', `validating ${document.fileName}`); + + const parser = findParser(document.mimeType, document.fileName); + if (!parser) { + throw ApplicationFailure.nonRetryable( + `Unsupported document type: ${document.mimeType} (${document.fileName})`, + 'UnsupportedType', + ); + } + try { + const blob = await streamToBuffer(await storage.getObject(bucket, version.storageKey)); + if (blob.length === 0) { + throw ApplicationFailure.nonRetryable('Stored file is empty', 'EmptyFile'); + } + } catch (error) { + if (error instanceof ApplicationFailure) throw error; + throw new Error(`Stored object unreadable: ${(error as Error).message}`); + } + + await prisma.document.update({ + where: { id: document.id }, + data: { status: 'PROCESSING' }, + }); + + return { + versionId: version.id, + version: version.version, + organizationId: document.organizationId, + storageBucket: document.storageBucket, + storageKey: version.storageKey, + fileName: document.fileName, + mimeType: document.mimeType, + fileSizeBytes: document.fileSizeBytes, + }; + }, + + /** PARSE + CLEAN + METADATA — extract text, persist derived artifacts. */ + async extractText(input: IngestionInput): Promise { + const { document, version } = await getDocument(input.documentId); + await trackStage(input.workflowId, 'PARSE', `parsing with ${document.mimeType} parser`); + + const parser = findParser(document.mimeType, document.fileName); + if (!parser) { + throw ApplicationFailure.nonRetryable('Parser vanished between stages', 'UnsupportedType'); + } + const blob = await streamToBuffer(await storage.getObject(bucket, version.storageKey)); + const parsed = await parser.parse(blob, { + fileName: document.fileName, + mimeType: document.mimeType, + }); + + await trackStage(input.workflowId, 'CLEAN', 'normalizing extracted text'); + const text = cleanText(parsed.text); + if (text.length === 0) { + throw ApplicationFailure.nonRetryable('Document produced no text', 'EmptyText'); + } + // Re-anchor section offsets onto the cleaned text. + const sections: DocumentSection[] = []; + let cursor = 0; + for (const section of parsed.sections) { + const at = text.indexOf(section.heading, cursor); + if (at === -1) continue; + if (sections.length > 0) sections[sections.length - 1]!.endOffset = at; + sections.push({ ...section, startOffset: at, endOffset: text.length }); + cursor = at + section.heading.length; + } + + await trackStage(input.workflowId, 'METADATA', 'building document metadata'); + const metadata = buildDocumentMetadata(parsed, text, { + fileName: document.fileName, + mimeType: document.mimeType, + fileSizeBytes: document.fileSizeBytes, + }); + + const textKey = derivedKey(version.storageKey, 'text.txt'); + const sectionsKey = derivedKey(version.storageKey, 'sections.json'); + const textBuffer = Buffer.from(text, 'utf8'); + await storage.putObject(bucket, textKey, textBuffer, textBuffer.length, { + 'Content-Type': 'text/plain; charset=utf-8', + }); + const sectionsBuffer = Buffer.from(JSON.stringify(sections), 'utf8'); + await storage.putObject(bucket, sectionsKey, sectionsBuffer, sectionsBuffer.length, { + 'Content-Type': 'application/json', + }); + + const title = String(metadata.title ?? document.title); + await prisma.document.update({ + where: { id: document.id }, + data: { + title, + language: String(metadata.language ?? 'en'), + metadata: metadata as Prisma.InputJsonValue, + }, + }); + await prisma.documentVersion.update({ + where: { id: version.id }, + data: { metadata: metadata as Prisma.InputJsonValue }, + }); + + log.info('extracted text', { documentId: document.id, chars: text.length }); + return { + textKey, + sectionsKey, + characterCount: text.length, + sectionCount: sections.length, + title, + }; + }, + + /** CHUNK — heading-aware splitting; idempotent per version. */ + async chunkText( + input: IngestionInput & { textKey: string; sectionsKey: string }, + ): Promise { + const { document, version } = await getDocument(input.documentId); + await trackStage(input.workflowId, 'CHUNK', 'chunking document'); + + const text = (await streamToBuffer(await storage.getObject(bucket, input.textKey))).toString( + 'utf8', + ); + const sections = JSON.parse( + (await streamToBuffer(await storage.getObject(bucket, input.sectionsKey))).toString('utf8'), + ) as DocumentSection[]; + + const chunks = chunkDocument(text, sections, ctx.knowledge.chunking); + if (chunks.length === 0) { + throw ApplicationFailure.nonRetryable('Chunker produced no chunks', 'NoChunks'); + } + + await prisma.$transaction([ + prisma.chunk.deleteMany({ where: { versionId: version.id } }), + prisma.chunk.createMany({ + data: chunks.map((chunk) => ({ + index: chunk.index, + content: chunk.content, + tokenCount: chunk.tokenCount, + heading: chunk.heading, + section: chunk.section, + startOffset: chunk.startOffset, + endOffset: chunk.endOffset, + documentId: document.id, + versionId: version.id, + organizationId: document.organizationId, + })), + }), + ]); + + await trackStage(input.workflowId, 'CHUNK', `created ${chunks.length} chunks`, { + chunkCount: chunks.length, + }); + return { chunkCount: chunks.length }; + }, + + /** EMBED + INDEX — vectors into the per-organization Qdrant collection. */ + async embedAndIndexChunks(input: IngestionInput): Promise { + const { document, version } = await getDocument(input.documentId); + await trackStage( + input.workflowId, + 'EMBED', + `embedding with ${embeddings.name}/${embeddings.model}`, + ); + + const chunks = await prisma.chunk.findMany({ + where: { versionId: version.id, deletedAt: null }, + orderBy: { index: 'asc' }, + }); + const tags = await prisma.documentTag.findMany({ + where: { documentId: document.id }, + include: { tag: true }, + }); + const tagSlugs = tags.map((t) => t.tag.slug); + + const collection = collectionForOrganization(document.organizationId); + const existing = await qdrant.collectionExists(collection); + if (!existing.exists) { + await qdrant.createCollection(collection, { + vectors: { size: embeddings.dimension, distance: 'Cosine' }, + }); + } + + const vectors = await embedAll( + embeddings, + chunks.map((c) => c.content), + ); + + await trackStage(input.workflowId, 'INDEX', `indexing ${vectors.length} vectors`); + await qdrant.upsert(collection, { + wait: true, + points: chunks.map((chunk, i) => ({ + id: chunk.id, + vector: vectors[i]!, + payload: { + chunkId: chunk.id, + documentId: document.id, + versionId: version.id, + organizationId: document.organizationId, + projectId: document.projectId, + folderId: document.folderId, + index: chunk.index, + heading: chunk.heading, + title: document.title, + fileName: document.fileName, + mimeType: document.mimeType, + tags: tagSlugs, + content: chunk.content, + }, + })), + }); + + await prisma.$transaction([ + prisma.embeddingReference.deleteMany({ + where: { chunkId: { in: chunks.map((c) => c.id) } }, + }), + prisma.embeddingReference.createMany({ + data: chunks.map((chunk) => ({ + chunkId: chunk.id, + documentId: document.id, + collection, + pointId: chunk.id, + provider: embeddings.name, + model: embeddings.model, + dimension: embeddings.dimension, + organizationId: document.organizationId, + })), + }), + ]); + + await trackStage(input.workflowId, 'PERSIST', 'embedding references stored', { + embeddingCount: chunks.length, + }); + return { embeddingCount: chunks.length, collection }; + }, + + /** COMPLETE / FAILED — terminal bookkeeping for document + job. */ + async finalizeIngestion(input: FinalizeInput): Promise { + await prisma.document.updateMany({ + where: { id: input.documentId }, + data: { status: input.success ? 'READY' : 'FAILED' }, + }); + await prisma.processingJob.updateMany({ + where: { workflowId: input.workflowId }, + data: { + status: input.success ? 'COMPLETED' : 'FAILED', + stage: input.success ? 'COMPLETE' : undefined, + error: input.error ?? null, + completedAt: new Date(), + logs: { + push: { + stage: input.success ? 'COMPLETE' : 'FAILED', + message: input.success + ? `ingestion complete (${input.chunkCount ?? 0} chunks, ${input.embeddingCount ?? 0} vectors)` + : `ingestion failed: ${input.error ?? 'unknown error'}`, + at: new Date().toISOString(), + }, + }, + }, + }); + }, + }; +} + +export type KnowledgeActivities = ReturnType; diff --git a/packages/activities/src/knowledge.context.ts b/packages/activities/src/knowledge.context.ts new file mode 100644 index 0000000..5eb8ddf --- /dev/null +++ b/packages/activities/src/knowledge.context.ts @@ -0,0 +1,46 @@ +import { PrismaClient } from '@prisma/client'; +import { QdrantClient } from '@qdrant/js-client-rest'; +import { + createEmbeddingProvider, + type ChunkOptions, + type EmbeddingConfig, + type EmbeddingProvider, +} from '@company-brain/knowledge'; +import type { ActivityContext } from './context.js'; + +export interface KnowledgeConfig { + embedding: EmbeddingConfig; + chunking: Partial; +} + +/** + * Extended context for knowledge-pipeline activities: adds the database, + * the vector store and the embedding provider on top of the base clients. + */ +export interface KnowledgeActivityContext extends ActivityContext { + prisma: PrismaClient; + qdrant: QdrantClient; + embeddings: EmbeddingProvider; + knowledge: KnowledgeConfig; +} + +export function createKnowledgeActivityContext( + base: ActivityContext, + knowledge: KnowledgeConfig, +): KnowledgeActivityContext { + const prisma = new PrismaClient(); + const qdrant = new QdrantClient({ url: base.config.qdrantUrl }); + const embeddings = createEmbeddingProvider(knowledge.embedding); + + return { + ...base, + prisma, + qdrant, + embeddings, + knowledge, + close: async () => { + await base.close(); + await prisma.$disconnect(); + }, + }; +} diff --git a/packages/auth/eslint.config.mjs b/packages/auth/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/packages/auth/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/packages/auth/package.json b/packages/auth/package.json new file mode 100644 index 0000000..27928a1 --- /dev/null +++ b/packages/auth/package.json @@ -0,0 +1,24 @@ +{ + "name": "@company-brain/auth", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "vitest run" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/node": "^22.14.1", + "eslint": "^9.24.0", + "typescript": "^5.8.3", + "vitest": "^3.1.1" + } +} diff --git a/packages/auth/src/auth.test.ts b/packages/auth/src/auth.test.ts new file mode 100644 index 0000000..35625c4 --- /dev/null +++ b/packages/auth/src/auth.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import { decryptSecret, encryptSecret, parseEncryptionKey } from './crypto.js'; +import { buildAuthorizationUrl, signState, verifyState, OAuthError } from './oauth2.js'; + +const KEY = parseEncryptionKey('a'.repeat(64)); + +describe('secret encryption (AES-256-GCM)', () => { + it('round-trips a refresh token', () => { + const token = '1//0abcdefg-refresh-token-material'; + const encrypted = encryptSecret(token, KEY); + expect(encrypted).toMatch(/^v1:/); + expect(encrypted).not.toContain(token); + expect(decryptSecret(encrypted, KEY)).toBe(token); + }); + + it('produces unique ciphertexts (random IV)', () => { + expect(encryptSecret('same', KEY)).not.toBe(encryptSecret('same', KEY)); + }); + + it('rejects tampered ciphertext', () => { + const encrypted = encryptSecret('secret', KEY); + const parts = encrypted.split(':'); + parts[3] = parts[3]!.slice(0, -4) + 'AAAA'; + expect(() => decryptSecret(parts.join(':'), KEY)).toThrow(); + }); + + it('rejects the wrong key', () => { + const other = parseEncryptionKey('b'.repeat(64)); + const encrypted = encryptSecret('secret', KEY); + expect(() => decryptSecret(encrypted, other)).toThrow(); + }); + + it('validates key format', () => { + expect(() => parseEncryptionKey('short')).toThrow(/64 hex/); + }); +}); + +describe('OAuth2 helpers', () => { + const config = { + clientId: 'client-123', + clientSecret: 'secret', + redirectUri: 'http://localhost:4000/cb', + endpoints: { + authorizationUrl: 'https://accounts.google.com/o/oauth2/v2/auth', + tokenUrl: 'https://oauth2.googleapis.com/token', + }, + }; + + it('builds a correct authorization URL', () => { + const url = new URL( + buildAuthorizationUrl(config, { + scopes: ['openid', 'email'], + state: 'the-state', + extraParams: { access_type: 'offline', prompt: 'consent' }, + }), + ); + expect(url.origin + url.pathname).toBe('https://accounts.google.com/o/oauth2/v2/auth'); + expect(url.searchParams.get('client_id')).toBe('client-123'); + expect(url.searchParams.get('scope')).toBe('openid email'); + expect(url.searchParams.get('state')).toBe('the-state'); + expect(url.searchParams.get('access_type')).toBe('offline'); + expect(url.searchParams.get('response_type')).toBe('code'); + }); +}); + +describe('signed OAuth state', () => { + const payload = { organizationId: 'org-1', userId: 'user-1', provider: 'google-workspace' }; + + it('round-trips and preserves the payload', () => { + const state = signState(payload, 'state-secret'); + const verified = verifyState(state, 'state-secret'); + expect(verified).toMatchObject(payload); + expect(verified.nonce).toHaveLength(16); + }); + + it('rejects a tampered payload', () => { + const state = signState(payload, 'state-secret'); + const [body, sig] = state.split('.'); + const evil = Buffer.from( + JSON.stringify({ + ...JSON.parse(Buffer.from(body!, 'base64url').toString()), + organizationId: 'other-org', + }), + ).toString('base64url'); + expect(() => verifyState(`${evil}.${sig}`, 'state-secret')).toThrow(OAuthError); + }); + + it('rejects the wrong secret and expired states', () => { + const state = signState(payload, 'state-secret'); + expect(() => verifyState(state, 'other-secret')).toThrow(/signature/i); + expect(() => verifyState(state, 'state-secret', -1)).toThrow(/expired/i); + }); +}); diff --git a/packages/auth/src/crypto.ts b/packages/auth/src/crypto.ts new file mode 100644 index 0000000..dbd299b --- /dev/null +++ b/packages/auth/src/crypto.ts @@ -0,0 +1,46 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; + +const ALGORITHM = 'aes-256-gcm'; +const VERSION = 'v1'; + +/** + * Secret-at-rest encryption for OAuth refresh tokens and similar + * credentials. AES-256-GCM (authenticated encryption); output format is + * `v1:::` (base64url parts) so the scheme can be + * rotated later without a migration. + * + * The key must be 32 bytes, hex-encoded (64 chars) — generate with: + * openssl rand -hex 32 + */ +export function parseEncryptionKey(hexKey: string): Buffer { + if (!/^[0-9a-fA-F]{64}$/.test(hexKey)) { + throw new Error('TOKEN_ENCRYPTION_KEY must be 64 hex characters (openssl rand -hex 32)'); + } + return Buffer.from(hexKey, 'hex'); +} + +export function encryptSecret(plaintext: string, key: Buffer): string { + const iv = randomBytes(12); + const cipher = createCipheriv(ALGORITHM, key, iv); + const ciphertext = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); + const tag = cipher.getAuthTag(); + return [ + VERSION, + iv.toString('base64url'), + tag.toString('base64url'), + ciphertext.toString('base64url'), + ].join(':'); +} + +export function decryptSecret(encrypted: string, key: Buffer): string { + const [version, ivB64, tagB64, dataB64] = encrypted.split(':'); + if (version !== VERSION || !ivB64 || !tagB64 || !dataB64) { + throw new Error('Unrecognized encrypted secret format'); + } + const decipher = createDecipheriv(ALGORITHM, key, Buffer.from(ivB64, 'base64url')); + decipher.setAuthTag(Buffer.from(tagB64, 'base64url')); + return Buffer.concat([ + decipher.update(Buffer.from(dataB64, 'base64url')), + decipher.final(), + ]).toString('utf8'); +} diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts new file mode 100644 index 0000000..ad4b26d --- /dev/null +++ b/packages/auth/src/index.ts @@ -0,0 +1,16 @@ +export { encryptSecret, decryptSecret, parseEncryptionKey } from './crypto.js'; +export { + buildAuthorizationUrl, + exchangeAuthorizationCode, + refreshAccessToken, + revokeToken, + signState, + verifyState, + OAuthError, +} from './oauth2.js'; +export type { + OAuthClientConfig, + OAuthEndpoints, + OAuthStatePayload, + TokenResponse, +} from './oauth2.js'; diff --git a/packages/auth/src/oauth2.ts b/packages/auth/src/oauth2.ts new file mode 100644 index 0000000..6569963 --- /dev/null +++ b/packages/auth/src/oauth2.ts @@ -0,0 +1,176 @@ +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; + +/** + * Provider-agnostic OAuth 2.0 authorization-code helpers. Each connector + * supplies its endpoints; this module owns the wire format so every + * provider (Google, Slack, Microsoft, …) reuses the same flow. + */ + +export interface OAuthEndpoints { + authorizationUrl: string; + tokenUrl: string; + revocationUrl?: string; +} + +export interface OAuthClientConfig { + clientId: string; + clientSecret: string; + redirectUri: string; + endpoints: OAuthEndpoints; +} + +export interface TokenResponse { + accessToken: string; + refreshToken?: string; + expiresInSeconds: number; + scope?: string; + tokenType: string; + /** OIDC id_token when openid scope was granted. */ + idToken?: string; +} + +interface RawTokenResponse { + access_token: string; + refresh_token?: string; + expires_in?: number; + scope?: string; + token_type?: string; + id_token?: string; + error?: string; + error_description?: string; +} + +export class OAuthError extends Error { + constructor( + message: string, + readonly code: string, + readonly status?: number, + ) { + super(message); + } +} + +export function buildAuthorizationUrl( + config: OAuthClientConfig, + options: { + scopes: string[]; + state: string; + /** offline access + refresh-token issuance (Google: access_type/prompt). */ + extraParams?: Record; + }, +): string { + const url = new URL(config.endpoints.authorizationUrl); + url.searchParams.set('client_id', config.clientId); + url.searchParams.set('redirect_uri', config.redirectUri); + url.searchParams.set('response_type', 'code'); + url.searchParams.set('scope', options.scopes.join(' ')); + url.searchParams.set('state', options.state); + for (const [key, value] of Object.entries(options.extraParams ?? {})) { + url.searchParams.set(key, value); + } + return url.toString(); +} + +async function tokenRequest( + config: OAuthClientConfig, + params: Record, +): Promise { + const response = await fetch(config.endpoints.tokenUrl, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + client_id: config.clientId, + client_secret: config.clientSecret, + ...params, + }).toString(), + }); + const body = (await response.json().catch(() => ({}))) as RawTokenResponse; + if (!response.ok || body.error || !body.access_token) { + throw new OAuthError( + body.error_description ?? body.error ?? `Token endpoint returned ${response.status}`, + body.error ?? 'token_request_failed', + response.status, + ); + } + return { + accessToken: body.access_token, + refreshToken: body.refresh_token, + expiresInSeconds: body.expires_in ?? 3600, + scope: body.scope, + tokenType: body.token_type ?? 'Bearer', + idToken: body.id_token, + }; +} + +/** Exchange an authorization code for tokens. */ +export function exchangeAuthorizationCode( + config: OAuthClientConfig, + code: string, +): Promise { + return tokenRequest(config, { + grant_type: 'authorization_code', + code, + redirect_uri: config.redirectUri, + }); +} + +/** Mint a fresh access token from a stored refresh token. */ +export function refreshAccessToken( + config: OAuthClientConfig, + refreshToken: string, +): Promise { + return tokenRequest(config, { grant_type: 'refresh_token', refresh_token: refreshToken }); +} + +/** Revoke a refresh or access token at the provider. */ +export async function revokeToken(config: OAuthClientConfig, token: string): Promise { + if (!config.endpoints.revocationUrl) return; + await fetch(config.endpoints.revocationUrl, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ token }).toString(), + }); +} + +// ── Signed state (CSRF protection for the redirect round-trip) ── + +export interface OAuthStatePayload { + organizationId: string; + userId: string; + provider: string; + nonce: string; + issuedAt: number; +} + +export function signState( + payload: Omit, + secret: string, +): string { + const full: OAuthStatePayload = { + ...payload, + nonce: randomBytes(8).toString('hex'), + issuedAt: Date.now(), + }; + const body = Buffer.from(JSON.stringify(full)).toString('base64url'); + const signature = createHmac('sha256', secret).update(body).digest('base64url'); + return `${body}.${signature}`; +} + +export function verifyState( + state: string, + secret: string, + maxAgeMs = 15 * 60 * 1000, +): OAuthStatePayload { + const [body, signature] = state.split('.'); + if (!body || !signature) throw new OAuthError('Malformed state', 'invalid_state'); + const expected = createHmac('sha256', secret).update(body).digest(); + const given = Buffer.from(signature, 'base64url'); + if (expected.length !== given.length || !timingSafeEqual(expected, given)) { + throw new OAuthError('State signature mismatch', 'invalid_state'); + } + const payload = JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) as OAuthStatePayload; + if (Date.now() - payload.issuedAt > maxAgeMs) { + throw new OAuthError('State expired — restart the connect flow', 'state_expired'); + } + return payload; +} diff --git a/packages/auth/tsconfig.json b/packages/auth/tsconfig.json new file mode 100644 index 0000000..7bfd764 --- /dev/null +++ b/packages/auth/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@company-brain/config/tsconfig/node", + "include": ["src"] +} diff --git a/packages/connectors/core/eslint.config.mjs b/packages/connectors/core/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/packages/connectors/core/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/packages/connectors/core/package.json b/packages/connectors/core/package.json new file mode 100644 index 0000000..c9b8506 --- /dev/null +++ b/packages/connectors/core/package.json @@ -0,0 +1,24 @@ +{ + "name": "@company-brain/connector-core", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "vitest run" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/node": "^22.14.1", + "eslint": "^9.24.0", + "typescript": "^5.8.3", + "vitest": "^3.1.1" + } +} diff --git a/packages/connectors/core/src/base.ts b/packages/connectors/core/src/base.ts new file mode 100644 index 0000000..3f8f12e --- /dev/null +++ b/packages/connectors/core/src/base.ts @@ -0,0 +1,60 @@ +import type { + Connector, + ConnectorContext, + ConnectorDescriptor, + DiscoveryResult, + HealthResult, + IncrementalSyncResult, + SyncPage, +} from './types.js'; + +/** + * Convenience base class: implements the boring parts (health via + * validate, connect via discover, no-op refresh/disconnect) so concrete + * connectors focus on provider API calls. + */ +export abstract class BaseConnector implements Connector { + abstract readonly descriptor: ConnectorDescriptor; + + abstract discover(ctx: ConnectorContext): Promise; + abstract validate(ctx: ConnectorContext): Promise; + abstract sync( + ctx: ConnectorContext, + service: string, + pageCursor?: string | null, + ): Promise; + abstract incrementalSync( + ctx: ConnectorContext, + service: string, + cursor: string, + ): Promise; + + /** Default connect: a discovery run against the fresh grant. */ + connect(ctx: ConnectorContext): Promise { + return this.discover(ctx); + } + + /** Default disconnect: nothing provider-side; platform revokes tokens. */ + async disconnect(_ctx: ConnectorContext): Promise { + // no-op by default + } + + /** Default refresh: nothing cached. */ + async refresh(_ctx: ConnectorContext): Promise { + // no-op by default + } + + /** Default health: every declared service inherits validate()'s answer. */ + async health(ctx: ConnectorContext): Promise { + let valid = false; + try { + valid = await this.validate(ctx); + } catch { + valid = false; + } + const services = Object.fromEntries( + this.descriptor.services.map((s) => [s, valid ? ('up' as const) : ('unauthorized' as const)]), + ); + return { healthy: valid, services, checkedAt: new Date().toISOString() }; + } +} diff --git a/packages/connectors/core/src/core.test.ts b/packages/connectors/core/src/core.test.ts new file mode 100644 index 0000000..1c30e5f --- /dev/null +++ b/packages/connectors/core/src/core.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import { BaseConnector } from './base.js'; +import { ConnectorRegistry } from './registry.js'; +import { RateLimitError, TokenExpiredError, ProviderApiError } from './errors.js'; +import type { + ConnectorContext, + ConnectorDescriptor, + DiscoveryResult, + IncrementalSyncResult, + SyncPage, +} from './types.js'; + +class FakeConnector extends BaseConnector { + readonly descriptor: ConnectorDescriptor = { + provider: 'fake', + displayName: 'Fake', + authType: 'oauth2', + scopes: ['read'], + services: ['files', 'mail'], + }; + valid = true; + + async discover(): Promise { + return { workspace: { domain: 'fake.test' }, services: { files: { available: true } } }; + } + async validate(): Promise { + return this.valid; + } + async sync(): Promise { + return { resources: [], nextPageCursor: null }; + } + async incrementalSync(): Promise { + return { changes: [], nextCursor: 'c1' }; + } +} + +const ctx: ConnectorContext = { + connectorId: 'c', + organizationId: 'o', + getAccessToken: async () => 'token', +}; + +describe('ConnectorRegistry', () => { + it('registers, resolves and lists providers', () => { + const registry = new ConnectorRegistry(); + registry.register('fake', () => new FakeConnector()); + expect(registry.has('fake')).toBe(true); + expect(registry.list()).toEqual(['fake']); + expect(registry.get('fake').descriptor.displayName).toBe('Fake'); + // Singleton per provider + expect(registry.get('fake')).toBe(registry.get('fake')); + }); + + it('rejects duplicates and unknown providers', () => { + const registry = new ConnectorRegistry(); + registry.register('fake', () => new FakeConnector()); + expect(() => registry.register('fake', () => new FakeConnector())).toThrow(/already/); + expect(() => registry.get('nope')).toThrow(/Unknown/); + }); +}); + +describe('BaseConnector defaults', () => { + it('connect() delegates to discover()', async () => { + const connector = new FakeConnector(); + const result = await connector.connect(ctx); + expect(result.workspace.domain).toBe('fake.test'); + }); + + it('health() reflects validate() across declared services', async () => { + const connector = new FakeConnector(); + const healthy = await connector.health(ctx); + expect(healthy).toMatchObject({ healthy: true, services: { files: 'up', mail: 'up' } }); + + connector.valid = false; + const unhealthy = await connector.health(ctx); + expect(unhealthy.healthy).toBe(false); + expect(unhealthy.services.files).toBe('unauthorized'); + }); +}); + +describe('typed errors', () => { + it('carries retryability semantics', () => { + expect(new TokenExpiredError().retryable).toBe(false); + expect(new RateLimitError('slow down', 5000)).toMatchObject({ + retryable: true, + retryAfterMs: 5000, + code: 'RATE_LIMITED', + }); + expect(new ProviderApiError('boom', 502)).toMatchObject({ retryable: true, status: 502 }); + }); +}); diff --git a/packages/connectors/core/src/errors.ts b/packages/connectors/core/src/errors.ts new file mode 100644 index 0000000..447bc45 --- /dev/null +++ b/packages/connectors/core/src/errors.ts @@ -0,0 +1,63 @@ +/** + * Typed connector errors. Temporal activities map `retryable` onto retry + * policies: non-retryable errors fail fast (revoked grant), retryable + * ones back off (rate limits, transient API failures). + */ +export class ConnectorError extends Error { + constructor( + message: string, + readonly code: string, + readonly retryable: boolean, + ) { + super(message); + this.name = new.target.name; + } +} + +/** Access/refresh token no longer valid — user must reconnect. */ +export class TokenExpiredError extends ConnectorError { + constructor(message = 'OAuth grant expired or revoked') { + super(message, 'TOKEN_EXPIRED', false); + } +} + +/** Provider rate limit hit — retry after backoff. */ +export class RateLimitError extends ConnectorError { + constructor( + message = 'Provider rate limit exceeded', + readonly retryAfterMs: number = 30_000, + ) { + super(message, 'RATE_LIMITED', true); + } +} + +/** Daily/project quota exhausted — retrying soon will not help much. */ +export class QuotaExceededError extends ConnectorError { + constructor(message = 'Provider quota exceeded') { + super(message, 'QUOTA_EXCEEDED', true); + } +} + +/** The grant lacks a scope or the resource is forbidden. */ +export class PermissionDeniedError extends ConnectorError { + constructor(message = 'Permission denied by provider') { + super(message, 'PERMISSION_DENIED', false); + } +} + +/** Incremental cursor invalidated by the provider — full resync required. */ +export class CursorExpiredError extends ConnectorError { + constructor(message = 'Sync cursor expired') { + super(message, 'CURSOR_EXPIRED', false); + } +} + +/** Any other provider API failure (5xx, network) — retryable. */ +export class ProviderApiError extends ConnectorError { + constructor( + message: string, + readonly status?: number, + ) { + super(message, 'PROVIDER_API_ERROR', true); + } +} diff --git a/packages/connectors/core/src/index.ts b/packages/connectors/core/src/index.ts new file mode 100644 index 0000000..3d0d786 --- /dev/null +++ b/packages/connectors/core/src/index.ts @@ -0,0 +1,28 @@ +export type { + ChangeKind, + Connector, + ConnectorContext, + ConnectorDescriptor, + ConnectorFactory, + DiscoveryResult, + HealthResult, + IncrementalSyncResult, + PermissionRole, + PrincipalType, + ResourceChange, + SyncedPermission, + SyncedResource, + SyncPage, + WorkspaceIdentity, +} from './types.js'; +export { BaseConnector } from './base.js'; +export { ConnectorRegistry } from './registry.js'; +export { + ConnectorError, + CursorExpiredError, + PermissionDeniedError, + ProviderApiError, + QuotaExceededError, + RateLimitError, + TokenExpiredError, +} from './errors.js'; diff --git a/packages/connectors/core/src/registry.ts b/packages/connectors/core/src/registry.ts new file mode 100644 index 0000000..16ab356 --- /dev/null +++ b/packages/connectors/core/src/registry.ts @@ -0,0 +1,37 @@ +import type { Connector, ConnectorFactory } from './types.js'; + +/** + * Provider registry. Adding a connector to the platform is: + * registry.register('slack', () => new SlackConnector()); + * The API and workers resolve connectors exclusively through this. + */ +export class ConnectorRegistry { + private readonly factories = new Map(); + private readonly instances = new Map(); + + register(provider: string, factory: ConnectorFactory): void { + if (this.factories.has(provider)) { + throw new Error(`Connector already registered: ${provider}`); + } + this.factories.set(provider, factory); + } + + get(provider: string): Connector { + let instance = this.instances.get(provider); + if (!instance) { + const factory = this.factories.get(provider); + if (!factory) throw new Error(`Unknown connector provider: ${provider}`); + instance = factory(); + this.instances.set(provider, instance); + } + return instance; + } + + has(provider: string): boolean { + return this.factories.has(provider); + } + + list(): string[] { + return [...this.factories.keys()]; + } +} diff --git a/packages/connectors/core/src/types.ts b/packages/connectors/core/src/types.ts new file mode 100644 index 0000000..15a8fbf --- /dev/null +++ b/packages/connectors/core/src/types.ts @@ -0,0 +1,148 @@ +/** + * The Connector SDK: every external system (Google Workspace, Slack, + * GitHub, Notion, Microsoft 365, …) is one implementation of the + * Connector interface below. The platform — API, Temporal workflows, + * storage — only ever programs against these types. + */ + +export type PrincipalType = 'USER' | 'GROUP' | 'DOMAIN' | 'ANYONE'; +export type PermissionRole = 'OWNER' | 'EDITOR' | 'COMMENTER' | 'VIEWER'; + +export interface SyncedPermission { + externalPermissionId?: string; + principalType: PrincipalType; + principalEmail?: string | null; + domain?: string | null; + role: PermissionRole; +} + +/** Normalized resource metadata — metadata ONLY, never content. */ +export interface SyncedResource { + externalId: string; + /** Platform resource type, e.g. GOOGLE_DOC, EMAIL_THREAD, CALENDAR_EVENT. */ + type: string; + title?: string | null; + mimeType?: string | null; + url?: string | null; + ownerEmail?: string | null; + parentExternalId?: string | null; + driveId?: string | null; + sizeBytes?: number | null; + checksum?: string | null; + version?: string | null; + externalCreatedAt?: string | null; + externalUpdatedAt?: string | null; + trashed?: boolean; + permissions?: SyncedPermission[]; + /** Service-specific extras (labels, attendees, worksheets, …). */ + metadata?: Record; +} + +export type ChangeKind = 'created' | 'updated' | 'deleted' | 'permission_changed'; + +export interface ResourceChange { + externalId: string; + service: string; + changeType: ChangeKind; + /** Present unless the resource was deleted. */ + resource?: SyncedResource; + occurredAt?: string; +} + +/** One page of a full sync — activities persist a page per call. */ +export interface SyncPage { + resources: SyncedResource[]; + nextPageCursor: string | null; + /** + * On the LAST page a connector may return the cursor from which + * incremental sync should start (e.g. Drive changes startPageToken). + */ + incrementalCursor?: string | null; +} + +export interface IncrementalSyncResult { + changes: ResourceChange[]; + /** Cursor to store for the next incremental run. */ + nextCursor: string; + /** True when the provider invalidated the cursor — full resync needed. */ + cursorExpired?: boolean; +} + +export interface WorkspaceIdentity { + externalId?: string | null; + domain?: string | null; + name?: string | null; + adminEmail?: string | null; + metadata?: Record; +} + +export interface DiscoveryResult { + workspace: WorkspaceIdentity; + /** Which provider services this grant can reach. */ + services: Record; +} + +export interface HealthResult { + healthy: boolean; + services: Record; + checkedAt: string; +} + +/** + * Runtime dependencies handed to a connector per call. The platform owns + * credential storage and refresh; connectors just ask for a live token. + */ +export interface ConnectorContext { + connectorId: string; + organizationId: string; + /** Always returns a currently-valid access token (auto-refreshed). */ + getAccessToken(): Promise; + log?(level: 'debug' | 'info' | 'warn' | 'error', message: string, context?: object): void; +} + +export interface ConnectorDescriptor { + /** Stable id, e.g. "google-workspace". */ + provider: string; + displayName: string; + authType: 'oauth2' | 'api_key' | 'app_install'; + /** Least-privilege scopes requested at connect time. */ + scopes: string[]; + /** Sync services the connector exposes, e.g. drive, gmail, calendar. */ + services: string[]; +} + +/** The contract every connector implements. */ +export interface Connector { + readonly descriptor: ConnectorDescriptor; + + /** Verify the fresh grant and identify the workspace behind it. */ + connect(ctx: ConnectorContext): Promise; + + /** Provider-side cleanup before the platform revokes credentials. */ + disconnect(ctx: ConnectorContext): Promise; + + /** Are the stored credentials still usable? */ + validate(ctx: ConnectorContext): Promise; + + /** Per-service reachability report. */ + health(ctx: ConnectorContext): Promise; + + /** Enumerate workspace structure without downloading content. */ + discover(ctx: ConnectorContext): Promise; + + /** Refresh cached provider state (quotas, service directory). */ + refresh(ctx: ConnectorContext): Promise; + + /** One page of a full metadata sync for one service. */ + sync(ctx: ConnectorContext, service: string, pageCursor?: string | null): Promise; + + /** Consume provider change feeds from a stored cursor. */ + incrementalSync( + ctx: ConnectorContext, + service: string, + cursor: string, + ): Promise; +} + +/** Constructor signature used by the registry. */ +export type ConnectorFactory = () => Connector; diff --git a/packages/connectors/core/tsconfig.json b/packages/connectors/core/tsconfig.json new file mode 100644 index 0000000..7bfd764 --- /dev/null +++ b/packages/connectors/core/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@company-brain/config/tsconfig/node", + "include": ["src"] +} diff --git a/packages/connectors/google/eslint.config.mjs b/packages/connectors/google/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/packages/connectors/google/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/packages/connectors/google/package.json b/packages/connectors/google/package.json new file mode 100644 index 0000000..cacf2fc --- /dev/null +++ b/packages/connectors/google/package.json @@ -0,0 +1,27 @@ +{ + "name": "@company-brain/connector-google", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@company-brain/connector-core": "workspace:*" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/node": "^22.14.1", + "eslint": "^9.24.0", + "typescript": "^5.8.3", + "vitest": "^3.1.1" + } +} diff --git a/packages/connectors/google/src/config.ts b/packages/connectors/google/src/config.ts new file mode 100644 index 0000000..1b633b6 --- /dev/null +++ b/packages/connectors/google/src/config.ts @@ -0,0 +1,55 @@ +/** Google Workspace OAuth + service configuration. Least privilege only. */ + +export const GOOGLE_PROVIDER = 'google-workspace'; + +export const GOOGLE_OAUTH_ENDPOINTS = { + authorizationUrl: 'https://accounts.google.com/o/oauth2/v2/auth', + tokenUrl: 'https://oauth2.googleapis.com/token', + revocationUrl: 'https://oauth2.googleapis.com/revoke', +} as const; + +/** + * Read-only, least-privilege scopes. Additional scopes are requested + * incrementally (include_granted_scopes) only when a service needs them. + */ +export const GOOGLE_SCOPES = [ + 'openid', + 'email', + 'profile', + 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/drive.metadata.readonly', + 'https://www.googleapis.com/auth/documents.readonly', + 'https://www.googleapis.com/auth/spreadsheets.readonly', + 'https://www.googleapis.com/auth/presentations.readonly', + 'https://www.googleapis.com/auth/calendar.readonly', + 'https://www.googleapis.com/auth/gmail.readonly', +] as const; + +/** + * Offline access (refresh token) + forced consent so a refresh token is + * issued on every connect, + incremental authorization. + */ +export const GOOGLE_AUTH_PARAMS = { + access_type: 'offline', + prompt: 'consent', + include_granted_scopes: 'true', +} as const; + +export const GOOGLE_SERVICES = [ + 'drive', + 'docs', + 'sheets', + 'slides', + 'gmail', + 'calendar', + 'permissions', +] as const; +export type GoogleService = (typeof GOOGLE_SERVICES)[number]; + +export const GOOGLE_MIME = { + doc: 'application/vnd.google-apps.document', + sheet: 'application/vnd.google-apps.spreadsheet', + slides: 'application/vnd.google-apps.presentation', + folder: 'application/vnd.google-apps.folder', + pdf: 'application/pdf', +} as const; diff --git a/packages/connectors/google/src/connector.ts b/packages/connectors/google/src/connector.ts new file mode 100644 index 0000000..ec2d9ff --- /dev/null +++ b/packages/connectors/google/src/connector.ts @@ -0,0 +1,193 @@ +import { + BaseConnector, + type ConnectorContext, + type ConnectorDescriptor, + type DiscoveryResult, + type HealthResult, + type IncrementalSyncResult, + type SyncPage, +} from '@company-brain/connector-core'; +import { GOOGLE_MIME, GOOGLE_PROVIDER, GOOGLE_SCOPES, GOOGLE_SERVICES } from './config.js'; +import { googleGet } from './http.js'; +import { driveAbout, driveIncrementalSync, driveSyncPage } from './services/drive.js'; +import { gmailIncrementalSync, gmailSyncPage } from './services/gmail.js'; +import { calendarIncrementalSync, calendarSyncPage } from './services/calendar.js'; + +const SHEETS_API = 'https://sheets.googleapis.com/v4/spreadsheets'; + +interface WorksheetProps { + sheets?: Array<{ + properties?: { + sheetId?: number; + title?: string; + gridProperties?: { rowCount?: number; columnCount?: number }; + }; + }>; +} + +/** + * Google Workspace connector: Drive, Docs, Sheets, Slides, Gmail and + * Calendar metadata synchronization over least-privilege read-only scopes. + * + * Sync model: + * - drive → shared drives + all files (with permissions), Changes API cursor + * - docs/sheets/slides → mime-filtered Drive views (Sheets enriched with + * worksheet metadata); share the drive change feed + * - gmail → message metadata, History API cursor + * - calendar → calendars + events, per-calendar syncToken cursor + * - permissions → permission-bearing file pages from Drive + */ +export class GoogleWorkspaceConnector extends BaseConnector { + readonly descriptor: ConnectorDescriptor = { + provider: GOOGLE_PROVIDER, + displayName: 'Google Workspace', + authType: 'oauth2', + scopes: [...GOOGLE_SCOPES], + services: [...GOOGLE_SERVICES], + }; + + async validate(ctx: ConnectorContext): Promise { + try { + await googleGet(ctx, 'https://openidconnect.googleapis.com/v1/userinfo'); + return true; + } catch { + return false; + } + } + + async discover(ctx: ConnectorContext): Promise { + const identity = await driveAbout(ctx); + const services: DiscoveryResult['services'] = {}; + + const probes: Array<[string, () => Promise]> = [ + [ + 'drive', + () => googleGet(ctx, 'https://www.googleapis.com/drive/v3/about', { fields: 'user' }), + ], + ['gmail', () => googleGet(ctx, 'https://gmail.googleapis.com/gmail/v1/users/me/profile')], + [ + 'calendar', + () => + googleGet(ctx, 'https://www.googleapis.com/calendar/v3/users/me/calendarList', { + maxResults: 1, + }), + ], + ]; + for (const [service, probe] of probes) { + try { + await probe(); + services[service] = { available: true }; + } catch (error) { + services[service] = { available: false, detail: (error as Error).message }; + } + } + // Docs/Sheets/Slides/permissions ride on Drive access. + for (const derived of ['docs', 'sheets', 'slides', 'permissions']) { + services[derived] = services.drive ?? { available: false }; + } + + return { + workspace: { + externalId: identity.email, + domain: identity.domain, + name: identity.domain ? `Google Workspace (${identity.domain})` : 'Google Workspace', + adminEmail: identity.email, + metadata: { displayName: identity.name }, + }, + services, + }; + } + + override async health(ctx: ConnectorContext): Promise { + const discovery = await this.discover(ctx).catch(() => null); + if (!discovery) { + return { + healthy: false, + services: Object.fromEntries(this.descriptor.services.map((s) => [s, 'unauthorized'])), + checkedAt: new Date().toISOString(), + }; + } + const services = Object.fromEntries( + this.descriptor.services.map((s) => [ + s, + discovery.services[s]?.available ? ('up' as const) : ('down' as const), + ]), + ); + return { + healthy: Object.values(services).every((s) => s === 'up'), + services, + checkedAt: new Date().toISOString(), + }; + } + + async sync( + ctx: ConnectorContext, + service: string, + pageCursor?: string | null, + ): Promise { + switch (service) { + case 'drive': + return driveSyncPage(ctx, pageCursor); + case 'docs': + return driveSyncPage(ctx, pageCursor ?? 'files|', GOOGLE_MIME.doc); + case 'slides': + return driveSyncPage(ctx, pageCursor ?? 'files|', GOOGLE_MIME.slides); + case 'permissions': + // Permission data rides on the file pages themselves. + return driveSyncPage(ctx, pageCursor ?? 'files|'); + case 'sheets': { + const page = await driveSyncPage(ctx, pageCursor ?? 'files|', GOOGLE_MIME.sheet); + // Enrich with worksheet structure (metadata only). + for (const resource of page.resources) { + try { + const detail = await googleGet( + ctx, + `${SHEETS_API}/${resource.externalId}`, + { fields: 'sheets.properties(sheetId,title,gridProperties(rowCount,columnCount))' }, + ); + resource.metadata = { + ...resource.metadata, + worksheets: (detail.sheets ?? []).map((s) => ({ + id: s.properties?.sheetId, + title: s.properties?.title, + rows: s.properties?.gridProperties?.rowCount, + columns: s.properties?.gridProperties?.columnCount, + })), + }; + } catch { + // Worksheet enrichment is best-effort; Drive metadata stands. + } + } + return page; + } + case 'gmail': + return gmailSyncPage(ctx, pageCursor); + case 'calendar': + return calendarSyncPage(ctx, pageCursor); + default: + throw new Error(`Unknown Google service: ${service}`); + } + } + + async incrementalSync( + ctx: ConnectorContext, + service: string, + cursor: string, + ): Promise { + switch (service) { + case 'drive': + case 'docs': + case 'sheets': + case 'slides': + case 'permissions': + // One Drive change feed covers every drive-derived view. + return driveIncrementalSync(ctx, cursor); + case 'gmail': + return gmailIncrementalSync(ctx, cursor); + case 'calendar': + return calendarIncrementalSync(ctx, cursor); + default: + throw new Error(`Unknown Google service: ${service}`); + } + } +} diff --git a/packages/connectors/google/src/google.test.ts b/packages/connectors/google/src/google.test.ts new file mode 100644 index 0000000..a946bbb --- /dev/null +++ b/packages/connectors/google/src/google.test.ts @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + CursorExpiredError, + RateLimitError, + TokenExpiredError, + type ConnectorContext, +} from '@company-brain/connector-core'; +import { + mapCalendarEvent, + mapDriveFile, + mapGmailMessage, + mapPermission, + resourceTypeForMime, +} from './mappers.js'; +import { googleGet } from './http.js'; +import { GoogleWorkspaceConnector } from './connector.js'; +import { driveSyncPage } from './services/drive.js'; + +const ctx: ConnectorContext = { + connectorId: 'c1', + organizationId: 'o1', + getAccessToken: async () => 'test-access-token', +}; + +function mockFetchOnce(status: number, body: unknown, headers: Record = {}) { + return vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json', ...headers }, + }), + ); +} + +afterEach(() => vi.restoreAllMocks()); + +describe('resource type mapping', () => { + it('maps Google mime types to platform types', () => { + expect(resourceTypeForMime('application/vnd.google-apps.document')).toBe('GOOGLE_DOC'); + expect(resourceTypeForMime('application/vnd.google-apps.spreadsheet')).toBe('GOOGLE_SHEET'); + expect(resourceTypeForMime('application/vnd.google-apps.presentation')).toBe('GOOGLE_SLIDES'); + expect(resourceTypeForMime('application/vnd.google-apps.folder')).toBe('FOLDER'); + expect(resourceTypeForMime('application/pdf')).toBe('PDF'); + expect(resourceTypeForMime('image/png')).toBe('DRIVE_FILE'); + }); +}); + +describe('drive file mapper', () => { + it('maps a full drive file with permissions', () => { + const resource = mapDriveFile({ + id: 'f1', + name: 'Q3 Plan', + mimeType: 'application/vnd.google-apps.document', + webViewLink: 'https://docs.google.com/document/d/f1', + parents: ['folder9'], + version: '42', + headRevisionId: 'rev-9', + createdTime: '2026-01-01T00:00:00Z', + modifiedTime: '2026-06-01T00:00:00Z', + owners: [{ emailAddress: 'ada@acme.test' }], + permissions: [ + { id: 'p1', type: 'user', emailAddress: 'ada@acme.test', role: 'owner' }, + { id: 'p2', type: 'domain', domain: 'acme.test', role: 'reader' }, + { id: 'p3', type: 'user', emailAddress: 'bob@acme.test', role: 'writer' }, + ], + }); + expect(resource).toMatchObject({ + externalId: 'f1', + type: 'GOOGLE_DOC', + title: 'Q3 Plan', + ownerEmail: 'ada@acme.test', + parentExternalId: 'folder9', + version: 'rev-9', + }); + expect(resource.permissions).toHaveLength(3); + expect(resource.permissions![0]).toMatchObject({ role: 'OWNER', principalType: 'USER' }); + expect(resource.permissions![1]).toMatchObject({ role: 'VIEWER', principalType: 'DOMAIN' }); + expect(resource.permissions![2]).toMatchObject({ role: 'EDITOR' }); + }); + + it('drops unknown permission roles', () => { + expect(mapPermission({ role: 'published_reader' })).toBeNull(); + }); +}); + +describe('gmail mapper', () => { + it('extracts headers, labels and attachment metadata', () => { + const resource = mapGmailMessage({ + id: 'm1', + threadId: 't1', + labelIds: ['INBOX', 'IMPORTANT'], + internalDate: '1770000000000', + sizeEstimate: 2048, + historyId: 'h77', + payload: { + headers: [ + { name: 'Subject', value: 'Quarterly numbers' }, + { name: 'From', value: 'cfo@acme.test' }, + { name: 'To', value: 'team@acme.test' }, + ], + parts: [{ filename: 'report.pdf', mimeType: 'application/pdf', body: { size: 9000 } }], + }, + }); + expect(resource).toMatchObject({ + externalId: 'm1', + type: 'EMAIL', + title: 'Quarterly numbers', + ownerEmail: 'cfo@acme.test', + parentExternalId: 't1', + version: 'h77', + }); + const meta = resource.metadata as { labels: string[]; attachments: unknown[] }; + expect(meta.labels).toContain('INBOX'); + expect(meta.attachments).toHaveLength(1); + }); +}); + +describe('calendar event mapper', () => { + it('maps attendees, meeting link and cancellation', () => { + const resource = mapCalendarEvent( + { + id: 'e1', + status: 'cancelled', + summary: 'Weekly sync', + hangoutLink: 'https://meet.google.com/xyz', + organizer: { email: 'pm@acme.test' }, + attendees: [{ email: 'dev@acme.test', responseStatus: 'accepted' }], + start: { dateTime: '2026-07-15T10:00:00Z' }, + end: { dateTime: '2026-07-15T10:30:00Z' }, + recurringEventId: 'recur-1', + }, + 'primary', + ); + expect(resource).toMatchObject({ + externalId: 'primary:e1', + type: 'CALENDAR_EVENT', + parentExternalId: 'primary', + trashed: true, // cancelled + ownerEmail: 'pm@acme.test', + }); + const meta = resource.metadata as { meetingLink: string; attendees: unknown[] }; + expect(meta.meetingLink).toBe('https://meet.google.com/xyz'); + expect(meta.attendees).toHaveLength(1); + }); +}); + +describe('google HTTP error mapping', () => { + it('maps 401 to TokenExpiredError', async () => { + mockFetchOnce(401, { error: { message: 'Invalid Credentials' } }); + await expect(googleGet(ctx, 'https://example.googleapis.com/x')).rejects.toBeInstanceOf( + TokenExpiredError, + ); + }); + + it('maps 429 to RateLimitError with retry-after', async () => { + mockFetchOnce(429, { error: { message: 'Rate limit' } }, { 'retry-after': '7' }); + const error = await googleGet(ctx, 'https://example.googleapis.com/x').catch((e) => e); + expect(error).toBeInstanceOf(RateLimitError); + expect((error as RateLimitError).retryAfterMs).toBe(7000); + }); + + it('maps 410 to CursorExpiredError (expired sync tokens)', async () => { + mockFetchOnce(410, { error: { message: 'Sync token is no longer valid' } }); + await expect(googleGet(ctx, 'https://example.googleapis.com/x')).rejects.toBeInstanceOf( + CursorExpiredError, + ); + }); + + it('sends the bearer token and query params', async () => { + const spy = mockFetchOnce(200, { ok: true }); + await googleGet(ctx, 'https://example.googleapis.com/x', { pageSize: 5 }); + const [url, init] = spy.mock.calls[0]!; + expect(String(url)).toContain('pageSize=5'); + expect((init!.headers as Record).authorization).toBe( + 'Bearer test-access-token', + ); + }); +}); + +describe('drive full sync paging (mocked API)', () => { + it('walks shared drives, then files, then returns the change cursor', async () => { + // Page 1: shared drives. + mockFetchOnce(200, { drives: [{ id: 'sd1', name: 'Team Drive' }] }); + const page1 = await driveSyncPage(ctx, null); + expect(page1.resources[0]).toMatchObject({ externalId: 'sd1', type: 'SHARED_DRIVE' }); + expect(page1.nextPageCursor).toBe('files|'); + + // Page 2: files with a next page token. + mockFetchOnce(200, { + files: [{ id: 'f1', name: 'Doc', mimeType: 'application/vnd.google-apps.document' }], + nextPageToken: 'tok2', + }); + const page2 = await driveSyncPage(ctx, page1.nextPageCursor); + expect(page2.resources[0]!.externalId).toBe('f1'); + expect(page2.nextPageCursor).toBe('files|tok2'); + + // Page 3: last file page → startPageToken fetched for incremental sync. + const spy = vi.spyOn(globalThis, 'fetch'); + spy.mockResolvedValueOnce(new Response(JSON.stringify({ files: [] }), { status: 200 })); + spy.mockResolvedValueOnce( + new Response(JSON.stringify({ startPageToken: 'start-42' }), { status: 200 }), + ); + const page3 = await driveSyncPage(ctx, page2.nextPageCursor); + expect(page3.nextPageCursor).toBeNull(); + expect(page3.incrementalCursor).toBe('start-42'); + }); +}); + +describe('connector descriptor', () => { + it('declares least-privilege scopes and all services', () => { + const connector = new GoogleWorkspaceConnector(); + expect(connector.descriptor.provider).toBe('google-workspace'); + expect(connector.descriptor.scopes).toContain('https://www.googleapis.com/auth/drive.readonly'); + expect(connector.descriptor.scopes.every((s) => !s.includes('write'))).toBe(true); + expect(connector.descriptor.services).toEqual( + expect.arrayContaining(['drive', 'docs', 'sheets', 'slides', 'gmail', 'calendar']), + ); + }); +}); diff --git a/packages/connectors/google/src/http.ts b/packages/connectors/google/src/http.ts new file mode 100644 index 0000000..d64e050 --- /dev/null +++ b/packages/connectors/google/src/http.ts @@ -0,0 +1,57 @@ +import { + CursorExpiredError, + PermissionDeniedError, + ProviderApiError, + QuotaExceededError, + RateLimitError, + TokenExpiredError, + type ConnectorContext, +} from '@company-brain/connector-core'; + +interface GoogleErrorBody { + error?: { + code?: number; + message?: string; + errors?: Array<{ reason?: string }>; + status?: string; + }; +} + +/** + * Authenticated GET against a Google API, mapping Google's error taxonomy + * onto the SDK's typed errors so Temporal retry policies behave correctly. + */ +export async function googleGet( + ctx: ConnectorContext, + url: string, + params: Record = {}, +): Promise { + const target = new URL(url); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) target.searchParams.set(key, String(value)); + } + const token = await ctx.getAccessToken(); + const response = await fetch(target, { headers: { authorization: `Bearer ${token}` } }); + + if (response.ok) return (await response.json()) as T; + + const body = (await response.json().catch(() => ({}))) as GoogleErrorBody; + const reason = body.error?.errors?.[0]?.reason ?? body.error?.status ?? ''; + const message = body.error?.message ?? `Google API ${response.status} at ${target.pathname}`; + + if (response.status === 401) throw new TokenExpiredError(message); + if ( + response.status === 429 || + reason === 'rateLimitExceeded' || + reason === 'userRateLimitExceeded' + ) { + const retryAfter = Number(response.headers.get('retry-after') ?? 30) * 1000; + throw new RateLimitError(message, retryAfter); + } + if (reason === 'quotaExceeded' || reason === 'dailyLimitExceeded') { + throw new QuotaExceededError(message); + } + if (response.status === 403) throw new PermissionDeniedError(message); + if (response.status === 410) throw new CursorExpiredError(message); // expired sync tokens + throw new ProviderApiError(message, response.status); +} diff --git a/packages/connectors/google/src/index.ts b/packages/connectors/google/src/index.ts new file mode 100644 index 0000000..bd81427 --- /dev/null +++ b/packages/connectors/google/src/index.ts @@ -0,0 +1,28 @@ +export { GoogleWorkspaceConnector } from './connector.js'; +export { + GOOGLE_AUTH_PARAMS, + GOOGLE_MIME, + GOOGLE_OAUTH_ENDPOINTS, + GOOGLE_PROVIDER, + GOOGLE_SCOPES, + GOOGLE_SERVICES, + type GoogleService, +} from './config.js'; +export { googleGet } from './http.js'; +export { + mapDriveFile, + mapGmailMessage, + mapCalendar, + mapCalendarEvent, + mapPermission, + mapSharedDrive, + resourceTypeForMime, +} from './mappers.js'; +export type { + GoogleDriveFile, + GoogleDrivePermission, + GmailMessageMeta, + GoogleCalendarEntry, + GoogleCalendarEvent, + GoogleSharedDrive, +} from './mappers.js'; diff --git a/packages/connectors/google/src/mappers.ts b/packages/connectors/google/src/mappers.ts new file mode 100644 index 0000000..9bf0c42 --- /dev/null +++ b/packages/connectors/google/src/mappers.ts @@ -0,0 +1,243 @@ +import type { + PermissionRole, + PrincipalType, + SyncedPermission, + SyncedResource, +} from '@company-brain/connector-core'; +import { GOOGLE_MIME } from './config.js'; + +// ── Raw Google payload shapes (only the fields we request) ────── + +export interface GoogleDriveFile { + id: string; + name?: string; + mimeType?: string; + webViewLink?: string; + parents?: string[]; + driveId?: string; + size?: string; + md5Checksum?: string; + version?: string; + createdTime?: string; + modifiedTime?: string; + trashed?: boolean; + owners?: Array<{ emailAddress?: string }>; + lastModifyingUser?: { emailAddress?: string }; + permissions?: GoogleDrivePermission[]; + headRevisionId?: string; +} + +export interface GoogleDrivePermission { + id?: string; + type?: string; // user | group | domain | anyone + emailAddress?: string; + domain?: string; + role?: string; // owner | organizer | fileOrganizer | writer | commenter | reader +} + +export interface GoogleSharedDrive { + id: string; + name?: string; + createdTime?: string; +} + +export interface GmailMessageMeta { + id: string; + threadId?: string; + labelIds?: string[]; + internalDate?: string; + sizeEstimate?: number; + historyId?: string; + payload?: { + headers?: Array<{ name?: string; value?: string }>; + parts?: Array<{ filename?: string; mimeType?: string; body?: { size?: number } }>; + }; +} + +export interface GoogleCalendarEntry { + id: string; + summary?: string; + description?: string; + timeZone?: string; + primary?: boolean; + accessRole?: string; +} + +export interface GoogleCalendarEvent { + id: string; + status?: string; // confirmed | tentative | cancelled + summary?: string; + htmlLink?: string; + location?: string; + hangoutLink?: string; + created?: string; + updated?: string; + recurringEventId?: string; + recurrence?: string[]; + organizer?: { email?: string }; + attendees?: Array<{ email?: string; responseStatus?: string; organizer?: boolean }>; + start?: { date?: string; dateTime?: string }; + end?: { date?: string; dateTime?: string }; + conferenceData?: { entryPoints?: Array<{ uri?: string; entryPointType?: string }> }; +} + +// ── Mapping to platform resources ─────────────────────────────── + +export function resourceTypeForMime(mimeType: string | undefined): string { + switch (mimeType) { + case GOOGLE_MIME.doc: + return 'GOOGLE_DOC'; + case GOOGLE_MIME.sheet: + return 'GOOGLE_SHEET'; + case GOOGLE_MIME.slides: + return 'GOOGLE_SLIDES'; + case GOOGLE_MIME.folder: + return 'FOLDER'; + case GOOGLE_MIME.pdf: + return 'PDF'; + default: + return 'DRIVE_FILE'; + } +} + +const ROLE_MAP: Record = { + owner: 'OWNER', + organizer: 'OWNER', + fileOrganizer: 'EDITOR', + writer: 'EDITOR', + commenter: 'COMMENTER', + reader: 'VIEWER', +}; + +export function mapPermission(p: GoogleDrivePermission): SyncedPermission | null { + const role = ROLE_MAP[p.role ?? '']; + if (!role) return null; + const type = (p.type ?? 'user').toUpperCase(); + const principalType: PrincipalType = + type === 'GROUP' || type === 'DOMAIN' || type === 'ANYONE' ? (type as PrincipalType) : 'USER'; + return { + externalPermissionId: p.id, + principalType, + principalEmail: p.emailAddress ?? null, + domain: p.domain ?? null, + role, + }; +} + +export function mapDriveFile(file: GoogleDriveFile): SyncedResource { + return { + externalId: file.id, + type: resourceTypeForMime(file.mimeType), + title: file.name ?? null, + mimeType: file.mimeType ?? null, + url: file.webViewLink ?? null, + ownerEmail: file.owners?.[0]?.emailAddress ?? null, + parentExternalId: file.parents?.[0] ?? null, + driveId: file.driveId ?? null, + sizeBytes: file.size ? Number(file.size) : null, + checksum: file.md5Checksum ?? null, + version: file.headRevisionId ?? file.version ?? null, + externalCreatedAt: file.createdTime ?? null, + externalUpdatedAt: file.modifiedTime ?? null, + trashed: file.trashed ?? false, + permissions: (file.permissions ?? []) + .map(mapPermission) + .filter((p): p is SyncedPermission => p !== null), + metadata: { + lastModifiedBy: file.lastModifyingUser?.emailAddress, + }, + }; +} + +export function mapSharedDrive(drive: GoogleSharedDrive): SyncedResource { + return { + externalId: drive.id, + type: 'SHARED_DRIVE', + title: drive.name ?? null, + url: `https://drive.google.com/drive/folders/${drive.id}`, + externalCreatedAt: drive.createdTime ?? null, + metadata: {}, + }; +} + +function header(message: GmailMessageMeta, name: string): string | undefined { + return message.payload?.headers?.find((h) => h.name?.toLowerCase() === name.toLowerCase())?.value; +} + +export function mapGmailMessage(message: GmailMessageMeta): SyncedResource { + const attachments = (message.payload?.parts ?? []) + .filter((part) => part.filename) + .map((part) => ({ + filename: part.filename, + mimeType: part.mimeType, + sizeBytes: part.body?.size, + })); + const internal = message.internalDate ? new Date(Number(message.internalDate)) : null; + return { + externalId: message.id, + type: 'EMAIL', + title: header(message, 'Subject') ?? '(no subject)', + url: `https://mail.google.com/mail/u/0/#all/${message.id}`, + ownerEmail: header(message, 'From') ?? null, + parentExternalId: message.threadId ?? null, + sizeBytes: message.sizeEstimate ?? null, + version: message.historyId ?? null, + externalCreatedAt: internal?.toISOString() ?? null, + externalUpdatedAt: internal?.toISOString() ?? null, + metadata: { + threadId: message.threadId, + labels: message.labelIds ?? [], + from: header(message, 'From'), + to: header(message, 'To'), + cc: header(message, 'Cc'), + date: header(message, 'Date'), + messageIdHeader: header(message, 'Message-ID'), + attachments, + }, + }; +} + +export function mapCalendar(calendar: GoogleCalendarEntry): SyncedResource { + return { + externalId: calendar.id, + type: 'CALENDAR', + title: calendar.summary ?? calendar.id, + metadata: { + description: calendar.description, + timeZone: calendar.timeZone, + primary: calendar.primary ?? false, + accessRole: calendar.accessRole, + }, + }; +} + +export function mapCalendarEvent(event: GoogleCalendarEvent, calendarId: string): SyncedResource { + const meetingLink = + event.hangoutLink ?? + event.conferenceData?.entryPoints?.find((e) => e.entryPointType === 'video')?.uri; + return { + externalId: `${calendarId}:${event.id}`, + type: 'CALENDAR_EVENT', + title: event.summary ?? '(no title)', + url: event.htmlLink ?? null, + ownerEmail: event.organizer?.email ?? null, + parentExternalId: calendarId, + externalCreatedAt: event.created ?? null, + externalUpdatedAt: event.updated ?? null, + trashed: event.status === 'cancelled', + metadata: { + status: event.status, + location: event.location, + meetingLink, + start: event.start?.dateTime ?? event.start?.date, + end: event.end?.dateTime ?? event.end?.date, + recurringEventId: event.recurringEventId, + recurrence: event.recurrence, + attendees: (event.attendees ?? []).map((a) => ({ + email: a.email, + responseStatus: a.responseStatus, + organizer: a.organizer ?? false, + })), + }, + }; +} diff --git a/packages/connectors/google/src/services/calendar.ts b/packages/connectors/google/src/services/calendar.ts new file mode 100644 index 0000000..e498f16 --- /dev/null +++ b/packages/connectors/google/src/services/calendar.ts @@ -0,0 +1,146 @@ +import type { + ConnectorContext, + IncrementalSyncResult, + ResourceChange, + SyncPage, + SyncedResource, +} from '@company-brain/connector-core'; +import { googleGet } from '../http.js'; +import { + mapCalendar, + mapCalendarEvent, + type GoogleCalendarEntry, + type GoogleCalendarEvent, +} from '../mappers.js'; + +const CAL = 'https://www.googleapis.com/calendar/v3'; + +interface CalendarListResponse { + items?: GoogleCalendarEntry[]; + nextPageToken?: string; +} +interface EventListResponse { + items?: GoogleCalendarEvent[]; + nextPageToken?: string; + nextSyncToken?: string; +} + +/** + * Full-sync cursor walks calendars first, then each calendar's events: + * JSON `{ phase, calendarIds, index, pageToken, syncTokens }`. On the last + * page the collected per-calendar syncTokens become the incremental cursor. + */ +interface CalendarCursor { + phase: 'calendars' | 'events'; + calendarIds: string[]; + index: number; + pageToken?: string; + syncTokens: Record; +} + +export async function calendarSyncPage( + ctx: ConnectorContext, + pageCursor?: string | null, +): Promise { + const cursor: CalendarCursor = pageCursor + ? (JSON.parse(pageCursor) as CalendarCursor) + : { phase: 'calendars', calendarIds: [], index: 0, syncTokens: {} }; + + if (cursor.phase === 'calendars') { + const response = await googleGet(ctx, `${CAL}/users/me/calendarList`, { + maxResults: 100, + pageToken: cursor.pageToken, + }); + const calendars = response.items ?? []; + const resources = calendars.map(mapCalendar); + const next: CalendarCursor = response.nextPageToken + ? { + ...cursor, + pageToken: response.nextPageToken, + calendarIds: [...cursor.calendarIds, ...calendars.map((c) => c.id)], + } + : { + phase: 'events', + calendarIds: [...cursor.calendarIds, ...calendars.map((c) => c.id)], + index: 0, + syncTokens: cursor.syncTokens, + }; + return { resources, nextPageCursor: JSON.stringify(next) }; + } + + // Events phase — one page of one calendar per call. + const calendarId = cursor.calendarIds[cursor.index]; + if (!calendarId) { + return { + resources: [], + nextPageCursor: null, + incrementalCursor: JSON.stringify(cursor.syncTokens), + }; + } + const response = await googleGet( + ctx, + `${CAL}/calendars/${encodeURIComponent(calendarId)}/events`, + { + maxResults: 100, + pageToken: cursor.pageToken, + singleEvents: false, + showDeleted: true, + }, + ); + const resources: SyncedResource[] = (response.items ?? []).map((e) => + mapCalendarEvent(e, calendarId), + ); + + let next: CalendarCursor; + if (response.nextPageToken) { + next = { ...cursor, pageToken: response.nextPageToken }; + } else { + const syncTokens = { ...cursor.syncTokens }; + if (response.nextSyncToken) syncTokens[calendarId] = response.nextSyncToken; + next = { ...cursor, index: cursor.index + 1, pageToken: undefined, syncTokens }; + if (next.index >= next.calendarIds.length) { + return { resources, nextPageCursor: null, incrementalCursor: JSON.stringify(syncTokens) }; + } + } + return { resources, nextPageCursor: JSON.stringify(next) }; +} + +/** + * Incremental sync: cursor is a JSON map calendarId → syncToken. Each + * calendar is queried with its token; expired tokens (410) surface as + * CursorExpiredError from the HTTP layer and trigger a full resync. + */ +export async function calendarIncrementalSync( + ctx: ConnectorContext, + cursor: string, +): Promise { + const syncTokens = JSON.parse(cursor) as Record; + const changes: ResourceChange[] = []; + const nextTokens: Record = { ...syncTokens }; + + for (const [calendarId, token] of Object.entries(syncTokens)) { + let pageToken: string | undefined; + for (let page = 0; page < 10; page += 1) { + const response = await googleGet( + ctx, + `${CAL}/calendars/${encodeURIComponent(calendarId)}/events`, + pageToken ? { pageToken, maxResults: 100 } : { syncToken: token, maxResults: 100 }, + ); + for (const event of response.items ?? []) { + const resource = mapCalendarEvent(event, calendarId); + changes.push({ + externalId: resource.externalId, + service: 'calendar', + changeType: event.status === 'cancelled' ? 'deleted' : 'updated', + resource: event.status === 'cancelled' ? undefined : resource, + occurredAt: event.updated, + }); + } + if (response.nextSyncToken) nextTokens[calendarId] = response.nextSyncToken; + if (!response.nextPageToken) break; + pageToken = response.nextPageToken; + } + } + + return { changes, nextCursor: JSON.stringify(nextTokens) }; +} diff --git a/packages/connectors/google/src/services/drive.ts b/packages/connectors/google/src/services/drive.ts new file mode 100644 index 0000000..6764897 --- /dev/null +++ b/packages/connectors/google/src/services/drive.ts @@ -0,0 +1,146 @@ +import type { + ConnectorContext, + IncrementalSyncResult, + ResourceChange, + SyncPage, +} from '@company-brain/connector-core'; +import { googleGet } from '../http.js'; +import { + mapDriveFile, + mapSharedDrive, + type GoogleDriveFile, + type GoogleSharedDrive, +} from '../mappers.js'; + +const DRIVE = 'https://www.googleapis.com/drive/v3'; +const FILE_FIELDS = + 'id,name,mimeType,webViewLink,parents,driveId,size,md5Checksum,version,headRevisionId,createdTime,modifiedTime,trashed,owners(emailAddress),lastModifyingUser(emailAddress),permissions(id,type,emailAddress,domain,role)'; + +interface FileListResponse { + files?: GoogleDriveFile[]; + nextPageToken?: string; +} +interface DriveListResponse { + drives?: GoogleSharedDrive[]; + nextPageToken?: string; +} +interface StartPageTokenResponse { + startPageToken: string; +} +interface ChangesResponse { + changes?: Array<{ + fileId?: string; + removed?: boolean; + time?: string; + file?: GoogleDriveFile; + }>; + nextPageToken?: string; + newStartPageToken?: string; +} + +/** + * Full-sync page cursor: `drives` page(s) first, then `files` pages. + * Encoded as `:` so a single opaque string round-trips + * through the SyncCursor table. + */ +export async function driveSyncPage( + ctx: ConnectorContext, + pageCursor?: string | null, + mimeFilter?: string, +): Promise { + const [phase, token] = pageCursor ? pageCursor.split('|', 2) : ['drives', '']; + + if (phase === 'drives' && !mimeFilter) { + const response = await googleGet(ctx, `${DRIVE}/drives`, { + pageSize: 100, + pageToken: token || undefined, + }); + const resources = (response.drives ?? []).map(mapSharedDrive); + return { + resources, + nextPageCursor: response.nextPageToken ? `drives|${response.nextPageToken}` : 'files|', + }; + } + + const response = await googleGet(ctx, `${DRIVE}/files`, { + pageSize: 100, + pageToken: token || undefined, + fields: `nextPageToken, files(${FILE_FIELDS})`, + includeItemsFromAllDrives: true, + supportsAllDrives: true, + q: mimeFilter ? `mimeType='${mimeFilter}' and trashed=false` : 'trashed=false', + }); + const resources = (response.files ?? []).map(mapDriveFile); + + if (response.nextPageToken) { + return { resources, nextPageCursor: `files|${response.nextPageToken}` }; + } + // Last page: hand back the change-feed anchor for incremental sync. + const start = await googleGet(ctx, `${DRIVE}/changes/startPageToken`, { + supportsAllDrives: true, + }); + return { resources, nextPageCursor: null, incrementalCursor: start.startPageToken }; +} + +/** Incremental sync via the Drive Changes API. */ +export async function driveIncrementalSync( + ctx: ConnectorContext, + cursor: string, +): Promise { + const changes: ResourceChange[] = []; + let pageToken = cursor; + let newStartPageToken = cursor; + + for (let page = 0; page < 20; page += 1) { + const response = await googleGet(ctx, `${DRIVE}/changes`, { + pageToken, + pageSize: 100, + fields: `nextPageToken,newStartPageToken,changes(fileId,removed,time,file(${FILE_FIELDS}))`, + includeItemsFromAllDrives: true, + supportsAllDrives: true, + }); + for (const change of response.changes ?? []) { + if (!change.fileId) continue; + if (change.removed || change.file?.trashed) { + changes.push({ + externalId: change.fileId, + service: 'drive', + changeType: 'deleted', + occurredAt: change.time, + }); + } else if (change.file) { + changes.push({ + externalId: change.fileId, + service: 'drive', + changeType: 'updated', + resource: mapDriveFile(change.file), + occurredAt: change.time, + }); + } + } + if (response.newStartPageToken) newStartPageToken = response.newStartPageToken; + if (!response.nextPageToken) break; + pageToken = response.nextPageToken; + } + + return { changes, nextCursor: newStartPageToken }; +} + +/** Drive "about" — identifies the account/workspace behind the grant. */ +export async function driveAbout(ctx: ConnectorContext): Promise<{ + email: string | null; + domain: string | null; + name: string | null; +}> { + const about = await googleGet<{ user?: { emailAddress?: string; displayName?: string } }>( + ctx, + `${DRIVE}/about`, + { fields: 'user(emailAddress,displayName)' }, + ); + const email = about.user?.emailAddress ?? null; + return { + email, + domain: email?.includes('@') ? email.split('@')[1]! : null, + name: about.user?.displayName ?? null, + }; +} diff --git a/packages/connectors/google/src/services/gmail.ts b/packages/connectors/google/src/services/gmail.ts new file mode 100644 index 0000000..ad1d65d --- /dev/null +++ b/packages/connectors/google/src/services/gmail.ts @@ -0,0 +1,118 @@ +import type { + ConnectorContext, + IncrementalSyncResult, + ResourceChange, + SyncPage, +} from '@company-brain/connector-core'; +import { googleGet } from '../http.js'; +import { mapGmailMessage, type GmailMessageMeta } from '../mappers.js'; + +const GMAIL = 'https://gmail.googleapis.com/gmail/v1/users/me'; +const METADATA_HEADERS = ['From', 'To', 'Cc', 'Subject', 'Date', 'Message-ID']; + +interface MessageListResponse { + messages?: Array<{ id: string; threadId?: string }>; + nextPageToken?: string; +} +interface HistoryResponse { + history?: Array<{ + messagesAdded?: Array<{ message?: { id?: string } }>; + messagesDeleted?: Array<{ message?: { id?: string } }>; + labelsAdded?: Array<{ message?: { id?: string } }>; + labelsRemoved?: Array<{ message?: { id?: string } }>; + }>; + nextPageToken?: string; + historyId?: string; +} + +async function getMessageMeta(ctx: ConnectorContext, id: string): Promise { + const params = new URLSearchParams({ format: 'metadata' }); + for (const h of METADATA_HEADERS) params.append('metadataHeaders', h); + return googleGet(ctx, `${GMAIL}/messages/${id}?${params.toString()}`); +} + +/** Full sync: pages of message metadata (never bodies). */ +export async function gmailSyncPage( + ctx: ConnectorContext, + pageCursor?: string | null, +): Promise { + const list = await googleGet(ctx, `${GMAIL}/messages`, { + maxResults: 50, + pageToken: pageCursor || undefined, + }); + + const resources = []; + for (const stub of list.messages ?? []) { + resources.push(mapGmailMessage(await getMessageMeta(ctx, stub.id))); + } + + if (list.nextPageToken) { + return { resources, nextPageCursor: list.nextPageToken }; + } + // Anchor incremental sync at the mailbox's current historyId. + const profile = await googleGet<{ historyId?: string }>(ctx, `${GMAIL}/profile`); + return { resources, nextPageCursor: null, incrementalCursor: profile.historyId ?? null }; +} + +/** Incremental sync via the Gmail History API (historyId cursor). */ +export async function gmailIncrementalSync( + ctx: ConnectorContext, + cursor: string, +): Promise { + const changes: ResourceChange[] = []; + const seen = new Set(); + let pageToken: string | undefined; + let latestHistoryId = cursor; + + for (let page = 0; page < 10; page += 1) { + const response = await googleGet(ctx, `${GMAIL}/history`, { + startHistoryId: cursor, + pageToken, + maxResults: 100, + }); + if (response.historyId) latestHistoryId = response.historyId; + + for (const entry of response.history ?? []) { + for (const added of entry.messagesAdded ?? []) { + const id = added.message?.id; + if (!id || seen.has(id)) continue; + seen.add(id); + try { + changes.push({ + externalId: id, + service: 'gmail', + changeType: 'created', + resource: mapGmailMessage(await getMessageMeta(ctx, id)), + }); + } catch { + // Message may already be gone (spam purge) — skip. + } + } + for (const deleted of entry.messagesDeleted ?? []) { + const id = deleted.message?.id; + if (!id || seen.has(`del:${id}`)) continue; + seen.add(`del:${id}`); + changes.push({ externalId: id, service: 'gmail', changeType: 'deleted' }); + } + for (const relabeled of [...(entry.labelsAdded ?? []), ...(entry.labelsRemoved ?? [])]) { + const id = relabeled.message?.id; + if (!id || seen.has(id)) continue; + seen.add(id); + try { + changes.push({ + externalId: id, + service: 'gmail', + changeType: 'updated', + resource: mapGmailMessage(await getMessageMeta(ctx, id)), + }); + } catch { + // ignore vanished messages + } + } + } + if (!response.nextPageToken) break; + pageToken = response.nextPageToken; + } + + return { changes, nextCursor: latestHistoryId }; +} diff --git a/packages/connectors/google/tsconfig.json b/packages/connectors/google/tsconfig.json new file mode 100644 index 0000000..7bfd764 --- /dev/null +++ b/packages/connectors/google/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@company-brain/config/tsconfig/node", + "include": ["src"] +} diff --git a/packages/events/eslint.config.mjs b/packages/events/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/packages/events/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/packages/events/package.json b/packages/events/package.json new file mode 100644 index 0000000..542084d --- /dev/null +++ b/packages/events/package.json @@ -0,0 +1,27 @@ +{ + "name": "@company-brain/events", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "ioredis": "^5.6.1" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/node": "^22.14.1", + "eslint": "^9.24.0", + "typescript": "^5.8.3", + "vitest": "^3.1.1" + } +} diff --git a/packages/events/src/bus.ts b/packages/events/src/bus.ts new file mode 100644 index 0000000..f89bfd2 --- /dev/null +++ b/packages/events/src/bus.ts @@ -0,0 +1,67 @@ +import { randomUUID } from 'node:crypto'; +import type { Redis } from 'ioredis'; +import type { EventType, PlatformEvent, ResourceRef } from './types.js'; + +export const EVENT_CHANNEL = 'brain:events'; +export const EVENT_STREAM = 'brain:events:stream'; +/** Keep roughly this many recent events in the durable stream. */ +const STREAM_MAX_LENGTH = 100_000; + +export interface EmitInput { + type: EventType; + organizationId: string; + connectorId: string; + provider: string; + resource?: ResourceRef; + payload?: Record; +} + +/** + * Redis-backed event bus. Every event is: + * 1. XADDed to a capped stream (durable — consumers can replay), and + * 2. PUBLISHed to a channel (realtime fan-out). + * Future phases consume with XREADGROUP without any changes here. + */ +export class EventBus { + constructor(private readonly redis: Redis) {} + + buildEvent(input: EmitInput): PlatformEvent { + return { + id: randomUUID(), + type: input.type, + occurredAt: new Date().toISOString(), + organizationId: input.organizationId, + connectorId: input.connectorId, + provider: input.provider, + resource: input.resource, + payload: input.payload, + }; + } + + async publish(input: EmitInput): Promise { + const event = this.buildEvent(input); + const serialized = JSON.stringify(event); + await this.redis + .multi() + .xadd(EVENT_STREAM, 'MAXLEN', '~', STREAM_MAX_LENGTH, '*', 'event', serialized) + .publish(EVENT_CHANNEL, serialized) + .exec(); + return event; + } + + /** Realtime subscription (dedicated connection required by Redis). */ + async subscribe( + subscriber: Redis, + handler: (event: PlatformEvent) => void | Promise, + ): Promise { + await subscriber.subscribe(EVENT_CHANNEL); + subscriber.on('message', (channel, message) => { + if (channel !== EVENT_CHANNEL) return; + try { + void handler(JSON.parse(message) as PlatformEvent); + } catch { + // Malformed event — ignore; durable copy remains in the stream. + } + }); + } +} diff --git a/packages/events/src/events.test.ts b/packages/events/src/events.test.ts new file mode 100644 index 0000000..af1d3d8 --- /dev/null +++ b/packages/events/src/events.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { EventBus } from './bus.js'; +import { EVENT_TYPES } from './types.js'; + +describe('event bus', () => { + it('builds well-formed platform events', () => { + const bus = new EventBus(null as never); + const event = bus.buildEvent({ + type: 'resource.document.updated', + organizationId: 'org-1', + connectorId: 'conn-1', + provider: 'google-workspace', + resource: { externalId: 'f1', type: 'GOOGLE_DOC', title: 'Plan' }, + }); + expect(event.id).toMatch(/[0-9a-f-]{36}/); + expect(new Date(event.occurredAt).getTime()).toBeGreaterThan(0); + expect(event).toMatchObject({ + type: 'resource.document.updated', + organizationId: 'org-1', + provider: 'google-workspace', + }); + }); + + it('declares the full connector event vocabulary', () => { + for (const required of [ + 'connector.connected', + 'connector.disconnected', + 'resource.document.created', + 'resource.document.updated', + 'resource.document.deleted', + 'resource.sheet.updated', + 'resource.email.received', + 'resource.calendar.updated', + 'resource.permission.changed', + 'sync.started', + 'sync.completed', + 'sync.failed', + ]) { + expect(EVENT_TYPES).toContain(required); + } + }); +}); diff --git a/packages/events/src/index.ts b/packages/events/src/index.ts new file mode 100644 index 0000000..efca7c5 --- /dev/null +++ b/packages/events/src/index.ts @@ -0,0 +1,4 @@ +export { EventBus, EVENT_CHANNEL, EVENT_STREAM } from './bus.js'; +export type { EmitInput } from './bus.js'; +export { EVENT_TYPES } from './types.js'; +export type { EventType, PlatformEvent, ResourceRef } from './types.js'; diff --git a/packages/events/src/types.ts b/packages/events/src/types.ts new file mode 100644 index 0000000..19510d5 --- /dev/null +++ b/packages/events/src/types.ts @@ -0,0 +1,49 @@ +/** + * Internal platform events. Connectors emit these as they synchronize; + * future phases (ingestion, memory, meeting intelligence) subscribe to + * them instead of polling the database. + */ + +export const EVENT_TYPES = [ + // Connector lifecycle + 'connector.connected', + 'connector.disconnected', + 'connector.error', + // Sync lifecycle + 'sync.started', + 'sync.completed', + 'sync.failed', + // Resource changes + 'resource.document.created', + 'resource.document.updated', + 'resource.document.deleted', + 'resource.sheet.updated', + 'resource.slides.updated', + 'resource.file.created', + 'resource.file.updated', + 'resource.file.deleted', + 'resource.email.received', + 'resource.calendar.updated', + 'resource.permission.changed', +] as const; + +export type EventType = (typeof EVENT_TYPES)[number]; + +export interface ResourceRef { + externalId: string; + type: string; + title?: string | null; +} + +export interface PlatformEvent> { + /** Unique event id (uuid). */ + id: string; + type: EventType; + /** ISO timestamp of emission. */ + occurredAt: string; + organizationId: string; + connectorId: string; + provider: string; + resource?: ResourceRef; + payload?: TPayload; +} diff --git a/packages/events/tsconfig.json b/packages/events/tsconfig.json new file mode 100644 index 0000000..7bfd764 --- /dev/null +++ b/packages/events/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@company-brain/config/tsconfig/node", + "include": ["src"] +} diff --git a/packages/knowledge/eslint.config.mjs b/packages/knowledge/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/packages/knowledge/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/packages/knowledge/package.json b/packages/knowledge/package.json new file mode 100644 index 0000000..8a22b3d --- /dev/null +++ b/packages/knowledge/package.json @@ -0,0 +1,33 @@ +{ + "name": "@company-brain/knowledge", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "csv-parse": "^5.6.0", + "html-to-text": "^9.0.5", + "mammoth": "^1.9.0", + "marked": "^15.0.7", + "pdf-parse": "^1.1.1" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/html-to-text": "^9.0.4", + "@types/node": "^22.14.1", + "@types/pdf-parse": "^1.1.4", + "eslint": "^9.24.0", + "typescript": "^5.8.3", + "vitest": "^3.1.1" + } +} diff --git a/packages/knowledge/src/chunker.test.ts b/packages/knowledge/src/chunker.test.ts new file mode 100644 index 0000000..2808583 --- /dev/null +++ b/packages/knowledge/src/chunker.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest'; +import { chunkDocument } from './chunker.js'; +import { estimateTokens } from './tokens.js'; +import type { DocumentSection } from './types.js'; + +const paragraph = (words: number, word = 'lorem') => Array(words).fill(word).join(' '); + +describe('chunkDocument', () => { + it('returns a single chunk for short text', () => { + const chunks = chunkDocument('Hello world, this is a short document.', []); + expect(chunks).toHaveLength(1); + expect(chunks[0]!.index).toBe(0); + expect(chunks[0]!.heading).toBeNull(); + }); + + it('splits long text into multiple chunks within the token budget', () => { + const text = Array.from({ length: 12 }, () => paragraph(80)).join('\n\n'); + const chunks = chunkDocument(text, [], { chunkSize: 150, chunkOverlap: 20, maxTokens: 200 }); + expect(chunks.length).toBeGreaterThan(2); + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(230); // budget + carried overlap + } + }); + + it('carries sentence overlap between consecutive chunks', () => { + const sentences = Array.from({ length: 40 }, (_, i) => `Sentence number ${i} ends here.`); + const text = sentences.join(' '); + const chunks = chunkDocument(text, [], { chunkSize: 60, chunkOverlap: 20, maxTokens: 80 }); + expect(chunks.length).toBeGreaterThan(1); + const tail = chunks[0]!.content.slice(-30); + expect(chunks[1]!.content.startsWith(tail.split(' ').slice(-2).join(' '))).toBe(false); // sanity + // The second chunk should repeat some trailing content of the first. + const overlapFound = sentences.some( + (s) => chunks[0]!.content.includes(s) && chunks[1]!.content.includes(s), + ); + expect(overlapFound).toBe(true); + }); + + it('respects section boundaries and attaches headings', () => { + const intro = paragraph(30, 'intro'); + const bodyA = paragraph(30, 'alpha'); + const bodyB = paragraph(30, 'beta'); + const text = `${intro}\n\nSection A\n${bodyA}\n\nSection B\n${bodyB}`; + const sections: DocumentSection[] = [ + { + heading: 'Section A', + level: 1, + startOffset: text.indexOf('Section A'), + endOffset: text.indexOf('Section B'), + }, + { + heading: 'Section B', + level: 1, + startOffset: text.indexOf('Section B'), + endOffset: text.length, + }, + ]; + const chunks = chunkDocument(text, sections, { chunkSize: 500, chunkOverlap: 0 }); + const headings = chunks.map((c) => c.heading); + expect(headings).toContain(null); // preamble + expect(headings).toContain('Section A'); + expect(headings).toContain('Section B'); + const chunkA = chunks.find((c) => c.heading === 'Section A')!; + expect(chunkA.content).toContain('alpha'); + expect(chunkA.content).not.toContain('beta'); + }); + + it('hard-splits pathological single-sentence paragraphs', () => { + const text = paragraph(2000, 'word'); // no sentence punctuation + const chunks = chunkDocument(text, [], { chunkSize: 200, chunkOverlap: 0, maxTokens: 250 }); + expect(chunks.length).toBeGreaterThan(3); + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(250); + } + }); + + it('assigns sequential indexes', () => { + const text = Array.from({ length: 8 }, () => paragraph(100)).join('\n\n'); + const chunks = chunkDocument(text, [], { chunkSize: 120, chunkOverlap: 10 }); + expect(chunks.map((c) => c.index)).toEqual(chunks.map((_, i) => i)); + }); +}); + +describe('estimateTokens', () => { + it('grows with text length and never returns 0 for content', () => { + expect(estimateTokens('')).toBe(0); + expect(estimateTokens('word')).toBeGreaterThan(0); + expect(estimateTokens(paragraph(200))).toBeGreaterThan(estimateTokens(paragraph(50))); + }); +}); diff --git a/packages/knowledge/src/chunker.ts b/packages/knowledge/src/chunker.ts new file mode 100644 index 0000000..cc389d2 --- /dev/null +++ b/packages/knowledge/src/chunker.ts @@ -0,0 +1,172 @@ +import { estimateTokens } from './tokens.js'; +import { + DEFAULT_CHUNK_OPTIONS, + type ChunkOptions, + type DocumentSection, + type TextChunk, +} from './types.js'; + +interface Paragraph { + text: string; + startOffset: number; + endOffset: number; +} + +function splitParagraphs(text: string, base: number): Paragraph[] { + const paragraphs: Paragraph[] = []; + let cursor = 0; + for (const part of text.split(/\n{2,}/)) { + const start = text.indexOf(part, cursor); + if (part.trim().length > 0) { + paragraphs.push({ + text: part.trim(), + startOffset: base + start, + endOffset: base + start + part.length, + }); + } + cursor = start + part.length; + } + return paragraphs; +} + +/** Split an oversized paragraph on sentence boundaries (fallback: words). */ +function splitOversized(paragraph: Paragraph, maxTokens: number): Paragraph[] { + const sentences = paragraph.text.match(/[^.!?\n]+[.!?\n]*/g) ?? [paragraph.text]; + const parts: Paragraph[] = []; + let buffer = ''; + let offset = paragraph.startOffset; + const flush = () => { + if (!buffer.trim()) return; + parts.push({ text: buffer.trim(), startOffset: offset, endOffset: offset + buffer.length }); + offset += buffer.length; + buffer = ''; + }; + for (const sentence of sentences) { + if (estimateTokens(buffer + sentence) > maxTokens && buffer) flush(); + if (estimateTokens(sentence) > maxTokens) { + // Pathological sentence — hard-split on words. + flush(); + let piece = ''; + for (const word of sentence.split(/\s+/)) { + if (estimateTokens(piece + ' ' + word) > maxTokens && piece) { + buffer = piece; + flush(); + piece = ''; + } + piece = piece ? `${piece} ${word}` : word; + } + buffer = piece; + flush(); + } else { + buffer += sentence; + } + } + flush(); + return parts; +} + +/** Tail of a chunk reused as overlap for the next one. */ +function overlapTail(content: string, overlapTokens: number): string { + if (overlapTokens <= 0) return ''; + const sentences = content.match(/[^.!?\n]+[.!?\n]*/g) ?? [content]; + let tail = ''; + for (let i = sentences.length - 1; i >= 0; i -= 1) { + const candidate = sentences[i] + tail; + if (estimateTokens(candidate) > overlapTokens) break; + tail = candidate; + } + return tail.trim(); +} + +/** + * Configurable, section/heading-aware chunker. + * + * Walks the document section by section (when `respectSections` is on), + * packs paragraphs up to `chunkSize` tokens, hard-caps at `maxTokens`, + * and prefixes each follow-up chunk with a sentence-aligned overlap of + * `chunkOverlap` tokens for retrieval continuity. + */ +export function chunkDocument( + text: string, + sections: DocumentSection[], + options: Partial = {}, +): TextChunk[] { + const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options }; + const regions: Array<{ heading: string | null; body: string; base: number }> = []; + + if (opts.respectSections && sections.length > 0) { + const ordered = [...sections].sort((a, b) => a.startOffset - b.startOffset); + const first = ordered[0]!; + if (first.startOffset > 0) { + regions.push({ heading: null, body: text.slice(0, first.startOffset), base: 0 }); + } + for (const section of ordered) { + regions.push({ + heading: section.heading, + body: text.slice(section.startOffset, section.endOffset), + base: section.startOffset, + }); + } + } else { + regions.push({ heading: null, body: text, base: 0 }); + } + + const chunks: TextChunk[] = []; + let index = 0; + + for (const region of regions) { + const paragraphs = splitParagraphs(region.body, region.base).flatMap((p) => + estimateTokens(p.text) > opts.maxTokens ? splitOversized(p, opts.maxTokens) : [p], + ); + + let content = ''; + let start = -1; + let end = -1; + // Distinguishes real new paragraphs from carried-over overlap text so + // the remainder flush never emits an overlap-only chunk (and never + // drops a short document). + let hasFreshContent = false; + + const emit = () => { + const trimmed = content.trim(); + if (!trimmed) return; + chunks.push({ + index: index++, + content: trimmed, + tokenCount: estimateTokens(trimmed), + heading: region.heading, + section: region.heading, + startOffset: start, + endOffset: end, + }); + const tail = overlapTail(trimmed, opts.chunkOverlap); + content = tail ? tail + '\n' : ''; + start = end; + hasFreshContent = false; + }; + + for (const paragraph of paragraphs) { + const candidate = content ? `${content}\n${paragraph.text}` : paragraph.text; + if (estimateTokens(candidate) > opts.chunkSize && hasFreshContent) emit(); + if (start === -1 || !content.trim()) start = paragraph.startOffset; + content = content ? `${content}\n${paragraph.text}` : paragraph.text; + end = paragraph.endOffset; + hasFreshContent = true; + } + // Flush the region remainder (overlap-only remainders are skipped). + if (hasFreshContent && content.trim()) { + const trimmed = content.trim(); + chunks.push({ + index: index++, + content: trimmed, + tokenCount: estimateTokens(trimmed), + heading: region.heading, + section: region.heading, + startOffset: start, + endOffset: end, + }); + } + } + + return chunks; +} diff --git a/packages/knowledge/src/cleaner.ts b/packages/knowledge/src/cleaner.ts new file mode 100644 index 0000000..a9c7691 --- /dev/null +++ b/packages/knowledge/src/cleaner.ts @@ -0,0 +1,20 @@ +// eslint-disable-next-line no-control-regex +const CONTROL_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g; +// eslint-disable-next-line no-misleading-character-class -- individual zero-width code points, intentionally not a grapheme +const ZERO_WIDTH = /[\u200B\u200C\u200D\u2060\uFEFF]/g; + +/** + * Normalizes parser output into stable, chunkable text: + * unifies line endings, strips control characters and zero-width marks, + * collapses horizontal whitespace and limits consecutive blank lines. + */ +export function cleanText(raw: string): string { + return raw + .replace(/\r\n?/g, '\n') + .replace(CONTROL_CHARS, '') + .replace(ZERO_WIDTH, '') + .replace(/[ \t]+/g, ' ') + .replace(/ ?\n ?/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} diff --git a/packages/knowledge/src/embeddings.test.ts b/packages/knowledge/src/embeddings.test.ts new file mode 100644 index 0000000..8d72e14 --- /dev/null +++ b/packages/knowledge/src/embeddings.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { createEmbeddingProvider, embedAll } from './embeddings/index.js'; +import { LocalHashEmbeddingProvider } from './embeddings/local.provider.js'; + +describe('embedding provider factory', () => { + it('creates the local provider without keys', () => { + const provider = createEmbeddingProvider({ provider: 'local', dimension: 256 }); + expect(provider.name).toBe('local'); + expect(provider.dimension).toBe(256); + }); + + it('requires API keys for remote providers', () => { + expect(() => createEmbeddingProvider({ provider: 'openai' })).toThrow(/OPENAI_API_KEY/); + expect(() => createEmbeddingProvider({ provider: 'gemini' })).toThrow(/GEMINI_API_KEY/); + expect(() => createEmbeddingProvider({ provider: 'voyage' })).toThrow(/VOYAGE_API_KEY/); + }); + + it('configures remote providers with defaults', () => { + const openai = createEmbeddingProvider({ provider: 'openai', apiKey: 'k' }); + expect(openai.model).toBe('text-embedding-3-small'); + expect(openai.dimension).toBe(1536); + const voyage = createEmbeddingProvider({ provider: 'voyage', apiKey: 'k' }); + expect(voyage.dimension).toBe(512); + }); +}); + +describe('local hash embeddings', () => { + const provider = new LocalHashEmbeddingProvider(128); + + it('is deterministic', async () => { + const [a] = await provider.embed(['the quick brown fox']); + const [b] = await provider.embed(['the quick brown fox']); + expect(a).toEqual(b); + }); + + it('produces L2-normalized vectors of the right size', async () => { + const [vec] = await provider.embed(['normalize me please']); + expect(vec).toHaveLength(128); + const norm = Math.sqrt(vec!.reduce((s, v) => s + v * v, 0)); + expect(norm).toBeCloseTo(1, 6); + }); + + it('scores related texts higher than unrelated ones', async () => { + const [query, related, unrelated] = await provider.embed([ + 'database connection pooling settings', + 'configure the database connection pool size', + 'grandma baked chocolate cookies yesterday', + ]); + const dot = (x: number[], y: number[]) => x.reduce((s, v, i) => s + v * y[i]!, 0); + expect(dot(query!, related!)).toBeGreaterThan(dot(query!, unrelated!)); + }); + + it('batches through embedAll', async () => { + const texts = Array.from({ length: 10 }, (_, i) => `text number ${i}`); + const vectors = await embedAll(provider, texts, 3); + expect(vectors).toHaveLength(10); + }); +}); diff --git a/packages/knowledge/src/embeddings/gemini.provider.ts b/packages/knowledge/src/embeddings/gemini.provider.ts new file mode 100644 index 0000000..395afd0 --- /dev/null +++ b/packages/knowledge/src/embeddings/gemini.provider.ts @@ -0,0 +1,38 @@ +import type { EmbeddingProvider } from './types.js'; + +interface GeminiBatchResponse { + embeddings: Array<{ values: number[] }>; +} + +export class GeminiEmbeddingProvider implements EmbeddingProvider { + readonly name = 'gemini' as const; + + constructor( + private readonly apiKey: string, + readonly model = 'text-embedding-004', + readonly dimension = 768, + ) {} + + async embed(texts: string[]): Promise { + const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents`; + const response = await fetch(url, { + method: 'POST', + headers: { + 'x-goog-api-key': this.apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + requests: texts.map((text) => ({ + model: `models/${this.model}`, + content: { parts: [{ text }] }, + outputDimensionality: this.dimension, + })), + }), + }); + if (!response.ok) { + throw new Error(`Gemini embeddings failed (${response.status}): ${await response.text()}`); + } + const body = (await response.json()) as GeminiBatchResponse; + return body.embeddings.map((e) => e.values); + } +} diff --git a/packages/knowledge/src/embeddings/index.ts b/packages/knowledge/src/embeddings/index.ts new file mode 100644 index 0000000..1e51130 --- /dev/null +++ b/packages/knowledge/src/embeddings/index.ts @@ -0,0 +1,54 @@ +import { LocalHashEmbeddingProvider } from './local.provider.js'; +import { OpenAIEmbeddingProvider } from './openai.provider.js'; +import { GeminiEmbeddingProvider } from './gemini.provider.js'; +import { VoyageEmbeddingProvider } from './voyage.provider.js'; +import type { EmbeddingConfig, EmbeddingProvider } from './types.js'; + +export const DEFAULT_EMBEDDING_BATCH_SIZE = 64; + +/** + * Provider factory — the single place that knows concrete providers. + * Everything else programs against EmbeddingProvider. + */ +export function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider { + switch (config.provider) { + case 'local': + return new LocalHashEmbeddingProvider(config.dimension ?? 384); + case 'openai': { + if (!config.apiKey) throw new Error('OPENAI_API_KEY is required for the openai provider'); + return new OpenAIEmbeddingProvider(config.apiKey, config.model, config.dimension); + } + case 'gemini': { + if (!config.apiKey) throw new Error('GEMINI_API_KEY is required for the gemini provider'); + return new GeminiEmbeddingProvider(config.apiKey, config.model, config.dimension); + } + case 'voyage': { + if (!config.apiKey) throw new Error('VOYAGE_API_KEY is required for the voyage provider'); + return new VoyageEmbeddingProvider(config.apiKey, config.model, config.dimension); + } + default: + throw new Error(`Unknown embedding provider: ${String(config.provider)}`); + } +} + +/** Embed any number of texts, batching per provider limits. */ +export async function embedAll( + provider: EmbeddingProvider, + texts: string[], + batchSize = DEFAULT_EMBEDDING_BATCH_SIZE, +): Promise { + const vectors: number[][] = []; + for (let i = 0; i < texts.length; i += batchSize) { + const batch = texts.slice(i, i + batchSize); + vectors.push(...(await provider.embed(batch))); + } + return vectors; +} + +export { + LocalHashEmbeddingProvider, + OpenAIEmbeddingProvider, + GeminiEmbeddingProvider, + VoyageEmbeddingProvider, +}; +export type { EmbeddingConfig, EmbeddingProvider, EmbeddingProviderName } from './types.js'; diff --git a/packages/knowledge/src/embeddings/local.provider.ts b/packages/knowledge/src/embeddings/local.provider.ts new file mode 100644 index 0000000..9fbba6a --- /dev/null +++ b/packages/knowledge/src/embeddings/local.provider.ts @@ -0,0 +1,45 @@ +import { createHash } from 'node:crypto'; +import type { EmbeddingProvider } from './types.js'; + +/** + * Deterministic feature-hashing embeddings. No network, no keys — meant for + * development, tests and offline environments. Word unigrams + bigrams are + * hashed into `dimension` buckets with a signed contribution, then the + * vector is L2-normalized so cosine similarity behaves sensibly. + * + * Not a semantic model: it captures lexical overlap only. Swap the provider + * via EMBEDDINGS_PROVIDER for real semantics. + */ +export class LocalHashEmbeddingProvider implements EmbeddingProvider { + readonly name = 'local' as const; + readonly model = 'feature-hash-v1'; + + constructor(readonly dimension: number = 384) {} + + async embed(texts: string[]): Promise { + return texts.map((text) => this.embedOne(text)); + } + + private embedOne(text: string): number[] { + const vector = new Array(this.dimension).fill(0); + const words = text + .toLowerCase() + .split(/\W+/) + .filter((w) => w.length > 1); + + const add = (feature: string, weight: number) => { + const digest = createHash('sha1').update(feature).digest(); + const bucket = digest.readUInt32BE(0) % this.dimension; + const sign = (digest[4]! & 1) === 0 ? 1 : -1; + vector[bucket]! += sign * weight; + }; + + for (let i = 0; i < words.length; i += 1) { + add(words[i]!, 1); + if (i + 1 < words.length) add(`${words[i]}_${words[i + 1]}`, 0.5); + } + + const norm = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0)) || 1; + return vector.map((v) => v / norm); + } +} diff --git a/packages/knowledge/src/embeddings/openai.provider.ts b/packages/knowledge/src/embeddings/openai.provider.ts new file mode 100644 index 0000000..8793485 --- /dev/null +++ b/packages/knowledge/src/embeddings/openai.provider.ts @@ -0,0 +1,31 @@ +import type { EmbeddingProvider } from './types.js'; + +interface OpenAIEmbeddingResponse { + data: Array<{ index: number; embedding: number[] }>; +} + +export class OpenAIEmbeddingProvider implements EmbeddingProvider { + readonly name = 'openai' as const; + + constructor( + private readonly apiKey: string, + readonly model = 'text-embedding-3-small', + readonly dimension = 1536, + ) {} + + async embed(texts: string[]): Promise { + const response = await fetch('https://api.openai.com/v1/embeddings', { + method: 'POST', + headers: { + authorization: `Bearer ${this.apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ model: this.model, input: texts, dimensions: this.dimension }), + }); + if (!response.ok) { + throw new Error(`OpenAI embeddings failed (${response.status}): ${await response.text()}`); + } + const body = (await response.json()) as OpenAIEmbeddingResponse; + return body.data.sort((a, b) => a.index - b.index).map((d) => d.embedding); + } +} diff --git a/packages/knowledge/src/embeddings/types.ts b/packages/knowledge/src/embeddings/types.ts new file mode 100644 index 0000000..d56f7c0 --- /dev/null +++ b/packages/knowledge/src/embeddings/types.ts @@ -0,0 +1,25 @@ +export type EmbeddingProviderName = 'local' | 'openai' | 'gemini' | 'voyage'; + +export interface EmbeddingConfig { + provider: EmbeddingProviderName; + /** Provider-specific model id; each provider has a sensible default. */ + model?: string; + /** Vector dimension. Required for 'local'; validated for the rest. */ + dimension?: number; + apiKey?: string; + /** Max texts per provider API call. */ + batchSize?: number; +} + +/** + * Provider abstraction: the rest of the system only ever sees this + * interface. New providers (e.g. a future local ONNX model) implement it + * and register in embeddings/index.ts. + */ +export interface EmbeddingProvider { + readonly name: EmbeddingProviderName; + readonly model: string; + readonly dimension: number; + /** Embed a batch of texts; returns one vector per input, same order. */ + embed(texts: string[]): Promise; +} diff --git a/packages/knowledge/src/embeddings/voyage.provider.ts b/packages/knowledge/src/embeddings/voyage.provider.ts new file mode 100644 index 0000000..f7408fd --- /dev/null +++ b/packages/knowledge/src/embeddings/voyage.provider.ts @@ -0,0 +1,31 @@ +import type { EmbeddingProvider } from './types.js'; + +interface VoyageEmbeddingResponse { + data: Array<{ index: number; embedding: number[] }>; +} + +export class VoyageEmbeddingProvider implements EmbeddingProvider { + readonly name = 'voyage' as const; + + constructor( + private readonly apiKey: string, + readonly model = 'voyage-3-lite', + readonly dimension = 512, + ) {} + + async embed(texts: string[]): Promise { + const response = await fetch('https://api.voyageai.com/v1/embeddings', { + method: 'POST', + headers: { + authorization: `Bearer ${this.apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ model: this.model, input: texts }), + }); + if (!response.ok) { + throw new Error(`Voyage embeddings failed (${response.status}): ${await response.text()}`); + } + const body = (await response.json()) as VoyageEmbeddingResponse; + return body.data.sort((a, b) => a.index - b.index).map((d) => d.embedding); + } +} diff --git a/packages/knowledge/src/index.ts b/packages/knowledge/src/index.ts new file mode 100644 index 0000000..0cd7f14 --- /dev/null +++ b/packages/knowledge/src/index.ts @@ -0,0 +1,26 @@ +export * from './types.js'; +export { cleanText } from './cleaner.js'; +export { estimateTokens } from './tokens.js'; +export { chunkDocument } from './chunker.js'; +export { buildDocumentMetadata, detectLanguage, extractKeywords } from './metadata.js'; +export type { FileInfo } from './metadata.js'; +export { + findParser, + isSupported, + SUPPORTED_MIME_TYPES, + SUPPORTED_EXTENSIONS, +} from './parsers/index.js'; +export { + createEmbeddingProvider, + embedAll, + DEFAULT_EMBEDDING_BATCH_SIZE, + LocalHashEmbeddingProvider, + OpenAIEmbeddingProvider, + GeminiEmbeddingProvider, + VoyageEmbeddingProvider, +} from './embeddings/index.js'; +export type { + EmbeddingConfig, + EmbeddingProvider, + EmbeddingProviderName, +} from './embeddings/index.js'; diff --git a/packages/knowledge/src/metadata.ts b/packages/knowledge/src/metadata.ts new file mode 100644 index 0000000..361d8d2 --- /dev/null +++ b/packages/knowledge/src/metadata.ts @@ -0,0 +1,77 @@ +import type { ExtractedMetadata, ParsedDocument } from './types.js'; + +const STOPWORDS = new Set( + ( + 'a an and are as at be but by for from has have i if in into is it its of on or ' + + 'not no so such that the their then there these they this to was were will with ' + + 'we you your our us he she his her them what when where which who how why can ' + + 'could should would may might must do does did done also more most other some any' + ).split(' '), +); + +const LANGUAGE_MARKERS: Record = { + en: ['the', 'and', 'of', 'to', 'is', 'that', 'with'], + es: ['el', 'la', 'los', 'las', 'que', 'para', 'una'], + fr: ['le', 'la', 'les', 'des', 'est', 'dans', 'pour'], + de: ['der', 'die', 'das', 'und', 'ist', 'nicht', 'mit'], +}; + +/** Cheap stopword-vote language detection — 'en' fallback. */ +export function detectLanguage(text: string): string { + const words = text.toLowerCase().slice(0, 20_000).split(/\W+/); + const counts = new Map(); + for (const [lang, markers] of Object.entries(LANGUAGE_MARKERS)) { + counts.set(lang, words.filter((w) => markers.includes(w)).length); + } + const best = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]; + return best && best[1] > 2 ? best[0] : 'en'; +} + +/** Top-N keywords by term frequency, ignoring stopwords and short tokens. */ +export function extractKeywords(text: string, limit = 12): string[] { + const frequencies = new Map(); + for (const raw of text.toLowerCase().split(/\W+/)) { + if (raw.length < 4 || STOPWORDS.has(raw) || /^\d+$/.test(raw)) continue; + frequencies.set(raw, (frequencies.get(raw) ?? 0) + 1); + } + return [...frequencies.entries()] + .filter(([, count]) => count > 1) + .sort((a, b) => b[1] - a[1]) + .slice(0, limit) + .map(([word]) => word); +} + +export interface FileInfo { + fileName: string; + mimeType: string; + fileSizeBytes: number; +} + +/** + * Merges parser-supplied metadata with derived fields (title fallback, + * language, keywords, heading list) into the document metadata record. + */ +export function buildDocumentMetadata( + parsed: ParsedDocument, + cleanedText: string, + file: FileInfo, +): ExtractedMetadata { + const headings = parsed.sections.map((s) => s.heading).slice(0, 100); + const title = + parsed.metadata.title?.trim() || + headings[0] || + file.fileName.replace(/\.[^.]+$/, '').replace(/[-_]+/g, ' '); + + return { + ...parsed.metadata, + title, + language: parsed.metadata.language ?? detectLanguage(cleanedText), + keywords: extractKeywords(cleanedText), + headings, + sectionCount: parsed.sections.length, + fileName: file.fileName, + mimeType: file.mimeType, + fileSizeBytes: file.fileSizeBytes, + characterCount: cleanedText.length, + }; +} diff --git a/packages/knowledge/src/parsers.test.ts b/packages/knowledge/src/parsers.test.ts new file mode 100644 index 0000000..25ef01a --- /dev/null +++ b/packages/knowledge/src/parsers.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; +import { findParser, isSupported } from './parsers/index.js'; +import { markdownParser } from './parsers/markdown.parser.js'; +import { htmlParser } from './parsers/html.parser.js'; +import { csvParser } from './parsers/csv.parser.js'; +import { jsonParser } from './parsers/json.parser.js'; +import { textParser } from './parsers/text.parser.js'; + +const ctx = (fileName: string, mimeType: string) => ({ fileName, mimeType }); + +describe('parser registry', () => { + it('resolves parsers by MIME type', () => { + expect(findParser('application/pdf', 'x.bin')?.name).toBe('pdf'); + expect(findParser('text/markdown', 'x')?.name).toBe('markdown'); + expect(findParser('text/html', 'x')?.name).toBe('html'); + expect(findParser('text/csv', 'x')?.name).toBe('csv'); + expect(findParser('application/json', 'x')?.name).toBe('json'); + expect(findParser('text/plain', 'x')?.name).toBe('text'); + expect( + findParser('application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'x') + ?.name, + ).toBe('docx'); + }); + + it('falls back to file extension when MIME is generic', () => { + expect(findParser('application/octet-stream', 'notes.md')?.name).toBe('markdown'); + expect(findParser('application/octet-stream', 'doc.pdf')?.name).toBe('pdf'); + expect(findParser('application/octet-stream', 'file.unknown')).toBeNull(); + }); + + it('ignores MIME parameters', () => { + expect(findParser('text/html; charset=utf-8', 'x')?.name).toBe('html'); + }); + + it('reports support', () => { + expect(isSupported('text/csv', 'data.csv')).toBe(true); + expect(isSupported('video/mp4', 'movie.mp4')).toBe(false); + }); +}); + +describe('markdown parser', () => { + it('extracts text, headings and tables', async () => { + const md = + '# Title\n\nIntro text.\n\n## Section A\n\nBody A.\n\n| h1 | h2 |\n|---|---|\n| a | b |\n'; + const parsed = await markdownParser.parse(Buffer.from(md), ctx('a.md', 'text/markdown')); + expect(parsed.metadata.title).toBe('Title'); + expect(parsed.metadata.tableCount).toBe(1); + expect(parsed.sections.map((s) => s.heading)).toEqual(['Title', 'Section A']); + expect(parsed.text).toContain('Body A.'); + expect(parsed.text).toContain('a | b'); + }); +}); + +describe('html parser', () => { + it('extracts title, text and heading structure', async () => { + const html = + 'Page Title

Main

Hello world.

Sub

More.

'; + const parsed = await htmlParser.parse(Buffer.from(html), ctx('a.html', 'text/html')); + expect(parsed.metadata.title).toBe('Page Title'); + expect(parsed.text).toContain('Hello world.'); + expect(parsed.text).not.toContain('ignored'); + expect(parsed.sections.map((s) => s.heading)).toEqual(['Main', 'Sub']); + expect(parsed.sections[1]!.level).toBe(2); + }); +}); + +describe('csv parser', () => { + it('renders rows as labelled fields', async () => { + const csv = 'name,role\nAda,Engineer\nGrace,Admiral\n'; + const parsed = await csvParser.parse(Buffer.from(csv), ctx('team.csv', 'text/csv')); + expect(parsed.metadata.rowCount).toBe(2); + expect(parsed.metadata.columns).toEqual(['name', 'role']); + expect(parsed.text).toContain('name: Ada'); + expect(parsed.text).toContain('role: Admiral'); + }); +}); + +describe('json parser', () => { + it('flattens nested structures to searchable lines', async () => { + const json = JSON.stringify({ app: { name: 'brain', ports: [3000, 4000] } }); + const parsed = await jsonParser.parse(Buffer.from(json), ctx('cfg.json', 'application/json')); + expect(parsed.text).toContain('app.name: brain'); + expect(parsed.text).toContain('app.ports[1]: 4000'); + }); + + it('handles JSON lines', async () => { + const jsonl = '{"a":1}\n{"a":2}\n'; + const parsed = await jsonParser.parse(Buffer.from(jsonl), ctx('x.jsonl', 'application/json')); + expect(parsed.text).toContain('a: 1'); + expect(parsed.text).toContain('a: 2'); + }); +}); + +describe('text parser', () => { + it('detects ALL-CAPS and numbered headings', async () => { + const txt = 'INTRODUCTION\n\nSome intro.\n\n1. First Steps\n\nDetails here.\n'; + const parsed = await textParser.parse(Buffer.from(txt), ctx('a.txt', 'text/plain')); + expect(parsed.sections.map((s) => s.heading)).toEqual(['INTRODUCTION', '1. First Steps']); + }); +}); diff --git a/packages/knowledge/src/parsers/csv.parser.ts b/packages/knowledge/src/parsers/csv.parser.ts new file mode 100644 index 0000000..46ad852 --- /dev/null +++ b/packages/knowledge/src/parsers/csv.parser.ts @@ -0,0 +1,37 @@ +import { parse } from 'csv-parse/sync'; +import type { DocumentParser, ParsedDocument } from '../types.js'; + +/** + * CSV parser: renders rows as "header: value" lines grouped per record so + * embeddings capture column semantics, not just raw cells. + */ +export const csvParser: DocumentParser = { + name: 'csv', + mimeTypes: ['text/csv', 'application/csv'], + extensions: ['.csv', '.tsv'], + async parse(buffer, context): Promise { + const delimiter = context.fileName.endsWith('.tsv') ? '\t' : ','; + const records = parse(buffer, { + delimiter, + columns: true, + skip_empty_lines: true, + relax_column_count: true, + trim: true, + }) as Record[]; + + const headers = records.length > 0 ? Object.keys(records[0]!) : []; + const rows = records.map((record, i) => { + const fields = headers + .map((h) => `${h}: ${record[h] ?? ''}`) + .filter((line) => !line.endsWith(': ')) + .join('\n'); + return `Row ${i + 1}\n${fields}`; + }); + + return { + text: rows.join('\n\n'), + sections: [], + metadata: { tableCount: 1, rowCount: records.length, columns: headers }, + }; + }, +}; diff --git a/packages/knowledge/src/parsers/docx.parser.ts b/packages/knowledge/src/parsers/docx.parser.ts new file mode 100644 index 0000000..019a814 --- /dev/null +++ b/packages/knowledge/src/parsers/docx.parser.ts @@ -0,0 +1,52 @@ +import mammoth from 'mammoth'; +import { htmlToText } from 'html-to-text'; +import type { DocumentParser, DocumentSection, ParsedDocument } from '../types.js'; + +const HEADING_RE = /]*>([\s\S]*?)<\/h\1>/gi; +const TABLE_RE = /]/gi; +const TAG_RE = /<[^>]+>/g; + +/** + * DOCX parser: mammoth converts to semantic HTML (preserving heading levels + * and tables), which we then reduce to text + section map. + */ +export const docxParser: DocumentParser = { + name: 'docx', + mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], + extensions: ['.docx'], + async parse(buffer): Promise { + const { value: html } = await mammoth.convertToHtml({ buffer }); + const text = htmlToText(html, { wordwrap: false }); + + const sections: DocumentSection[] = []; + const found: Array<{ heading: string; level: number }> = []; + for (const match of html.matchAll(HEADING_RE)) { + const heading = match[2]!.replace(TAG_RE, '').replace(/\s+/g, ' ').trim(); + if (heading) found.push({ heading, level: Number(match[1]) }); + } + let cursor = 0; + const offsets: number[] = []; + for (const { heading } of found) { + const at = text.indexOf(heading, cursor); + offsets.push(at === -1 ? cursor : at); + if (at !== -1) cursor = at + heading.length; + } + found.forEach((h, i) => { + sections.push({ + heading: h.heading, + level: h.level, + startOffset: offsets[i]!, + endOffset: offsets[i + 1] ?? text.length, + }); + }); + + return { + text, + sections, + metadata: { + title: found.find((h) => h.level === 1)?.heading, + tableCount: html.match(TABLE_RE)?.length ?? 0, + }, + }; + }, +}; diff --git a/packages/knowledge/src/parsers/html.parser.ts b/packages/knowledge/src/parsers/html.parser.ts new file mode 100644 index 0000000..59cb9c8 --- /dev/null +++ b/packages/knowledge/src/parsers/html.parser.ts @@ -0,0 +1,58 @@ +import { htmlToText } from 'html-to-text'; +import type { DocumentParser, DocumentSection, ParsedDocument } from '../types.js'; + +const TITLE_RE = /]*>([^<]*)<\/title>/i; +const HEADING_RE = /]*>([\s\S]*?)<\/h\1>/gi; +const TABLE_RE = /]/gi; +const TAG_RE = /<[^>]+>/g; + +/** + * HTML parser: html-to-text for robust text extraction, plus lightweight + * scans for the title, heading structure and table count. + */ +export const htmlParser: DocumentParser = { + name: 'html', + mimeTypes: ['text/html', 'application/xhtml+xml'], + extensions: ['.html', '.htm', '.xhtml'], + async parse(buffer): Promise { + const html = buffer.toString('utf8'); + const text = htmlToText(html, { + wordwrap: false, + selectors: [ + { selector: 'a', options: { ignoreHref: true } }, + { selector: 'img', format: 'skip' }, + { selector: 'nav', format: 'skip' }, + { selector: 'script', format: 'skip' }, + { selector: 'style', format: 'skip' }, + ], + }); + + const title = TITLE_RE.exec(html)?.[1]?.trim() || undefined; + const tableCount = html.match(TABLE_RE)?.length ?? 0; + + const sections: DocumentSection[] = []; + const found: Array<{ heading: string; level: number }> = []; + for (const match of html.matchAll(HEADING_RE)) { + const heading = match[2]!.replace(TAG_RE, '').replace(/\s+/g, ' ').trim(); + if (heading) found.push({ heading, level: Number(match[1]) }); + } + // Map headings onto the extracted text by search order. + let cursor = 0; + const offsets: number[] = []; + for (const { heading } of found) { + const at = text.indexOf(heading, cursor); + offsets.push(at === -1 ? cursor : at); + if (at !== -1) cursor = at + heading.length; + } + found.forEach((h, i) => { + sections.push({ + heading: h.heading, + level: h.level, + startOffset: offsets[i]!, + endOffset: offsets[i + 1] ?? text.length, + }); + }); + + return { text, sections, metadata: { title, tableCount } }; + }, +}; diff --git a/packages/knowledge/src/parsers/index.ts b/packages/knowledge/src/parsers/index.ts new file mode 100644 index 0000000..e33dd6f --- /dev/null +++ b/packages/knowledge/src/parsers/index.ts @@ -0,0 +1,41 @@ +import { extname } from 'node:path'; +import type { DocumentParser } from '../types.js'; +import { textParser } from './text.parser.js'; +import { markdownParser } from './markdown.parser.js'; +import { htmlParser } from './html.parser.js'; +import { csvParser } from './csv.parser.js'; +import { jsonParser } from './json.parser.js'; +import { pdfParser } from './pdf.parser.js'; +import { docxParser } from './docx.parser.js'; + +/** + * Parser registry. Pluggable: to support a new format, implement + * DocumentParser and add it here — nothing else in the pipeline changes. + */ +const PARSERS: DocumentParser[] = [ + pdfParser, + docxParser, + markdownParser, + htmlParser, + csvParser, + jsonParser, + textParser, +]; + +export const SUPPORTED_MIME_TYPES = PARSERS.flatMap((p) => p.mimeTypes); +export const SUPPORTED_EXTENSIONS = PARSERS.flatMap((p) => p.extensions); + +/** Resolve a parser by MIME type first, file extension second. */ +export function findParser(mimeType: string, fileName: string): DocumentParser | null { + const normalizedMime = mimeType.split(';')[0]!.trim().toLowerCase(); + const byMime = PARSERS.find((p) => p.mimeTypes.includes(normalizedMime)); + if (byMime) return byMime; + const ext = extname(fileName).toLowerCase(); + return PARSERS.find((p) => p.extensions.includes(ext)) ?? null; +} + +export function isSupported(mimeType: string, fileName: string): boolean { + return findParser(mimeType, fileName) !== null; +} + +export { textParser, markdownParser, htmlParser, csvParser, jsonParser, pdfParser, docxParser }; diff --git a/packages/knowledge/src/parsers/json.parser.ts b/packages/knowledge/src/parsers/json.parser.ts new file mode 100644 index 0000000..113d6e8 --- /dev/null +++ b/packages/knowledge/src/parsers/json.parser.ts @@ -0,0 +1,42 @@ +import type { DocumentParser, ParsedDocument } from '../types.js'; + +/** Flattens nested JSON into "path: value" lines for searchable text. */ +function flatten(value: unknown, path: string, lines: string[], depth: number): void { + if (depth > 20) return; + if (value === null || typeof value !== 'object') { + lines.push(`${path || 'value'}: ${String(value)}`); + return; + } + if (Array.isArray(value)) { + value.forEach((item, i) => flatten(item, `${path}[${i}]`, lines, depth + 1)); + return; + } + for (const [key, child] of Object.entries(value as Record)) { + flatten(child, path ? `${path}.${key}` : key, lines, depth + 1); + } +} + +export const jsonParser: DocumentParser = { + name: 'json', + mimeTypes: ['application/json'], + extensions: ['.json', '.jsonl', '.ndjson'], + async parse(buffer): Promise { + const raw = buffer.toString('utf8'); + const lines: string[] = []; + try { + const parsed: unknown = JSON.parse(raw); + flatten(parsed, '', lines, 0); + } catch { + // JSON Lines: one object per line. + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + try { + flatten(JSON.parse(line), '', lines, 0); + } catch { + lines.push(line); + } + } + } + return { text: lines.join('\n'), sections: [], metadata: {} }; + }, +}; diff --git a/packages/knowledge/src/parsers/markdown.parser.ts b/packages/knowledge/src/parsers/markdown.parser.ts new file mode 100644 index 0000000..d98d045 --- /dev/null +++ b/packages/knowledge/src/parsers/markdown.parser.ts @@ -0,0 +1,66 @@ +import { marked, type Token, type Tokens } from 'marked'; +import type { DocumentParser, DocumentSection, ParsedDocument } from '../types.js'; + +function tokenToText(token: Token): string { + switch (token.type) { + case 'heading': + case 'paragraph': + case 'text': + return 'text' in token ? String(token.text) : ''; + case 'code': + return token.text; + case 'list': + return (token as Tokens.List).items.map((item) => `- ${item.text}`).join('\n'); + case 'table': { + const table = token as Tokens.Table; + const header = table.header.map((cell) => cell.text).join(' | '); + const rows = table.rows.map((row) => row.map((cell) => cell.text).join(' | ')); + return [header, ...rows].join('\n'); + } + case 'blockquote': + return 'text' in token ? String(token.text) : ''; + default: + return 'raw' in token ? '' : ''; + } +} + +/** + * Markdown parser built on marked's lexer: converts block tokens to plain + * text while recording heading structure and table count. + */ +export const markdownParser: DocumentParser = { + name: 'markdown', + mimeTypes: ['text/markdown', 'text/x-markdown'], + extensions: ['.md', '.markdown', '.mdx'], + async parse(buffer): Promise { + const tokens = marked.lexer(buffer.toString('utf8')); + const parts: string[] = []; + const headings: Array<{ heading: string; level: number; offset: number }> = []; + let tableCount = 0; + let offset = 0; + let title: string | undefined; + + for (const token of tokens) { + if (token.type === 'space') continue; + if (token.type === 'table') tableCount += 1; + const text = tokenToText(token).trim(); + if (!text) continue; + if (token.type === 'heading') { + headings.push({ heading: text, level: token.depth, offset }); + if (!title && token.depth === 1) title = text; + } + parts.push(text); + offset += text.length + 2; // joined with \n\n below + } + + const fullText = parts.join('\n\n'); + const sections: DocumentSection[] = headings.map((h, i) => ({ + heading: h.heading, + level: h.level, + startOffset: h.offset, + endOffset: headings[i + 1]?.offset ?? fullText.length, + })); + + return { text: fullText, sections, metadata: { title, tableCount } }; + }, +}; diff --git a/packages/knowledge/src/parsers/pdf.parser.ts b/packages/knowledge/src/parsers/pdf.parser.ts new file mode 100644 index 0000000..8e30d04 --- /dev/null +++ b/packages/knowledge/src/parsers/pdf.parser.ts @@ -0,0 +1,29 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference -- ambient module decl must be pulled into dependents' programs +/// +import pdfParse from 'pdf-parse/lib/pdf-parse.js'; +import type { DocumentParser, ParsedDocument } from '../types.js'; + +/** + * PDF parser via pdf-parse (pdf.js under the hood). PDFs carry no reliable + * structural headings, so sections come from the text-level heuristics in + * the chunker; we surface document info (title/author/dates) and page count. + */ +export const pdfParser: DocumentParser = { + name: 'pdf', + mimeTypes: ['application/pdf'], + extensions: ['.pdf'], + async parse(buffer): Promise { + const result = await pdfParse(buffer); + const info = (result.info ?? {}) as Record; + return { + text: result.text, + sections: [], + metadata: { + title: typeof info.Title === 'string' && info.Title.trim() ? info.Title : undefined, + author: typeof info.Author === 'string' && info.Author.trim() ? info.Author : undefined, + creationDate: typeof info.CreationDate === 'string' ? info.CreationDate : undefined, + pageCount: result.numpages, + }, + }; + }, +}; diff --git a/packages/knowledge/src/parsers/text.parser.ts b/packages/knowledge/src/parsers/text.parser.ts new file mode 100644 index 0000000..5f31372 --- /dev/null +++ b/packages/knowledge/src/parsers/text.parser.ts @@ -0,0 +1,40 @@ +import type { DocumentParser, DocumentSection, ParsedDocument } from '../types.js'; + +/** + * Heading heuristic for plain text: short lines in ALL CAPS or with + * "1." / "1.2" numeric prefixes followed by a blank line. + */ +function detectSections(text: string): DocumentSection[] { + const sections: DocumentSection[] = []; + const lines = text.split('\n'); + let offset = 0; + const candidates: Array<{ heading: string; offset: number }> = []; + + for (const line of lines) { + const trimmed = line.trim(); + const isCaps = /^[A-Z][A-Z0-9 .,'&-]{3,60}$/.test(trimmed) && trimmed === trimmed.toUpperCase(); + const isNumbered = /^\d+(\.\d+)*[.)]?\s+\S{3,}/.test(trimmed) && trimmed.length <= 80; + if (isCaps || isNumbered) candidates.push({ heading: trimmed, offset }); + offset += line.length + 1; + } + for (let i = 0; i < candidates.length; i += 1) { + const current = candidates[i]!; + sections.push({ + heading: current.heading, + level: 1, + startOffset: current.offset, + endOffset: candidates[i + 1]?.offset ?? text.length, + }); + } + return sections; +} + +export const textParser: DocumentParser = { + name: 'text', + mimeTypes: ['text/plain'], + extensions: ['.txt', '.text', '.log'], + async parse(buffer): Promise { + const text = buffer.toString('utf8'); + return { text, sections: detectSections(text), metadata: {} }; + }, +}; diff --git a/packages/knowledge/src/pdf-parse.d.ts b/packages/knowledge/src/pdf-parse.d.ts new file mode 100644 index 0000000..78f92c5 --- /dev/null +++ b/packages/knowledge/src/pdf-parse.d.ts @@ -0,0 +1,17 @@ +// The pdf-parse root entry runs debug code when imported directly, so we +// import the library file — which ships no types. Minimal declaration here. +declare module 'pdf-parse/lib/pdf-parse.js' { + interface PdfParseResult { + numpages: number; + numrender: number; + info: Record | null; + metadata: unknown; + text: string; + version: string; + } + function pdfParse( + data: Buffer | Uint8Array, + options?: Record, + ): Promise; + export default pdfParse; +} diff --git a/packages/knowledge/src/tokens.ts b/packages/knowledge/src/tokens.ts new file mode 100644 index 0000000..c5ffb32 --- /dev/null +++ b/packages/knowledge/src/tokens.ts @@ -0,0 +1,10 @@ +/** + * Fast token estimate (≈ GPT-family BPE): mean of the character-based + * (chars/4) and word-based (words×1.33) heuristics. Good enough for chunk + * budgeting; exact counts belong to the embedding provider. + */ +export function estimateTokens(text: string): number { + if (!text) return 0; + const words = text.split(/\s+/).filter(Boolean).length; + return Math.max(1, Math.ceil((text.length / 4 + words * 1.33) / 2)); +} diff --git a/packages/knowledge/src/types.ts b/packages/knowledge/src/types.ts new file mode 100644 index 0000000..ab252d9 --- /dev/null +++ b/packages/knowledge/src/types.ts @@ -0,0 +1,71 @@ +/** A heading-delimited region of the parsed document. */ +export interface DocumentSection { + heading: string; + /** Heading depth, 1 = top level. */ + level: number; + /** Character offsets into the cleaned text. */ + startOffset: number; + endOffset: number; +} + +/** Metadata a parser could recover from the raw file. */ +export interface ExtractedMetadata { + title?: string; + author?: string; + creationDate?: string; + language?: string; + pageCount?: number; + tableCount?: number; + headings?: string[]; + keywords?: string[]; + [key: string]: unknown; +} + +/** Uniform output of every parser. */ +export interface ParsedDocument { + /** Linear plain text (pre-clean; the pipeline cleans it afterwards). */ + text: string; + sections: DocumentSection[]; + metadata: ExtractedMetadata; +} + +export interface ParseContext { + fileName: string; + mimeType: string; +} + +/** Pluggable parser contract — register implementations in parsers/index.ts. */ +export interface DocumentParser { + name: string; + mimeTypes: string[]; + extensions: string[]; + parse(buffer: Buffer, context: ParseContext): Promise; +} + +export interface ChunkOptions { + /** Target chunk size in tokens (approximate). */ + chunkSize: number; + /** Tokens carried over from the previous chunk. */ + chunkOverlap: number; + /** Never emit a chunk above this hard token ceiling. */ + maxTokens: number; + /** Start a new chunk at section/heading boundaries. */ + respectSections: boolean; +} + +export const DEFAULT_CHUNK_OPTIONS: ChunkOptions = { + chunkSize: 400, + chunkOverlap: 60, + maxTokens: 512, + respectSections: true, +}; + +export interface TextChunk { + index: number; + content: string; + tokenCount: number; + heading: string | null; + section: string | null; + startOffset: number; + endOffset: number; +} diff --git a/packages/knowledge/tsconfig.json b/packages/knowledge/tsconfig.json new file mode 100644 index 0000000..7bfd764 --- /dev/null +++ b/packages/knowledge/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@company-brain/config/tsconfig/node", + "include": ["src"] +} diff --git a/packages/types/src/auth.ts b/packages/types/src/auth.ts index 29e5902..c783cc0 100644 --- a/packages/types/src/auth.ts +++ b/packages/types/src/auth.ts @@ -14,6 +14,11 @@ export const PERMISSIONS = [ 'user:read', 'apikey:manage', 'audit:read', + 'knowledge:read', + 'knowledge:write', + 'knowledge:manage', + 'connector:read', + 'connector:manage', ] as const; export type Permission = (typeof PERMISSIONS)[number]; @@ -27,10 +32,33 @@ export const ROLE_PERMISSIONS: Record = { 'user:read', 'apikey:manage', 'audit:read', + 'knowledge:read', + 'knowledge:write', + 'knowledge:manage', + 'connector:read', + 'connector:manage', ], - MANAGER: ['org:read', 'project:manage', 'project:read', 'user:read', 'audit:read'], - EMPLOYEE: ['org:read', 'project:read', 'user:read'], - SERVICE: ['org:read', 'project:read'], + MANAGER: [ + 'org:read', + 'project:manage', + 'project:read', + 'user:read', + 'audit:read', + 'knowledge:read', + 'knowledge:write', + 'knowledge:manage', + 'connector:read', + 'connector:manage', + ], + EMPLOYEE: [ + 'org:read', + 'project:read', + 'user:read', + 'knowledge:read', + 'knowledge:write', + 'connector:read', + ], + SERVICE: ['org:read', 'project:read', 'knowledge:read', 'connector:read'], }; export interface AccessTokenPayload { diff --git a/packages/workflows/src/connector-contract.ts b/packages/workflows/src/connector-contract.ts new file mode 100644 index 0000000..6236dfb --- /dev/null +++ b/packages/workflows/src/connector-contract.ts @@ -0,0 +1,61 @@ +/** + * Activity contract for connector sync workflows. The connector-worker + * implements this interface (checked with `satisfies`); workflows proxy + * against it. Keeping the contract here avoids a workflows → worker + * dependency while staying fully typed on both sides. + */ + +export interface StartSyncJobInput { + connectorId: string; + service: string; + type: 'INITIAL' | 'INCREMENTAL' | 'DISCOVERY' | 'PERMISSION' | 'MANUAL'; + workflowId: string; +} + +export interface SyncPageInput { + connectorId: string; + service: string; + jobId: string; + pageCursor: string | null; +} + +export interface SyncPageOutput { + nextPageCursor: string | null; + resourceCount: number; +} + +export interface CompleteSyncJobInput { + jobId: string; + connectorId: string; + status: 'COMPLETED' | 'FAILED' | 'PARTIAL'; + error?: string; +} + +export interface IncrementalSyncInput { + connectorId: string; + service: string; + jobId: string; +} + +export interface IncrementalSyncOutput { + changeCount: number; + /** No cursor stored yet — service still needs its initial full sync. */ + cursorMissing?: boolean; + /** Provider invalidated the cursor — schedule a full resync. */ + cursorExpired?: boolean; +} + +export interface DiscoverWorkspaceOutput { + domain: string | null; + adminEmail: string | null; + services: Record; +} + +export interface ConnectorActivitiesContract { + discoverWorkspace(input: { connectorId: string }): Promise; + startSyncJob(input: StartSyncJobInput): Promise<{ jobId: string }>; + syncServicePage(input: SyncPageInput): Promise; + completeSyncJob(input: CompleteSyncJobInput): Promise; + runIncrementalSync(input: IncrementalSyncInput): Promise; + markConnectorSynced(input: { connectorId: string; nextSyncInMinutes: number }): Promise; +} diff --git a/packages/workflows/src/constants.ts b/packages/workflows/src/constants.ts index 46f85b2..a22366f 100644 --- a/packages/workflows/src/constants.ts +++ b/packages/workflows/src/constants.ts @@ -5,6 +5,7 @@ */ export const TASK_QUEUES = { core: 'brain-core', + connectors: 'brain-connectors', } as const; export type TaskQueue = (typeof TASK_QUEUES)[keyof typeof TASK_QUEUES]; @@ -13,5 +14,15 @@ export const WORKFLOW_TYPES = { hello: 'helloWorkflow', healthCheck: 'healthCheckWorkflow', storage: 'storageWorkflow', + documentIngestion: 'documentIngestionWorkflow', + workspaceInitialSync: 'workspaceInitialSyncWorkflow', + incrementalSync: 'incrementalSyncWorkflow', + driveSync: 'driveSyncWorkflow', + docsSync: 'docsSyncWorkflow', + sheetsSync: 'sheetsSyncWorkflow', + slidesSync: 'slidesSyncWorkflow', + emailSync: 'emailSyncWorkflow', + calendarSync: 'calendarSyncWorkflow', + permissionSync: 'permissionSyncWorkflow', } as const; export type WorkflowType = (typeof WORKFLOW_TYPES)[keyof typeof WORKFLOW_TYPES]; diff --git a/packages/workflows/src/definitions.ts b/packages/workflows/src/definitions.ts index 04fbe8e..369ed4c 100644 --- a/packages/workflows/src/definitions.ts +++ b/packages/workflows/src/definitions.ts @@ -14,3 +14,24 @@ export const getStatusQuery = defineQuery('getStatus'); /** healthCheckWorkflow — latest report (null until the first check lands). */ export const getReportQuery = defineQuery('getReport'); + +/** documentIngestionWorkflow — live pipeline progress. */ +export interface IngestionProgress { + documentId: string; + stage: 'VALIDATE' | 'PARSE' | 'CHUNK' | 'EMBED' | 'COMPLETE'; + chunkCount: number; + embeddingCount: number; + error: string | null; +} +export const getIngestionProgressQuery = defineQuery('getIngestionProgress'); + +/** connector sync workflows — live page/resource counters. */ +export interface ConnectorSyncProgress { + connectorId: string; + service: string; + pages: number; + resources: number; + done: boolean; + error: string | null; +} +export const getSyncProgressQuery = defineQuery('getSyncProgress'); diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index d5dfd74..bf156db 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -7,7 +7,45 @@ export { helloWorkflow, type HelloWorkflowResult } from './workflows/hello.workflow.js'; export { healthCheckWorkflow } from './workflows/health-check.workflow.js'; export { storageWorkflow } from './workflows/storage.workflow.js'; +export { + documentIngestionWorkflow, + type DocumentIngestionInput, + type DocumentIngestionResult, +} from './workflows/document-ingestion.workflow.js'; -export { skipDelaySignal, getStatusQuery, getReportQuery } from './definitions.js'; +export { + workspaceInitialSyncWorkflow, + incrementalSyncWorkflow, + driveSyncWorkflow, + docsSyncWorkflow, + sheetsSyncWorkflow, + slidesSyncWorkflow, + emailSyncWorkflow, + calendarSyncWorkflow, + permissionSyncWorkflow, + type ServiceSyncInput, + type ServiceSyncResult, + type WorkspaceInitialSyncResult, +} from './workflows/connector-sync.workflow.js'; +export type { + ConnectorActivitiesContract, + StartSyncJobInput, + SyncPageInput, + SyncPageOutput, + CompleteSyncJobInput, + IncrementalSyncInput, + IncrementalSyncOutput, + DiscoverWorkspaceOutput, +} from './connector-contract.js'; + +export { + skipDelaySignal, + getStatusQuery, + getReportQuery, + getIngestionProgressQuery, + getSyncProgressQuery, + type IngestionProgress, + type ConnectorSyncProgress, +} from './definitions.js'; export { TASK_QUEUES, WORKFLOW_TYPES, type TaskQueue, type WorkflowType } from './constants.js'; export { DEFAULT_RETRY_POLICY, QUICK_RETRY_POLICY } from './retry-policies.js'; diff --git a/packages/workflows/src/workflows/connector-sync.workflow.ts b/packages/workflows/src/workflows/connector-sync.workflow.ts new file mode 100644 index 0000000..cd411e9 --- /dev/null +++ b/packages/workflows/src/workflows/connector-sync.workflow.ts @@ -0,0 +1,234 @@ +import { executeChild, proxyActivities, setHandler, workflowInfo } from '@temporalio/workflow'; +import type { ConnectorActivitiesContract } from '../connector-contract.js'; +import { DEFAULT_RETRY_POLICY } from '../retry-policies.js'; +import { getSyncProgressQuery, type ConnectorSyncProgress } from '../definitions.js'; + +// Sync pages call external provider APIs: patient retries for rate limits, +// fail fast on revoked grants / expired cursors. +const activities = proxyActivities({ + startToCloseTimeout: '5 minutes', + retry: { + ...DEFAULT_RETRY_POLICY, + maximumAttempts: 6, + nonRetryableErrorTypes: ['TOKEN_EXPIRED', 'PERMISSION_DENIED', 'CURSOR_EXPIRED', 'NotFound'], + }, +}); + +const bookkeeping = proxyActivities({ + startToCloseTimeout: '30 seconds', + retry: { ...DEFAULT_RETRY_POLICY, maximumAttempts: 8 }, +}); + +export interface ServiceSyncInput { + connectorId: string; + type?: 'INITIAL' | 'MANUAL'; +} + +export interface ServiceSyncResult { + connectorId: string; + service: string; + status: 'COMPLETED' | 'FAILED'; + resourceCount: number; + error: string | null; +} + +/** + * Shared engine for every per-service full sync workflow: open a SyncJob, + * page through the connector's sync() until the cursor is exhausted, then + * close the job. Progress is queryable; every page is an independent + * retryable activity. + */ +async function runServiceSync( + service: string, + input: ServiceSyncInput, +): Promise { + const { workflowId } = workflowInfo(); + const progress: ConnectorSyncProgress = { + connectorId: input.connectorId, + service, + pages: 0, + resources: 0, + done: false, + error: null, + }; + setHandler(getSyncProgressQuery, () => progress); + + const { jobId } = await bookkeeping.startSyncJob({ + connectorId: input.connectorId, + service, + type: input.type ?? 'INITIAL', + workflowId, + }); + + try { + let pageCursor: string | null = null; + do { + const page: { nextPageCursor: string | null; resourceCount: number } = + await activities.syncServicePage({ + connectorId: input.connectorId, + service, + jobId, + pageCursor, + }); + pageCursor = page.nextPageCursor; + progress.pages += 1; + progress.resources += page.resourceCount; + } while (pageCursor !== null); + + await bookkeeping.completeSyncJob({ + jobId, + connectorId: input.connectorId, + status: 'COMPLETED', + }); + progress.done = true; + return { + connectorId: input.connectorId, + service, + status: 'COMPLETED', + resourceCount: progress.resources, + error: null, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + progress.error = message; + await bookkeeping.completeSyncJob({ + jobId, + connectorId: input.connectorId, + status: 'FAILED', + error: message, + }); + return { + connectorId: input.connectorId, + service, + status: 'FAILED', + resourceCount: progress.resources, + error: message, + }; + } +} + +// ── Per-service workflows (independent + individually retryable) ── + +export async function driveSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('drive', input); +} +export async function docsSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('docs', input); +} +export async function sheetsSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('sheets', input); +} +export async function slidesSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('slides', input); +} +export async function emailSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('gmail', input); +} +export async function calendarSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('calendar', input); +} +export async function permissionSyncWorkflow(input: ServiceSyncInput): Promise { + return runServiceSync('permissions', input); +} + +// ── Orchestrators ───────────────────────────────────────────────── + +export interface WorkspaceInitialSyncResult { + connectorId: string; + services: ServiceSyncResult[]; +} + +/** + * Runs after OAuth connect: identifies the workspace, then fans out one + * child workflow per service. Children are independent — one failing + * service (e.g. Gmail disabled) never blocks the others. + */ +export async function workspaceInitialSyncWorkflow(input: { + connectorId: string; +}): Promise { + await activities.discoverWorkspace({ connectorId: input.connectorId }); + + const { workflowId } = workflowInfo(); + const children = [ + driveSyncWorkflow, + docsSyncWorkflow, + sheetsSyncWorkflow, + slidesSyncWorkflow, + emailSyncWorkflow, + calendarSyncWorkflow, + permissionSyncWorkflow, + ].map((child) => + executeChild(child, { + args: [{ connectorId: input.connectorId, type: 'INITIAL' as const }], + workflowId: `${workflowId}-${child.name}`, + }), + ); + + const settled = await Promise.allSettled(children); + const services = settled.map((result, i) => + result.status === 'fulfilled' + ? result.value + : { + connectorId: input.connectorId, + service: ['drive', 'docs', 'sheets', 'slides', 'gmail', 'calendar', 'permissions'][i]!, + status: 'FAILED' as const, + resourceCount: 0, + error: String(result.reason), + }, + ); + + await bookkeeping.markConnectorSynced({ connectorId: input.connectorId, nextSyncInMinutes: 15 }); + return { connectorId: input.connectorId, services }; +} + +/** + * Cron-scheduled change detection: consumes provider change feeds + * (Drive Changes API, Gmail History API, Calendar syncTokens) using the + * stored cursors. Never re-downloads everything. + */ +export async function incrementalSyncWorkflow(input: { connectorId: string }): Promise<{ + connectorId: string; + changes: Record; +}> { + const { workflowId } = workflowInfo(); + const changes: Record = {}; + + // Drive change feed covers docs/sheets/slides/permissions too. + for (const service of ['drive', 'gmail', 'calendar']) { + const { jobId } = await bookkeeping.startSyncJob({ + connectorId: input.connectorId, + service, + type: 'INCREMENTAL', + workflowId: `${workflowId}-${service}`, + }); + try { + const result = await activities.runIncrementalSync({ + connectorId: input.connectorId, + service, + jobId, + }); + changes[service] = result.changeCount; + await bookkeeping.completeSyncJob({ + jobId, + connectorId: input.connectorId, + status: result.cursorExpired || result.cursorMissing ? 'PARTIAL' : 'COMPLETED', + error: result.cursorExpired + ? 'cursor expired — full resync required' + : result.cursorMissing + ? 'no cursor — initial sync has not completed' + : undefined, + }); + } catch (error) { + changes[service] = 0; + await bookkeeping.completeSyncJob({ + jobId, + connectorId: input.connectorId, + status: 'FAILED', + error: error instanceof Error ? error.message : String(error), + }); + } + } + + await bookkeeping.markConnectorSynced({ connectorId: input.connectorId, nextSyncInMinutes: 15 }); + return { connectorId: input.connectorId, changes }; +} diff --git a/packages/workflows/src/workflows/document-ingestion.workflow.ts b/packages/workflows/src/workflows/document-ingestion.workflow.ts new file mode 100644 index 0000000..b5a5ac5 --- /dev/null +++ b/packages/workflows/src/workflows/document-ingestion.workflow.ts @@ -0,0 +1,109 @@ +import { proxyActivities, setHandler, workflowInfo, log } from '@temporalio/workflow'; +import type { KnowledgeActivities } from '@company-brain/activities'; +import { DEFAULT_RETRY_POLICY } from '../retry-policies.js'; +import { getIngestionProgressQuery, type IngestionProgress } from '../definitions.js'; + +// Parsing can be CPU-heavy on large files; embedding may call slow external +// APIs — give the pipeline generous per-attempt budgets and let Temporal +// retry transient failures with backoff. +const pipeline = proxyActivities({ + startToCloseTimeout: '5 minutes', + retry: { + ...DEFAULT_RETRY_POLICY, + nonRetryableErrorTypes: [ + 'NotFound', + 'NoVersion', + 'UnsupportedType', + 'EmptyFile', + 'EmptyText', + 'NoChunks', + ], + }, +}); + +const finalize = proxyActivities({ + startToCloseTimeout: '30 seconds', + retry: { ...DEFAULT_RETRY_POLICY, maximumAttempts: 8 }, +}); + +export interface DocumentIngestionInput { + documentId: string; +} + +export interface DocumentIngestionResult { + documentId: string; + status: 'READY' | 'FAILED'; + chunkCount: number; + embeddingCount: number; + collection: string | null; + error: string | null; +} + +/** + * Ingestion pipeline: + * VALIDATE → PARSE → CLEAN → CHUNK → METADATA → EMBED → INDEX → PERSIST → COMPLETE. + * + * Every stage is a retryable activity; the workflow itself only sequences + * them and records progress (queryable via getIngestionProgress). A failed + * stage — after retries exhaust — still runs finalizeIngestion so the + * document and its ProcessingJob always land in a terminal state. + */ +export async function documentIngestionWorkflow( + input: DocumentIngestionInput, +): Promise { + const { workflowId } = workflowInfo(); + const io = { documentId: input.documentId, workflowId }; + + const progress: IngestionProgress = { + documentId: input.documentId, + stage: 'VALIDATE', + chunkCount: 0, + embeddingCount: 0, + error: null, + }; + setHandler(getIngestionProgressQuery, () => progress); + + try { + await pipeline.validateDocument(io); + + progress.stage = 'PARSE'; + const extracted = await pipeline.extractText(io); + + progress.stage = 'CHUNK'; + const { chunkCount } = await pipeline.chunkText({ + ...io, + textKey: extracted.textKey, + sectionsKey: extracted.sectionsKey, + }); + progress.chunkCount = chunkCount; + + progress.stage = 'EMBED'; + const { embeddingCount, collection } = await pipeline.embedAndIndexChunks(io); + progress.embeddingCount = embeddingCount; + + progress.stage = 'COMPLETE'; + await finalize.finalizeIngestion({ ...io, success: true, chunkCount, embeddingCount }); + + return { + documentId: input.documentId, + status: 'READY', + chunkCount, + embeddingCount, + collection, + error: null, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + progress.error = message; + log.error('document ingestion failed', { documentId: input.documentId, error: message }); + await finalize.finalizeIngestion({ ...io, success: false, error: message }); + return { + documentId: input.documentId, + status: 'FAILED', + chunkCount: progress.chunkCount, + embeddingCount: progress.embeddingCount, + collection: null, + error: message, + }; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5921bcc..488eff5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,21 @@ importers: '@company-brain/activities': specifier: workspace:* version: link:../../packages/activities + '@company-brain/auth': + specifier: workspace:* + version: link:../../packages/auth + '@company-brain/connector-core': + specifier: workspace:* + version: link:../../packages/connectors/core + '@company-brain/connector-google': + specifier: workspace:* + version: link:../../packages/connectors/google + '@company-brain/events': + specifier: workspace:* + version: link:../../packages/events + '@company-brain/knowledge': + specifier: workspace:* + version: link:../../packages/knowledge '@company-brain/types': specifier: workspace:* version: link:../../packages/types @@ -53,6 +68,9 @@ importers: '@fastify/helmet': specifier: ^13.0.1 version: 13.1.0 + '@fastify/multipart': + specifier: ^10.1.0 + version: 10.1.0 '@fastify/rate-limit': specifier: ^10.2.2 version: 10.3.0 @@ -187,6 +205,15 @@ importers: packages/activities: dependencies: + '@company-brain/knowledge': + specifier: workspace:* + version: link:../knowledge + '@prisma/client': + specifier: ^6.19.3 + version: 6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3) + '@qdrant/js-client-rest': + specifier: ^1.18.0 + version: 1.18.0(typescript@5.9.3) '@temporalio/activity': specifier: ^1.20.2 version: 1.20.2 @@ -210,6 +237,24 @@ importers: specifier: ^5.8.3 version: 5.9.3 + packages/auth: + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../config + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) + packages/config: dependencies: '@eslint/js': @@ -222,6 +267,108 @@ importers: specifier: ^8.30.0 version: 8.63.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + packages/connectors/core: + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../../config + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) + + packages/connectors/google: + dependencies: + '@company-brain/connector-core': + specifier: workspace:* + version: link:../core + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../../config + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) + + packages/events: + dependencies: + ioredis: + specifier: ^5.11.1 + version: 5.11.1 + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../config + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) + + packages/knowledge: + dependencies: + csv-parse: + specifier: ^5.6.0 + version: 5.6.0 + html-to-text: + specifier: ^9.0.5 + version: 9.0.5 + mammoth: + specifier: ^1.9.0 + version: 1.12.0 + marked: + specifier: ^15.0.7 + version: 15.0.12 + pdf-parse: + specifier: ^1.1.1 + version: 1.1.4 + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../config + '@types/html-to-text': + specifier: ^9.0.4 + version: 9.0.4 + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + '@types/pdf-parse': + specifier: ^1.1.4 + version: 1.1.5 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^3.1.1 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) + packages/types: devDependencies: '@company-brain/config': @@ -299,6 +446,64 @@ importers: specifier: ^5.8.3 version: 5.9.3 + services/connector-worker: + dependencies: + '@company-brain/auth': + specifier: workspace:* + version: link:../../packages/auth + '@company-brain/connector-core': + specifier: workspace:* + version: link:../../packages/connectors/core + '@company-brain/connector-google': + specifier: workspace:* + version: link:../../packages/connectors/google + '@company-brain/events': + specifier: workspace:* + version: link:../../packages/events + '@company-brain/workflows': + specifier: workspace:* + version: link:../../packages/workflows + '@prisma/client': + specifier: ^6.6.0 + version: 6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3) + '@temporalio/activity': + specifier: ^1.20.2 + version: 1.20.2 + '@temporalio/worker': + specifier: ^1.20.2 + version: 1.20.2(tslib@2.8.1) + dotenv: + specifier: ^16.5.0 + version: 16.6.1 + ioredis: + specifier: ^5.11.1 + version: 5.11.1 + pino: + specifier: ^9.6.0 + version: 9.14.0 + zod: + specifier: ^3.24.3 + version: 3.25.76 + devDependencies: + '@company-brain/config': + specifier: workspace:* + version: link:../../packages/config + '@types/node': + specifier: ^22.14.1 + version: 22.20.1 + eslint: + specifier: ^9.24.0 + version: 9.39.5(jiti@2.7.0) + pino-pretty: + specifier: ^13.0.0 + version: 13.1.3 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + typescript: + specifier: ^5.8.3 + version: 5.9.3 + services/temporal-worker: dependencies: '@company-brain/activities': @@ -665,6 +870,9 @@ packages: '@fastify/ajv-compiler@4.0.5': resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} + '@fastify/busboy@3.2.0': + resolution: {integrity: sha512-m9FVDXU3GT2ITSe0UaMA5rU3QkfC/UXtCU8y0gSN/GugTqtVldOBWIB5V6V3sbmenVZUIpU6f+mPEO2+m5iTaA==} + '@fastify/compress@8.3.1': resolution: {integrity: sha512-BUpItLr6MUX9e9ukg5Y6xekyA/7pBFG8QWtFCrUDm9ctoBc3R2/nA16yOaOWtVoccpXGjdDEYA/MxAb5+8cxag==} @@ -674,6 +882,9 @@ packages: '@fastify/cors@11.3.0': resolution: {integrity: sha512-ggQGua+xHv1MvePbPr0v//xLYEsCXbWspquXCJS9Ot5YoRXq8J8ZWzHnxDBVnbtXosvistXo6LtNzOJswf64Fw==} + '@fastify/deepmerge@3.2.1': + resolution: {integrity: sha512-N5Oqvltoa2r9z1tbx4xjky0oRR60v+T47Ic4J1ukoVQcptLOrIdRnCSdTGmOmajZuHVKlTnfcmrjyqsGEW1ztA==} + '@fastify/error@4.2.0': resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} @@ -689,6 +900,9 @@ packages: '@fastify/merge-json-schemas@0.2.1': resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} + '@fastify/multipart@10.1.0': + resolution: {integrity: sha512-b2r6CovmLQvLFJ5HJDtxVigZjcO9TwkRGslhiNQxsGJwilm5B3eqvXPOVLudC44zXMXJITpQ/iEATIE7zoXqcQ==} + '@fastify/proxy-addr@5.1.0': resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} @@ -1343,6 +1557,9 @@ packages: cpu: [x64] os: [win32] + '@selderee/plugin-htmlparser2@0.11.0': + resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -1486,6 +1703,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/html-to-text@9.0.4': + resolution: {integrity: sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ==} + '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -1498,6 +1718,9 @@ packages: '@types/node@22.20.1': resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==} + '@types/pdf-parse@1.1.5': + resolution: {integrity: sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==} + '@types/react-dom@19.2.3': resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} peerDependencies: @@ -1639,6 +1862,10 @@ packages: '@webassemblyjs/wast-printer@1.14.1': resolution: {integrity: sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==} + '@xmldom/xmldom@0.8.13': + resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} + engines: {node: '>=10.0.0'} + '@xtuc/ieee754@1.2.0': resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} @@ -1732,6 +1959,9 @@ packages: arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1781,6 +2011,9 @@ packages: block-stream2@2.1.0: resolution: {integrity: sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==} + bluebird@3.4.7: + resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} + brace-expansion@1.1.16: resolution: {integrity: sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==} @@ -2000,6 +2233,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + csv-parse@5.6.0: + resolution: {integrity: sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==} + dargs@8.1.0: resolution: {integrity: sha512-wAV9QHOsNbwnWdNW2FYvE1P56wtgSbM+3SZcdGiWQILwVjACCXDCI3Ai8QlCjMDB8YK5zySiXZYBiwGmNY3lnw==} engines: {node: '>=12'} @@ -2031,6 +2267,10 @@ packages: resolution: {integrity: sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==} engines: {node: '>=16.0.0'} + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + defu@6.1.7: resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} @@ -2056,9 +2296,25 @@ packages: didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} + dingbat-to-unicode@1.0.1: + resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==} + dlv@1.1.3: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + dot-prop@5.3.0: resolution: {integrity: sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==} engines: {node: '>=8'} @@ -2067,6 +2323,9 @@ packages: resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} engines: {node: '>=12'} + duck@0.1.12: + resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==} + duplexify@3.7.1: resolution: {integrity: sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==} @@ -2099,6 +2358,10 @@ packages: resolution: {integrity: sha512-rpsZEGT1jFuve6QlpyRp9ckQ+kN61hvF9BzCPyMdaKTm8UJce96KBn3sorXOFXlzjPrs3Vc4T1NsSroZ3PxlFw==} engines: {node: '>=10.13.0'} + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} @@ -2408,6 +2671,13 @@ packages: help-me@5.0.0: resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + html-to-text@9.0.5: + resolution: {integrity: sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==} + engines: {node: '>=14'} + + htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + http-errors@2.0.1: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} @@ -2440,6 +2710,9 @@ packages: resolution: {integrity: sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==} engines: {node: '>= 4'} + immediate@3.0.6: + resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -2582,6 +2855,9 @@ packages: resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} engines: {node: '>=12', npm: '>=6'} + jszip@3.10.1: + resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@2.0.1: resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} @@ -2591,10 +2867,16 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + leac@0.6.0: + resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lie@3.3.0: + resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} + light-my-request@6.6.0: resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} @@ -2681,6 +2963,9 @@ packages: long@5.3.2: resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lop@0.4.2: + resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==} + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} @@ -2700,6 +2985,16 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + mammoth@1.12.0: + resolution: {integrity: sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==} + engines: {node: '>=12.0.0'} + hasBin: true + + marked@15.0.12: + resolution: {integrity: sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==} + engines: {node: '>= 18'} + hasBin: true + memfs@4.64.0: resolution: {integrity: sha512-Kw72fgY7Wn+sD8KmtNWSafl1dz0UvAsE/PHs3YVfLiaZuA3HxNm9sRLqAu0ATiBGJvME1PxZXbBZPv5GycDeAw==} peerDependencies: @@ -2868,6 +3163,9 @@ packages: node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-ensure@0.0.0: + resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==} + node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} @@ -2921,6 +3219,9 @@ packages: openapi-types@12.1.3: resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + option@0.2.4: + resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -2941,6 +3242,9 @@ packages: resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + pako@1.0.11: + resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -2949,6 +3253,9 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + parseley@0.12.1: + resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -2961,6 +3268,10 @@ packages: resolution: {integrity: sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==} engines: {node: '>=14.0.0'} + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -2983,6 +3294,13 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pdf-parse@1.1.4: + resolution: {integrity: sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ==} + engines: {node: '>=6.8.1'} + + peberminta@0.9.0: + resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==} + peek-stream@1.1.3: resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} @@ -3271,6 +3589,9 @@ packages: secure-json-parse@4.1.0: resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + selderee@0.11.0: + resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==} + semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -3279,6 +3600,9 @@ packages: set-cookie-parser@2.7.2: resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + setimmediate@1.0.5: + resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -3341,6 +3665,9 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -3575,6 +3902,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + underscore@1.13.8: + resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -3729,6 +4059,10 @@ packages: resolution: {integrity: sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==} engines: {node: '>=4.0.0'} + xmlbuilder@10.1.1: + resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==} + engines: {node: '>=4.0'} + xmlbuilder@11.0.1: resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} engines: {node: '>=4.0'} @@ -4034,6 +4368,8 @@ snapshots: ajv-formats: 3.0.1(ajv@8.20.0) fast-uri: 3.1.3 + '@fastify/busboy@3.2.0': {} + '@fastify/compress@8.3.1': dependencies: '@fastify/accept-negotiator': 2.0.1 @@ -4055,6 +4391,8 @@ snapshots: fastify-plugin: 6.0.0 toad-cache: 3.7.4 + '@fastify/deepmerge@3.2.1': {} + '@fastify/error@4.2.0': {} '@fastify/fast-json-stringify-compiler@5.1.0': @@ -4072,6 +4410,14 @@ snapshots: dependencies: dequal: 2.0.3 + '@fastify/multipart@10.1.0': + dependencies: + '@fastify/busboy': 3.2.0 + '@fastify/deepmerge': 3.2.1 + '@fastify/error': 4.2.0 + fastify-plugin: 6.0.0 + secure-json-parse: 4.1.0 + '@fastify/proxy-addr@5.1.0': dependencies: '@fastify/forwarded': 3.0.1 @@ -4594,6 +4940,11 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@selderee/plugin-htmlparser2@0.11.0': + dependencies: + domhandler: 5.0.3 + selderee: 0.11.0 + '@standard-schema/spec@1.1.0': {} '@swc/core-darwin-arm64@1.15.43': @@ -4758,6 +5109,8 @@ snapshots: '@types/estree@1.0.9': {} + '@types/html-to-text@9.0.4': {} + '@types/json-schema@7.0.15': {} '@types/jsonwebtoken@9.0.10': @@ -4771,6 +5124,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/pdf-parse@1.1.5': + dependencies: + '@types/node': 22.20.1 + '@types/react-dom@19.2.3(@types/react@19.2.17)': dependencies: '@types/react': 19.2.17 @@ -4988,6 +5345,8 @@ snapshots: '@webassemblyjs/ast': 1.14.1 '@xtuc/long': 4.2.2 + '@xmldom/xmldom@0.8.13': {} + '@xtuc/ieee754@1.2.0': {} '@xtuc/long@4.2.2': {} @@ -5065,6 +5424,10 @@ snapshots: arg@5.0.2: {} + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} array-ify@1.0.0: {} @@ -5103,6 +5466,8 @@ snapshots: dependencies: readable-stream: 3.6.2 + bluebird@3.4.7: {} + brace-expansion@1.1.16: dependencies: balanced-match: 1.0.2 @@ -5315,6 +5680,8 @@ snapshots: csstype@3.2.3: {} + csv-parse@5.6.0: {} + dargs@8.1.0: {} dateformat@4.6.3: {} @@ -5331,6 +5698,8 @@ snapshots: deepmerge-ts@7.1.5: {} + deepmerge@4.3.1: {} + defu@6.1.7: {} denque@2.1.0: {} @@ -5346,14 +5715,38 @@ snapshots: didyoumean@1.2.2: {} + dingbat-to-unicode@1.0.1: {} + dlv@1.1.3: {} + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dot-prop@5.3.0: dependencies: is-obj: 2.0.0 dotenv@16.6.1: {} + duck@0.1.12: + dependencies: + underscore: 1.13.8 + duplexify@3.7.1: dependencies: end-of-stream: 1.4.5 @@ -5394,6 +5787,8 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.3.3 + entities@4.5.0: {} + env-paths@2.2.1: {} environment@1.1.0: {} @@ -5785,6 +6180,21 @@ snapshots: help-me@5.0.0: {} + html-to-text@9.0.5: + dependencies: + '@selderee/plugin-htmlparser2': 0.11.0 + deepmerge: 4.3.1 + dom-serializer: 2.0.0 + htmlparser2: 8.0.2 + selderee: 0.11.0 + + htmlparser2@8.0.2: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 4.5.0 + http-errors@2.0.1: dependencies: depd: 2.0.0 @@ -5809,6 +6219,8 @@ snapshots: ignore@7.0.6: {} + immediate@3.0.6: {} + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -5935,6 +6347,13 @@ snapshots: ms: 2.1.3 semver: 7.8.5 + jszip@3.10.1: + dependencies: + lie: 3.3.0 + pako: 1.0.11 + readable-stream: 2.3.8 + setimmediate: 1.0.5 + jwa@2.0.1: dependencies: buffer-equal-constant-time: 1.0.1 @@ -5950,11 +6369,17 @@ snapshots: dependencies: json-buffer: 3.0.1 + leac@0.6.0: {} + levn@0.4.1: dependencies: prelude-ls: 1.2.1 type-check: 0.4.0 + lie@3.3.0: + dependencies: + immediate: 3.0.6 + light-my-request@6.6.0: dependencies: cookie: 1.1.1 @@ -6041,6 +6466,12 @@ snapshots: long@5.3.2: {} + lop@0.4.2: + dependencies: + duck: 0.1.12 + option: 0.2.4 + underscore: 1.13.8 + loupe@3.2.1: {} lru-cache@11.5.2: {} @@ -6055,6 +6486,21 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + mammoth@1.12.0: + dependencies: + '@xmldom/xmldom': 0.8.13 + argparse: 1.0.10 + base64-js: 1.5.1 + bluebird: 3.4.7 + dingbat-to-unicode: 1.0.1 + jszip: 3.10.1 + lop: 0.4.2 + path-is-absolute: 1.0.1 + underscore: 1.13.8 + xmlbuilder: 10.1.1 + + marked@15.0.12: {} + memfs@4.64.0(tslib@2.8.1): dependencies: '@jsonjoy.com/fs-core': 4.64.0(tslib@2.8.1) @@ -6199,6 +6645,8 @@ snapshots: node-abort-controller@3.1.1: {} + node-ensure@0.0.0: {} + node-fetch-native@1.6.7: {} node-gyp-build-optional-packages@5.2.2: @@ -6242,6 +6690,8 @@ snapshots: openapi-types@12.1.3: {} + option@0.2.4: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -6267,6 +6717,8 @@ snapshots: dependencies: p-limit: 4.0.0 + pako@1.0.11: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -6278,12 +6730,19 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + parseley@0.12.1: + dependencies: + leac: 0.6.0 + peberminta: 0.9.0 + path-exists@4.0.0: {} path-exists@5.0.0: {} path-expression-matcher@1.6.2: {} + path-is-absolute@1.0.1: {} + path-key@3.1.1: {} path-key@4.0.0: {} @@ -6299,6 +6758,12 @@ snapshots: pathval@2.0.1: {} + pdf-parse@1.1.4: + dependencies: + node-ensure: 0.0.0 + + peberminta@0.9.0: {} + peek-stream@1.1.3: dependencies: buffer-from: 1.1.2 @@ -6619,10 +7084,16 @@ snapshots: secure-json-parse@4.1.0: {} + selderee@0.11.0: + dependencies: + parseley: 0.12.1 + semver@7.8.5: {} set-cookie-parser@2.7.2: {} + setimmediate@1.0.5: {} + setprototypeof@1.2.0: {} sharp@0.34.5: @@ -6702,6 +7173,8 @@ snapshots: split2@4.2.0: {} + sprintf-js@1.0.3: {} + stackback@0.0.2: {} standard-as-callback@2.1.0: {} @@ -6927,6 +7400,8 @@ snapshots: typescript@5.9.3: {} + underscore@1.13.8: {} + undici-types@6.21.0: {} undici@6.27.0: {} @@ -7105,6 +7580,8 @@ snapshots: sax: 1.6.0 xmlbuilder: 11.0.1 + xmlbuilder@10.1.1: {} + xmlbuilder@11.0.1: {} xtend@4.0.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f11f236..3947277 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - 'apps/*' - 'packages/*' + - 'packages/connectors/*' - 'services/*' # Single ioredis instance across the workspace so BullMQ and our own code diff --git a/services/connector-worker/eslint.config.mjs b/services/connector-worker/eslint.config.mjs new file mode 100644 index 0000000..6163fbb --- /dev/null +++ b/services/connector-worker/eslint.config.mjs @@ -0,0 +1,3 @@ +import { nodeConfig } from '@company-brain/config/eslint'; + +export default nodeConfig; diff --git a/services/connector-worker/package.json b/services/connector-worker/package.json new file mode 100644 index 0000000..63ca988 --- /dev/null +++ b/services/connector-worker/package.json @@ -0,0 +1,36 @@ +{ + "name": "@company-brain/connector-worker", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/index.ts", + "build": "tsc -p tsconfig.build.json", + "start": "node dist/index.js", + "lint": "eslint src", + "typecheck": "tsc --noEmit", + "test": "echo 'no tests' && exit 0" + }, + "dependencies": { + "@company-brain/auth": "workspace:*", + "@company-brain/connector-core": "workspace:*", + "@company-brain/connector-google": "workspace:*", + "@company-brain/events": "workspace:*", + "@company-brain/workflows": "workspace:*", + "@prisma/client": "^6.6.0", + "@temporalio/activity": "^1.20.2", + "@temporalio/worker": "^1.20.2", + "dotenv": "^16.5.0", + "ioredis": "^5.6.1", + "pino": "^9.6.0", + "zod": "^3.24.3" + }, + "devDependencies": { + "@company-brain/config": "workspace:*", + "@types/node": "^22.14.1", + "eslint": "^9.24.0", + "pino-pretty": "^13.0.0", + "tsx": "^4.19.3", + "typescript": "^5.8.3" + } +} diff --git a/services/connector-worker/src/activities.ts b/services/connector-worker/src/activities.ts new file mode 100644 index 0000000..4b12081 --- /dev/null +++ b/services/connector-worker/src/activities.ts @@ -0,0 +1,538 @@ +import { ApplicationFailure } from '@temporalio/activity'; +import type { Prisma } from '@prisma/client'; +import { + ConnectorError, + CursorExpiredError, + type ResourceChange, + type SyncedResource, +} from '@company-brain/connector-core'; +import type { EventType } from '@company-brain/events'; +import type { + CompleteSyncJobInput, + ConnectorActivitiesContract, + DiscoverWorkspaceOutput, + IncrementalSyncInput, + IncrementalSyncOutput, + StartSyncJobInput, + SyncPageInput, + SyncPageOutput, +} from '@company-brain/workflows'; +import { PROVIDER_IDS, type WorkerContext } from './context.js'; + +/** Services that own an incremental cursor (drive feed covers doc views). */ +const CURSOR_SERVICES = new Set(['drive', 'gmail', 'calendar']); + +function eventTypeFor(resourceType: string, kind: ResourceChange['changeType']): EventType { + if (kind === 'permission_changed') return 'resource.permission.changed'; + if (resourceType === 'EMAIL' || resourceType === 'EMAIL_THREAD') return 'resource.email.received'; + if (resourceType === 'CALENDAR' || resourceType === 'CALENDAR_EVENT') + return 'resource.calendar.updated'; + if (resourceType === 'GOOGLE_SHEET' && kind === 'updated') return 'resource.sheet.updated'; + if (resourceType === 'GOOGLE_SLIDES' && kind === 'updated') return 'resource.slides.updated'; + if (resourceType === 'GOOGLE_DOC') { + return kind === 'created' + ? 'resource.document.created' + : kind === 'deleted' + ? 'resource.document.deleted' + : 'resource.document.updated'; + } + return kind === 'created' + ? 'resource.file.created' + : kind === 'deleted' + ? 'resource.file.deleted' + : 'resource.file.updated'; +} + +/** Map SDK errors onto Temporal failures with the right retry semantics. */ +function toTemporalFailure(error: unknown): never { + if (error instanceof ConnectorError && !error.retryable) { + throw ApplicationFailure.nonRetryable(error.message, error.code); + } + throw error; +} + +export function createConnectorActivities(ctx: WorkerContext) { + const { prisma, events, registry, tokens } = ctx; + + async function requireConnector(connectorId: string) { + const connector = await prisma.connector.findFirst({ + where: { id: connectorId, deletedAt: null }, + }); + if (!connector) { + throw ApplicationFailure.nonRetryable(`Connector ${connectorId} not found`, 'NotFound'); + } + return connector; + } + + function providerConnector(providerEnum: string) { + const providerId = PROVIDER_IDS[providerEnum]; + if (!providerId) { + throw ApplicationFailure.nonRetryable(`No implementation for ${providerEnum}`, 'NotFound'); + } + return registry.get(providerId); + } + + async function logEvent( + connectorId: string, + organizationId: string, + level: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR', + event: string, + message: string, + context?: Record, + ): Promise { + await prisma.connectorLog.create({ + data: { + connectorId, + organizationId, + level, + event, + message, + context: (context ?? {}) as Prisma.InputJsonValue, + }, + }); + } + + interface PersistCounts { + created: number; + updated: number; + permissionChanges: number; + unchanged: number; + } + + /** Idempotent metadata upsert + permission diff + version + events. */ + async function persistResource( + connector: { id: string; organizationId: string; provider: string }, + service: string, + resource: SyncedResource, + counts: PersistCounts, + emitEvents: boolean, + ): Promise { + const providerId = PROVIDER_IDS[connector.provider] ?? connector.provider; + const existing = await prisma.externalResource.findUnique({ + where: { + connectorId_externalId: { connectorId: connector.id, externalId: resource.externalId }, + }, + include: { permissions: { where: { deletedAt: null } } }, + }); + + const data = { + type: resource.type as never, + status: (resource.trashed ? 'TRASHED' : 'ACTIVE') as never, + title: resource.title ?? null, + mimeType: resource.mimeType ?? null, + url: resource.url ?? null, + ownerEmail: resource.ownerEmail ?? null, + parentExternalId: resource.parentExternalId ?? null, + driveId: resource.driveId ?? null, + sizeBytes: resource.sizeBytes != null ? BigInt(resource.sizeBytes) : null, + checksum: resource.checksum ?? null, + version: resource.version ?? null, + externalCreatedAt: resource.externalCreatedAt ? new Date(resource.externalCreatedAt) : null, + externalUpdatedAt: resource.externalUpdatedAt ? new Date(resource.externalUpdatedAt) : null, + metadata: (resource.metadata ?? {}) as Prisma.InputJsonValue, + deletedAt: null, + }; + + let row = existing; + let changeKind: ResourceChange['changeType'] | null = null; + + if (!existing) { + row = await prisma.externalResource.create({ + data: { + ...data, + connectorId: connector.id, + externalId: resource.externalId, + organizationId: connector.organizationId, + }, + include: { permissions: true }, + }); + changeKind = 'created'; + counts.created += 1; + } else { + const changed = + existing.version !== data.version || + existing.checksum !== data.checksum || + existing.title !== data.title || + existing.status !== data.status || + (existing.externalUpdatedAt?.getTime() ?? 0) !== (data.externalUpdatedAt?.getTime() ?? 0); + if (changed) { + row = await prisma.externalResource.update({ + where: { id: existing.id }, + data, + include: { permissions: { where: { deletedAt: null } } }, + }); + changeKind = 'updated'; + counts.updated += 1; + } else { + counts.unchanged += 1; + } + } + + // Version history. + if (resource.version && row) { + await prisma.resourceVersion.upsert({ + where: { resourceId_version: { resourceId: row.id, version: resource.version } }, + create: { + resourceId: row.id, + version: resource.version, + checksum: resource.checksum ?? null, + sizeBytes: resource.sizeBytes != null ? BigInt(resource.sizeBytes) : null, + modifiedByEmail: (resource.metadata?.lastModifiedBy as string | undefined) ?? null, + externalModifiedAt: resource.externalUpdatedAt + ? new Date(resource.externalUpdatedAt) + : null, + organizationId: connector.organizationId, + }, + update: {}, + }); + } + + // Permission diff (only when the connector supplied permissions). + if (resource.permissions && row) { + const nextKeys = new Set( + resource.permissions.map( + (p) => + `${p.externalPermissionId ?? ''}|${p.principalType}|${p.principalEmail ?? p.domain ?? ''}|${p.role}`, + ), + ); + const currentKeys = new Set( + (existing?.permissions ?? []).map( + (p) => + `${p.externalPermissionId ?? ''}|${p.principalType}|${p.principalEmail ?? p.domain ?? ''}|${p.role}`, + ), + ); + const differs = + nextKeys.size !== currentKeys.size || [...nextKeys].some((k) => !currentKeys.has(k)); + if (differs) { + await prisma.$transaction([ + prisma.resourcePermission.deleteMany({ where: { resourceId: row.id } }), + prisma.resourcePermission.createMany({ + data: resource.permissions.map((p) => ({ + resourceId: row!.id, + externalPermissionId: p.externalPermissionId ?? null, + principalType: p.principalType, + principalEmail: p.principalEmail ?? null, + domain: p.domain ?? null, + role: p.role, + organizationId: connector.organizationId, + })), + }), + ]); + if (existing) { + counts.permissionChanges += 1; + if (!changeKind) changeKind = 'permission_changed'; + } + } + } + + if (changeKind && row) { + await prisma.externalChange.create({ + data: { + connectorId: connector.id, + resourceId: row.id, + externalId: resource.externalId, + service, + changeType: (changeKind === 'permission_changed' + ? 'PERMISSION_CHANGED' + : changeKind.toUpperCase()) as never, + organizationId: connector.organizationId, + payload: { title: resource.title, type: resource.type } as Prisma.InputJsonValue, + }, + }); + if (emitEvents) { + await events.publish({ + type: eventTypeFor(resource.type, changeKind), + organizationId: connector.organizationId, + connectorId: connector.id, + provider: providerId, + resource: { externalId: resource.externalId, type: resource.type, title: resource.title }, + }); + } + } + } + + async function markResourceDeleted( + connector: { id: string; organizationId: string; provider: string }, + service: string, + externalId: string, + ): Promise { + const existing = await prisma.externalResource.findUnique({ + where: { connectorId_externalId: { connectorId: connector.id, externalId } }, + }); + if (!existing || existing.status === 'DELETED') return; + await prisma.externalResource.update({ + where: { id: existing.id }, + data: { status: 'DELETED', deletedAt: new Date() }, + }); + await prisma.externalChange.create({ + data: { + connectorId: connector.id, + resourceId: existing.id, + externalId, + service, + changeType: 'DELETED', + organizationId: connector.organizationId, + }, + }); + await events.publish({ + type: eventTypeFor(existing.type, 'deleted'), + organizationId: connector.organizationId, + connectorId: connector.id, + provider: PROVIDER_IDS[connector.provider] ?? connector.provider, + resource: { externalId, type: existing.type, title: existing.title }, + }); + } + + const activities = { + async discoverWorkspace(input: { connectorId: string }): Promise { + const connector = await requireConnector(input.connectorId); + const impl = providerConnector(connector.provider); + const cctx = ctx.connectorContext(connector.id, connector.organizationId); + + try { + const discovery = await impl.discover(cctx); + await prisma.workspace.upsert({ + where: { connectorId: connector.id }, + create: { + connectorId: connector.id, + organizationId: connector.organizationId, + externalId: discovery.workspace.externalId ?? null, + domain: discovery.workspace.domain ?? null, + name: discovery.workspace.name ?? null, + adminEmail: discovery.workspace.adminEmail ?? null, + metadata: (discovery.workspace.metadata ?? {}) as Prisma.InputJsonValue, + }, + update: { + externalId: discovery.workspace.externalId ?? null, + domain: discovery.workspace.domain ?? null, + name: discovery.workspace.name ?? null, + adminEmail: discovery.workspace.adminEmail ?? null, + metadata: (discovery.workspace.metadata ?? {}) as Prisma.InputJsonValue, + }, + }); + await prisma.connector.update({ + where: { id: connector.id }, + data: { status: 'SYNCING', error: null }, + }); + await logEvent( + connector.id, + connector.organizationId, + 'INFO', + 'discovery.completed', + `workspace discovered: ${discovery.workspace.domain ?? 'unknown domain'}`, + { services: discovery.services }, + ); + return { + domain: discovery.workspace.domain ?? null, + adminEmail: discovery.workspace.adminEmail ?? null, + services: Object.fromEntries( + Object.entries(discovery.services).map(([k, v]) => [k, v.available]), + ), + }; + } catch (error) { + await logEvent( + connector.id, + connector.organizationId, + 'ERROR', + 'discovery.failed', + (error as Error).message, + ); + toTemporalFailure(error); + } + }, + + async startSyncJob(input: StartSyncJobInput): Promise<{ jobId: string }> { + const connector = await requireConnector(input.connectorId); + const job = await prisma.syncJob.create({ + data: { + connectorId: connector.id, + organizationId: connector.organizationId, + type: input.type, + service: input.service, + status: 'RUNNING', + workflowId: input.workflowId, + startedAt: new Date(), + stats: { discovered: 0, created: 0, updated: 0, deleted: 0, permissionChanges: 0 }, + }, + }); + await events.publish({ + type: 'sync.started', + organizationId: connector.organizationId, + connectorId: connector.id, + provider: PROVIDER_IDS[connector.provider] ?? connector.provider, + payload: { service: input.service, jobType: input.type, jobId: job.id }, + }); + return { jobId: job.id }; + }, + + async syncServicePage(input: SyncPageInput): Promise { + const connector = await requireConnector(input.connectorId); + const impl = providerConnector(connector.provider); + const cctx = ctx.connectorContext(connector.id, connector.organizationId); + + try { + const page = await impl.sync(cctx, input.service, input.pageCursor); + const counts = { created: 0, updated: 0, permissionChanges: 0, unchanged: 0 }; + for (const resource of page.resources) { + await persistResource(connector, input.service, resource, counts, true); + } + + // Store the incremental anchor delivered with the last page. + if (page.incrementalCursor && CURSOR_SERVICES.has(input.service)) { + await prisma.syncCursor.upsert({ + where: { + connectorId_service_resourceScope: { + connectorId: connector.id, + service: input.service, + resourceScope: '', + }, + }, + create: { + connectorId: connector.id, + organizationId: connector.organizationId, + service: input.service, + resourceScope: '', + cursor: page.incrementalCursor, + }, + update: { cursor: page.incrementalCursor }, + }); + } + + // Accumulate job stats. + const job = await prisma.syncJob.findUnique({ where: { id: input.jobId } }); + const stats = (job?.stats ?? {}) as Record; + await prisma.syncJob.update({ + where: { id: input.jobId }, + data: { + stats: { + discovered: (stats.discovered ?? 0) + page.resources.length, + created: (stats.created ?? 0) + counts.created, + updated: (stats.updated ?? 0) + counts.updated, + deleted: stats.deleted ?? 0, + permissionChanges: (stats.permissionChanges ?? 0) + counts.permissionChanges, + } as Prisma.InputJsonValue, + }, + }); + + return { nextPageCursor: page.nextPageCursor, resourceCount: page.resources.length }; + } catch (error) { + await logEvent( + connector.id, + connector.organizationId, + 'ERROR', + `sync.${input.service}.page_failed`, + (error as Error).message, + { pageCursor: input.pageCursor }, + ); + toTemporalFailure(error); + } + }, + + async completeSyncJob(input: CompleteSyncJobInput): Promise { + const connector = await requireConnector(input.connectorId); + const job = await prisma.syncJob.update({ + where: { id: input.jobId }, + data: { status: input.status, error: input.error ?? null, completedAt: new Date() }, + }); + await logEvent( + connector.id, + connector.organizationId, + input.status === 'COMPLETED' ? 'INFO' : 'WARN', + `sync.${job.service ?? 'unknown'}.${input.status.toLowerCase()}`, + input.error ?? `${job.service} sync ${input.status.toLowerCase()}`, + { stats: job.stats, jobId: job.id }, + ); + await events.publish({ + type: input.status === 'COMPLETED' ? 'sync.completed' : 'sync.failed', + organizationId: connector.organizationId, + connectorId: connector.id, + provider: PROVIDER_IDS[connector.provider] ?? connector.provider, + payload: { service: job.service, jobId: job.id, stats: job.stats, error: input.error }, + }); + }, + + async runIncrementalSync(input: IncrementalSyncInput): Promise { + const connector = await requireConnector(input.connectorId); + const impl = providerConnector(connector.provider); + const cctx = ctx.connectorContext(connector.id, connector.organizationId); + + const cursorRow = await prisma.syncCursor.findUnique({ + where: { + connectorId_service_resourceScope: { + connectorId: connector.id, + service: input.service, + resourceScope: '', + }, + }, + }); + if (!cursorRow) return { changeCount: 0, cursorMissing: true }; + + try { + const result = await impl.incrementalSync(cctx, input.service, cursorRow.cursor); + const counts = { created: 0, updated: 0, permissionChanges: 0, unchanged: 0 }; + let deleted = 0; + + for (const change of result.changes) { + if (change.changeType === 'deleted') { + await markResourceDeleted(connector, change.service, change.externalId); + deleted += 1; + } else if (change.resource) { + await persistResource(connector, change.service, change.resource, counts, true); + } + } + + await prisma.syncCursor.update({ + where: { id: cursorRow.id }, + data: { cursor: result.nextCursor }, + }); + await prisma.syncJob.update({ + where: { id: input.jobId }, + data: { + stats: { + discovered: result.changes.length, + created: counts.created, + updated: counts.updated, + deleted, + permissionChanges: counts.permissionChanges, + } as Prisma.InputJsonValue, + }, + }); + return { changeCount: result.changes.length }; + } catch (error) { + if (error instanceof CursorExpiredError) { + await prisma.syncCursor.delete({ where: { id: cursorRow.id } }); + await logEvent( + connector.id, + connector.organizationId, + 'WARN', + `sync.${input.service}.cursor_expired`, + 'provider invalidated the sync cursor — full resync required', + ); + return { changeCount: 0, cursorExpired: true }; + } + toTemporalFailure(error); + } + }, + + async markConnectorSynced(input: { + connectorId: string; + nextSyncInMinutes: number; + }): Promise { + const connector = await requireConnector(input.connectorId); + if (connector.status === 'REVOKED' || connector.status === 'DISCONNECTED') return; + await prisma.connector.update({ + where: { id: input.connectorId }, + data: { + status: 'CONNECTED', + lastSyncAt: new Date(), + nextSyncAt: new Date(Date.now() + input.nextSyncInMinutes * 60_000), + }, + }); + }, + } satisfies ConnectorActivitiesContract & Record unknown>; + + void tokens; // token manager is reached through connectorContext + return activities; +} + +export type ConnectorActivities = ReturnType; diff --git a/services/connector-worker/src/config.ts b/services/connector-worker/src/config.ts new file mode 100644 index 0000000..b8a56d6 --- /dev/null +++ b/services/connector-worker/src/config.ts @@ -0,0 +1,68 @@ +import { resolve } from 'node:path'; +import { config as loadDotenv } from 'dotenv'; +import { z } from 'zod'; + +loadDotenv({ path: resolve(process.cwd(), '.env') }); +loadDotenv({ path: resolve(process.cwd(), '../../.env') }); + +const envSchema = z.object({ + NODE_ENV: z.enum(['development', 'test', 'production']).default('development'), + LOG_LEVEL: z.enum(['fatal', 'error', 'warn', 'info', 'debug', 'trace']).default('info'), + + TEMPORAL_ADDRESS: z.string().default('localhost:7233'), + TEMPORAL_NAMESPACE: z.string().default('company-brain'), + CONNECTOR_TASK_QUEUE: z.string().default('brain-connectors'), + CONNECTOR_WORKER_HEALTH_PORT: z.coerce.number().int().positive().default(4101), + + DATABASE_URL: z.string().url(), + + REDIS_HOST: z.string().default('localhost'), + REDIS_PORT: z.coerce.number().int().positive().default(6379), + REDIS_PASSWORD: z.string().optional().default(''), + + // 64 hex chars — openssl rand -hex 32. Required: refresh tokens at rest. + TOKEN_ENCRYPTION_KEY: z.string().regex(/^[0-9a-fA-F]{64}$/, { + message: 'TOKEN_ENCRYPTION_KEY must be 64 hex chars (openssl rand -hex 32)', + }), + + GOOGLE_CLIENT_ID: z.string().default(''), + GOOGLE_CLIENT_SECRET: z.string().default(''), + GOOGLE_REDIRECT_URI: z + .string() + .url() + .default('http://localhost:4000/api/v1/connectors/google/callback'), +}); + +const parsed = envSchema.safeParse(process.env); +if (!parsed.success) { + const details = parsed.error.issues + .map((issue) => ` - ${issue.path.join('.')}: ${issue.message}`) + .join('\n'); + throw new Error(`Invalid connector-worker environment:\n${details}`); +} +const env = parsed.data; + +export const config = { + env: env.NODE_ENV, + isProduction: env.NODE_ENV === 'production', + logLevel: env.LOG_LEVEL, + temporal: { + address: env.TEMPORAL_ADDRESS, + namespace: env.TEMPORAL_NAMESPACE, + taskQueue: env.CONNECTOR_TASK_QUEUE, + healthPort: env.CONNECTOR_WORKER_HEALTH_PORT, + connect: { maxAttempts: 10, initialDelayMs: 1000, maxDelayMs: 15_000 }, + shutdownGraceTimeMs: 10_000, + }, + redis: { + host: env.REDIS_HOST, + port: env.REDIS_PORT, + password: env.REDIS_PASSWORD || undefined, + }, + encryptionKeyHex: env.TOKEN_ENCRYPTION_KEY, + google: { + clientId: env.GOOGLE_CLIENT_ID, + clientSecret: env.GOOGLE_CLIENT_SECRET, + redirectUri: env.GOOGLE_REDIRECT_URI, + }, +} as const; diff --git a/services/connector-worker/src/context.ts b/services/connector-worker/src/context.ts new file mode 100644 index 0000000..1fe029c --- /dev/null +++ b/services/connector-worker/src/context.ts @@ -0,0 +1,56 @@ +import { PrismaClient } from '@prisma/client'; +import { Redis } from 'ioredis'; +import { EventBus } from '@company-brain/events'; +import { ConnectorRegistry, type ConnectorContext } from '@company-brain/connector-core'; +import { GoogleWorkspaceConnector, GOOGLE_PROVIDER } from '@company-brain/connector-google'; +import { config } from './config.js'; +import { TokenManager } from './token-manager.js'; + +/** Prisma provider enum → registry provider id. */ +export const PROVIDER_IDS: Record = { + GOOGLE_WORKSPACE: GOOGLE_PROVIDER, +}; + +export interface WorkerContext { + prisma: PrismaClient; + redis: Redis; + events: EventBus; + registry: ConnectorRegistry; + tokens: TokenManager; + connectorContext(connectorId: string, organizationId: string): ConnectorContext; + close(): Promise; +} + +export function createWorkerContext(): WorkerContext { + const prisma = new PrismaClient(); + const redis = new Redis({ + host: config.redis.host, + port: config.redis.port, + password: config.redis.password, + maxRetriesPerRequest: 2, + lazyConnect: true, + }); + redis.on('error', () => {}); + + const registry = new ConnectorRegistry(); + registry.register(GOOGLE_PROVIDER, () => new GoogleWorkspaceConnector()); + + const tokens = new TokenManager(prisma); + + return { + prisma, + redis, + events: new EventBus(redis), + registry, + tokens, + connectorContext: (connectorId, organizationId) => ({ + connectorId, + organizationId, + getAccessToken: () => tokens.getAccessToken(connectorId), + }), + close: async () => { + redis.disconnect(); + await prisma.$disconnect(); + }, + }; +} diff --git a/services/connector-worker/src/index.ts b/services/connector-worker/src/index.ts new file mode 100644 index 0000000..4a88bb9 --- /dev/null +++ b/services/connector-worker/src/index.ts @@ -0,0 +1,94 @@ +import { createRequire } from 'node:module'; +import { createServer } from 'node:http'; +import { NativeConnection, Worker } from '@temporalio/worker'; +import { pino } from 'pino'; +import { config } from './config.js'; +import { createWorkerContext } from './context.js'; +import { createConnectorActivities } from './activities.js'; + +const logger = pino({ + level: config.logLevel, + ...(config.isProduction + ? {} + : { + transport: { + target: 'pino-pretty', + options: { translateTime: 'HH:MM:ss', ignore: 'pid,hostname' }, + }, + }), +}); + +const require = createRequire(import.meta.url); + +async function connectWithRetry(): Promise { + const { address, connect } = config.temporal; + let delay: number = connect.initialDelayMs; + for (let attempt = 1; ; attempt += 1) { + try { + return await NativeConnection.connect({ address }); + } catch (error) { + if (attempt >= connect.maxAttempts) throw error; + logger.warn({ address, attempt }, 'temporal not reachable yet — retrying'); + await new Promise((r) => setTimeout(r, delay)); + delay = Math.min(delay * 2, connect.maxDelayMs); + } + } +} + +async function main(): Promise { + const connection = await connectWithRetry(); + const context = createWorkerContext(); + + const worker = await Worker.create({ + connection, + namespace: config.temporal.namespace, + taskQueue: config.temporal.taskQueue, + workflowsPath: require.resolve('@company-brain/workflows'), + activities: createConnectorActivities(context), + shutdownGraceTime: config.temporal.shutdownGraceTimeMs, + }); + + const startedAt = Date.now(); + const healthServer = createServer((request, response) => { + if (request.url !== '/health') { + response.writeHead(404).end(); + return; + } + const state = worker.getState(); + const healthy = state === 'RUNNING'; + response.writeHead(healthy ? 200 : 503, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + status: healthy ? 'healthy' : 'degraded', + worker: state, + taskQueue: config.temporal.taskQueue, + namespace: config.temporal.namespace, + uptimeSeconds: Math.round((Date.now() - startedAt) / 1000), + }), + ); + }); + healthServer.listen(config.temporal.healthPort, () => + logger.info({ port: config.temporal.healthPort }, 'connector-worker health endpoint listening'), + ); + + process.on('SIGINT', () => worker.shutdown()); + process.on('SIGTERM', () => worker.shutdown()); + + logger.info( + { taskQueue: config.temporal.taskQueue, namespace: config.temporal.namespace }, + 'connector worker starting', + ); + try { + await worker.run(); + } finally { + healthServer.close(); + await context.close(); + await connection.close(); + } + logger.info('connector worker stopped cleanly'); +} + +main().catch((error) => { + logger.error({ err: error }, 'connector worker crashed'); + process.exit(1); +}); diff --git a/services/connector-worker/src/token-manager.ts b/services/connector-worker/src/token-manager.ts new file mode 100644 index 0000000..ac9d589 --- /dev/null +++ b/services/connector-worker/src/token-manager.ts @@ -0,0 +1,101 @@ +import type { PrismaClient } from '@prisma/client'; +import { + decryptSecret, + encryptSecret, + refreshAccessToken, + OAuthError, + type OAuthClientConfig, +} from '@company-brain/auth'; +import { TokenExpiredError } from '@company-brain/connector-core'; +import { GOOGLE_OAUTH_ENDPOINTS } from '@company-brain/connector-google'; +import { config } from './config.js'; +import { parseEncryptionKey } from '@company-brain/auth'; + +interface CachedToken { + accessToken: string; + expiresAt: number; +} + +/** + * Automatic token refresh with in-memory caching. Refresh tokens live + * encrypted in OAuthCredential; access tokens exist only in this process's + * memory and are never persisted or exposed. An `invalid_grant` response + * marks the credential REVOKED and flips the connector to REVOKED so the + * UI can offer a reconnect flow. + */ +export class TokenManager { + private readonly cache = new Map(); + private readonly key = parseEncryptionKey(config.encryptionKeyHex); + + constructor(private readonly prisma: PrismaClient) {} + + private oauthConfigFor(provider: string): OAuthClientConfig { + if (provider === 'GOOGLE_WORKSPACE') { + return { + clientId: config.google.clientId, + clientSecret: config.google.clientSecret, + redirectUri: config.google.redirectUri, + endpoints: GOOGLE_OAUTH_ENDPOINTS, + }; + } + throw new Error(`No OAuth configuration for provider ${provider}`); + } + + encryptRefreshToken(token: string): string { + return encryptSecret(token, this.key); + } + + async getAccessToken(connectorId: string): Promise { + const cached = this.cache.get(connectorId); + if (cached && cached.expiresAt - 60_000 > Date.now()) return cached.accessToken; + + const credential = await this.prisma.oAuthCredential.findFirst({ + where: { connectorId, status: 'ACTIVE', deletedAt: null }, + orderBy: { createdAt: 'desc' }, + include: { connector: { select: { provider: true } } }, + }); + if (!credential) { + throw new TokenExpiredError('No active credential — reconnect the workspace'); + } + + const refreshToken = decryptSecret(credential.encryptedRefreshToken, this.key); + try { + const tokens = await refreshAccessToken( + this.oauthConfigFor(credential.connector.provider), + refreshToken, + ); + const expiresAt = Date.now() + tokens.expiresInSeconds * 1000; + this.cache.set(connectorId, { accessToken: tokens.accessToken, expiresAt }); + await this.prisma.oAuthCredential.update({ + where: { id: credential.id }, + data: { + lastRefreshedAt: new Date(), + accessTokenExpiresAt: new Date(expiresAt), + // Token rotation: some providers issue a new refresh token. + ...(tokens.refreshToken + ? { encryptedRefreshToken: this.encryptRefreshToken(tokens.refreshToken) } + : {}), + }, + }); + return tokens.accessToken; + } catch (error) { + if (error instanceof OAuthError && error.code === 'invalid_grant') { + await this.prisma.oAuthCredential.update({ + where: { id: credential.id }, + data: { status: 'REVOKED' }, + }); + await this.prisma.connector.update({ + where: { id: connectorId }, + data: { status: 'REVOKED', error: 'OAuth grant revoked — reconnect required' }, + }); + this.cache.delete(connectorId); + throw new TokenExpiredError('OAuth grant revoked by the provider'); + } + throw error; + } + } + + invalidate(connectorId: string): void { + this.cache.delete(connectorId); + } +} diff --git a/services/connector-worker/tsconfig.build.json b/services/connector-worker/tsconfig.build.json new file mode 100644 index 0000000..cc5774c --- /dev/null +++ b/services/connector-worker/tsconfig.build.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false + } +} diff --git a/services/connector-worker/tsconfig.json b/services/connector-worker/tsconfig.json new file mode 100644 index 0000000..37dcfff --- /dev/null +++ b/services/connector-worker/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"], + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/services/temporal-worker/src/config.ts b/services/temporal-worker/src/config.ts index b109d9b..67eef1d 100644 --- a/services/temporal-worker/src/config.ts +++ b/services/temporal-worker/src/config.ts @@ -1,7 +1,7 @@ import { resolve } from 'node:path'; import { config as loadDotenv } from 'dotenv'; import { z } from 'zod'; -import type { ActivityConfig } from '@company-brain/activities'; +import type { ActivityConfig, KnowledgeConfig } from '@company-brain/activities'; loadDotenv({ path: resolve(process.cwd(), '.env') }); loadDotenv({ path: resolve(process.cwd(), '../../.env') }); @@ -32,6 +32,16 @@ const envSchema = z.object({ STORAGE_DEFAULT_BUCKET: z.string().default('company-brain'), QDRANT_URL: z.string().url().default('http://localhost:6333'), + + EMBEDDINGS_PROVIDER: z.enum(['local', 'openai', 'gemini', 'voyage']).default('local'), + EMBEDDINGS_MODEL: z.string().optional(), + EMBEDDINGS_DIMENSION: z.coerce.number().int().positive().optional(), + OPENAI_API_KEY: z.string().optional(), + GEMINI_API_KEY: z.string().optional(), + VOYAGE_API_KEY: z.string().optional(), + + CHUNK_SIZE: z.coerce.number().int().positive().default(400), + CHUNK_OVERLAP: z.coerce.number().int().nonnegative().default(60), }); const parsed = envSchema.safeParse(process.env); @@ -67,6 +77,26 @@ const activityConfig: ActivityConfig = { }, }; +const providerKeys = { + local: undefined, + openai: env.OPENAI_API_KEY, + gemini: env.GEMINI_API_KEY, + voyage: env.VOYAGE_API_KEY, +} as const; + +const knowledgeConfig: KnowledgeConfig = { + embedding: { + provider: env.EMBEDDINGS_PROVIDER, + model: env.EMBEDDINGS_MODEL, + dimension: env.EMBEDDINGS_DIMENSION, + apiKey: providerKeys[env.EMBEDDINGS_PROVIDER], + }, + chunking: { + chunkSize: env.CHUNK_SIZE, + chunkOverlap: env.CHUNK_OVERLAP, + }, +}; + export const config = { env: env.NODE_ENV, isProduction: env.NODE_ENV === 'production', @@ -86,6 +116,7 @@ export const config = { shutdownGraceTimeMs: 10_000, }, activities: activityConfig, + knowledge: knowledgeConfig, } as const; export type WorkerConfig = typeof config; diff --git a/services/temporal-worker/src/index.ts b/services/temporal-worker/src/index.ts index 3392c8a..b6af3a6 100644 --- a/services/temporal-worker/src/index.ts +++ b/services/temporal-worker/src/index.ts @@ -1,7 +1,12 @@ import { createRequire } from 'node:module'; import { Worker } from '@temporalio/worker'; import { pino } from 'pino'; -import { createActivities, createActivityContext } from '@company-brain/activities'; +import { + createActivities, + createActivityContext, + createKnowledgeActivities, + createKnowledgeActivityContext, +} from '@company-brain/activities'; import { config } from './config.js'; import { connectWithRetry } from './connection.js'; import { startHealthServer } from './health-server.js'; @@ -25,7 +30,8 @@ async function main(): Promise { let connectionStatus: 'connected' | 'disconnected' = 'connected'; // Long-lived clients shared by every activity invocation. - const activityContext = createActivityContext(config.activities); + const baseContext = createActivityContext(config.activities); + const activityContext = createKnowledgeActivityContext(baseContext, config.knowledge); const worker = await Worker.create({ connection, @@ -35,7 +41,10 @@ async function main(): Promise { // @company-brain/workflows is bundled and registered by export name. workflowsPath: require.resolve('@company-brain/workflows'), // The activity registry: plain functions closed over shared clients. - activities: createActivities(activityContext), + activities: { + ...createActivities(activityContext), + ...createKnowledgeActivities(activityContext), + }, // In-flight activities get this long to finish on shutdown before // being cancelled (their tasks are then retried by another worker). shutdownGraceTime: config.temporal.shutdownGraceTimeMs,