feat: implement Phase 2 vector storage toolbox

CLAUDE.md

@@ -2,7 +2,7 @@
 
 ## Project Overview
 
-Pluggable RAG-enabled chatbot for scrape-and-analyze. Two-person team: one frontend, one backend.
+Vector storage toolbox — standalone service that receives pre-chunked, pre-embedded article data from external services and stores it in PostgreSQL + pgvector. Will also serve chat/retrieval APIs for frontend UI. Two-person team: one frontend, one backend.
 
 ## SDD (Spec-Driven Development) Workflow
 
@@ -12,11 +12,11 @@ Pluggable RAG-enabled chatbot for scrape-and-analyze. Two-person team: one front
 
 1. **Spec-first**: Before writing or changing any code, read and update `specs/` first
 2. **Spec files**:
-   - `specs/chat-api.md` — API contract (frontend + backend shared)
-   - `specs/rag-pipeline.md` — RAG internals (backend only)
-   - `specs/integration.md` — How to mount into scrape-and-analyze
+   - `specs/toolbox-api.md` — API contract (shared with external services)
+   - `specs/rag-pipeline.md` — Storage internals (backend only)
+   - `specs/integration.md` — How external services integrate with the toolbox
 3. **Contracts = spec in code**: `src/chatbot_plugin/contracts/` contains Pydantic models that mirror the spec. If spec changes, update contracts first
-4. **API changes must start from spec**: Change `specs/chat-api.md` → update `contracts/` → update router/service → tests pass
+4. **API changes must start from spec**: Change `specs/toolbox-api.md` → update `contracts/` → update router/service → tests pass
 5. **Notify the other person**: If you change an API shape, the other developer needs to know. Spec is the communication channel
 6. **Contract tests must pass**: `src/tests/contracts/` verifies Pydantic models match spec. These are non-negotiable
 
@@ -29,45 +29,40 @@ Pluggable RAG-enabled chatbot for scrape-and-analyze. Two-person team: one front
 4. Implement routers/ + service/ (make tests green)
 ```
 
-### Frontend-Backend Boundary
+### Scrape-and-Analyze Boundary
 
-- Frontend only needs `specs/chat-api.md` — do not look at backend implementation
+- External services only need `specs/toolbox-api.md` — do not look at backend implementation
 - Backend can change internals freely as long as API shape (spec) is unchanged
 - Disputes are resolved by reading the spec, not the code
 
 ## Project Structure
 
 ```
 specs/                           # SDD spec files (source of truth)
+  toolbox-api.md                 # API contract (POST /chunks)
+  rag-pipeline.md                # Storage internals (backend only)
+  integration.md                 # How external services integrate
 src/chatbot_plugin/
   contracts/                     # Pydantic models = spec in code
-    requests.py                  # ChatMessageRequest, SearchRequest, IndexRequest
-    responses.py                 # ChatMessageResponse, SearchResponse, etc.
-  llm/                           # LLM provider infrastructure
-    config.py                    # load_providers() — reads providers.toml
-    bootstrap.py                 # build_llm_service() factory
-    base_provider.py             # BaseProvider ABC + async tenacity retry
-    claude_provider.py           # Anthropic AsyncAnthropic
-    gemini_provider.py           # Google genai.Client + asyncio.to_thread
-    openrouter_provider.py       # httpx.AsyncClient
-    resilient_llm_service.py     # ProviderHandler + fallback chain
-    rate_limit/                  # Async rate limiting
-      quota_strategy.py          # ABC
-      sliding_window_strategy.py # asyncio.Lock + deque
-      no_op_strategy.py          # No-op passthrough
+    requests.py                  # ArticleInfo, ChunkData, StoreChunksRequest
+    responses.py                 # StoreChunksResponse
+  models/                        # SQLAlchemy ORM models
+    article.py                   # Article + DeclarativeBase
+    chunk.py                     # ArticleChunk (pgvector Vector column)
   config.py                      # CHATBOT_* env vars
+  db.py                          # Async engine + session factory
+  main.py                        # Standalone FastAPI app + lifespan
   routers.py                     # FastAPI endpoints
-  service.py                     # Business logic
-providers.example.toml           # Provider config template (commit this)
-providers.toml                   # Actual provider config (gitignored)
+  service.py                     # Business logic (ToolboxService)
 src/tests/
   contracts/                     # Contract conformance tests
   routers/                       # API endpoint tests
-  llm/                           # LLM provider tests
   test_service.py                # Unit tests
+alembic/                         # Database migrations
 ```
 
 ## Commands
 
+- **Run server:** `uvicorn chatbot_plugin.main:app --reload`
 - **Test:** `uv run pytest src/tests/ -v`
 - **Coverage:** `uv run pytest src/tests/ --cov=chatbot_plugin --cov-report=html`

alembic/env.py

@@ -8,7 +8,7 @@
 from sqlalchemy import pool
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from chatbot_plugin.models.article import Base
+from chatbot_plugin.models import Base  # noqa: F401 — models register themselves on Base.metadata
 
 config = context.config
 if config.config_file_name is not None:

alembic/versions/002_toolbox_schema.py

@@ -0,0 +1,99 @@
+"""Toolbox schema: simplified articles + article_chunks
+
+Revision ID: 002
+Revises: 001
+Create Date: 2026-05-28
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+
+revision: str = "002"
+down_revision: Union[str, None] = "001"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Drop old indexes + columns
+    op.execute("DROP INDEX IF EXISTS idx_articles_tsv")
+    op.drop_index("idx_articles_topic_id", table_name="articles")
+    op.drop_index("idx_articles_scraped_at", table_name="articles")
+    op.drop_index("idx_articles_source", table_name="articles")
+
+    # Rebuild articles table with minimal schema
+    op.drop_table("articles")
+
+    op.create_table(
+        "articles",
+        sa.Column("id", UUID(as_uuid=True), primary_key=True),
+        sa.Column("url", sa.String(), nullable=False),
+        sa.Column("title", sa.String(), nullable=True),
+        sa.Column("source", sa.String(), nullable=True),
+        sa.Column("metadata", JSONB, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+    )
+    op.create_index("idx_articles_source", "articles", ["source"])
+    op.create_index("idx_articles_url", "articles", ["url"])
+    op.create_unique_constraint("articles_url_key", "articles", ["url"])
+
+    # Create article_chunks table
+    op.create_table(
+        "article_chunks",
+        sa.Column("id", UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("article_id", UUID(as_uuid=True), sa.ForeignKey("articles.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("chunk_index", sa.Integer(), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("dense_vector", sa.Text(), nullable=True),
+        sa.Column("sparse_vector", JSONB, nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+    )
+
+    # pgvector: enable extension + cast dense_vector to proper vector type
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+    op.execute("ALTER TABLE article_chunks ALTER COLUMN dense_vector TYPE vector(1024) USING dense_vector::vector")
+
+    op.create_unique_constraint("uq_article_chunk_idx", "article_chunks", ["article_id", "chunk_index"])
+    op.create_index("idx_chunks_article_id", "article_chunks", ["article_id"])
+
+    # HNSW index for dense similarity search (required by spec)
+    op.execute(
+        "CREATE INDEX hnsw_chunks_dense ON article_chunks USING hnsw (dense_vector vector_cosine_ops)"
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("article_chunks")
+    op.drop_table("articles")
+
+    # Recreate old articles table (simplified, just for rollback compat)
+    op.create_table(
+        "articles",
+        sa.Column("id", UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()")),
+        sa.Column("url", sa.String(), nullable=False, unique=True),
+        sa.Column("url_hash", sa.String(), nullable=False),
+        sa.Column("source", sa.String(), nullable=True),
+        sa.Column("title", sa.String(), nullable=True),
+        sa.Column("content", sa.Text(), nullable=True),
+        sa.Column("published_at", sa.String(), nullable=True),
+        sa.Column("scraped_at", sa.String(), nullable=True),
+        sa.Column("metadata", JSONB, nullable=True),
+        sa.Column("correlation_id", UUID(as_uuid=True), nullable=True),
+        sa.Column("topic_id", UUID(as_uuid=True), nullable=True),
+        sa.Column(
+            "search_tsv",
+            sa.Text(),  # simplified placeholder
+            sa.Computed(
+                "to_tsvector('english', coalesce(title,'') || ' ' || coalesce(content,''))",
+                persisted=True,
+            ),
+            nullable=True,
+        ),
+    )
+    op.create_index("idx_articles_source", "articles", ["source"])
+    op.create_index("idx_articles_scraped_at", "articles", ["scraped_at"])
+    op.create_index("idx_articles_topic_id", "articles", ["topic_id"])

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "chatbot-plugin"
 version = "0.1.0"
-description = "Chatbot plugin for scrape-and-analyze — pluggable conversational AI over scraped articles"
+description = "Vector storage toolbox for scrape-and-analyze — chunk storage and retrieval"
 requires-python = ">=3.11"
 dependencies = [
     "fastapi",
@@ -11,11 +11,8 @@ dependencies = [
     "structlog",
     "sqlalchemy[asyncio]>=2.0",
     "asyncpg",
-    "anthropic",
-    "google-genai",
-    "httpx",
-    "tenacity",
     "alembic",
+    "pgvector",
 ]
 
 [dependency-groups]