diff --git a/README.md b/README.md index c299cf0..47bc452 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,16 @@ VITE_API_BASE_URL=https://.onrender.com/api The included `render.yaml` defines a backend web service and a frontend static site for the same repo. +### RAG Retrieval & Source Ranking Configuration + +During document Q&A queries, LocalMind ranks retrieved text segments to provide accurate citations and relevant context: + +1. **Selection & Ranking**: Matching reference segments are selected using **Cosine Similarity** (`"hnsw:space": "cosine"` configured in ChromaDB). Chunks are ranked by similarity score, placing the most relevant fragments at the top of the LLM context. +2. **Citations**: The ranked segments are processed into a structured list of unique document sources and text previews, which are rendered as inline citation links in the UI. +3. **Production Configuration**: The retrieval process is influenced by the following settings in the `app_settings` SQLite database table (adjustable via the Settings Panel in the UI): + - **`rag_top_k`** (Default: `4`, Range: `1-10`): Controls the number of top-ranked retrieved source chunks injected into the context window. + - **`rag_chunk_overlap`** (Default: `50`, Range: `0-200` characters): Sets the character overlap bounds between adjacent document chunks to maintain continuous context links during document ingestion. + ## macOS Install Troubleshooting ### `node` or `npm` not found diff --git a/backend/models/schemas.py b/backend/models/schemas.py index 41c8666..08584c5 100644 --- a/backend/models/schemas.py +++ b/backend/models/schemas.py @@ -136,6 +136,7 @@ class ExportFormat(str, Enum): markdown = "markdown" json = "json" txt = "txt" + pdf = "pdf" class SessionRenameItem(BaseModel): diff --git a/backend/requirements.txt b/backend/requirements.txt index 4c8ef29..74b1d67 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -18,3 +18,4 @@ jsonschema>=4.17.0 psutil grapheme==0.6.0 pre-commit==4.6.0 +fpdf2==2.8.7 diff --git a/backend/requirements_fixed.txt b/backend/requirements_fixed.txt index 38f6283..c3a8df1 100644 --- a/backend/requirements_fixed.txt +++ b/backend/requirements_fixed.txt @@ -13,3 +13,4 @@ python-dotenv==1.0.1 httpx==0.27.0 pytest==8.3.0 pytest-asyncio==0.24.0 +fpdf2==2.8.7 diff --git a/backend/routes/export.py b/backend/routes/export.py index 3d21673..a017ad7 100644 --- a/backend/routes/export.py +++ b/backend/routes/export.py @@ -13,6 +13,57 @@ router = APIRouter() +def generate_pdf(title: str, subtitle: str, messages: list) -> bytes: + # Check Whisper status + whisper_status = "Whisper Engine: Active" + try: + import whisper + except (ImportError, ModuleNotFoundError, TypeError): + import logging + logging.getLogger(__name__).warning("Whisper package not found. Falling back to text-only processing for PDF export.") + whisper_status = "Whisper Engine: Offline (Audio features unavailable)" + + from fpdf import FPDF + pdf = FPDF() + pdf.add_page() + + title_latin1 = title.encode("latin-1", errors="replace").decode("latin-1") + pdf.set_font("helvetica", "B", 16) + pdf.cell(0, 10, title_latin1, new_x="LMARGIN", new_y="NEXT") + + sub = f"{subtitle} | {whisper_status}" + sub_latin1 = sub.encode("latin-1", errors="replace").decode("latin-1") + pdf.set_font("helvetica", "I", 10) + pdf.cell(0, 8, sub_latin1, new_x="LMARGIN", new_y="NEXT") + pdf.ln(5) + + for m in messages: + role = "YOU" if m["role"] == "user" else "LOCALMIND" + pdf.set_font("helvetica", "B", 11) + pdf.cell(0, 6, f"[{role}]", new_x="LMARGIN", new_y="NEXT") + + pdf.set_font("helvetica", "", 10) + content = m["content"].strip() + content_latin1 = content.encode("latin-1", errors="replace").decode("latin-1") + pdf.multi_cell(0, 5, content_latin1, new_x="LMARGIN", new_y="NEXT") + + if m["role"] == "assistant" and m.get("sources"): + source_names = [] + for src in m["sources"]: + if isinstance(src, dict): + source_names.append(src.get("source", "")) + else: + source_names.append(str(src)) + source_names = [s for s in source_names if s] + if source_names: + pdf.set_font("helvetica", "I", 9) + pdf.cell(0, 5, f"Sources: {', '.join(source_names)}", new_x="LMARGIN", new_y="NEXT") + + pdf.ln(5) + + return bytes(pdf.output()) + + class ExportMessagesRequest(BaseModel): message_ids: List[str] format: ExportFormat @@ -107,22 +158,27 @@ async def export_session(session_id: str, fmt: ExportFormat): safe_title = "localmind_chat" if fmt == ExportFormat.json: - content = export_session_json(session, messages, ts) + content = export_session_json(session, messages, ts).encode("utf-8") media = "application/json" filename = f"{safe_title}.json" elif fmt == ExportFormat.markdown: - content = export_session_markdown(session, messages, ts) + content = export_session_markdown(session, messages, ts).encode("utf-8") media = "text/markdown" filename = f"{safe_title}.md" + elif fmt == ExportFormat.pdf: + content = generate_pdf(title, f"Exported: {ts} | Model: {session.get('model', '?')}", messages) + media = "application/pdf" + filename = f"{safe_title}.pdf" + else: # txt - content = export_session_txt(session, messages, ts) + content = export_session_txt(session, messages, ts).encode("utf-8") media = "text/plain" filename = f"{safe_title}.txt" return Response( - content=content.encode("utf-8"), + content=content, media_type=media, headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) @@ -195,7 +251,7 @@ async def export_messages(req: ExportMessagesRequest): safe_title = f"localmind_messages_{ts.replace(' ', '_').replace(':', '-')}" if req.format == ExportFormat.json: - content = json.dumps({"messages": messages, "exported_at": ts}, indent=2, ensure_ascii=False) + content = json.dumps({"messages": messages, "exported_at": ts}, indent=2, ensure_ascii=False).encode("utf-8") media = "application/json" filename = f"{safe_title}.json" @@ -219,10 +275,15 @@ async def export_messages(req: ExportMessagesRequest): lines.append(msg_block + "\n") lines.append("\n\n---\n\n") - content = "\n".join(lines) + content = "\n".join(lines).encode("utf-8") media = "text/markdown" filename = f"{safe_title}.md" + elif req.format == ExportFormat.pdf: + content = generate_pdf("LocalMind – Exported Messages", f"Exported: {ts}", messages) + media = "application/pdf" + filename = f"{safe_title}.pdf" + else: lines = ["LocalMind Export — Selected Messages", f"Exported: {ts}", "=" * 50, ""] for m in messages: @@ -241,12 +302,12 @@ async def export_messages(req: ExportMessagesRequest): msg_block += f"\nSources: {', '.join(source_names)}" lines += [msg_block, ""] - content = "\n".join(lines) + content = "\n".join(lines).encode("utf-8") media = "text/plain" filename = f"{safe_title}.txt" return Response( - content=content.encode("utf-8"), + content=content, media_type=media, headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) \ No newline at end of file diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 8cec224..e47b869 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -228,6 +228,15 @@ def init_db(): success INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')) ); + + CREATE TABLE IF NOT EXISTS export_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT, + format TEXT NOT NULL, + export_type TEXT NOT NULL, + details TEXT, + created_at TEXT DEFAULT (datetime('now')) + ); CREATE TABLE IF NOT EXISTS shared_sessions ( id TEXT PRIMARY KEY, @@ -445,6 +454,28 @@ def get_messages_full(session_id: str) -> list[dict]: ] +def get_messages_by_ids(message_ids: list[str]) -> list[dict]: + if not message_ids: + return [] + placeholders = ",".join("?" for _ in message_ids) + with get_db() as conn: + rows = conn.execute( + f"SELECT id, role, content, sources, created_at, benchmarks FROM messages WHERE id IN ({placeholders}) ORDER BY created_at ASC", + message_ids, + ).fetchall() + return [ + { + "id": r["id"], + "role": r["role"], + "content": r["content"], + "sources": json.loads(r["sources"] or "[]"), + "created_at": r["created_at"], + "benchmarks": json.loads(r["benchmarks"] or "{}") + } + for r in rows + ] + + def clear_messages(session_id: str): with get_db() as conn: cur = conn.execute("DELETE FROM messages WHERE session_id=?", (session_id,)) @@ -556,6 +587,25 @@ def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = ) +# ─── Export logs ───────────────────────────────────────────── + +def get_export_logs(limit: int = 50) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM export_logs ORDER BY created_at DESC LIMIT ?", + (limit,) + ).fetchall() + return [dict(r) for r in rows] + +def log_export(session_id: str | None, format: str, export_type: str, details: str = None): + with get_db() as conn: + conn.execute( + "INSERT INTO export_logs (session_id, format, export_type, details) VALUES (?,?,?,?)", + (session_id, format, export_type, details), + ) + + + # ─── Shareable Sessions (Issue #270) ───────────────────────── def create_shared_session(session_id: str) -> str: diff --git a/backend/tests/test_pdf_export.py b/backend/tests/test_pdf_export.py new file mode 100644 index 0000000..0f3c5a6 --- /dev/null +++ b/backend/tests/test_pdf_export.py @@ -0,0 +1,66 @@ +import tempfile +import sys +from unittest.mock import patch, MagicMock +from fastapi.testclient import TestClient +import services.db_service as db + +# Setup temporary database for PDF export tests +_tmp = tempfile.mktemp(suffix=".db") +db.DB_PATH = _tmp +db.init_db() + +from app import app +client = TestClient(app) + +def test_pdf_export_session_success_whisper_present(): + # 1. Create a session and save a message + db.create_session("sess_pdf_1", title="PDF Whisper Present Chat", model="llama3", language="en") + db.save_message("sess_pdf_1", "user", "Hello PDF") + db.save_message("sess_pdf_1", "assistant", "Response PDF", [{"source": "report.pdf", "chunk": 0}]) + + # Mock whisper presence + mock_whisper = MagicMock() + with patch.dict(sys.modules, {"whisper": mock_whisper}): + response = client.get("/api/export/sess_pdf_1/pdf") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "Content-Disposition" in response.headers + assert "attachment; filename=" in response.headers["Content-Disposition"] + assert "pdf_whisper_present_chat.pdf" in response.headers["Content-Disposition"] + assert len(response.content) > 0 + # Check standard PDF header prefix + assert response.content.startswith(b"%PDF") + +def test_pdf_export_session_success_whisper_missing(): + # 1. Create a session and save a message + db.create_session("sess_pdf_2", title="PDF Whisper Missing Chat", model="llama3", language="en") + db.save_message("sess_pdf_2", "user", "Hello PDF 2") + db.save_message("sess_pdf_2", "assistant", "Response PDF 2") + + # Mock whisper missing + with patch.dict(sys.modules, {"whisper": None}): + sys.modules.pop("whisper", None) + response = client.get("/api/export/sess_pdf_2/pdf") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "pdf_whisper_missing_chat.pdf" in response.headers["Content-Disposition"] + assert response.content.startswith(b"%PDF") + +def test_pdf_export_messages_success_whisper_missing(): + # Save test messages and export them + db.create_session("sess_pdf_3", title="PDF Message Export", model="llama3") + db.save_message("sess_pdf_3", "user", "Standalone message to export") + # Retrieve messages to get the correct message ID from the database + messages = db.get_messages_full("sess_pdf_3") + db_msg_id = str(messages[0]["id"]) + + with patch.dict(sys.modules, {"whisper": None}): + sys.modules.pop("whisper", None) + response = client.post( + "/api/export/messages", + json={"message_ids": [db_msg_id], "format": "pdf"} + ) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "localmind_messages_" in response.headers["Content-Disposition"] + assert response.content.startswith(b"%PDF")