diff --git a/README.md b/README.md index c299cf0..47bc452 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,16 @@ VITE_API_BASE_URL=https://.onrender.com/api The included `render.yaml` defines a backend web service and a frontend static site for the same repo. +### RAG Retrieval & Source Ranking Configuration + +During document Q&A queries, LocalMind ranks retrieved text segments to provide accurate citations and relevant context: + +1. **Selection & Ranking**: Matching reference segments are selected using **Cosine Similarity** (`"hnsw:space": "cosine"` configured in ChromaDB). Chunks are ranked by similarity score, placing the most relevant fragments at the top of the LLM context. +2. **Citations**: The ranked segments are processed into a structured list of unique document sources and text previews, which are rendered as inline citation links in the UI. +3. **Production Configuration**: The retrieval process is influenced by the following settings in the `app_settings` SQLite database table (adjustable via the Settings Panel in the UI): + - **`rag_top_k`** (Default: `4`, Range: `1-10`): Controls the number of top-ranked retrieved source chunks injected into the context window. + - **`rag_chunk_overlap`** (Default: `50`, Range: `0-200` characters): Sets the character overlap bounds between adjacent document chunks to maintain continuous context links during document ingestion. + ## macOS Install Troubleshooting ### `node` or `npm` not found diff --git a/backend/models/schemas.py b/backend/models/schemas.py index 41c8666..08584c5 100644 --- a/backend/models/schemas.py +++ b/backend/models/schemas.py @@ -136,6 +136,7 @@ class ExportFormat(str, Enum): markdown = "markdown" json = "json" txt = "txt" + pdf = "pdf" class SessionRenameItem(BaseModel): diff --git a/backend/requirements.txt b/backend/requirements.txt index 4c8ef29..74b1d67 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -18,3 +18,4 @@ jsonschema>=4.17.0 psutil grapheme==0.6.0 pre-commit==4.6.0 +fpdf2==2.8.7 diff --git a/backend/requirements_fixed.txt b/backend/requirements_fixed.txt index 38f6283..c3a8df1 100644 --- a/backend/requirements_fixed.txt +++ b/backend/requirements_fixed.txt @@ -13,3 +13,4 @@ python-dotenv==1.0.1 httpx==0.27.0 pytest==8.3.0 pytest-asyncio==0.24.0 +fpdf2==2.8.7 diff --git a/backend/routes/export.py b/backend/routes/export.py index 3d21673..af2aeae 100644 --- a/backend/routes/export.py +++ b/backend/routes/export.py @@ -13,6 +13,57 @@ router = APIRouter() +def generate_pdf(title: str, subtitle: str, messages: list) -> bytes: + # Check Whisper status + whisper_status = "Whisper Engine: Active" + try: + import whisper + except (ImportError, ModuleNotFoundError, TypeError): + import logging + logging.getLogger(__name__).warning("Whisper package not found. Falling back to text-only processing for PDF export.") + whisper_status = "Whisper Engine: Offline (Audio features unavailable)" + + from fpdf import FPDF + pdf = FPDF() + pdf.add_page() + + title_latin1 = title.encode("latin-1", errors="replace").decode("latin-1") + pdf.set_font("helvetica", "B", 16) + pdf.cell(0, 10, title_latin1, new_x="LMARGIN", new_y="NEXT") + + sub = f"{subtitle} | {whisper_status}" + sub_latin1 = sub.encode("latin-1", errors="replace").decode("latin-1") + pdf.set_font("helvetica", "I", 10) + pdf.cell(0, 8, sub_latin1, new_x="LMARGIN", new_y="NEXT") + pdf.ln(5) + + for m in messages: + role = "YOU" if m["role"] == "user" else "LOCALMIND" + pdf.set_font("helvetica", "B", 11) + pdf.cell(0, 6, f"[{role}]", new_x="LMARGIN", new_y="NEXT") + + pdf.set_font("helvetica", "", 10) + content = m["content"].strip() + content_latin1 = content.encode("latin-1", errors="replace").decode("latin-1") + pdf.multi_cell(0, 5, content_latin1, new_x="LMARGIN", new_y="NEXT") + + if m["role"] == "assistant" and m.get("sources"): + source_names = [] + for src in m["sources"]: + if isinstance(src, dict): + source_names.append(src.get("source", "")) + else: + source_names.append(str(src)) + source_names = [s for s in source_names if s] + if source_names: + pdf.set_font("helvetica", "I", 9) + pdf.cell(0, 5, f"Sources: {', '.join(source_names)}", new_x="LMARGIN", new_y="NEXT") + + pdf.ln(5) + + return bytes(pdf.output()) + + class ExportMessagesRequest(BaseModel): message_ids: List[str] format: ExportFormat @@ -90,163 +141,285 @@ def export_session_txt(session: dict, messages: list, ts: str) -> str: return "\n".join(lines) -@router.get("/{session_id}/{fmt}") -async def export_session(session_id: str, fmt: ExportFormat): - session = db_service.get_session(session_id) - if not session: - raise HTTPException(404, "Session not found") - - messages = db_service.get_messages_full(session_id) - title = session.get("title", "LocalMind Chat") - ts = datetime.now().strftime("%Y-%m-%d %H:%M") +@router.get("/logs") +async def get_export_logs(limit: int = 50): + """Fetch recent export logs for the audit UI / backend verification.""" + try: + logs = db_service.get_export_logs(limit) + return {"logs": logs} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to fetch logs: {str(e)}") - # FIXED (#83): Clean the title to make it safe for a filename - safe_title = re.sub(r'[^\w\s-]', '', title).strip().lower() - safe_title = re.sub(r'[\s-]+', '_', safe_title) - if not safe_title: - safe_title = "localmind_chat" - if fmt == ExportFormat.json: - content = export_session_json(session, messages, ts) - media = "application/json" - filename = f"{safe_title}.json" - - elif fmt == ExportFormat.markdown: - content = export_session_markdown(session, messages, ts) - media = "text/markdown" - filename = f"{safe_title}.md" - - else: # txt - content = export_session_txt(session, messages, ts) - media = "text/plain" - filename = f"{safe_title}.txt" - - return Response( - content=content.encode("utf-8"), - media_type=media, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - -@router.post("/sessions") -async def export_sessions(req: BulkSessionExportRequest): - valid_exports = [] - for session_id in req.session_ids: +@router.get("/{session_id}/{fmt}") +async def export_session(session_id: str, fmt: ExportFormat): + import time + start_time = time.perf_counter() + success = False + error_msg = None + messages_count = 0 + try: session = db_service.get_session(session_id) if not session: - continue - messages = db_service.get_messages_full(session_id) - valid_exports.append((session, messages)) + error_msg = "Session not found" + raise HTTPException(404, "Session not found") - if not valid_exports: - raise HTTPException( - status_code=404, - detail="No valid sessions found for the given IDs" + messages = db_service.get_messages_full(session_id) + messages_count = len(messages) + title = session.get("title", "LocalMind Chat") + ts = datetime.now().strftime("%Y-%m-%d %H:%M") + + # FIXED (#83): Clean the title to make it safe for a filename + safe_title = re.sub(r'[^\w\s-]', '', title).strip().lower() + safe_title = re.sub(r'[\s-]+', '_', safe_title) + if not safe_title: + safe_title = "localmind_chat" + + if fmt == ExportFormat.json: + content = export_session_json(session, messages, ts).encode("utf-8") + media = "application/json" + filename = f"{safe_title}.json" + + elif fmt == ExportFormat.markdown: + content = export_session_markdown(session, messages, ts).encode("utf-8") + media = "text/markdown" + filename = f"{safe_title}.md" + + elif fmt == ExportFormat.pdf: + content = generate_pdf(title, f"Exported: {ts} | Model: {session.get('model', '?')}", messages) + media = "application/pdf" + filename = f"{safe_title}.pdf" + + else: # txt + content = export_session_txt(session, messages, ts).encode("utf-8") + media = "text/plain" + filename = f"{safe_title}.txt" + + success = True + return Response( + content=content, + media_type=media, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) - - ts = datetime.now().strftime("%Y-%m-%d %H:%M") - - if req.format == "json": - sessions_data = [] - for session, messages in valid_exports: - sessions_data.append({ - "session": session, - "messages": messages - }) - payload = { - "exported_at": ts, - "sessions": sessions_data + except Exception as e: + if not error_msg: + if isinstance(e, HTTPException): + error_msg = e.detail + else: + error_msg = str(e) + raise + finally: + duration_ms = int((time.perf_counter() - start_time) * 1000) + details_dict = { + "success": success, + "duration_ms": duration_ms, + "messages_count": messages_count, + "error": error_msg } - content = json.dumps(payload, indent=2, ensure_ascii=False) - media = "application/json" - ext = "json" - - elif req.format == "markdown": - session_markdowns = [export_session_markdown(session, messages, ts) for session, messages in valid_exports] - content = "\n\n---\n\n".join(session_markdowns) - media = "text/markdown" - ext = "md" + try: + db_service.log_export( + session_id=session_id, + format=fmt.value, + export_type="single", + details=json.dumps(details_dict) + ) + except Exception as log_err: + import logging + logging.getLogger(__name__).error(f"Audit log failed: {log_err}") - else: # txt - session_txts = [export_session_txt(session, messages, ts) for session, messages in valid_exports] - content = "\n\n==================================================\n\n".join(session_txts) - media = "text/plain" - ext = "txt" - filename = f"localmind_bulk_export_{ts.replace(' ', '_').replace(':', '-')}.{ext}" - - return Response( - content=content.encode("utf-8"), - media_type=media, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) +@router.post("/sessions") +async def export_sessions(req: BulkSessionExportRequest): + import time + start_time = time.perf_counter() + success = False + error_msg = None + sessions_count = 0 + try: + valid_exports = [] + for session_id in req.session_ids: + session = db_service.get_session(session_id) + if not session: + continue + messages = db_service.get_messages_full(session_id) + valid_exports.append((session, messages)) + + if not valid_exports: + error_msg = "No valid sessions found for the given IDs" + raise HTTPException( + status_code=404, + detail=error_msg + ) + + sessions_count = len(valid_exports) + ts = datetime.now().strftime("%Y-%m-%d %H:%M") + + if req.format == "json": + sessions_data = [] + for session, messages in valid_exports: + sessions_data.append({ + "session": session, + "messages": messages + }) + payload = { + "exported_at": ts, + "sessions": sessions_data + } + content = json.dumps(payload, indent=2, ensure_ascii=False) + media = "application/json" + ext = "json" + + elif req.format == "markdown": + session_markdowns = [export_session_markdown(session, messages, ts) for session, messages in valid_exports] + content = "\n\n---\n\n".join(session_markdowns) + media = "text/markdown" + ext = "md" + + else: # txt + session_txts = [export_session_txt(session, messages, ts) for session, messages in valid_exports] + content = "\n\n==================================================\n\n".join(session_txts) + media = "text/plain" + ext = "txt" + + filename = f"localmind_bulk_export_{ts.replace(' ', '_').replace(':', '-')}.{ext}" + + success = True + return Response( + content=content.encode("utf-8"), + media_type=media, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + except Exception as e: + if not error_msg: + if isinstance(e, HTTPException): + error_msg = e.detail + else: + error_msg = str(e) + raise + finally: + duration_ms = int((time.perf_counter() - start_time) * 1000) + details_dict = { + "success": success, + "duration_ms": duration_ms, + "sessions_count": sessions_count, + "error": error_msg + } + try: + db_service.log_export( + session_id=None, + format=req.format, + export_type="bulk", + details=json.dumps(details_dict) + ) + except Exception as log_err: + import logging + logging.getLogger(__name__).error(f"Audit log failed: {log_err}") @router.post("/messages") async def export_messages(req: ExportMessagesRequest): - messages = db_service.get_messages_by_ids(req.message_ids) - if not messages: - raise HTTPException(404, "No messages found for the given IDs") - - messages.sort(key=lambda m: m.get("timestamp", "")) - ts = datetime.now().strftime("%Y-%m-%d %H:%M") - - # FIXED (#83): Use a shared default title format for custom standalone message groups - safe_title = f"localmind_messages_{ts.replace(' ', '_').replace(':', '-')}" - - if req.format == ExportFormat.json: - content = json.dumps({"messages": messages, "exported_at": ts}, indent=2, ensure_ascii=False) - media = "application/json" - filename = f"{safe_title}.json" - - elif req.format == ExportFormat.markdown: - lines = ["# LocalMind – Exported Messages\n", f"*Exported: {ts}*\n\n---\n"] - for m in messages: - role_label = "**You**" if m["role"] == "user" else "**LocalMind**" - msg_block = f"{role_label}\n\n{m['content'].strip()}" - - if m["role"] == "assistant" and m.get("sources"): - source_names = [] - for src in m["sources"]: - if isinstance(src, dict): - source_names.append(src.get("source", "")) - else: - source_names.append(str(src)) - source_names = [s for s in source_names if s] - if source_names: - msg_block += f"\n\n*Sources: {', '.join(source_names)}*" - - lines.append(msg_block + "\n") - lines.append("\n\n---\n\n") - - content = "\n".join(lines) - media = "text/markdown" - filename = f"{safe_title}.md" - - else: - lines = ["LocalMind Export — Selected Messages", f"Exported: {ts}", "=" * 50, ""] - for m in messages: - role = "YOU" if m["role"] == "user" else "LOCALMIND" - msg_block = f"[{role}]\n{m['content'].strip()}" - - if m["role"] == "assistant" and m.get("sources"): - source_names = [] - for src in m["sources"]: - if isinstance(src, dict): - source_names.append(src.get("source", "")) - else: - source_names.append(str(src)) - source_names = [s for s in source_names if s] - if source_names: - msg_block += f"\nSources: {', '.join(source_names)}" + import time + start_time = time.perf_counter() + success = False + error_msg = None + messages_count = 0 + try: + messages = db_service.get_messages_by_ids(req.message_ids) + if not messages: + error_msg = "No messages found for the given IDs" + raise HTTPException(404, error_msg) + + messages_count = len(messages) + messages.sort(key=lambda m: m.get("timestamp", "")) + ts = datetime.now().strftime("%Y-%m-%d %H:%M") + + # FIXED (#83): Use a shared default title format for custom standalone message groups + safe_title = f"localmind_messages_{ts.replace(' ', '_').replace(':', '-')}" + + if req.format == ExportFormat.json: + content = json.dumps({"messages": messages, "exported_at": ts}, indent=2, ensure_ascii=False).encode("utf-8") + media = "application/json" + filename = f"{safe_title}.json" + + elif req.format == ExportFormat.markdown: + lines = ["# LocalMind – Exported Messages\n", f"*Exported: {ts}*\n\n---\n"] + for m in messages: + role_label = "**You**" if m["role"] == "user" else "**LocalMind**" + msg_block = f"{role_label}\n\n{m['content'].strip()}" + + if m["role"] == "assistant" and m.get("sources"): + source_names = [] + for src in m["sources"]: + if isinstance(src, dict): + source_names.append(src.get("source", "")) + else: + source_names.append(str(src)) + source_names = [s for s in source_names if s] + if source_names: + msg_block += f"\n\n*Sources: {', '.join(source_names)}*" + + lines.append(msg_block + "\n") + lines.append("\n\n---\n\n") + + content = "\n".join(lines).encode("utf-8") + media = "text/markdown" + filename = f"{safe_title}.md" + + elif req.format == ExportFormat.pdf: + content = generate_pdf("LocalMind – Exported Messages", f"Exported: {ts}", messages) + media = "application/pdf" + filename = f"{safe_title}.pdf" + + else: + lines = ["LocalMind Export — Selected Messages", f"Exported: {ts}", "=" * 50, ""] + for m in messages: + role = "YOU" if m["role"] == "user" else "LOCALMIND" + msg_block = f"[{role}]\n{m['content'].strip()}" - lines += [msg_block, ""] - content = "\n".join(lines) - media = "text/plain" - filename = f"{safe_title}.txt" - - return Response( - content=content.encode("utf-8"), - media_type=media, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) \ No newline at end of file + if m["role"] == "assistant" and m.get("sources"): + source_names = [] + for src in m["sources"]: + if isinstance(src, dict): + source_names.append(src.get("source", "")) + else: + source_names.append(str(src)) + source_names = [s for s in source_names if s] + if source_names: + msg_block += f"\nSources: {', '.join(source_names)}" + + lines += [msg_block, ""] + content = "\n".join(lines).encode("utf-8") + media = "text/plain" + filename = f"{safe_title}.txt" + + success = True + return Response( + content=content, + media_type=media, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + except Exception as e: + if not error_msg: + if isinstance(e, HTTPException): + error_msg = e.detail + else: + error_msg = str(e) + raise + finally: + duration_ms = int((time.perf_counter() - start_time) * 1000) + details_dict = { + "success": success, + "duration_ms": duration_ms, + "messages_count": messages_count, + "error": error_msg + } + try: + db_service.log_export( + session_id=None, + format=req.format.value, + export_type="messages", + details=json.dumps(details_dict) + ) + except Exception as log_err: + import logging + logging.getLogger(__name__).error(f"Audit log failed: {log_err}") \ No newline at end of file diff --git a/backend/routes/upload.py b/backend/routes/upload.py index a19d86e..aaf0513 100644 --- a/backend/routes/upload.py +++ b/backend/routes/upload.py @@ -1,5 +1,6 @@ """Upload routes — /api/upload""" import os +import asyncio from pathlib import Path from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Query, BackgroundTasks from models.schemas import UploadResponse @@ -110,24 +111,39 @@ async def delete_document(doc_id: int): @router.get("/preview") async def preview_document(filename: str = Query(...), session_id: str = Query(...)): file_path = os.path.join(".", "data", "uploads", session_id, filename) - - if not os.path.exists(file_path): - raise HTTPException(status_code=404, detail="Document storage path not found.") - ext = Path(filename).suffix.lower() - try: - # If it's a standard clear-text format file layout, parse it cleanly - if ext in [".txt", ".md", ".csv", ".html", ".srt", ".vtt"]: - with open(file_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read(50000) # Capped at 50k characters for UI fast loading optimization - return {"content": content} - - # For non-trivial binary sets (PDFs/DOCX), offer a safe notice for version 1 - else: - return { - "content": f"[Binary Formatter Notice]\nPreviews for '{ext}' files are natively processed. Content indexed safely for Chat retrieval contexts." - } + max_attempts = 3 + backoff = 0.5 + + for attempt in range(1, max_attempts + 1): + try: + if not os.path.exists(file_path): + raise FileNotFoundError("Document storage path not found.") + + # If it's a standard clear-text format file layout, parse it cleanly + if ext in [".txt", ".md", ".csv", ".html", ".srt", ".vtt"]: + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read(50000) # Capped at 50k characters for UI fast loading optimization + return {"content": content} + + # For non-trivial binary sets (PDFs/DOCX), offer a safe notice for version 1 + else: + return { + "content": f"[Binary Formatter Notice]\nPreviews for '{ext}' files are natively processed. Content indexed safely for Chat retrieval contexts." + } + + except (FileNotFoundError, PermissionError, OSError) as e: + if attempt == max_attempts: + if isinstance(e, FileNotFoundError): + raise HTTPException(status_code=404, detail="Document storage path not found.") + raise HTTPException(status_code=500, detail=f"Failed to read file contents: {str(e)}") - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to read file contents: {str(e)}") + logger.warning( + "preview_retry route=/preview session=%s filename=%s error=%s attempt=%d/%d", + session_id, filename, str(e), attempt, max_attempts + ) + await asyncio.sleep(backoff) + backoff *= 2 + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to read file contents: {str(e)}") diff --git a/backend/services/db_service.py b/backend/services/db_service.py index 8cec224..583338a 100644 --- a/backend/services/db_service.py +++ b/backend/services/db_service.py @@ -228,6 +228,15 @@ def init_db(): success INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')) ); + + CREATE TABLE IF NOT EXISTS export_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT, + format TEXT NOT NULL, + export_type TEXT NOT NULL, + details TEXT, + created_at TEXT DEFAULT (datetime('now')) + ); CREATE TABLE IF NOT EXISTS shared_sessions ( id TEXT PRIMARY KEY, @@ -445,6 +454,28 @@ def get_messages_full(session_id: str) -> list[dict]: ] +def get_messages_by_ids(message_ids: list[str]) -> list[dict]: + if not message_ids: + return [] + placeholders = ",".join("?" for _ in message_ids) + with get_db() as conn: + rows = conn.execute( + f"SELECT id, role, content, sources, created_at, benchmarks FROM messages WHERE id IN ({placeholders}) ORDER BY created_at ASC", + message_ids, + ).fetchall() + return [ + { + "id": r["id"], + "role": r["role"], + "content": r["content"], + "sources": json.loads(r["sources"] or "[]"), + "created_at": r["created_at"], + "benchmarks": json.loads(r["benchmarks"] or "{}") + } + for r in rows + ] + + def clear_messages(session_id: str): with get_db() as conn: cur = conn.execute("DELETE FROM messages WHERE session_id=?", (session_id,)) @@ -556,6 +587,25 @@ def log_plugin(session_id: str, plugin: str, inp: str, out: str, success: bool = ) +# ─── Export logs ───────────────────────────────────────────── + +def get_export_logs(limit: int = 50) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM export_logs ORDER BY id DESC LIMIT ?", + (limit,) + ).fetchall() + return [dict(r) for r in rows] + +def log_export(session_id: str | None, format: str, export_type: str, details: str = None): + with get_db() as conn: + conn.execute( + "INSERT INTO export_logs (session_id, format, export_type, details) VALUES (?,?,?,?)", + (session_id, format, export_type, details), + ) + + + # ─── Shareable Sessions (Issue #270) ───────────────────────── def create_shared_session(session_id: str) -> str: diff --git a/backend/tests/test_export_audit.py b/backend/tests/test_export_audit.py new file mode 100644 index 0000000..80ac84e --- /dev/null +++ b/backend/tests/test_export_audit.py @@ -0,0 +1,130 @@ +import tempfile +import json +from unittest.mock import patch +from fastapi.testclient import TestClient +import services.db_service as db +from app import app + +# Setup temporary database +_tmp = tempfile.mktemp(suffix=".db") +db.DB_PATH = _tmp +db.init_db() + +client = TestClient(app) + + +def test_export_session_audit_success(): + # 1. Create a test session + db.create_session("audit_sess_1", title="Audit Single Session", model="llama3", language="en") + db.save_message("audit_sess_1", "user", "Message 1") + db.save_message("audit_sess_1", "assistant", "Response 1") + + # 2. Call single session export + response = client.get("/api/export/audit_sess_1/json") + assert response.status_code == 200 + + # 3. Retrieve export logs + logs_res = client.get("/api/export/logs?limit=5") + assert logs_res.status_code == 200 + logs = logs_res.json()["logs"] + + assert len(logs) >= 1 + latest_log = logs[0] + assert latest_log["session_id"] == "audit_sess_1" + assert latest_log["format"] == "json" + assert latest_log["export_type"] == "single" + + details = json.loads(latest_log["details"]) + assert details["success"] is True + assert details["messages_count"] == 2 + assert details["error"] is None + assert "duration_ms" in details + + +def test_bulk_export_audit_success(): + # 1. Create sessions + db.create_session("audit_bulk_1", title="Bulk Session A") + db.create_session("audit_bulk_2", title="Bulk Session B") + + # 2. Call bulk export + response = client.post( + "/api/export/sessions", + json={"session_ids": ["audit_bulk_1", "audit_bulk_2"], "format": "json"} + ) + assert response.status_code == 200 + + # 3. Check log + logs_res = client.get("/api/export/logs?limit=5") + assert logs_res.status_code == 200 + logs = logs_res.json()["logs"] + + latest_log = logs[0] + assert latest_log["session_id"] is None + assert latest_log["format"] == "json" + assert latest_log["export_type"] == "bulk" + + details = json.loads(latest_log["details"]) + assert details["success"] is True + assert details["sessions_count"] == 2 + assert details["error"] is None + + +def test_export_messages_audit_success(): + # 1. Create a session and message + db.create_session("audit_msg_sess", title="Msg Session") + db.save_message("audit_msg_sess", "user", "Unique Msg A") + + # Get message ID + msgs = db.get_messages_full("audit_msg_sess") + msg_id = msgs[0]["id"] + + # 2. Call messages export + response = client.post( + "/api/export/messages", + json={"message_ids": [str(msg_id)], "format": "markdown"} + ) + assert response.status_code == 200 + + # 3. Check log + logs_res = client.get("/api/export/logs?limit=5") + assert logs_res.status_code == 200 + logs = logs_res.json()["logs"] + + latest_log = logs[0] + assert latest_log["format"] == "markdown" + assert latest_log["export_type"] == "messages" + + details = json.loads(latest_log["details"]) + assert details["success"] is True + assert details["messages_count"] == 1 + + +def test_export_audit_failure_logged(): + # 1. Export nonexistent session + response = client.get("/api/export/nonexistent_session/json") + assert response.status_code == 404 + + # 2. Check that the failure was logged + logs_res = client.get("/api/export/logs?limit=5") + assert logs_res.status_code == 200 + logs = logs_res.json()["logs"] + + latest_log = logs[0] + assert latest_log["session_id"] == "nonexistent_session" + assert latest_log["export_type"] == "single" + + details = json.loads(latest_log["details"]) + assert details["success"] is False + assert details["error"] == "Session not found" + + +def test_audit_logging_resilience_to_failures(): + # If the logging function itself raises an exception, the export should NOT fail. + # 1. Setup session + db.create_session("audit_resilient_sess", title="Resilient Session") + + # 2. Mock db_service.log_export to raise an exception + with patch("services.db_service.log_export", side_effect=Exception("Database lock error/disk full")): + response = client.get("/api/export/audit_resilient_sess/json") + # The export should still succeed even though audit logging raised an exception + assert response.status_code == 200 diff --git a/backend/tests/test_pdf_export.py b/backend/tests/test_pdf_export.py new file mode 100644 index 0000000..0f3c5a6 --- /dev/null +++ b/backend/tests/test_pdf_export.py @@ -0,0 +1,66 @@ +import tempfile +import sys +from unittest.mock import patch, MagicMock +from fastapi.testclient import TestClient +import services.db_service as db + +# Setup temporary database for PDF export tests +_tmp = tempfile.mktemp(suffix=".db") +db.DB_PATH = _tmp +db.init_db() + +from app import app +client = TestClient(app) + +def test_pdf_export_session_success_whisper_present(): + # 1. Create a session and save a message + db.create_session("sess_pdf_1", title="PDF Whisper Present Chat", model="llama3", language="en") + db.save_message("sess_pdf_1", "user", "Hello PDF") + db.save_message("sess_pdf_1", "assistant", "Response PDF", [{"source": "report.pdf", "chunk": 0}]) + + # Mock whisper presence + mock_whisper = MagicMock() + with patch.dict(sys.modules, {"whisper": mock_whisper}): + response = client.get("/api/export/sess_pdf_1/pdf") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "Content-Disposition" in response.headers + assert "attachment; filename=" in response.headers["Content-Disposition"] + assert "pdf_whisper_present_chat.pdf" in response.headers["Content-Disposition"] + assert len(response.content) > 0 + # Check standard PDF header prefix + assert response.content.startswith(b"%PDF") + +def test_pdf_export_session_success_whisper_missing(): + # 1. Create a session and save a message + db.create_session("sess_pdf_2", title="PDF Whisper Missing Chat", model="llama3", language="en") + db.save_message("sess_pdf_2", "user", "Hello PDF 2") + db.save_message("sess_pdf_2", "assistant", "Response PDF 2") + + # Mock whisper missing + with patch.dict(sys.modules, {"whisper": None}): + sys.modules.pop("whisper", None) + response = client.get("/api/export/sess_pdf_2/pdf") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "pdf_whisper_missing_chat.pdf" in response.headers["Content-Disposition"] + assert response.content.startswith(b"%PDF") + +def test_pdf_export_messages_success_whisper_missing(): + # Save test messages and export them + db.create_session("sess_pdf_3", title="PDF Message Export", model="llama3") + db.save_message("sess_pdf_3", "user", "Standalone message to export") + # Retrieve messages to get the correct message ID from the database + messages = db.get_messages_full("sess_pdf_3") + db_msg_id = str(messages[0]["id"]) + + with patch.dict(sys.modules, {"whisper": None}): + sys.modules.pop("whisper", None) + response = client.post( + "/api/export/messages", + json={"message_ids": [db_msg_id], "format": "pdf"} + ) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/pdf" + assert "localmind_messages_" in response.headers["Content-Disposition"] + assert response.content.startswith(b"%PDF") diff --git a/backend/tests/test_preview_retry.py b/backend/tests/test_preview_retry.py new file mode 100644 index 0000000..839523a --- /dev/null +++ b/backend/tests/test_preview_retry.py @@ -0,0 +1,113 @@ +import os +import tempfile +import shutil +import asyncio +from unittest.mock import patch, MagicMock +from fastapi.testclient import TestClient +import pytest + +import services.db_service as db +from app import app + +_tmp = tempfile.mktemp(suffix=".db") +db.DB_PATH = _tmp +db.init_db() + +client = TestClient(app) + +@pytest.fixture(autouse=True) +def setup_teardown(): + # Setup: ensure uploads folder exists + os.makedirs("./data/uploads", exist_ok=True) + yield + # Teardown: clean up uploads and temp database + if os.path.exists("./data/uploads"): + shutil.rmtree("./data/uploads") + if os.path.exists(_tmp): + try: + os.remove(_tmp) + except Exception: + pass + +def test_preview_success(): + session_id = "session_success" + filename = "success.txt" + dir_path = f"./data/uploads/{session_id}" + os.makedirs(dir_path, exist_ok=True) + file_path = os.path.join(dir_path, filename) + + with open(file_path, "w", encoding="utf-8") as f: + f.write("Hello, LocalMind!") + + response = client.get(f"/api/upload/preview?filename={filename}&session_id={session_id}") + assert response.status_code == 200 + assert response.json()["content"] == "Hello, LocalMind!" + +def test_preview_transient_failure_then_success(): + session_id = "session_transient" + filename = "transient.txt" + dir_path = f"./data/uploads/{session_id}" + os.makedirs(dir_path, exist_ok=True) + file_path = os.path.join(dir_path, filename) + + with open(file_path, "w", encoding="utf-8") as f: + f.write("Recovered Content") + + call_count = 0 + original_open = open + + def mock_open(file, *args, **kwargs): + nonlocal call_count + if filename in str(file): + call_count += 1 + if call_count == 1: + raise PermissionError("Simulated temporary sharing lock") + return original_open(file, *args, **kwargs) + + with patch("builtins.open", side_effect=mock_open): + # We also mock asyncio.sleep to run the test instantly + with patch("asyncio.sleep", return_value=None) as mock_sleep: + response = client.get(f"/api/upload/preview?filename={filename}&session_id={session_id}") + assert response.status_code == 200 + assert response.json()["content"] == "Recovered Content" + assert call_count == 2 + mock_sleep.assert_called_once() + +def test_preview_retry_exhaustion(): + session_id = "session_exhaust" + filename = "locked.txt" + dir_path = f"./data/uploads/{session_id}" + os.makedirs(dir_path, exist_ok=True) + file_path = os.path.join(dir_path, filename) + + with open(file_path, "w", encoding="utf-8") as f: + f.write("Locked Content") + + call_count = 0 + def mock_open(file, *args, **kwargs): + nonlocal call_count + if filename in str(file): + call_count += 1 + raise PermissionError("Always locked") + raise FileNotFoundError() + + with patch("builtins.open", side_effect=mock_open): + with patch("asyncio.sleep", return_value=None) as mock_sleep: + response = client.get(f"/api/upload/preview?filename={filename}&session_id={session_id}") + assert response.status_code == 500 + assert "Failed to read file contents" in response.json()["detail"] + assert call_count == 3 + assert mock_sleep.call_count == 2 + +def test_preview_file_not_found(): + session_id = "session_not_found" + filename = "nonexistent.txt" + + # Check that os.path.exists is called 3 times (due to retries) and then returns 404 + with patch("os.path.exists", return_value=False) as mock_exists: + with patch("asyncio.sleep", return_value=None) as mock_sleep: + response = client.get(f"/api/upload/preview?filename={filename}&session_id={session_id}") + assert response.status_code == 404 + assert "Document storage path not found" in response.json()["detail"] + assert mock_exists.call_count == 3 + assert mock_sleep.call_count == 2 diff --git a/frontend/src/utils/api.js b/frontend/src/utils/api.js index 75bcda5..a412bfe 100644 --- a/frontend/src/utils/api.js +++ b/frontend/src/utils/api.js @@ -56,8 +56,26 @@ export const exportSession = (id, fmt) => window.open(`${BASE}/export/${id}/${fm export const deleteDocument = (docId) => req(`/upload/${docId}`, { method: "DELETE" }); // --- Issue #265: Fetch Read-Only Plain Text Document Preview --- -export const previewDocument = (filename, sessionId) => - req(`/upload/preview?filename=${encodeURIComponent(filename)}&session_id=${encodeURIComponent(sessionId)}`); +export const previewDocument = async (filename, sessionId, maxAttempts = 3, initialDelay = 500) => { + let attempt = 1; + let delay = initialDelay; + while (true) { + try { + return await req(`/upload/preview?filename=${encodeURIComponent(filename)}&session_id=${encodeURIComponent(sessionId)}`); + } catch (e) { + if (attempt >= maxAttempts) { + throw e; + } + if (e.message && (e.message.toLowerCase().includes("not found") || e.message.includes("404"))) { + throw e; + } + console.warn(`Preview fetch failed: ${e.message}. Retrying in ${delay}ms... (Attempt ${attempt}/${maxAttempts})`); + await new Promise(resolve => setTimeout(resolve, delay)); + attempt++; + delay *= 2; + } + } +}; // Prompt Templates export const getPromptTemplates = () => req("/prompt-templates/");