diff --git a/ai_core/services/document_parser.py b/ai_core/services/document_parser.py index 1ec517c5..002e7d93 100644 --- a/ai_core/services/document_parser.py +++ b/ai_core/services/document_parser.py @@ -4,36 +4,44 @@ overlapping chunks suitable for embedding and semantic search. """ +import io import logging from pathlib import Path +from typing import Union logger = logging.getLogger("ai_core") -def extract_text(file_path: str) -> str: +def extract_text(file_input: Union[str, io.IOBase]) -> str: """Extract plain text content from a document file. Args: - file_path: Absolute path to the document file. + file_input: Either a file path (str) or file-like object (e.g., UploadedFile, BytesIO). Returns: Extracted text as a single string. Returns empty string if the file format is unsupported or extraction fails. """ - ext = Path(file_path).suffix.lower() + # Determine file extension + if isinstance(file_input, str): + ext = Path(file_input).suffix.lower() + else: + # File object - try to get name attribute, fallback to stream detection + name = getattr(file_input, 'name', '') + ext = Path(name).suffix.lower() if name else '' try: if ext == ".pdf": - return _extract_pdf(file_path) + return _extract_pdf(file_input) elif ext in (".doc", ".docx"): - return _extract_docx(file_path) + return _extract_docx(file_input) elif ext == ".pptx": - return _extract_pptx(file_path) + return _extract_pptx(file_input) else: logger.warning("Unsupported file format for text extraction: %s", ext) return "" except Exception as e: - logger.error("Failed to extract text from %s: %s", file_path, e) + logger.error("Failed to extract text from %s: %s", file_input, e) return "" @@ -73,11 +81,11 @@ def chunk_text( # --------------------------------------------------------------------------- -def _extract_pdf(file_path: str) -> str: +def _extract_pdf(file_input: Union[str, io.IOBase]) -> str: """Extract text from a PDF file using PyPDF2.""" from PyPDF2 import PdfReader - reader = PdfReader(file_path) + reader = PdfReader(file_input) pages: list[str] = [] for page in reader.pages: page_text = page.extract_text() @@ -89,11 +97,11 @@ def _extract_pdf(file_path: str) -> str: return text -def _extract_docx(file_path: str) -> str: +def _extract_docx(file_input: Union[str, io.IOBase]) -> str: """Extract text from a DOCX file using python-docx.""" from docx import Document - doc = Document(file_path) + doc = Document(file_input) paragraphs = [p.text for p in doc.paragraphs if p.text.strip()] text = "\n".join(paragraphs) @@ -101,11 +109,11 @@ def _extract_docx(file_path: str) -> str: return text -def _extract_pptx(file_path: str) -> str: +def _extract_pptx(file_input: Union[str, io.IOBase]) -> str: """Extract text from a PPTX file using python-pptx.""" from pptx import Presentation - prs = Presentation(file_path) + prs = Presentation(file_input) slides_text: list[str] = [] for slide in prs.slides: diff --git a/ai_core/services/rag.py b/ai_core/services/rag.py index 48f37285..09d1e528 100644 --- a/ai_core/services/rag.py +++ b/ai_core/services/rag.py @@ -70,6 +70,9 @@ def embed_texts(texts: list[str]) -> list[list[float]]: def embed_and_store_chunks(upload_id: int) -> int: """Parse a document, chunk it, embed the chunks, and store in the database. + + Streams text extraction from the file without saving to disk, then + deletes the original file after chunks are created. This function is idempotent — if chunks already exist for the given upload, it skips processing entirely (lazy indexing). @@ -85,35 +88,66 @@ def embed_and_store_chunks(upload_id: int) -> int: return 0 upload = Upload.objects.get(pk=upload_id) - file_path = upload.file.path - - # Extract and chunk text - raw_text = extract_text(file_path) + + # Stream extract text from file object (no temp file needed) + if not upload.file: + logger.warning("Upload %d has no file attached", upload_id) + return 0 + + # Open file object for extraction + try: + upload.file.open('rb') + raw_text = extract_text(upload.file) + upload.file.close() + except Exception as e: + logger.error("Failed to extract text from upload %d: %s", upload_id, e) + return 0 + if not raw_text: - logger.warning("No text extracted from upload %d (%s)", upload_id, file_path) + logger.warning("No text extracted from upload %d", upload_id) + # Delete file since it's useless + upload.file.delete(save=False) return 0 chunks = chunk_text(raw_text) if not chunks: logger.warning("No chunks generated from upload %d", upload_id) + # Delete file since it's useless + upload.file.delete(save=False) return 0 - # Batch embed all chunks in one API call - logger.info("Embedding %d chunks for upload %d", len(chunks), upload_id) - vectors = embed_texts(chunks) - - # Bulk create chunk records - chunk_objects = [ - DocumentChunk( - upload_id=upload_id, - chunk_index=i, - content=content, - embedding=vector, + # Batch embed chunks in smaller groups to avoid large requests + batch_size = 20 + logger.info("Embedding %d chunks for upload %d using batch_size=%d", len(chunks), upload_id, batch_size) + + chunk_objects = [] + for start in range(0, len(chunks), batch_size): + batch = chunks[start : start + batch_size] + logger.info( + "Embedding chunk batch %d-%d for upload %d", + start, + min(start + batch_size, len(chunks)) - 1, + upload_id, ) - for i, (content, vector) in enumerate(zip(chunks, vectors)) - ] + vectors = embed_texts(batch) + chunk_objects.extend( + DocumentChunk( + upload_id=upload_id, + chunk_index=i, + content=content, + embedding=vector, + ) + for i, (content, vector) in enumerate( + zip(batch, vectors), start=start + ) + ) + DocumentChunk.objects.bulk_create(chunk_objects) logger.info("Stored %d chunks for upload %d", len(chunk_objects), upload_id) + + # Delete the original file after successful chunking and embedding + logger.info("Deleting original file for upload %d (chunks stored in DB)", upload_id) + upload.file.delete(save=False) return len(chunk_objects) @@ -135,7 +169,7 @@ def _cosine_similarity(vec_a: np.ndarray, vec_b: np.ndarray) -> float: def search_chunks( query: str, upload_ids: list[int], - top_k: int = 3, + top_k: int = 10, ) -> str: """Find the most relevant document chunks for a given query. @@ -182,5 +216,9 @@ def search_chunks( len(scored), len(top_chunks), ) + + # Log the top chunks for debugging + for i, chunk in enumerate(top_chunks, 1): + logger.debug(f"Top chunk {i}: {chunk[:200]}...") return "\n\n---\n\n".join(top_chunks) diff --git a/ai_core/views.py b/ai_core/views.py index 3dfd40cc..3e26b901 100644 --- a/ai_core/views.py +++ b/ai_core/views.py @@ -106,6 +106,14 @@ def generate_lesson(request): service = YoloFarmLessonService(llm) else: service = LessonService(llm) + + # Log context being used + if context: + logger.info(f"Generating lesson with context: {len(context)} chars") + logger.debug(f"Context preview: {context[:500]}...") + else: + logger.warning(f"Generating lesson WITHOUT context for topic: {topic}") + html_content = service.generate( topic, requirements=requirements, context=context ) diff --git a/config/settings.py b/config/settings.py index 4306fba8..411efc5d 100644 --- a/config/settings.py +++ b/config/settings.py @@ -284,5 +284,6 @@ def gettext(s): 'API_SECRET': config('CLOUDINARY_API_SECRET'), } -# Tell Django to upload all media files to Cloudinary -DEFAULT_FILE_STORAGE = 'cloudinary_storage.storage.RawMediaCloudinaryStorage' +# Tell Django to upload all media files using default FileSystemStorage +# DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +# (using default FileSystemStorage which stores in MEDIA_ROOT) diff --git a/course/views.py b/course/views.py index 1666d983..65c7bab9 100644 --- a/course/views.py +++ b/course/views.py @@ -7,6 +7,8 @@ from django.utils.decorators import method_decorator from django.views.generic import CreateView from django_filters.views import FilterView +import threading +import logging from accounts.decorators import lecturer_required, student_required from accounts.models import Student @@ -29,6 +31,8 @@ ) from result.models import TakenCourse +logger = logging.getLogger("course") + # ######################################################## # Program Views @@ -271,6 +275,22 @@ def handle_file_upload(request, slug): upload.course = course upload.save() messages.success(request, f"{upload.title} has been uploaded.") + + # Process document asynchronously: extract text -> chunk -> embed -> delete file + def process_document(): + try: + from ai_core.services.rag import embed_and_store_chunks + chunk_count = embed_and_store_chunks(upload.id) + logger = logging.getLogger("course") + logger.info(f"Document {upload.id} processed: {chunk_count} chunks created") + except Exception as e: + logger = logging.getLogger("course") + logger.error(f"Failed to process document {upload.id}: {e}", exc_info=True) + + # Run in background thread so user gets immediate feedback + thread = threading.Thread(target=process_document, daemon=True) + thread.start() + return redirect("course_detail", slug=slug) messages.error(request, "Correct the error(s) below.") else: