Fix to upload large files

ai_core/services/document_parser.py

@@ -4,36 +4,44 @@
 overlapping chunks suitable for embedding and semantic search.
 """
 
+import io
 import logging
 from pathlib import Path
+from typing import Union
 
 logger = logging.getLogger("ai_core")
 
 
-def extract_text(file_path: str) -> str:
+def extract_text(file_input: Union[str, io.IOBase]) -> str:
     """Extract plain text content from a document file.
 
     Args:
-        file_path: Absolute path to the document file.
+        file_input: Either a file path (str) or file-like object (e.g., UploadedFile, BytesIO).
 
     Returns:
         Extracted text as a single string. Returns empty string
         if the file format is unsupported or extraction fails.
     """
-    ext = Path(file_path).suffix.lower()
+    # Determine file extension
+    if isinstance(file_input, str):
+        ext = Path(file_input).suffix.lower()
+    else:
+        # File object - try to get name attribute, fallback to stream detection
+        name = getattr(file_input, 'name', '')
+        ext = Path(name).suffix.lower() if name else ''
 
     try:
         if ext == ".pdf":
-            return _extract_pdf(file_path)
+            return _extract_pdf(file_input)
         elif ext in (".doc", ".docx"):
-            return _extract_docx(file_path)
+            return _extract_docx(file_input)
         elif ext == ".pptx":
-            return _extract_pptx(file_path)
+            return _extract_pptx(file_input)
         else:
             logger.warning("Unsupported file format for text extraction: %s", ext)
             return ""
     except Exception as e:
-        logger.error("Failed to extract text from %s: %s", file_path, e)
+        logger.error("Failed to extract text from %s: %s", file_input, e)
         return ""
 
 
@@ -73,11 +81,11 @@ def chunk_text(
 # ---------------------------------------------------------------------------
 
 
-def _extract_pdf(file_path: str) -> str:
+def _extract_pdf(file_input: Union[str, io.IOBase]) -> str:
     """Extract text from a PDF file using PyPDF2."""
     from PyPDF2 import PdfReader
 
-    reader = PdfReader(file_path)
+    reader = PdfReader(file_input)
     pages: list[str] = []
     for page in reader.pages:
         page_text = page.extract_text()
@@ -89,23 +97,23 @@ def _extract_pdf(file_path: str) -> str:
     return text
 
 
-def _extract_docx(file_path: str) -> str:
+def _extract_docx(file_input: Union[str, io.IOBase]) -> str:
     """Extract text from a DOCX file using python-docx."""
     from docx import Document
 
-    doc = Document(file_path)
+    doc = Document(file_input)
     paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
 
     text = "\n".join(paragraphs)
     logger.info("Extracted %d characters from DOCX (%d paragraphs)", len(text), len(paragraphs))
     return text
 
 
-def _extract_pptx(file_path: str) -> str:
+def _extract_pptx(file_input: Union[str, io.IOBase]) -> str:
     """Extract text from a PPTX file using python-pptx."""
     from pptx import Presentation
 
-    prs = Presentation(file_path)
+    prs = Presentation(file_input)
     slides_text: list[str] = []
 
     for slide in prs.slides:

ai_core/services/rag.py

@@ -70,6 +70,9 @@ def embed_texts(texts: list[str]) -> list[list[float]]:
 
 def embed_and_store_chunks(upload_id: int) -> int:
     """Parse a document, chunk it, embed the chunks, and store in the database.
+
+    Streams text extraction from the file without saving to disk, then
+    deletes the original file after chunks are created.
 
     This function is idempotent — if chunks already exist for the given
     upload, it skips processing entirely (lazy indexing).
@@ -85,35 +88,66 @@ def embed_and_store_chunks(upload_id: int) -> int:
         return 0
 
     upload = Upload.objects.get(pk=upload_id)
-    file_path = upload.file.path
-
-    # Extract and chunk text
-    raw_text = extract_text(file_path)
+
+    # Stream extract text from file object (no temp file needed)
+    if not upload.file:
+        logger.warning("Upload %d has no file attached", upload_id)
+        return 0
+
+    # Open file object for extraction
+    try:
+        upload.file.open('rb')
+        raw_text = extract_text(upload.file)
+        upload.file.close()
+    except Exception as e:
+        logger.error("Failed to extract text from upload %d: %s", upload_id, e)
+        return 0
+
     if not raw_text:
-        logger.warning("No text extracted from upload %d (%s)", upload_id, file_path)
+        logger.warning("No text extracted from upload %d", upload_id)
+        # Delete file since it's useless
+        upload.file.delete(save=False)
         return 0
 
     chunks = chunk_text(raw_text)
     if not chunks:
         logger.warning("No chunks generated from upload %d", upload_id)
+        # Delete file since it's useless
+        upload.file.delete(save=False)
         return 0
 
-    # Batch embed all chunks in one API call
-    logger.info("Embedding %d chunks for upload %d", len(chunks), upload_id)
-    vectors = embed_texts(chunks)
-
-    # Bulk create chunk records
-    chunk_objects = [
-        DocumentChunk(
-            upload_id=upload_id,
-            chunk_index=i,
-            content=content,
-            embedding=vector,
+    # Batch embed chunks in smaller groups to avoid large requests
+    batch_size = 20
+    logger.info("Embedding %d chunks for upload %d using batch_size=%d", len(chunks), upload_id, batch_size)
+
+    chunk_objects = []
+    for start in range(0, len(chunks), batch_size):
+        batch = chunks[start : start + batch_size]
+        logger.info(
+            "Embedding chunk batch %d-%d for upload %d",
+            start,
+            min(start + batch_size, len(chunks)) - 1,
+            upload_id,
         )
-        for i, (content, vector) in enumerate(zip(chunks, vectors))
-    ]
+        vectors = embed_texts(batch)
+        chunk_objects.extend(
+            DocumentChunk(
+                upload_id=upload_id,
+                chunk_index=i,
+                content=content,
+                embedding=vector,
+            )
+            for i, (content, vector) in enumerate(
+                zip(batch, vectors), start=start
+            )
+        )
+
     DocumentChunk.objects.bulk_create(chunk_objects)
     logger.info("Stored %d chunks for upload %d", len(chunk_objects), upload_id)
+
+    # Delete the original file after successful chunking and embedding
+    logger.info("Deleting original file for upload %d (chunks stored in DB)", upload_id)
+    upload.file.delete(save=False)
 
     return len(chunk_objects)
 
@@ -135,7 +169,7 @@ def _cosine_similarity(vec_a: np.ndarray, vec_b: np.ndarray) -> float:
 def search_chunks(
     query: str,
     upload_ids: list[int],
-    top_k: int = 3,
+    top_k: int = 10,
 ) -> str:
     """Find the most relevant document chunks for a given query.
 
@@ -182,5 +216,9 @@ def search_chunks(
         len(scored),
         len(top_chunks),
     )
+
+    # Log the top chunks for debugging
+    for i, chunk in enumerate(top_chunks, 1):
+        logger.debug(f"Top chunk {i}: {chunk[:200]}...")
 
     return "\n\n---\n\n".join(top_chunks)

ai_core/views.py

@@ -106,6 +106,14 @@ def generate_lesson(request):
             service = YoloFarmLessonService(llm)
         else:
             service = LessonService(llm)
+
+        # Log context being used
+        if context:
+            logger.info(f"Generating lesson with context: {len(context)} chars")
+            logger.debug(f"Context preview: {context[:500]}...")
+        else:
+            logger.warning(f"Generating lesson WITHOUT context for topic: {topic}")
+
         html_content = service.generate(
             topic, requirements=requirements, context=context
         )

config/settings.py

@@ -284,5 +284,6 @@ def gettext(s):
     'API_SECRET': config('CLOUDINARY_API_SECRET'),
 }
 
-# Tell Django to upload all media files to Cloudinary
-DEFAULT_FILE_STORAGE = 'cloudinary_storage.storage.RawMediaCloudinaryStorage'
+# Tell Django to upload all media files using default FileSystemStorage
+# DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
+# (using default FileSystemStorage which stores in MEDIA_ROOT)

course/views.py

@@ -7,6 +7,8 @@
 from django.utils.decorators import method_decorator
 from django.views.generic import CreateView
 from django_filters.views import FilterView
+import threading
+import logging
 
 from accounts.decorators import lecturer_required, student_required
 from accounts.models import Student
@@ -29,6 +31,8 @@
 )
 from result.models import TakenCourse
 
+logger = logging.getLogger("course")
+
 
 # ########################################################
 # Program Views
@@ -271,6 +275,22 @@ def handle_file_upload(request, slug):
             upload.course = course
             upload.save()
             messages.success(request, f"{upload.title} has been uploaded.")
+
+            # Process document asynchronously: extract text -> chunk -> embed -> delete file
+            def process_document():
+                try:
+                    from ai_core.services.rag import embed_and_store_chunks
+                    chunk_count = embed_and_store_chunks(upload.id)
+                    logger = logging.getLogger("course")
+                    logger.info(f"Document {upload.id} processed: {chunk_count} chunks created")
+                except Exception as e:
+                    logger = logging.getLogger("course")
+                    logger.error(f"Failed to process document {upload.id}: {e}", exc_info=True)
+
+            # Run in background thread so user gets immediate feedback
+            thread = threading.Thread(target=process_document, daemon=True)
+            thread.start()
+
             return redirect("course_detail", slug=slug)
         messages.error(request, "Correct the error(s) below.")
     else: