Implement evaluation

.env.template

@@ -8,6 +8,10 @@ HUGGINGFACE_EMBEDDING_MODEL=
 # For cohere re-ranking model
 COHERE_API_KEY=
 
+# Groq AI model
+GROQ_API_KEY=
+GROQ_MODEL=
+
 # Logging configuration
 LOG_NAME=study-buddy
 LOG_LEVEL=INFO

.gitignore

@@ -2,7 +2,6 @@
 __pycache__/
 *.py[oc]
 build/
-data/
 dist/
 logs/
 wheels/

app.py

@@ -1,8 +1,9 @@
-from controllers.app_controller import chunk_document, chunk_document_semantically, chunk_document_with_layout, query_ai_model, retrieve_and_query_ai_model
+from controllers.app_controller import chunk_document, chunk_document_semantically, chunk_document_with_layout, query_ai_model, retrieve_and_query_ai_model, evaluate_model, preload_chroma_db
 from fastapi import FastAPI
 from models.app_models import DocumentProcessRequest, QueryRequest
 from services.chroma_db_service import connect_to_chroma_db, disconnect_chroma_db, get_document_count
 from services.query_service import connect_to_google_ai
+from evaluation.evaluations import connect_to_groq_ai
 from utils.logger import log
 
 async def lifespan(app: FastAPI):
@@ -13,8 +14,10 @@ async def lifespan(app: FastAPI):
     log.info("Backend server starting up")
     app.state.chroma_db = connect_to_chroma_db()
     app.state.google_ai = connect_to_google_ai()
+    app.state.groq_ai = connect_to_groq_ai()
+
     yield
-    # Anything after yeild is for teardown / cleanup
+    # Anything after yield is for teardown / cleanup
     log.warning("Disconnecting from chroma DB")
     disconnect_chroma_db(app.state.chroma_db )
     log.warning("Backend server shutting down")
@@ -66,3 +69,11 @@ async def rag_query_ai_model(request: QueryRequest) -> dict:
     Retrieves and queries the model.
     """
     return retrieve_and_query_ai_model(request, app.state.google_ai, app.state.chroma_db)
+
+@app.post("/evaluate")
+async def evaluate():
+    return evaluate_model(app.state.google_ai, app.state.groq_ai)
+
+@app.post("/evaluate/preload")
+async def preload():
+    return preload_chroma_db()

controllers/app_controller.py

@@ -1,9 +1,10 @@
 from langchain_chroma import Chroma
 from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_groq import ChatGroq
 from models.app_models import DocumentProcessRequest, QueryRequest
 
 from services.document_chunking import layout_chunking, read_pdf_document, native_chunking, read_pdf_document_into_markdown, semantic_chunking
-from services.chroma_db_service import embed_and_add_document, multi_retrieve
+from services.chroma_db_service import embed_and_add_document, multi_retrieve, retrieve
 
 from services.query_service import query_google_ai
 from services.reranking import rerank
@@ -32,7 +33,7 @@ def chunk_document_semantically(process_request: DocumentProcessRequest, chroma_
     list_of_documents = read_pdf_document(process_request.document_path)
     semantic_chunks = semantic_chunking(list_of_documents)
     embed_and_add_document(semantic_chunks, chroma_db)
-    
+
 def chunk_document_with_layout(process_request: DocumentProcessRequest, chroma_db: Chroma) -> None:
     """
     Performs layout-aware chunking on the given document and stores it in the chroma vector database.
@@ -51,7 +52,7 @@ def query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI) ->
     """
     return query_google_ai(request.query, google_ai)
 
-def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI, chroma_db ) -> dict:
+def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI, chroma_db: Chroma ) -> dict:
     """
     Retrieves relevant documents and dends it as context to the model
     """
@@ -68,4 +69,49 @@ def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGene
     context = "\n\n".join(reranked_context)
     query_with_context = f"Context:\n{context}\n\nQuestion:\n{request.query}"
 
-    return query_google_ai(query_with_context, google_ai)
+    return query_google_ai(query_with_context, google_ai)
+
+################################################## EVALUATION FUNCTIONS ##################################################
+def evaluate_model(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq):  
+    from evaluation.eval_models import base_chunk_base_retrieve, base_chunk_multi_retrieve, base_chunk_rerank, layout_chunk_base_retrieve, layout_chunk_multi_retrieve, base_chunk_everything, plain
+    # Test base chunk + base retrieve
+    # base_results = base_chunk_base_retrieve(google_ai, groq_ai)
+
+    # Test base chunk + multi retrieve
+    # query_transformation_results = base_chunk_multi_retrieve(google_ai, groq_ai) 
+
+    # Test base chunk + reranking
+    # base_chunk_rerank_results = base_chunk_rerank(google_ai, groq_ai)
+
+    # Test layout chunk + base retrieve
+    # layout_chunk_base_retrieve_results = layout_chunk_base_retrieve(google_ai, groq_ai)
+
+    # Test layout chunk + multi retrieve
+    # layout_chunk_multi_retrieve_results = layout_chunk_multi_retrieve(google_ai, groq_ai)
+    # base_chunk_everything_result = base_chunk_everything(google_ai, groq_ai)
+
+    plain(google_ai, groq_ai)
+
+########### PRE-LOAD CHROMA DATABASE ###########
+
+def preload_chroma_db():
+    from evaluation.pre_loader import preload_native_chunking, preload_layout_chunking, combi_chunking
+
+    document_file_names = [
+        "Computational_cost_of_semantic_chunking.pdf",
+        "Reconstructing_Context.pdf",
+        "Accelerating_LLM_Inference.pdf",
+        "Beyond_Long_Context.pdf",
+        "LLM_Chunk_Filtering_Method.pdf",
+        "Long_Context_Modelling.pdf"
+    ]
+
+    document_paths = [f"static/{name}" for name in document_file_names]
+
+    for i, filepath in enumerate(document_paths):
+        process_request = DocumentProcessRequest(document_path=filepath)
+        normal_name = f"normal-{document_file_names[i]}"
+        layout_name = f"layout-{document_file_names[i]}"
+        preload_native_chunking(normal_name, process_request)
+        preload_layout_chunking(layout_name, process_request)
+        combi_chunking(process_request)

data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/length.bin

@@ -0,0 +1 @@
+invalid type: string "# Output: token-level sparsity mask **M**  \n1: Define chunks **C** _j_ _←_ [ **t** _b_ _j_ _, . . .,_ **t** _b_ _j_ +1 ], for _j_ = 0\nto _N_ _c_ _−_ 1\n2: Obtain chunk-level queries **Q** _c_ [ _j,_ :] and keys **K** _c_ [ _j,_ :\n] for each **C** _j_\n3: Compute chunk-level similarity: **S** _c_ _←_ **Q** _c_ **K** _[⊤]_ _c_ _[∈]_\nR _[N]_ _[c]_ _[×][N]_ _[c]_  

data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/length.bin

@@ -0,0 +1 @@
+The ablation study highlights redundancy filtering’s key role in ChunkRAG, with dynamic chunk merging and optimal similarity thresholds (validated at _θ_ = 0 _._ 8 ) balancing chunk reduction and relevance while preventing over-filtering. Future work could investigate domain-specific thresholds for varying chunk granularity needs and incorporate computational efficiency metrics to assess scalab

data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/length.bin

@@ -0,0 +1 @@
+invalid type: string "Retrieval (%). Datasets marked with * are stitched. Rows are sorted by the average number of sentences per document (before stitching) in ascending order for easier comparison.", expected f64d i64oolean chunker, we evaluated three hyperparameters: , similar to single-linkage; EPS, the maximum distance between two samples for them to be considered part of the same neighbor- ho