From 0d92cc4c10fd8909f002bb40deb2a315caf2cd77 Mon Sep 17 00:00:00 2001 From: Shalini Date: Wed, 25 Feb 2026 21:03:12 +0530 Subject: [PATCH 1/3] feat(engine): add query preprocessing pipeline before retrieval --- engine/preprocessing.py | 11 +++++++++++ engine/rag_engine.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 engine/preprocessing.py diff --git a/engine/preprocessing.py b/engine/preprocessing.py new file mode 100644 index 0000000..19d4d4b --- /dev/null +++ b/engine/preprocessing.py @@ -0,0 +1,11 @@ +def preprocess_query(query: str) -> str: + """ + Basic query preprocessing: + - Lowercases text + - Strips extra whitespace + """ + if not query: + return "" + + query = query.lower().strip() + return query \ No newline at end of file diff --git a/engine/rag_engine.py b/engine/rag_engine.py index 05735b8..0674e3f 100644 --- a/engine/rag_engine.py +++ b/engine/rag_engine.py @@ -12,7 +12,7 @@ import hashlib import streamlit as st import numpy as np - +from engine.preprocessing import preprocess_query try: import pdfplumber except Exception: @@ -205,6 +205,7 @@ def _emb_search(query: str, top_k: int = 3): return None def search_pdfs(query: str, top_k: int = 3): + query = preprocess_query(query) """ Default: if an embeddings engine is configured -> use it. Else if internal embeddings available -> use that. From a59bbb48bcb2502f4a5a75e268b0ecd4513fd041 Mon Sep 17 00:00:00 2001 From: Shalini Date: Mon, 2 Mar 2026 20:21:24 +0530 Subject: [PATCH 2/3] Fix missing preprocess_query import --- engine/rag_engine.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/engine/rag_engine.py b/engine/rag_engine.py index d282fe6..a00614e 100644 --- a/engine/rag_engine.py +++ b/engine/rag_engine.py @@ -14,6 +14,7 @@ import streamlit as st import numpy as np import logging +from engine.preprocessing import preprocess_query logger = logging.getLogger(__name__) try: @@ -203,9 +204,6 @@ def _emb_search(query: str, top_k: int = 3): # --- USE CACHED MODEL HERE --- model = load_embedding_model() - # --- USE CACHED MODEL HERE --- - model = load_embedding_model() - qvec = model.encode([query], convert_to_numpy=True)[0] scores = [] for d in _EMB_INDEX: From 2475955705504257166b7afc5bb611ad012c32f7 Mon Sep 17 00:00:00 2001 From: Shalini Date: Tue, 3 Mar 2026 11:12:29 +0530 Subject: [PATCH 3/3] fix: ensure preprocess_query exists for CI --- engine/preprocessing.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/engine/preprocessing.py b/engine/preprocessing.py index 4234beb..19d4d4b 100644 --- a/engine/preprocessing.py +++ b/engine/preprocessing.py @@ -1,20 +1,11 @@ -import re - -def normalize_query(query: str) -> str: +def preprocess_query(query: str) -> str: """ - Normalize user query before retrieval: - - Lowercase conversion - - Remove unnecessary punctuation - - Remove extra whitespace + Basic query preprocessing: + - Lowercases text + - Strips extra whitespace """ + if not query: + return "" - # Convert to lowercase - query = query.lower() - - # Remove punctuation (keep alphanumeric and spaces) - query = re.sub(r"[^\w\s]", "", query) - - # Remove extra whitespace - query = re.sub(r"\s+", " ", query).strip() - + query = query.lower().strip() return query \ No newline at end of file