diff --git a/.env.template b/.env.template index 40d0d2e..525b1e5 100644 --- a/.env.template +++ b/.env.template @@ -8,6 +8,10 @@ HUGGINGFACE_EMBEDDING_MODEL= # For cohere re-ranking model COHERE_API_KEY= +# Groq AI model +GROQ_API_KEY= +GROQ_MODEL= + # Logging configuration LOG_NAME=study-buddy LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore index 79bc60b..e49da7d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ __pycache__/ *.py[oc] build/ -data/ dist/ logs/ wheels/ diff --git a/app.py b/app.py index 01d8893..f7c0097 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,9 @@ -from controllers.app_controller import chunk_document, chunk_document_semantically, chunk_document_with_layout, query_ai_model, retrieve_and_query_ai_model +from controllers.app_controller import chunk_document, chunk_document_semantically, chunk_document_with_layout, query_ai_model, retrieve_and_query_ai_model, evaluate_model, preload_chroma_db from fastapi import FastAPI from models.app_models import DocumentProcessRequest, QueryRequest from services.chroma_db_service import connect_to_chroma_db, disconnect_chroma_db, get_document_count from services.query_service import connect_to_google_ai +from evaluation.evaluations import connect_to_groq_ai from utils.logger import log async def lifespan(app: FastAPI): @@ -13,8 +14,10 @@ async def lifespan(app: FastAPI): log.info("Backend server starting up") app.state.chroma_db = connect_to_chroma_db() app.state.google_ai = connect_to_google_ai() + app.state.groq_ai = connect_to_groq_ai() + yield - # Anything after yeild is for teardown / cleanup + # Anything after yield is for teardown / cleanup log.warning("Disconnecting from chroma DB") disconnect_chroma_db(app.state.chroma_db ) log.warning("Backend server shutting down") @@ -66,3 +69,11 @@ async def rag_query_ai_model(request: QueryRequest) -> dict: Retrieves and queries the model. """ return retrieve_and_query_ai_model(request, app.state.google_ai, app.state.chroma_db) + +@app.post("/evaluate") +async def evaluate(): + return evaluate_model(app.state.google_ai, app.state.groq_ai) + +@app.post("/evaluate/preload") +async def preload(): + return preload_chroma_db() \ No newline at end of file diff --git a/controllers/app_controller.py b/controllers/app_controller.py index 9fcc1eb..809b24c 100644 --- a/controllers/app_controller.py +++ b/controllers/app_controller.py @@ -1,9 +1,10 @@ from langchain_chroma import Chroma from langchain_google_genai import ChatGoogleGenerativeAI +from langchain_groq import ChatGroq from models.app_models import DocumentProcessRequest, QueryRequest from services.document_chunking import layout_chunking, read_pdf_document, native_chunking, read_pdf_document_into_markdown, semantic_chunking -from services.chroma_db_service import embed_and_add_document, multi_retrieve +from services.chroma_db_service import embed_and_add_document, multi_retrieve, retrieve from services.query_service import query_google_ai from services.reranking import rerank @@ -32,7 +33,7 @@ def chunk_document_semantically(process_request: DocumentProcessRequest, chroma_ list_of_documents = read_pdf_document(process_request.document_path) semantic_chunks = semantic_chunking(list_of_documents) embed_and_add_document(semantic_chunks, chroma_db) - + def chunk_document_with_layout(process_request: DocumentProcessRequest, chroma_db: Chroma) -> None: """ Performs layout-aware chunking on the given document and stores it in the chroma vector database. @@ -51,7 +52,7 @@ def query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI) -> """ return query_google_ai(request.query, google_ai) -def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI, chroma_db ) -> dict: +def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGenerativeAI, chroma_db: Chroma ) -> dict: """ Retrieves relevant documents and dends it as context to the model """ @@ -68,4 +69,49 @@ def retrieve_and_query_ai_model(request: QueryRequest, google_ai: ChatGoogleGene context = "\n\n".join(reranked_context) query_with_context = f"Context:\n{context}\n\nQuestion:\n{request.query}" - return query_google_ai(query_with_context, google_ai) \ No newline at end of file + return query_google_ai(query_with_context, google_ai) + +################################################## EVALUATION FUNCTIONS ################################################## +def evaluate_model(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + from evaluation.eval_models import base_chunk_base_retrieve, base_chunk_multi_retrieve, base_chunk_rerank, layout_chunk_base_retrieve, layout_chunk_multi_retrieve, base_chunk_everything, plain + # Test base chunk + base retrieve + # base_results = base_chunk_base_retrieve(google_ai, groq_ai) + + # Test base chunk + multi retrieve + # query_transformation_results = base_chunk_multi_retrieve(google_ai, groq_ai) + + # Test base chunk + reranking + # base_chunk_rerank_results = base_chunk_rerank(google_ai, groq_ai) + + # Test layout chunk + base retrieve + # layout_chunk_base_retrieve_results = layout_chunk_base_retrieve(google_ai, groq_ai) + + # Test layout chunk + multi retrieve + # layout_chunk_multi_retrieve_results = layout_chunk_multi_retrieve(google_ai, groq_ai) + # base_chunk_everything_result = base_chunk_everything(google_ai, groq_ai) + + plain(google_ai, groq_ai) + +########### PRE-LOAD CHROMA DATABASE ########### + +def preload_chroma_db(): + from evaluation.pre_loader import preload_native_chunking, preload_layout_chunking, combi_chunking + + document_file_names = [ + "Computational_cost_of_semantic_chunking.pdf", + "Reconstructing_Context.pdf", + "Accelerating_LLM_Inference.pdf", + "Beyond_Long_Context.pdf", + "LLM_Chunk_Filtering_Method.pdf", + "Long_Context_Modelling.pdf" + ] + + document_paths = [f"static/{name}" for name in document_file_names] + + for i, filepath in enumerate(document_paths): + process_request = DocumentProcessRequest(document_path=filepath) + normal_name = f"normal-{document_file_names[i]}" + layout_name = f"layout-{document_file_names[i]}" + preload_native_chunking(normal_name, process_request) + preload_layout_chunking(layout_name, process_request) + combi_chunking(process_request) diff --git a/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/data_level0.bin b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/data_level0.bin new file mode 100644 index 0000000..b56e9d9 Binary files /dev/null and b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/data_level0.bin differ diff --git a/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/header.bin b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/header.bin differ diff --git a/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/length.bin b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/length.bin new file mode 100644 index 0000000..393aac2 --- /dev/null +++ b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/length.bin @@ -0,0 +1 @@ +invalid type: string "# Output: token-level sparsity mask **M** \n1: Define chunks **C** _j_ _←_ [ **t** _b_ _j_ _, . . .,_ **t** _b_ _j_ +1 ], for _j_ = 0\nto _N_ _c_ _−_ 1\n2: Obtain chunk-level queries **Q** _c_ [ _j,_ :] and keys **K** _c_ [ _j,_ :\n] for each **C** _j_\n3: Compute chunk-level similarity: **S** _c_ _←_ **Q** _c_ **K** _[⊤]_ _c_ _[∈]_\nR _[N]_ _[c]_ _[×][N]_ _[c]_ \ No newline at end of file diff --git a/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/link_lists.bin b/data/chroma_langchain_db/0a50730c-6ee7-4618-a47a-37e2f25e2672/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/data_level0.bin b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/data_level0.bin new file mode 100644 index 0000000..d05ada2 Binary files /dev/null and b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/data_level0.bin differ diff --git a/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/header.bin b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/header.bin differ diff --git a/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/length.bin b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/length.bin new file mode 100644 index 0000000..90e3f38 Binary files /dev/null and b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/length.bin differ diff --git a/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/link_lists.bin b/data/chroma_langchain_db/1f0514bf-d16a-4185-9d27-3f561c9ff559/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/data_level0.bin b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/data_level0.bin new file mode 100644 index 0000000..e8a6134 Binary files /dev/null and b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/data_level0.bin differ diff --git a/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/header.bin b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/header.bin differ diff --git a/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/length.bin b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/length.bin new file mode 100644 index 0000000..9b8880c --- /dev/null +++ b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/length.bin @@ -0,0 +1 @@ +The ablation study highlights redundancy filtering’s key role in ChunkRAG, with dynamic chunk merging and optimal similarity thresholds (validated at _θ_ = 0 _._ 8 ) balancing chunk reduction and relevance while preventing over-filtering. Future work could investigate domain-specific thresholds for varying chunk granularity needs and incorporate computational efficiency metrics to assess scalab \ No newline at end of file diff --git a/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/link_lists.bin b/data/chroma_langchain_db/2c15cf21-caad-4aa5-ba92-a7233fa5cd9c/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/data_level0.bin b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/data_level0.bin new file mode 100644 index 0000000..9fd7258 Binary files /dev/null and b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/data_level0.bin differ diff --git a/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/header.bin b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/header.bin differ diff --git a/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/length.bin b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/length.bin new file mode 100644 index 0000000..70ec1b3 Binary files /dev/null and b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/length.bin differ diff --git a/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/link_lists.bin b/data/chroma_langchain_db/464ccb30-ac23-439f-a79e-a8f5c5f8e754/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/data_level0.bin b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/data_level0.bin new file mode 100644 index 0000000..3706c56 Binary files /dev/null and b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/data_level0.bin differ diff --git a/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/header.bin b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/header.bin differ diff --git a/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/length.bin b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/length.bin new file mode 100644 index 0000000..e3ceaeb Binary files /dev/null and b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/length.bin differ diff --git a/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/link_lists.bin b/data/chroma_langchain_db/4d369b52-4fef-4a7b-8e0f-95b3a78ccd48/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/data_level0.bin b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/data_level0.bin new file mode 100644 index 0000000..68771a9 Binary files /dev/null and b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/data_level0.bin differ diff --git a/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/header.bin b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/header.bin differ diff --git a/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/length.bin b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/length.bin new file mode 100644 index 0000000..d958972 Binary files /dev/null and b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/length.bin differ diff --git a/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/link_lists.bin b/data/chroma_langchain_db/656d55d1-b2a9-438a-8376-4502a34c327f/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/data_level0.bin b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/data_level0.bin new file mode 100644 index 0000000..8b1602f Binary files /dev/null and b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/data_level0.bin differ diff --git a/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/header.bin b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/header.bin differ diff --git a/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/length.bin b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/length.bin new file mode 100644 index 0000000..04e8696 Binary files /dev/null and b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/length.bin differ diff --git a/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/link_lists.bin b/data/chroma_langchain_db/7e1ea925-a1c1-431f-baea-20f990286090/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/data_level0.bin b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/data_level0.bin new file mode 100644 index 0000000..0aefa49 Binary files /dev/null and b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/data_level0.bin differ diff --git a/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/header.bin b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/header.bin differ diff --git a/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/length.bin b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/length.bin new file mode 100644 index 0000000..a38baad --- /dev/null +++ b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/length.bin @@ -0,0 +1 @@ +invalid type: string "Retrieval (%). Datasets marked with * are stitched. Rows are sorted by the average number of sentences per document (before stitching) in ascending order for easier comparison.", expected f64d i64oolean chunker, we evaluated three hyperparameters: , similar to single-linkage; EPS, the maximum distance between two samples for them to be considered part of the same neighbor- ho \ No newline at end of file diff --git a/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/link_lists.bin b/data/chroma_langchain_db/7e2e803d-9033-48d9-8fec-1589609f4ed0/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/data_level0.bin b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/data_level0.bin new file mode 100644 index 0000000..20aa711 Binary files /dev/null and b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/data_level0.bin differ diff --git a/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/header.bin b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/header.bin differ diff --git a/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/length.bin b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/length.bin new file mode 100644 index 0000000..f13c070 Binary files /dev/null and b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/length.bin differ diff --git a/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/link_lists.bin b/data/chroma_langchain_db/839ef489-eacd-4b11-937b-37ad1273765e/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/data_level0.bin b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/data_level0.bin new file mode 100644 index 0000000..c152003 Binary files /dev/null and b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/data_level0.bin differ diff --git a/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/header.bin b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/header.bin differ diff --git a/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/length.bin b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/length.bin new file mode 100644 index 0000000..cfbf0ae Binary files /dev/null and b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/length.bin differ diff --git a/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/link_lists.bin b/data/chroma_langchain_db/8fbb2aca-b54b-499c-b761-252e5731db4e/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/data_level0.bin b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/data_level0.bin new file mode 100644 index 0000000..5704a45 Binary files /dev/null and b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/data_level0.bin differ diff --git a/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/header.bin b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/header.bin differ diff --git a/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/length.bin b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/length.bin new file mode 100644 index 0000000..da1dfcf Binary files /dev/null and b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/length.bin differ diff --git a/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/link_lists.bin b/data/chroma_langchain_db/92e08ce5-c4d3-4e3f-a3dc-8c5dd0db7bbb/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/data_level0.bin b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/data_level0.bin new file mode 100644 index 0000000..38239bb Binary files /dev/null and b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/data_level0.bin differ diff --git a/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/header.bin b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/header.bin differ diff --git a/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/length.bin b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/length.bin new file mode 100644 index 0000000..a6a2e92 Binary files /dev/null and b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/length.bin differ diff --git a/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/link_lists.bin b/data/chroma_langchain_db/a00d4791-7317-4f12-9fc1-5adfb98a3e11/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/data_level0.bin b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/data_level0.bin new file mode 100644 index 0000000..d16e81c Binary files /dev/null and b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/data_level0.bin differ diff --git a/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/header.bin b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/header.bin differ diff --git a/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/length.bin b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/length.bin new file mode 100644 index 0000000..cccd219 Binary files /dev/null and b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/length.bin differ diff --git a/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/link_lists.bin b/data/chroma_langchain_db/a5ad1291-65ef-4d07-8b24-a7df834d988f/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/data/chroma_langchain_db/chroma.sqlite3 b/data/chroma_langchain_db/chroma.sqlite3 new file mode 100644 index 0000000..64b9c2d Binary files /dev/null and b/data/chroma_langchain_db/chroma.sqlite3 differ diff --git a/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/data_level0.bin b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/data_level0.bin new file mode 100644 index 0000000..50954ad Binary files /dev/null and b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/data_level0.bin differ diff --git a/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/header.bin b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/header.bin new file mode 100644 index 0000000..dff34e7 Binary files /dev/null and b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/header.bin differ diff --git a/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/length.bin b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/length.bin new file mode 100644 index 0000000..657d197 Binary files /dev/null and b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/length.bin differ diff --git a/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/link_lists.bin b/data/chroma_langchain_db/dc3f89e1-2233-4050-87e1-d49732e696c6/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/evaluation/eval_models.py b/evaluation/eval_models.py new file mode 100644 index 0000000..fda0247 --- /dev/null +++ b/evaluation/eval_models.py @@ -0,0 +1,393 @@ +from langchain_google_genai import ChatGoogleGenerativeAI +from langchain_groq import ChatGroq +from langchain_chroma import Chroma +from services.query_service import query_google_ai, query_transformation, query_google_ai_test +from services.chroma_db_service import connect_to_chroma_db, retrieve, multi_retrieve, rerank_retrieve +from services.reranking import rerank +from evaluation.questions import len_questions, get_question +from evaluation.evaluations import test_correctness, test_faithfulness +from utils.logger import log + +import json + +### For base RAG Model +def base_chunk_base_retrieve(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + result = [] + model = "base" + evaluation_data = [] + + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + if isinstance(file, list): + db_name = "normal-combi" + else: + db_name = f"normal-{file}" + db = connect_to_chroma_db(db_name) + context = retrieve(question, db) + query_with_context = f"Context:\n{context['context']}\n\nQuestion:\n{question}" + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context["context"], generated_answer, groq_ai) + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + + return result + +### For query transformation model +def base_chunk_multi_retrieve(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + db: Chroma + result = [] + model = "Query Transformation" + evaluation_data = [] + + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + if isinstance(file, list): + db_name = "normal-combi" + else: + db_name = f"normal-{file}" + db = connect_to_chroma_db(db_name) + queries = query_transformation(question, google_ai) + context = multi_retrieve(queries, db) + + query_with_context = f"Context:\n{context}\n\nQuestion:\n{question}" + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context, generated_answer, groq_ai) + + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + return result + +### For reranking model +def base_chunk_rerank(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + db: Chroma + result = [] + model = "rerank" + evaluation_data = [] + + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + if isinstance(file, list): + db_name = "normal-combi" + else: + db_name = f"normal-{file}" + db = connect_to_chroma_db(db_name) + + retrieved_documents = rerank_retrieve(question, db) + if retrieved_documents: + # Extract the string content from the list of Document objects + documents_to_rerank = [getattr(doc, "page_content", str(doc)) for doc in retrieved_documents] + + # Rerank the list of strings + reranked_context = rerank(question, documents_to_rerank) + else: + reranked_context = ["No relevant documents retrieved"] + + context = "\n\n".join(reranked_context) + query_with_context = f"Context:\n{context}\n\nQuestion:\n{question}" + + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context, generated_answer, groq_ai) + + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + return result + +### For layout chunking model +def layout_chunk_base_retrieve(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + db: Chroma + result = [] + model = "Layout" + evaluation_data = [] + + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + try: + if isinstance(file, list): + db_name = "layout-combi" + else: + db_name = f"layout-{file}" + db = connect_to_chroma_db(db_name) + + context = retrieve(question, db, k = 4) + query_with_context = f"Context:\n{context['context']}\n\nQuestion:\n{question}" + + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context["context"][:-3100], generated_answer, groq_ai) + + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + except Exception as e: + log.error(f"Failed to process question {i+1}: {e}") + evaluation_data.append({ + "question_id": i + 1, + "failed": True, + "context length": len(context["context"]), + "context": context + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + return result + +### For model with everything +def layout_chunk_multi_retrieve(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + db: Chroma + result = [] + model = "everything" + evaluation_data = [] + + for i in range(len_questions): + i=5 + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + try: + if isinstance(file, list): + db_name = "layout-combi" + else: + db_name = f"layout-{file}" + db = connect_to_chroma_db(db_name) + queries = query_transformation(question, google_ai) + context = multi_retrieve(queries, db, k=5) + + if context: # Only proceed if the context list is not empty + # Find the longest string (document) in the context list + longest_context_string = max(context, key=len) + + # Remove the longest string from the context list + try: + context.remove(longest_context_string) + log.info(f"Removed longest context string of length: {len(longest_context_string)}") + except ValueError: + # This handles the unlikely case where .remove() fails + log.warning("Longest context string was not found for removal.") + + longest_context_string = max(context, key=len) + # Remove the longest string from the context list + try: + context.remove(longest_context_string) + log.info(f"Removed longest context string of length: {len(longest_context_string)}") + except ValueError: + # This handles the unlikely case where .remove() fails + log.warning("Longest context string was not found for removal.") + + + if (context): + documents = [getattr(doc, "page_content", str(doc)) for doc in context] + reranked_context = rerank(question, documents) + else: + reranked_context = ["No relevant documents retrieved"] + + context = "\n\n".join(reranked_context) + query_with_context = f"Context:\n{context}\n\nQuestion:\n{question}" + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context, generated_answer, groq_ai) + + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + except Exception as e: + log.error(f"Failed to process question {i+1}: {e}") + evaluation_data.append({ + "question_id": i + 1, + "failed": True, + "context length": len(context), + "context": context + }) + finally: + break + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + return result + +def base_chunk_everything(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + db: Chroma + result = [] + model = "base_chunk_everything" + evaluation_data = [] + + # for i in range(len_questions): + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + try: + if isinstance(file, list): + db_name = "layout-combi" + else: + db_name = f"layout-{file}" + db = connect_to_chroma_db(db_name) + + db = connect_to_chroma_db(db_name) + queries = query_transformation(question, google_ai) + context = multi_retrieve(queries, db, k=5) + if (context): + documents = [getattr(doc, "page_content", str(doc)) for doc in context] + reranked_context = rerank(question, documents) + # Uncomment for manual removal of context + # longest_context = max(reranked_context, key=len) + # reranked_context.remove(longest_context) + else: + reranked_context = ["No relevant documents retrieved"] + context = "\n\n".join(reranked_context) + query_with_context = f"Context:\n{context}\n\nQuestion:\n{question}" + response = query_google_ai(query_with_context, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + faithfulness_score = test_faithfulness(question, context, generated_answer, groq_ai) + + result.append((correctness_score, faithfulness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "retrieved_context": context, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "faithfulness_score": faithfulness_score, + "model_tested": model + }) + except Exception as e: + log.error(f"Failed to process question {i+1}: {e}") + evaluation_data.append({ + "question_id": i + 1, + "failed": True, + "context length": len(context), + "context": context + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}_test.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) + return result + +def plain(google_ai: ChatGoogleGenerativeAI, groq_ai: ChatGroq): + result = [] + model = "plain" + evaluation_data = [] + + for i in range(len_questions): + file, file_path, question, ground_truth = get_question(i) + log.info(f"{model} model: evaluating question {i+1}") + + response = query_google_ai_test(question, google_ai) + generated_answer = response["response"].content + + correctness_score = test_correctness(question, ground_truth, generated_answer, groq_ai) + result.append((correctness_score)) + evaluation_data.append({ + "question_id": i + 1, + "document_path": file_path, + "question": question, + "ground_truth": ground_truth, + "generated_answer": generated_answer, + "correctness_score": correctness_score, + "model_tested": model + }) + + log.info(f"{model} model results: {result}") + evaluation_data.insert(0, result) + output_filename = f"evaluation/results/evaluation_results_{model}.json" + with open(output_filename, 'w') as f: + json.dump(evaluation_data, f, indent=4) diff --git a/evaluation/evaluations.py b/evaluation/evaluations.py new file mode 100644 index 0000000..0aeb8c7 --- /dev/null +++ b/evaluation/evaluations.py @@ -0,0 +1,82 @@ +from utils.utils import get_envvar +from utils.logger import log +from langchain_core.prompts import PromptTemplate +from langchain_groq import ChatGroq +from pydantic import SecretStr, BaseModel, Field + +ENV_GROQ_API_KEY="GROQ_API_KEY" +ENV_GROQ_MODEL="GROQ_MODEL" + +correctness_prompt = PromptTemplate( + input_variables=["question", "ground_truth", "generated_answer"], + template=""" + You are an expert evaluator. Your sole task is to determine the correctness score. + + Question: {question} + Ground Truth: {ground_truth} + Generated Answer: {generated_answer} + + Based on the comparison, you **MUST** provide your final assessment by scoring from 0.0 to 1.0, + where 1.0 is perfectly correct and 0.0 is completely incorrect. + + **You MUST output the score by calling the provided function (tool) named 'ResultScore' and nothing else.** + """ +) + +faithfulness_prompt = PromptTemplate( + input_variables=["question","context", "generated_answer"], + template=""" + Question: {question} + Context: {context} + Generated Answer: {generated_answer} + + Evaluate if the generated answer to the question can be deduced from the context. + Score of 0 or 1, where 1 is perfectly faithful *AND CAN BE DERIVED FROM THE CONTEXT* and 0 otherwise. + You don't mind if the answer is correct; all you care about is if the answer can be deduced from the context. + + Example: + Question: What is 2+2? + Context: 4. + Generated Answer: 4. + In this case, the context states '4', but it does not provide information to deduce the answer to 'What is 2+2?', so the score should be 0. + + **You MUST output the score by calling the provided function (tool) named 'ResultScore' and nothing else.** + """ + +) + +class ResultScore(BaseModel): + score: float = Field(..., description="The score of the result, ranging from 0 to 1 where 1 is the best possible score.") + +def connect_to_groq_ai() -> ChatGroq: + api_key = get_envvar(ENV_GROQ_API_KEY) + model = get_envvar(ENV_GROQ_MODEL) + try: + groq_ai= ChatGroq( + model=model, + api_key = SecretStr(api_key), + ) + log.info(f"Open AI model set: {model}") + return groq_ai + except Exception as err: + log.error(f"Could not create claude AI model due to, {err}") + +def test_correctness(question, ground_truth, generated_answer, groq_ai: ChatGroq) -> float: + correctness_chain = correctness_prompt | groq_ai.with_structured_output(ResultScore) + + result = correctness_chain.invoke({ + "question": question, + "ground_truth": ground_truth, + "generated_answer": generated_answer + }) + return result.score + +def test_faithfulness(question, context, generated_answer, groq_ai: ChatGroq) -> float: + faithfulness_chain = faithfulness_prompt | groq_ai.with_structured_output(ResultScore) + result = faithfulness_chain.invoke({ + "question": question, + "context": context, + "generated_answer": generated_answer + }) + return result.score + diff --git a/evaluation/pre_loader.py b/evaluation/pre_loader.py new file mode 100644 index 0000000..5196dd4 --- /dev/null +++ b/evaluation/pre_loader.py @@ -0,0 +1,20 @@ +from models.app_models import DocumentProcessRequest +from services.chroma_db_service import connect_to_chroma_db +from controllers.app_controller import chunk_document, chunk_document_with_layout + +def preload_native_chunking(db_name:str, request: DocumentProcessRequest): + db = connect_to_chroma_db(db_name) + chunk_document(request, db) + +def preload_layout_chunking(db_name:str, request: DocumentProcessRequest): + db = connect_to_chroma_db(db_name) + chunk_document_with_layout(request, db) + +def combi_chunking(request: DocumentProcessRequest): + db_name1= "normal-combi" + db_name2 = "layout-combi" + db1 = connect_to_chroma_db(db_name1) + db2 = connect_to_chroma_db(db_name2) + + chunk_document(request, db1) + chunk_document_with_layout(request, db2) \ No newline at end of file diff --git a/evaluation/questions.py b/evaluation/questions.py new file mode 100644 index 0000000..b71a184 --- /dev/null +++ b/evaluation/questions.py @@ -0,0 +1,54 @@ +questions = [ + { + "file" : "Computational_cost_of_semantic_chunking.pdf", + "file_path" : "static/Computational cost of semantic chunking.pdf", + "question" : "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth" : "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence" + }, + { + "file" : "Computational_cost_of_semantic_chunking.pdf", + "file_path" : "static/Computational cost of semantic chunking.pdf", + "question" : "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation" + }, + { + "file" : "Reconstructing_Context.pdf", + "file_path" : "static/Reconstructing Context.pdf", + "question" : "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth" : "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks." + }, + { + "file" : "Accelerating_LLM_Inference.pdf", + "file_path" : "static/Accelerating LLM Inference.pdf", + "question" : "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth" : "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary." + }, + { + "file" : "Beyond_Long_Context.pdf", + "file_path" : "static/Beyond Long Context.pdf", + "question" : "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth" : "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78%"+" reduction in token usage compared to wide-context processing in evaluations" + }, + { + "file" : "Accelerating_LLM_Inference.pdf", + "file_path" : "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64%" + " of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60%"+ " and peak memory usage by 35%." + }, + { + "file" : ["Reconstructing_Context.pdf", "Computational_cost_of_semantic_chunking.pdf", "Beyond_Long_Context.pdf"], + "file_path" : "multiple files", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies—including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)—and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing" + }, +] + +len_questions = len(questions) + +def get_question(index:int): + file = questions[index]["file"] + file_path = questions[index]["file_path"] + question = questions[index]["question"] + ground_truth = questions[index]["ground_truth"] + return file, file_path, question, ground_truth + diff --git a/evaluation/results/evaluation_results_Layout.json b/evaluation/results/evaluation_results_Layout.json new file mode 100644 index 0000000..9d9cc5b --- /dev/null +++ b/evaluation/results/evaluation_results_Layout.json @@ -0,0 +1,123 @@ +[ + [ + [ + 0.97, + 1.0 + ], + [ + 0.97, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.85, + 1.0 + ], + [ + 0.95, + 1.0 + ], + [ + 0.25, + 1.0 + ], + [ + 0.85, + 1.0 + ] + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "retrieved_context": { + "context": "# Abstract \nRecent advances in Retrieval-Augmented Generation (RAG) systems have popularized **se-**\n\n# Abstract \nRecent advances in Retrieval-Augmented Generation (RAG) systems have popularized **se-**\n\n# mantic chunking, which aims to improve retrieval performance by dividing documents into\nsemantically coherent segments. Despite its\ngrowing adoption, the actual benefits over simpler **fixed-size chunking**, where documents are\nsplit into consecutive, fixed-size segments, remain unclear. This study systematically evaluates the effectiveness of semantic chunking using three common retrieval-related tasks: document retrieval, evidence retrieval, and retrievalbased answer generation. The results show that\nthe computational costs associated with semantic chunking are not justified by consistent performance gains. These findings challenge the\nprevious assumptions about semantic chunking and highlight the need for more efficient\nchunking strategies in RAG systems.\n\n# mantic chunking, which aims to improve retrieval performance by dividing documents into\nsemantically coherent segments. Despite its\ngrowing adoption, the actual benefits over simpler **fixed-size chunking**, where documents are\nsplit into consecutive, fixed-size segments, remain unclear. This study systematically evaluates the effectiveness of semantic chunking using three common retrieval-related tasks: document retrieval, evidence retrieval, and retrievalbased answer generation. The results show that\nthe computational costs associated with semantic chunking are not justified by consistent performance gains. These findings challenge the\nprevious assumptions about semantic chunking and highlight the need for more efficient\nchunking strategies in RAG systems." + }, + "generated_answer": "Hello there! I can help you understand semantic chunking based on your notes.\n\n**Answer:**\nThe defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems is its approach to **dividing documents into semantically coherent segments**. This means that instead of just splitting text at arbitrary points, semantic chunking aims to keep related ideas and information together within each segment, making each chunk meaningful on its own.\n\n**Key Points/Summary:**\n* **Purpose:** To improve retrieval performance in RAG systems.\n* **Method:** Divides documents into segments that are semantically coherent.\n* **Goal:** Each segment contains related ideas, maintaining meaning.\n\n**Extra Tip:**\nThink of it like organizing a book: instead of cutting it into pages of equal length (fixed-size chunking), semantic chunking would be like keeping each chapter or even each major paragraph as a complete unit, because they discuss one main idea.", + "correctness_score": 0.97, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "retrieved_context": { + "context": "# Lack of Chunk Quality Measures As noted in\nSection 4, while the output chunks differed between methods, retrieval and generation performances were similar across chunkers. In addition \nto the influence of embedding models, the absence\nof direct chunk quality metrics likely contributed\nto this issue. Having ground-truth query-chunk relevance scores would provide more accurate evaluations than relying solely on document or evidence\nmapping.\n\n# Lack of Chunk Quality Measures As noted in\nSection 4, while the output chunks differed between methods, retrieval and generation performances were similar across chunkers. In addition \nto the influence of embedding models, the absence\nof direct chunk quality metrics likely contributed\nto this issue. Having ground-truth query-chunk relevance scores would provide more accurate evaluations than relying solely on document or evidence\nmapping.\n\n# 4 **Results** \n## 4.1 **Measuring and reporting performances** \nAs mentioned earlier, we used three proxy tasks\nthe study chunking. We cannot directly assess the\nquality of retrieval at the chunk level due to the lack\nof ground-truth at the chunk level. Instead, each\nretrieved chunk is mapped back to either the source\ndocument or the included evidence sentences. \nSince the number of relevant documents or evi\ndence sentences is not fixed (unlike the _k_ value for\nretrieved chunks), traditional metrics such as Recall@k and NDCG@k are not suitable. F1 provides\na balanced measure that accounts for both precision\nand recall under these circumstances. Therefore,\nwe use **F1@5** as the metric. For further details, see\nAppendix D.\nFor each dataset, results are reported based on\nthe best hyperparameter configuration for each\nchunker, determined by the average F1 score across\nall _k_ values. All results to be reported below are\nobtained using dunzhang/stella_en_1.5B_v5 as the\nembedder for being the best among those tested.\nIn the following subsections, **Bold** values indicate the best performance on the respective dataset.\nThe results for Answer Generation closely matched\nthose of Evidence Retrieval and are discussed in \nAppendix E.1. Additional analysis of hyperparameters is provided in Appendix B. Inspection of the\noutputs of different chunkers is provided in Appendix E.4.\n\n# 4 **Results** \n## 4.1 **Measuring and reporting performances** \nAs mentioned earlier, we used three proxy tasks\nthe study chunking. We cannot directly assess the\nquality of retrieval at the chunk level due to the lack\nof ground-truth at the chunk level. Instead, each\nretrieved chunk is mapped back to either the source\ndocument or the included evidence sentences. \nSince the number of relevant documents or evi\ndence sentences is not fixed (unlike the _k_ value for\nretrieved chunks), traditional metrics such as Recall@k and NDCG@k are not suitable. F1 provides\na balanced measure that accounts for both precision\nand recall under these circumstances. Therefore,\nwe use **F1@5** as the metric. For further details, see\nAppendix D.\nFor each dataset, results are reported based on\nthe best hyperparameter configuration for each\nchunker, determined by the average F1 score across\nall _k_ values. All results to be reported below are\nobtained using dunzhang/stella_en_1.5B_v5 as the\nembedder for being the best among those tested.\nIn the following subsections, **Bold** values indicate the best performance on the respective dataset.\nThe results for Answer Generation closely matched\nthose of Evidence Retrieval and are discussed in \nAppendix E.1. Additional analysis of hyperparameters is provided in Appendix B. Inspection of the\noutputs of different chunkers is provided in Appendix E.4.\n\n# 3 **Experiments** \nIn the absence of ground-truth chunk data, we designed three experiments to indirectly assess the\nquality of each chunker: document retrieval, evidence retrieval, and answer generation. Different\ndatasets and evaluation metrics were used for each \nexperiment to align with the specific task requirements. All documents were first split into sentences using SpaCy\u2019s en_core_web_sm model (Explosion, 2024) before being embedded and chunked. We tested three embedding models selected to\nrepresent a range of performances based on their\nrankings on the MTEB Leaderboard (Muennighoff\net al., 2022). See Appendix E.2 for details.\n\n# 3 **Experiments** \nIn the absence of ground-truth chunk data, we designed three experiments to indirectly assess the\nquality of each chunker: document retrieval, evidence retrieval, and answer generation. Different\ndatasets and evaluation metrics were used for each \nexperiment to align with the specific task requirements. All documents were first split into sentences using SpaCy\u2019s en_core_web_sm model (Explosion, 2024) before being embedded and chunked. We tested three embedding models selected to\nrepresent a range of performances based on their\nrankings on the MTEB Leaderboard (Muennighoff\net al., 2022). See Appendix E.2 for details.\n\n# E.4 **Chunk Inspection** \nWe examined the output chunks to (1) confirm that\ndifferent chunkers were functioning as intended,\nand (2) investigate the reasons behind performance\ndifferences. BEIR\u2019s HotpotQA dataset (Thakur\net al., 2021; Yang et al., 2018) was selected for its\nreasonably sized documents. We randomly sampled five documents, stitching the first four together\nto form a stitched document (Figure 6), and keeping the fifth as a normal document (Figure 7. The\ndocument IDs are: \n- Stitched: 44547136, 14115210, 5580754, \n54045118. \n- Normal: 30214079.\n\n# E.4 **Chunk Inspection** \nWe examined the output chunks to (1) confirm that\ndifferent chunkers were functioning as intended,\nand (2) investigate the reasons behind performance\ndifferences. BEIR\u2019s HotpotQA dataset (Thakur\net al., 2021; Yang et al., 2018) was selected for its\nreasonably sized documents. We randomly sampled five documents, stitching the first four together\nto form a stitched document (Figure 6), and keeping the fifth as a normal document (Figure 7. The\ndocument IDs are: \n- Stitched: 44547136, 14115210, 5580754, \n54045118. \n- Normal: 30214079.\n\n# 2 **Chunking Strategies** \nIn this paper, a document is first split into sentences\nwhich are then grouped into chunks. We evaluate\nthree chunking strategies, hereafter referred to as \nFigure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within\nthe sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk\nbreakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering\nsemantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance\nthresholds between consecutive sentences to maintain coherence. (c) Clustering-based Semantic Chunker groups\nsemantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. \n\u201cchunkers.\u201d\n\n# 2 **Chunking Strategies** \nIn this paper, a document is first split into sentences\nwhich are then grouped into chunks. We evaluate\nthree chunking strategies, hereafter referred to as \nFigure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within\nthe sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk\nbreakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering\nsemantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance\nthresholds between consecutive sentences to maintain coherence. (c) Clustering-based Semantic Chunker groups\nsemantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. \n\u201cchunkers.\u201d" + }, + "generated_answer": "In the study, since there was no direct \"ground-truth\" data available for chunk quality, three primary proxy tasks were designed to indirectly assess how well different chunking strategies performed. These tasks are:\n\n1. **Document Retrieval**: This task evaluates how effectively chunks contribute to retrieving relevant documents.\n2. **Evidence Retrieval**: This task focuses on the ability of chunks to help in retrieving specific pieces of evidence.\n3. **Answer Generation**: This task assesses how well the retrieved chunks support the generation of accurate answers.\n\n### Key Points/Summary:\n* Three proxy tasks were used to indirectly evaluate chunker quality due to the lack of direct ground-truth data.\n* The tasks were document retrieval, evidence retrieval, and answer generation.\n* Each task used different datasets and evaluation metrics tailored to its specific requirements.\n\n### Extra Tip:\nThink of these proxy tasks like using a thermometer to check if a cake is done when you can't see the inside directly. Each task measures a different aspect (like temperature, texture, or smell) to give an overall idea of the chunker's \"doneness\" or quality in a real-world application.", + "correctness_score": 0.97, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "retrieved_context": { + "context": "# 3 **Methodology** \nTo guide our study, we define the following research questions (RQs), aimed at\nevaluating different strategies for chunking and retrieval in RAG systems:\n\n# 3 **Methodology** \nTo guide our study, we define the following research questions (RQs), aimed at\nevaluating different strategies for chunking and retrieval in RAG systems:\n\n# ditional early chunking strategies, utilizing **different text segmenters**\nand embedding models to evaluate their impact on retrieval accuracy and\ndownstream performance in RAG systems.\n\n# ditional early chunking strategies, utilizing **different text segmenters**\nand embedding models to evaluate their impact on retrieval accuracy and\ndownstream performance in RAG systems.\n\n# \u2013 RQ#1 : Compares the effectiveness of **early versus late chunking** strategies, utilizing **different text segmenters** and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG\nsystems.\n\n# \u2013 RQ#1 : Compares the effectiveness of **early versus late chunking** strategies, utilizing **different text segmenters** and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG\nsystems.\n\n# 2 **Related Work** \n_Classic RAG._ A standard RAG workflow involves four main stages: document\nsegmentation, chunk embedding, indexing, and retrieval. During segmentation,\ndocuments are divided into manageable chunks. These chunks are then transformed into vector representations using encoder models, often normalized to \n1 `[https://www.anthropic.com/news/contextual-retrieval](https://www.anthropic.com/news/contextual-retrieval)`\n2 `[https://github.com/disi-unibo-nlp/rag-when-how-chunk](https://github.com/disi-unibo-nlp/rag-when-how-chunk)` \nReconstructing Context 3 \nensure unit magnitudes. The resulting embeddings are stored in indexed vector\ndatabases, enabling efficient approximate similarity searches. Retrieval involves\ncomparing query embeddings with the stored embeddings using metrics such as\ncosine similarity or Euclidean distance, which identify the most relevant chunks.\nSeminal works like [15] and [13] have demonstrated the effectiveness of RAG\nin tasks such as open-domain question answering. More recent studies, including [7], have introduced advancements in scalability and embedding techniques,\nfurther establishing RAG as a foundational framework for knowledge-intensive\napplications. \n_Document Segmentation._ Document segmentation is essential for processing long\ntexts in RAG workflows, with methods ranging from _fixed-size segmentation_ [7]\nto more adaptive techniques like _semantic segmentation_, [3] which detect semantic\nbreakpoints based on shifts in meaning. Recent advancements include _supervised_\n_segmentation models_ [14,12] and _segment-then-predict models_, trained end-to-end\nwithout explicit labels to optimize chunking for downstream task performance \n[17]. In 2024, _late chunking_ and _contextual retrieval_ introduced novel paradigms.\nBoth techniques have proven effective in retrieval benchmarks but remain largely\nuntested in integrated RAG workflows. Despite several RAG surveys [7,6,8],\nno prior work has compared these methods within a comprehensive evaluation\nframework. This study addresses this gap by holistically analyzing late chunking\nand contextual retrieval, offering actionable insights into their relative strengths\nand trade-offs.\n\n# 2 **Related Work** \n_Classic RAG._ A standard RAG workflow involves four main stages: document\nsegmentation, chunk embedding, indexing, and retrieval. During segmentation,\ndocuments are divided into manageable chunks. These chunks are then transformed into vector representations using encoder models, often normalized to \n1 `[https://www.anthropic.com/news/contextual-retrieval](https://www.anthropic.com/news/contextual-retrieval)`\n2 `[https://github.com/disi-unibo-nlp/rag-when-how-chunk](https://github.com/disi-unibo-nlp/rag-when-how-chunk)` \nReconstructing Context 3 \nensure unit magnitudes. The resulting embeddings are stored in indexed vector\ndatabases, enabling efficient approximate similarity searches. Retrieval involves\ncomparing query embeddings with the stored embeddings using metrics such as\ncosine similarity or Euclidean distance, which identify the most relevant chunks.\nSeminal works like [15] and [13] have demonstrated the effectiveness of RAG\nin tasks such as open-domain question answering. More recent studies, including [7], have introduced advancements in scalability and embedding techniques,\nfurther establishing RAG as a foundational framework for knowledge-intensive\napplications. \n_Document Segmentation._ Document segmentation is essential for processing long\ntexts in RAG workflows, with methods ranging from _fixed-size segmentation_ [7]\nto more adaptive techniques like _semantic segmentation_, [3] which detect semantic\nbreakpoints based on shifts in meaning. Recent advancements include _supervised_\n_segmentation models_ [14,12] and _segment-then-predict models_, trained end-to-end\nwithout explicit labels to optimize chunking for downstream task performance \n[17]. In 2024, _late chunking_ and _contextual retrieval_ introduced novel paradigms.\nBoth techniques have proven effective in retrieval benchmarks but remain largely\nuntested in integrated RAG workflows. Despite several RAG surveys [7,6,8],\nno prior work has compared these methods within a comprehensive evaluation\nframework. This study addresses this gap by holistically analyzing late chunking\nand contextual retrieval, offering actionable insights into their relative strengths\nand trade-offs.\n\n# \u2013\n_Loss of Context:_ dividing documents without considering semantic boundaries can result in chunks that lack sufficient context, impairing the model\u2019s\nability to generate accurate and coherent responses. \n# \u2013\n_Incomplete Information Retrieval:_ important information split across chunks\nmay not be effectively retrieved or integrated. \nTo address these issues, we analyse and compare two recent techniques\u2014\ncontextual retrieval [1] and late chunking [9]\u2014within a unified setup, evaluating\ntheir strengths and limitations in tackling challenges like context loss and incomplete information retrieval. Contextual retrieval preserves coherence by prepending LLM-generated context to chunks, while late chunking embeds entire documents to retain global context before segmenting.\nOur study rigorously assesses their impact on generation performance in\nquestion-answering tasks, finding that neither technique offers a definitive solution. This work highlights the trade-offs between these methods and provides\npractical guidance for optimizing RAG systems.\nTo further support the community, we release all code, prompts, and data\nunder the permissive MIT license, enabling full reproducibility and empowering\npractitioners to adapt and extend our work. [2]\n\n# \u2013\n_Loss of Context:_ dividing documents without considering semantic boundaries can result in chunks that lack sufficient context, impairing the model\u2019s\nability to generate accurate and coherent responses. \n# \u2013\n_Incomplete Information Retrieval:_ important information split across chunks\nmay not be effectively retrieved or integrated. \nTo address these issues, we analyse and compare two recent techniques\u2014\ncontextual retrieval [1] and late chunking [9]\u2014within a unified setup, evaluating\ntheir strengths and limitations in tackling challenges like context loss and incomplete information retrieval. Contextual retrieval preserves coherence by prepending LLM-generated context to chunks, while late chunking embeds entire documents to retain global context before segmenting.\nOur study rigorously assesses their impact on generation performance in\nquestion-answering tasks, finding that neither technique offers a definitive solution. This work highlights the trade-offs between these methods and provides\npractical guidance for optimizing RAG systems.\nTo further support the community, we release all code, prompts, and data\nunder the permissive MIT license, enabling full reproducibility and empowering\npractitioners to adapt and extend our work. [2]" + }, + "generated_answer": "Hello there! I can help you with that.\n\nBased on your notes, two advanced techniques aimed at preserving global context within RAG systems, beyond traditional early chunking, are **late chunking** and **contextual retrieval**.\n\n### Answer:\n\n1. **Late Chunking:** This technique involves embedding entire documents first to retain their global context. Only after the embedding process is complete are the documents segmented into chunks. This approach ensures that the broader meaning and relationships within the full document are captured before specific parts are isolated.\n2. **Contextual Retrieval:** This method focuses on preserving coherence by prepending additional context, often generated by a Large Language Model (LLM), to the retrieved chunks. By adding this LLM-generated context, the system aims to provide a more complete and coherent picture around the specific information being retrieved, helping to mitigate the loss of context that can occur when documents are simply divided.\n\nBoth of these techniques were introduced in 2024 as novel paradigms to address challenges like loss of context and incomplete information retrieval in RAG workflows.\n\n### Key Points/Summary:\n\n* **Late Chunking:** Embeds entire documents first to maintain global context, then segments them.\n* **Contextual Retrieval:** Prepends LLM-generated context to chunks to enhance coherence and provide a fuller picture.\n* Both are advanced strategies designed to overcome context loss in RAG systems.\n\n### Extra Tip:\nThink of \"late chunking\" like reading a whole book to understand the plot before highlighting specific passages. \"Contextual retrieval\" is like having a helpful friend summarize the surrounding chapters for you whenever you look at a highlighted passage, ensuring you don't miss the bigger picture!", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "retrieved_context": { + "context": "# ICAC We find a phenomenon during the model inference, as illustrated in Figure 3. The chunks attended to by tokens within a generated chunk exhibit substantial consistency, whereas chunk updates \nFigure 3: Attention visualization of chunk selection during the inference phase. The sample is derived from the passkey retrieval task. \n4 \npredominantly occur at chunk boundaries. We name this phenomenon the \u201d **I** ntra- **C** hunk **A** ttention\n\n# ICAC We find a phenomenon during the model inference, as illustrated in Figure 3. The chunks attended to by tokens within a generated chunk exhibit substantial consistency, whereas chunk updates \nFigure 3: Attention visualization of chunk selection during the inference phase. The sample is derived from the passkey retrieval task. \n4 \npredominantly occur at chunk boundaries. We name this phenomenon the \u201d **I** ntra- **C** hunk **A** ttention\n\n# tion Answering, which evaluates the model\u2019s capabilities in question understanding and information\nmatching, CommonsenseQA (Talmor et al., 2019) (5-shot), Social IQA (Sap et al., 2019) (15-shot),\nPIQA (Bisk et al., 2020) (25-shot); and **Reasoning** which evaluates the model\u2019s capabilities in logical abstraction and complex decision-making, HellaSwag (Zellers et al., 2019) (10-shot), WinoGrande (Sakaguchi et al., 2020) (25-shot), ARC-c/ARC-e (Clark et al., 2018) (25-shot). \n5 \nMethods SDQA MDQA Summary Few-shot Avg KV\nNQA Qasper MFQA HQA Musi 2WQA GR QMS SAM TREC\nQwen2.5-7B 33.76 51.40 57.17 56.76 29.40 38.09 31.57 24.40 46.73 72.00 44.13 100.00\nChunkLLM **31.09** **50.52** **56.95** **55.61** **29.41** **38.27** **31.25** **23.16** 46.53 **72.50** **43.53** 48.58\nSepLLM 26.43 50.18 50.29 47.25 22.83 36.34 28.31 21.14 46.71 71.50 40.10 53.17\nStrmLLM 28.88 50.37 50.47 51.60 24.55 37.76 30.36 22.27 **46.84** 72.00 41.51 68.50 \nLlama3.1-8B 40.88 51.72 58.47 52.68 35.80 42.52 30.64 24.44 47.10 73.00 45.73 100.00 \nChunkLLM **39.14** 49.93 53.45 **52.29** **34.40** **42.98** **31.05** **23.89** **47.20** 72.50 **44.68** 50.18\nSepLLM 36.23 **51.16** 51.81 47.70 27.83 40.85 27.12 21.84 46.89 72.00 42.34 54.56\nStrmLLM 35.94 50.96 **54.13** 50.54 30.95 41.83 27.50 23.06 46.23 **73.00** 43.42 69.25 \nTable 1: Experimental results on LongBench. Avg denotes average score. SDQA: single-document\nquestion answering, MDQA: multi-document question answering. The full names of the subtasks\nare shown in Appendix 6.2. \n3.2 M AIN R ESULTS \n3.2.1 R ESULTS ON L ONG B ENCH \nWe set the top-k ratio to 45% and the number of local chunks to 15 for ChunkLLM. The experimental\nresults on the LongBench using Qwen2.5-7B and Llama3.1-8B are presented in Table 1. Here,\n\u201dStrmLLM\u201d represents StreamingLLM (Xiao et al., 2024). We take Qwen2.5-7B as an example for\nanalysis, and the same conclusion holds for Llama3.1-8B. The following observations can be made:\n(1) In terms of overall average performance, ChunkLLM attains the optimal performance when\ncompared to SepLLM and StreamingLLM, with respective improvements of 3.43 and 2.02 (43.53\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM\ndemonstrates a remarkable improvement in long-context evaluation, which validates the advantage\nof ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably,\nin the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that\nthe core challenge of MDQA lies in the dispersion of critical information across distinct positions\nwithin the context, which places high demands on the model\u2019s context comprehension capability.\nSepLLM leverages separators as chunk features, which is plagued by constrained representational\ncapacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches\nthe representational capacity of chunks via attention distillation, which enhances the recall rates of\ncritical chunks. This, in turn, effectively boosts the model\u2019s long-context understanding capability.\n(3) ChunkLLM attains 98.64% of the vanilla model\u2019s performance while employing the minimum\nKV cache. Notably, relative to SepLLM and StreamingLLM, ChunkLLM reduces the KV cache\nusage rate by 4.59% and 19.92% (48.58% v.s. 53.17% v.s. 68.50%), respectively, findings that\nfurther substantiate the superiority of ChunkLLM in long-context scenarios. \n3.2.2 R ESULTS ON NIAH \nFigure 4: Needle-in-a-Haystack retrieval accuracy across context positions with 64k context length.\nThe last column represents the KV-cache utilization rate. \n6 \nWe set the top-k to 256 and the number of local chunks to 16 for ChunkLLM. As depicted in\nFigure 4, ChunkLLM outperforms SepLLM across all scenarios in the 64K-context NIAH evaluation conducted on Qwen2.5-7B and Llama3.1-8B, achieving superior performance. Notably, in\nscenarios where the context length exceeds 12K, SepLLM exhibits near-total loss of retrieval capability (visualized in red), whereas ChunkLLM retains performance comparable to the vanilla model.\nThis discrepancy is primarily attributed to ChunkLLM\u2019s attention distillation mechanism, which\nstrengthens the feature representational capacity of chunks. Consequently, during chunk selection,\nthe model effectively identifies critical chunks with higher query relevance, leading to improved inference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative\nto SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct\nexperiments with StreamingLLM, as shown in Appendix 6.3. \n3.2.3 I NFERENCE E FFICIENCY \n|1600
1400
1200 (s)
1000 time
800 Cost
600
400
200|Qwen2.5-7B
Ours|Col3|2300
2000
1700
1400
1100
800
500
200|Llama3.1-8B
Ours|Col6|\n|---|---|---|---|---|---|\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)|~~3.84x~~|~~3.84x~~|~~3.84x~~|~~4.~~|~~4.~~|\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)|||||| \nFigure 5: Comparison of inference time per 10k tokens in the generation process on PG19 test set. \nWe conduct runtime evaluations of Vanilla and ChunkLLM for 120K-token generation tasks on\nthe PG19(Rae et al., 2019) test set, with metrics recorded every 10K tokens. We set the top-k to\n256 and the number of local chunks to 16 for ChunkLLM. As shown in Figure 5, as the number\nof generated tokens increases, Vanilla\u2019s inference time rises linearly, while ChunkLLM maintains\npersistent stability in time consumption. In the 110K\u2013120K token generation phase, ChunkLLM\noutperforms Vanilla by speedups of 3.84\u00d7 and 4.48\u00d7, which corroborates the efficacy of the proposed\nICAC mechanism. During ChunkLLM\u2019s inference phase, chunk updates occur exclusively at chunk\nboundaries, minimizing the updates frequency and thereby boosting inference efficiency. We also\nconduct supplementary experiments using the FineWeb-Edu dataset, from which 1000 test corpora\nof 4k length are sampled. For the task of chunk boundary prediction, we evaluate its performance\nusing three key metrics: precision, recall, and F1-score. The calculated results are 98.31, 95.54,\nand 96.91, respectively. Such promising performance indicators serve to verify the reliability and\neffectiveness of our chunk boundary prediction task. \nWe conduct experiments where these models Length Methods PPL Total Time(s)\ngenerated 120K tokens, evaluating both total Qwen2.5-7B 14.41 10,684.31\ninference time and average perplexity (ppl) on 120K ChunkLLM 16.23 4,782.62\nthe PG19 test set, and results are summarizedin Table 2. Compared to the vanilla model, Llama3.1-8BChunkLLM 11.9312.89 145,,906.19963.83\nChunkLLM yields a slight enhancement in ppl Table 2: The perplexity and running time comparalongside a significant decrease in total infer- ison on the PG19 test set.\nence time. The underlying reason is that while\nthe vanilla model maintains semantic integrity, it incurs linearly increasing inference time as generated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up\ninference by leveraging its chunk selection and ICAC mechanisms. \nLength Methods PPL Total Time(s) \n120K \nQwen2.5-7B 14.41 10,684.31\nChunkLLM 16.23 4,782.62\nLlama3.1-8B 11.93 14,906.19\nChunkLLM 12.89 5,963.83 \nTable 2: The perplexity and running time comparison on the PG19 test set. \n3.2.4 R ESULTS ON S HORT T EXT \nThe experimental results for short texts are presented in Table 3. The following conclusions can\nbe drawn: (1) The overall average metrics of ChunkLLM on Qwen2.5-7B and Llama3.1-8B both\noutperform those of StreamingLLM and SepLLM, achieving 99.57% and 99.84% of the Vanilla\nmodel\u2019s performance, respectively. Notably, ChunkLLM attains optimal performance across 8 out of \n7 \nMethods General Knowledge Question Answering Reasoning Avg KV\nMMLU SciQ OQA CQA SIQA PIQA Heag WG ARC-c ARC-e\nQwen2.5-7B 74.25 97.00 52.80 84.52 58.44 81.72 80.24 77.27 63.82 87.21 75.73 100.00\nChunkLLM 72.51 96.60 **52.40** **84.68** **58.34** **81.77** 80.08 **76.87** **63.65** **87.21** **75.41** 45.47\nSepLLM 73.07 **96.70** 52.20 84.19 58.25 81.41 **80.10** 76.48 62.94 86.11 75.15 50.20\nStrmLLM **73.31** 96.60 52.00 84.28 58.19 81.39 79.76 76.64 62.79 86.36 75.13 45.14 \nLlama3.1-8B 65.30 97.60 48.00 74.28 54.04 83.19 81.76 80.03 57.85 84.55 72.66 100.00 \nChunkLLM **64.78** 97.30 **48.40** **74.45** **54.76** 82.75 **81.84** **79.01** 57.68 **84.55** **72.55** 45.04\nSepLLM 64.32 **97.40** 47.40 74.10 54.25 **83.03** 81.68 79.01 **57.93** 84.09 72.32 50.32\nStrmLLM 61.19 97.20 48.00 73.79 53.94 81.56 80.14 78.22 56.91 83.59 72.45 45.30 \nTable 3: Experimental results on short context benchmarks. \nthe 10 evaluation tasks, validating its efficacy in short-text task scenarios. (2) We perform statistical\nanalyses on the average utilization rate of the KV cache. In comparison with SepLLM, ChunkLLM\nachieves superior performance while consuming a lower volume of KV cache (45.47% v.s. 50.20%).\nSpecifically, on the Llama3.1-8B model, ChunkLLM not only exhibits the minimal KV cache usage\nbut also outperforms both SepLLM and StreamingLLM in terms of performance metrics. This\nfinding further validates the precision of ChunkLLM in chunk recall, achieving a balanced trade-off\nbetween performance and memory consumption. \n3.3 A BLATION S TUDY \n3.3.1 E FFECTIVENESS OF V OTE AND ICAC \nMethods SDQA MDQA Summary Few-shot Avg\nNQA Qasper MFQA HQA Musi 2WQA GR QMS SAM TREC\nQwen2.5-7B 33.76 51.40 57.17 56.76 29.40 38.09 31.57 24.40 46.73 72.00 44.13\nChunkLLM 31.09 50.52 **56.95** **55.61** **29.41** 38.27 31.25 23.16 46.53 **72.50** 43.53 \nw/o vote 30.78 45.00 51.53 54.96 28.00 38.17 30.82 23.49 46.58 70.00 41.93 \nw/o ICAC **31.36** **50.94** 56.84 55.61 28.50 **38.59** **31.86** **23.93** **46.66** 72.50 **43.68** \nTable 4: Ablation Study on LongBench, w/o vote: remove vote mechanism, w/o ICAC: remove\nICAC pattern. \nWe validate the proposed vote mechanism and\nICAC pattern based on the Qwen2.5-7B using the LongBench, with experimental results\nshown in Table 4. Removal of the vote mechanism leads to a 1.6 drop in overall performance\n(43.53 v.s. 41.93), which confirms the mechanism\u2019s efficacy, as it integrates inter-layer differences in chunk selection and minimizes interference arising from such discrepancies. We Figure 6: Inference efficiency of ICAC on PG19\nalso conduct a visual investigation into the re- test set.\ncall performance of top-k chunks across different layers, with comprehensive experimental results provided in Appendix 6.5. Conversely, removing ICAC results in a marginal 0.15 improvement in overall performance. This slight gain is\nattributed to the increased frequency of chunk selection updates during the inference phase. Frequent chunk selection, however, poses a limitation of low inference efficiency. As shown in Figure\n6, after ICAC is removed, the inference latency is 2.12 times higher than that of ChunkLLM in the\n110K\u2013120K token generation stage. Conversely, incorporating ICAC enables the model to maintain near-lossless performance alongside improved inference efficiency, which provides additional\nvalidation of ICAC\u2019s success. Appendix 6.4 shows a case study of the ICAC. \nFigure 6: Inference efficiency of ICAC on PG19 \ntest set. \n3.3.2 A NALYSIS OF F IXED C HUNKS AND S EMANTIC C HUNKS \nWe conduct an experimental analysis of the fixed chunk method (FixLLM) on the NIAH task. To\nensure consistent KV cache utilization and facilitate a fair comparison, FixLLM is configured with\na top-k of 384 and a local chunks of 24, while ChunkLLM is set to a top-k of 256 and a local\nchunks of 16. The experimental results are illustrated in Figure 7. As observed, under conditions \n8 \nFigure 7: Visualization of fixed chunks and semantic chunks in NIAH test. \nof approximately consistent KV cache utilization, FixLLM exhibits an 8.23 reduction (87.78 v.s.\n79.55) in accuracy relative to ChunkLLM on the 64K NIAH task. This discrepancy stems from\nthe semantic incompleteness of fixed chunks, which in turn compromises chunk selection during\nthe inference phase. In contrast, ChunkLLM leverages contextual semantic information to identify\nchunk boundaries, preserving the semantic integrity of chunks. \n4 R ELATED W ORK\n\n# tion Answering, which evaluates the model\u2019s capabilities in question understanding and information\nmatching, CommonsenseQA (Talmor et al., 2019) (5-shot), Social IQA (Sap et al., 2019) (15-shot),\nPIQA (Bisk et al., 2020) (25-shot); and **Reasoning** which evaluates the model\u2019s capabilities in logical abstraction and complex decision-making, HellaSwag (Zellers et al., 2019) (10-shot), WinoGrande (Sakaguchi et al., 2020) (25-shot), ARC-c/ARC-e (Clark et al., 2018) (25-shot). \n5 \nMethods SDQA MDQA Summary Few-shot Avg KV\nNQA Qasper MFQA HQA Musi 2WQA GR QMS SAM TREC\nQwen2.5-7B 33.76 51.40 57.17 56.76 29.40 38.09 31.57 24.40 46.73 72.00 44.13 100.00\nChunkLLM **31.09** **50.52** **56.95** **55.61** **29.41** **38.27** **31.25** **23.16** 46.53 **72.50** **43.53** 48.58\nSepLLM 26.43 50.18 50.29 47.25 22.83 36.34 28.31 21.14 46.71 71.50 40.10 53.17\nStrmLLM 28.88 50.37 50.47 51.60 24.55 37.76 30.36 22.27 **46.84** 72.00 41.51 68.50 \nLlama3.1-8B 40.88 51.72 58.47 52.68 35.80 42.52 30.64 24.44 47.10 73.00 45.73 100.00 \nChunkLLM **39.14** 49.93 53.45 **52.29** **34.40** **42.98** **31.05** **23.89** **47.20** 72.50 **44.68** 50.18\nSepLLM 36.23 **51.16** 51.81 47.70 27.83 40.85 27.12 21.84 46.89 72.00 42.34 54.56\nStrmLLM 35.94 50.96 **54.13** 50.54 30.95 41.83 27.50 23.06 46.23 **73.00** 43.42 69.25 \nTable 1: Experimental results on LongBench. Avg denotes average score. SDQA: single-document\nquestion answering, MDQA: multi-document question answering. The full names of the subtasks\nare shown in Appendix 6.2. \n3.2 M AIN R ESULTS \n3.2.1 R ESULTS ON L ONG B ENCH \nWe set the top-k ratio to 45% and the number of local chunks to 15 for ChunkLLM. The experimental\nresults on the LongBench using Qwen2.5-7B and Llama3.1-8B are presented in Table 1. Here,\n\u201dStrmLLM\u201d represents StreamingLLM (Xiao et al., 2024). We take Qwen2.5-7B as an example for\nanalysis, and the same conclusion holds for Llama3.1-8B. The following observations can be made:\n(1) In terms of overall average performance, ChunkLLM attains the optimal performance when\ncompared to SepLLM and StreamingLLM, with respective improvements of 3.43 and 2.02 (43.53\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM\ndemonstrates a remarkable improvement in long-context evaluation, which validates the advantage\nof ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably,\nin the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that\nthe core challenge of MDQA lies in the dispersion of critical information across distinct positions\nwithin the context, which places high demands on the model\u2019s context comprehension capability.\nSepLLM leverages separators as chunk features, which is plagued by constrained representational\ncapacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches\nthe representational capacity of chunks via attention distillation, which enhances the recall rates of\ncritical chunks. This, in turn, effectively boosts the model\u2019s long-context understanding capability.\n(3) ChunkLLM attains 98.64% of the vanilla model\u2019s performance while employing the minimum\nKV cache. Notably, relative to SepLLM and StreamingLLM, ChunkLLM reduces the KV cache\nusage rate by 4.59% and 19.92% (48.58% v.s. 53.17% v.s. 68.50%), respectively, findings that\nfurther substantiate the superiority of ChunkLLM in long-context scenarios. \n3.2.2 R ESULTS ON NIAH \nFigure 4: Needle-in-a-Haystack retrieval accuracy across context positions with 64k context length.\nThe last column represents the KV-cache utilization rate. \n6 \nWe set the top-k to 256 and the number of local chunks to 16 for ChunkLLM. As depicted in\nFigure 4, ChunkLLM outperforms SepLLM across all scenarios in the 64K-context NIAH evaluation conducted on Qwen2.5-7B and Llama3.1-8B, achieving superior performance. Notably, in\nscenarios where the context length exceeds 12K, SepLLM exhibits near-total loss of retrieval capability (visualized in red), whereas ChunkLLM retains performance comparable to the vanilla model.\nThis discrepancy is primarily attributed to ChunkLLM\u2019s attention distillation mechanism, which\nstrengthens the feature representational capacity of chunks. Consequently, during chunk selection,\nthe model effectively identifies critical chunks with higher query relevance, leading to improved inference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative\nto SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct\nexperiments with StreamingLLM, as shown in Appendix 6.3. \n3.2.3 I NFERENCE E FFICIENCY \n|1600
1400
1200 (s)
1000 time
800 Cost
600
400
200|Qwen2.5-7B
Ours|Col3|2300
2000
1700
1400
1100
800
500
200|Llama3.1-8B
Ours|Col6|\n|---|---|---|---|---|---|\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)|~~3.84x~~|~~3.84x~~|~~3.84x~~|~~4.~~|~~4.~~|\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)||||||\n|200
400
600
800
1000
1200
1400
1600
Cost time (s)|||||| \nFigure 5: Comparison of inference time per 10k tokens in the generation process on PG19 test set. \nWe conduct runtime evaluations of Vanilla and ChunkLLM for 120K-token generation tasks on\nthe PG19(Rae et al., 2019) test set, with metrics recorded every 10K tokens. We set the top-k to\n256 and the number of local chunks to 16 for ChunkLLM. As shown in Figure 5, as the number\nof generated tokens increases, Vanilla\u2019s inference time rises linearly, while ChunkLLM maintains\npersistent stability in time consumption. In the 110K\u2013120K token generation phase, ChunkLLM\noutperforms Vanilla by speedups of 3.84\u00d7 and 4.48\u00d7, which corroborates the efficacy of the proposed\nICAC mechanism. During ChunkLLM\u2019s inference phase, chunk updates occur exclusively at chunk\nboundaries, minimizing the updates frequency and thereby boosting inference efficiency. We also\nconduct supplementary experiments using the FineWeb-Edu dataset, from which 1000 test corpora\nof 4k length are sampled. For the task of chunk boundary prediction, we evaluate its performance\nusing three key metrics: precision, recall, and F1-score. The calculated results are 98.31, 95.54,\nand 96.91, respectively. Such promising performance indicators serve to verify the reliability and\neffectiveness of our chunk boundary prediction task. \nWe conduct experiments where these models Length Methods PPL Total Time(s)\ngenerated 120K tokens, evaluating both total Qwen2.5-7B 14.41 10,684.31\ninference time and average perplexity (ppl) on 120K ChunkLLM 16.23 4,782.62\nthe PG19 test set, and results are summarizedin Table 2. Compared to the vanilla model, Llama3.1-8BChunkLLM 11.9312.89 145,,906.19963.83\nChunkLLM yields a slight enhancement in ppl Table 2: The perplexity and running time comparalongside a significant decrease in total infer- ison on the PG19 test set.\nence time. The underlying reason is that while\nthe vanilla model maintains semantic integrity, it incurs linearly increasing inference time as generated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up\ninference by leveraging its chunk selection and ICAC mechanisms. \nLength Methods PPL Total Time(s) \n120K \nQwen2.5-7B 14.41 10,684.31\nChunkLLM 16.23 4,782.62\nLlama3.1-8B 11.93 14,906.19\nChunkLLM 12.89 5,963.83 \nTable 2: The perplexity and running time comparison on the PG19 test set. \n3.2.4 R ESULTS ON S HORT T EXT \nThe experimental results for short texts are presented in Table 3. The following conclusions can\nbe drawn: (1) The overall average metrics of ChunkLLM on Qwen2.5-7B and Llama3.1-8B both\noutperform those of StreamingLLM and SepLLM, achieving 99.57% and 99.84% of the Vanilla\nmodel\u2019s performance, respectively. Notably, ChunkLLM attains optimal performance across 8 out of \n7 \nMethods General Knowledge Question Answering Reasoning Avg KV\nMMLU SciQ OQA CQA SIQA PIQA Heag WG ARC-c ARC-e\nQwen2.5-7B 74.25 97.00 52.80 84.52 58.44 81.72 80.24 77.27 63.82 87.21 75.73 100.00\nChunkLLM 72.51 96.60 **52.40** **84.68** **58.34** **81.77** 80.08 **76.87** **63.65** **87.21** **75.41** 45.47\nSepLLM 73.07 **96.70** 52.20 84.19 58.25 81.41 **80.10** 76.48 62.94 86.11 75.15 50.20\nStrmLLM **73.31** 96.60 52.00 84.28 58.19 81.39 79.76 76.64 62.79 86.36 75.13 45.14 \nLlama3.1-8B 65.30 97.60 48.00 74.28 54.04 83.19 81.76 80.03 57.85 84.55 72.66 100.00 \nChunkLLM **64.78** 97.30 **48.40** **74.45** **54.76** 82.75 **81.84** **79.01** 57.68 **84.55** **72.55** 45.04\nSepLLM 64.32 **97.40** 47.40 74.10 54.25 **83.03** 81.68 79.01 **57.93** 84.09 72.32 50.32\nStrmLLM 61.19 97.20 48.00 73.79 53.94 81.56 80.14 78.22 56.91 83.59 72.45 45.30 \nTable 3: Experimental results on short context benchmarks. \nthe 10 evaluation tasks, validating its efficacy in short-text task scenarios. (2) We perform statistical\nanalyses on the average utilization rate of the KV cache. In comparison with SepLLM, ChunkLLM\nachieves superior performance while consuming a lower volume of KV cache (45.47% v.s. 50.20%).\nSpecifically, on the Llama3.1-8B model, ChunkLLM not only exhibits the minimal KV cache usage\nbut also outperforms both SepLLM and StreamingLLM in terms of performance metrics. This\nfinding further validates the precision of ChunkLLM in chunk recall, achieving a balanced trade-off\nbetween performance and memory consumption. \n3.3 A BLATION S TUDY \n3.3.1 E FFECTIVENESS OF V OTE AND ICAC \nMethods SDQA MDQA Summary Few-shot Avg\nNQA Qasper MFQA HQA Musi 2WQA GR QMS SAM TREC\nQwen2.5-7B 33.76 51.40 57.17 56.76 29.40 38.09 31.57 24.40 46.73 72.00 44.13\nChunkLLM 31.09 50.52 **56.95** **55.61** **29.41** 38.27 31.25 23.16 46.53 **72.50** 43.53 \nw/o vote 30.78 45.00 51.53 54.96 28.00 38.17 30.82 23.49 46.58 70.00 41.93 \nw/o ICAC **31.36** **50.94** 56.84 55.61 28.50 **38.59** **31.86** **23.93** **46.66** 72.50 **43.68** \nTable 4: Ablation Study on LongBench, w/o vote: remove vote mechanism, w/o ICAC: remove\nICAC pattern. \nWe validate the proposed vote mechanism and\nICAC pattern based on the Qwen2.5-7B using the LongBench, with experimental results\nshown in Table 4. Removal of the vote mechanism leads to a 1.6 drop in overall performance\n(43.53 v.s. 41.93), which confirms the mechanism\u2019s efficacy, as it integrates inter-layer differences in chunk selection and minimizes interference arising from such discrepancies. We Figure 6: Inference efficiency of ICAC on PG19\nalso conduct a visual investigation into the re- test set.\ncall performance of top-k chunks across different layers, with comprehensive experimental results provided in Appendix 6.5. Conversely, removing ICAC results in a marginal 0.15 improvement in overall performance. This slight gain is\nattributed to the increased frequency of chunk selection updates during the inference phase. Frequent chunk selection, however, poses a limitation of low inference efficiency. As shown in Figure\n6, after ICAC is removed, the inference latency is 2.12 times higher than that of ChunkLLM in the\n110K\u2013120K token generation stage. Conversely, incorporating ICAC enables the model to maintain near-lossless performance alongside improved inference efficiency, which provides additional\nvalidation of ICAC\u2019s success. Appendix 6.4 shows a case study of the ICAC. \nFigure 6: Inference efficiency of ICAC on PG19 \ntest set. \n3.3.2 A NALYSIS OF F IXED C HUNKS AND S EMANTIC C HUNKS \nWe conduct an experimental analysis of the fixed chunk method (FixLLM) on the NIAH task. To\nensure consistent KV cache utilization and facilitate a fair comparison, FixLLM is configured with\na top-k of 384 and a local chunks of 24, while ChunkLLM is set to a top-k of 256 and a local\nchunks of 16. The experimental results are illustrated in Figure 7. As observed, under conditions \n8 \nFigure 7: Visualization of fixed chunks and semantic chunks in NIAH test. \nof approximately consistent KV cache utilization, FixLLM exhibits an 8.23 reduction (87.78 v.s.\n79.55) in accuracy relative to ChunkLLM on the 64K NIAH task. This discrepancy stems from\nthe semantic incompleteness of fixed chunks, which in turn compromises chunk selection during\nthe inference phase. In contrast, ChunkLLM leverages contextual semantic information to identify\nchunk boundaries, preserving the semantic integrity of chunks. \n4 R ELATED W ORK" + }, + "generated_answer": "That's a great question about how long-context Large Language Models (LLMs) manage information!\n\n**Answer:**\nIn the context of long-context LLMs, the key phenomenon where chunks attended to by tokens within a generated chunk exhibit substantial consistency, and which ChunkLLM exploits to enhance inference efficiency, is called **Intra-Chunk Attention (ICAC)**. This phenomenon also notes that chunk updates predominantly occur at chunk boundaries.\n\n**Key Points/Summary:**\n* **Intra-Chunk Attention (ICAC):** This is the name given to the observed phenomenon.\n* **Consistency:** Tokens within a generated chunk consistently attend to the same set of chunks.\n* **Chunk Updates:** Changes to which chunks are being attended to primarily happen at the boundaries between chunks, rather than constantly throughout a chunk.\n* **Inference Efficiency:** ChunkLLM leverages this ICAC phenomenon to reduce the frequency of chunk selection updates during inference, which significantly boosts its efficiency.\n\n**Extra Tip:**\nImagine reading a book. When you're in the middle of a paragraph, your focus likely stays within that paragraph's ideas. You only really shift your \"attention\" to new major ideas when you reach the end of a paragraph or section. ICAC works similarly for LLMs, allowing them to maintain focus within a \"chunk\" of text and only update their attention more broadly when they hit a natural \"chunk boundary,\" saving computational effort!", + "correctness_score": 0.85, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "retrieved_context": { + "context": "## 1.1 **Related Work** \nEntity-aware retrieval has gained increasing attention within biomedical NLP and question-answering \ndomains. Early retrieval-augmented methods such as RAG [2] demonstrated the potential of embedding \nbased chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches \nleveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., \n2016) provided partial improvements but often failed to maintain contextual continuity across long clin \nical narratives. \nCLEAR [1] represented a significant advancement by introducing entity-centered retrieval aligned \nwith clinical semantics. Our work extends this line of research by operationalizing CLEAR within an \nend-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale \ndocument sets. \nRecent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., \n2023; Xiong et al., 2024) emphasize that retrieval strategies often outperform naive long-context prompt \ning, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation \nframework within this paradigm, focusing on realistic EHR-scale clinical notes. \n3\n\n## 1.1 **Related Work** \nEntity-aware retrieval has gained increasing attention within biomedical NLP and question-answering \ndomains. Early retrieval-augmented methods such as RAG [2] demonstrated the potential of embedding \nbased chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches \nleveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., \n2016) provided partial improvements but often failed to maintain contextual continuity across long clin \nical narratives. \nCLEAR [1] represented a significant advancement by introducing entity-centered retrieval aligned \nwith clinical semantics. Our work extends this line of research by operationalizing CLEAR within an \nend-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale \ndocument sets. \nRecent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., \n2023; Xiong et al., 2024) emphasize that retrieval strategies often outperform naive long-context prompt \ning, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation \nframework within this paradigm, focusing on realistic EHR-scale clinical notes. \n3\n\n# 1. **INTRODUCTION** \nElectronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta \ntion within FHIR DocumentReference resources, typically encoded as base64 attachments containing \nunstructured clinical notes. These documents, ranging from brief progress notes to comprehensive dis \ncharge summaries, present substantial challenges for automated question-answering systems that require \nsemantically accurate information extraction rather than statistical correlation-based retrieval common \nin traditional vector database approaches. \nContemporary approaches to clinical document processing have largely focused on two paradigms: \nzero-shot inference with large context windows that process entire documents but face computational \n2 \nconstraints and the \u201dlost in the middle\u201d problem, and chunk-based retrieval-augmented generation (RAG) \nsystems that utilize vector databases for semantic similarity search but often fail to capture critical clin \nical entity relationships and contextual dependencies essential for accurate medical information extrac \ntion. \nA growing body of evidence shows that merely expanding context windows does not guarantee \neffective use of information: performance often drops when relevant spans occur in the middle of long \ninputs (\u201clost in the middle\u201d) [3]. This motivates entity-aware retrieval that selectively centers clinically \nrelevant spans rather than relying on statistically similar but potentially off-target chunks. \nThe CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in \n2025 [1], introduced a novel approach that addresses these limitations through entity-aware, entity \ncentered retrieval strategies. The original study demonstrated significant performance improvements \n(F1 score of 0.90 vs 0.86 for traditional RAG) with substantial efficiency gains (71% token reduction, \n72% faster inference time) on clinical information extraction tasks, positioning CLEAR as a potentially \ntransformative approach for production EHR processing systems. \nTo validate these claims in realistic healthcare scenarios and provide a robust evaluation framework \nfor clinical NLP approaches, we developed the Clinical Notes Q&A Evaluation Platform. This com \nprehensive validation study implements and compares three fundamental approaches: (1) wide context \nprocessing for zero-shot inference with large language models, (2) traditional vector database chunking \nwith embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu \nmentReference processing. Our contributions include: systematic validation of CLEAR\u2019s performance \nclaims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis \nof retrieval strategy performance across clinical documents of varying complexity and length.\n\n# 1. **INTRODUCTION** \nElectronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta \ntion within FHIR DocumentReference resources, typically encoded as base64 attachments containing \nunstructured clinical notes. These documents, ranging from brief progress notes to comprehensive dis \ncharge summaries, present substantial challenges for automated question-answering systems that require \nsemantically accurate information extraction rather than statistical correlation-based retrieval common \nin traditional vector database approaches. \nContemporary approaches to clinical document processing have largely focused on two paradigms: \nzero-shot inference with large context windows that process entire documents but face computational \n2 \nconstraints and the \u201dlost in the middle\u201d problem, and chunk-based retrieval-augmented generation (RAG) \nsystems that utilize vector databases for semantic similarity search but often fail to capture critical clin \nical entity relationships and contextual dependencies essential for accurate medical information extrac \ntion. \nA growing body of evidence shows that merely expanding context windows does not guarantee \neffective use of information: performance often drops when relevant spans occur in the middle of long \ninputs (\u201clost in the middle\u201d) [3]. This motivates entity-aware retrieval that selectively centers clinically \nrelevant spans rather than relying on statistically similar but potentially off-target chunks. \nThe CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in \n2025 [1], introduced a novel approach that addresses these limitations through entity-aware, entity \ncentered retrieval strategies. The original study demonstrated significant performance improvements \n(F1 score of 0.90 vs 0.86 for traditional RAG) with substantial efficiency gains (71% token reduction, \n72% faster inference time) on clinical information extraction tasks, positioning CLEAR as a potentially \ntransformative approach for production EHR processing systems. \nTo validate these claims in realistic healthcare scenarios and provide a robust evaluation framework \nfor clinical NLP approaches, we developed the Clinical Notes Q&A Evaluation Platform. This com \nprehensive validation study implements and compares three fundamental approaches: (1) wide context \nprocessing for zero-shot inference with large language models, (2) traditional vector database chunking \nwith embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu \nmentReference processing. Our contributions include: systematic validation of CLEAR\u2019s performance \nclaims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis \nof retrieval strategy performance across clinical documents of varying complexity and length." + }, + "generated_answer": "Hello there! I can certainly help you understand that.\n\nIn the clinical domain, the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes is called **CLinical Entity Augmented Retrieval (CLEAR)**.\n\nThis approach was introduced by Lopez et al. in 2025 and represents a significant advancement. It addresses the limitations of earlier methods by focusing on \"entity-aware, entity-centered retrieval strategies\" that are aligned with clinical semantics.\n\n**Answer:**\nThe methodology in the clinical domain that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes is the **CLinical Entity Augmented Retrieval (CLEAR) methodology**.\n\nCLEAR introduced a novel approach that specifically focuses on entity-aware and entity-centered retrieval strategies, which are designed to handle the complexities and specific semantic requirements of clinical documentation. Unlike traditional vector database approaches that might rely on statistical correlations, CLEAR aims for semantically accurate information extraction.\n\nThe original study on CLEAR demonstrated notable improvements:\n* **Semantic Accuracy:** It achieved a significant performance improvement with an F1 score of 0.90, compared to 0.86 for traditional Retrieval-Augmented Generation (RAG) methods, on clinical information extraction tasks. This indicates better accuracy in understanding and extracting relevant clinical information.\n* **Computational Efficiency:** It also provided substantial efficiency gains, including a 71% reduction in tokens processed and a 72% faster inference time. This makes it a highly efficient approach for processing large volumes of clinical data.\n\nThese benefits position CLEAR as a potentially transformative approach for production EHR processing systems, as it effectively addresses challenges like the \"lost in the middle\" problem often encountered when processing long clinical narratives with other methods.\n\n**Key Points/Summary:**\n* **Name:** CLinical Entity Augmented Retrieval (CLEAR) methodology.\n* **Approach:** Uses entity-aware and entity-centered retrieval strategies specifically designed for clinical semantics.\n* **Benefits:** Achieves improved semantic accuracy (higher F1 score) and significant computational efficiency (reduced tokens and faster inference time).\n* **Purpose:** Addresses limitations of traditional methods in processing EHR notes for accurate information extraction.\n\n**Extra Tip:**\nThink of \"entity-aware\" retrieval like a smart librarian who doesn't just look for keywords, but understands the *meaning* of clinical terms (like specific diseases, medications, or symptoms) and their relationships within a patient's record. This allows for much more precise and relevant information retrieval compared to just matching similar-sounding text chunks!", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "retrieved_context": { + "context": "# Sparse Attention The sparse attention mechanism constructs sparse attention matrices by confining\nattention to predefined patterns, such as local windows or fixed-stride block patterns. Beltagy et al.\n(2020a) combined dilated local window attention with task-specific global attention. MoBA (Lu\net al., 2025) proposes an innovative mixed block attention mechanism. ESA (Wang et al., 2025)\nreduces computational costs by selecting tokens most critical to the current generation for attention\ncalculation. NSA (Yuan et al., 2025) combines coarse-grained token compression and fine-grained\ntoken selection. SepLLM (Chen et al., 2024) finds that the segment information between separators\ncan be effectively compressed into the separators themselves without causing significant information\nloss.\n\n# Sparse Attention The sparse attention mechanism constructs sparse attention matrices by confining\nattention to predefined patterns, such as local windows or fixed-stride block patterns. Beltagy et al.\n(2020a) combined dilated local window attention with task-specific global attention. MoBA (Lu\net al., 2025) proposes an innovative mixed block attention mechanism. ESA (Wang et al., 2025)\nreduces computational costs by selecting tokens most critical to the current generation for attention\ncalculation. NSA (Yuan et al., 2025) combines coarse-grained token compression and fine-grained\ntoken selection. SepLLM (Chen et al., 2024) finds that the segment information between separators\ncan be effectively compressed into the separators themselves without causing significant information\nloss.\n\n# Haojie Ouyang [1] **, Jianwei Lv** [2] **, Lei Ren** [2] **, Chen Wei** [2] **, Xiaojie Wang** [1] **, Fangxiang Feng** [1] \n1 School of Artificial Intelligence, Beijing University of Posts and Telecommunications\n2 Li Auto \nouyanghaojie@bupt.edu.cn \nA BSTRACT \nTransformer-based large models excel in natural language processing and computer vision, but face severe computational inefficiencies due to the self-attention\u2019s\nquadratic complexity with input tokens. Recently, researchers have proposed a\nseries of methods based on block selection and compression to alleviate this problem, but they either have issues with semantic incompleteness or poor traininginference efficiency. To comprehensively address these challenges, we propose\nChunkLLM, a lightweight and pluggable training framework. Specifically, we\nintroduce two components: QK Adapter (Q-Adapter and K-Adapter) and Chunk\nAdapter. The former is attached to each Transformer layer, serving dual purposes\nof feature compression and chunk attention acquisition. The latter operates at the\nbottommost layer of the model, functioning to detect chunk boundaries by leveraging contextual semantic information. During the training phase, the parameters\nof the backbone remain frozen, with only the QK Adapter and Chunk Adapter\nundergoing training. Notably, we design an attention distillation method for training the QK Adapter, which enhances the recall rate of key chunks. During the\ninference phase, chunk selection is triggered exclusively when the current token is\ndetected as a chunk boundary, thereby accelerating model inference. Experimental\nevaluations are conducted on a diverse set of long-text and short-text benchmark\ndatasets spanning multiple tasks. ChunkLLM not only attains comparable performance on short-text benchmarks but also maintains 98.64% of the performance\non long-context benchmarks while preserving a 48.58% key-value cache retention\nrate. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7 in comparison\nto the vanilla Transformer in the processing of 120K long texts. \n1 I NTRODUCTION \nTransformer-based large models (Vaswani et al., 2017) have demonstrated exceptional performance\nacross a diverse range of tasks, including natural language processing (Srivastava et al., 2025; Zhang\net al., 2024) and computer vision (Jiang et al., 2025). However, they have also faced significant challenges in terms of computational efficiency, particularly when scaling to larger structures and large\ncontext inputs. A core issue of efficiency limitations lies in the self-attention module, whose computational complexity is a quadratic relationship with the number of input tokens. Such deficiencies\nin computational efficiency exert a profound impact on both the training complexity and inference\nlatency of large models. \nEfficiency optimization of Transformer has emerged as a pivotal research domain, with efforts predominantly converging into three methodological paradigms. **Linear attention**, such as Mamba\n(Dao & Gu, 2024), RWKV (Peng et al., 2023b; 2024), and RetNet (Sun et al., 2023), seek to\napproximate and substitute the traditional softmax-based self-attention mechanism. However, the\nfundamental architectural disparities between linear attention and conventional attention mechanisms introduce non-trivial challenges: adapting pre-existing Transformer models to integrate linear attention often incurs prohibitive conversion costs(Mercat et al., 2024; Wang et al., 2024; Bick\net al., 2024), while alternative strategies necessitate end-to-end training of entirely new model from\nscratch(Li et al., 2025). Another optimization paradigm is **Sparse attention**, which leverages predefined structural constraints, such as sink-based attention mechanisms (Xiao et al., 2024) or sliding \n1 \nwindow attention mechanisms (Beltagy et al., 2020b), to exploit this sparsity. While these methods may yield certain effects, they often rely heavily on specific tasks, which can limit the overall\ngeneralization ability of the model. Dynamic sparse attention mechanisms (Tang et al., 2024; Jiang\net al., 2024; Liu et al., 2024) filter out subsets of tokens during the inference phase. Although such\nmethods can reduce the computational load of long sequences, they fail to significantly lower the\nhigh training costs of long-context models, making it difficult for large language models to efficiently scale to context-processing tasks with million-level token sizes. **Chunk Selective attention**,\na special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk\n(Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen\net al., 2024). Both approaches partition the input into discrete chunks: the former conducts partitioning with a fixed length, which gives rise to semantic incompleteness; the latter utilizes separators\nfor partitioning, yet ambiguities often emerge. For example, periods frequently occur in numerical\nvalues or abbreviations. Furthermore, during the inference phase, these methods necessitate chunk\nselection for each generated token, incurring additional computational overhead. It is thus evident\nthat existing efficient approaches still exhibit inherent limitations. \nTo address the aforementioned challenges, We propose ChunkLLM, which can be directly constructed by integrating two lightweight and trainable modules into existing LLMs: **QK Adapter**\nand **Chunk Adapter** . The Chunk Adapter connects to the output of the bottommost Transformer\nlayer and used for identify if a token is the last token of a chunk. The QK Adapter is in parallel with\nQ and K matrix at each Transformer layer. It maps full attention scores to chunk attention scores,\nand trained by a distillation approach. \nThe QK Adapter fulfills feature compression and the generation of chunk attention scores. To train\nthe QK Adapter, we propose an attention distillation approach designed to enhance the recall rate\nof key chunks. During training, LLM parameters are kept frozen, with the Kullback\u2013Leibler (KL)\ndivergence between chunk attention scores and full attention scores serving as a guidance signal for\noptimization. The Chunk Adapter determines whether a token corresponds to a chunk boundary by\nleveraging contextual semantic information. During the inference phase, we exploit the Intra-Chunk\nAttention Consistency (ICAC) pattern such that chunk selection is only updated when the current\ntoken is identified as a chunk boundary, which substantially enhances inference efficiency. Furthermore, ChunkLLM can achieve inference performance comparable to that of models optimized for\n120K context lengths, despite being trained solely on 4K context lengths, thereby substantially reducing the training overhead associated with 120K context scaling. Experimental results validate\nthat ChunkLLM yields a 4.48\u00d7 speedup relative to the vanilla Transformer when processing 120K\nlong texts. \nOur contributions are summarized as follows: \n- We introduce ChunkLLM that integrates two lightweight and pluggable components into\nexisting LLMs: the QK Adapter and the Chunk Adapter. The newly developed ChunkLLM only necessitates fine-tuning these lightweight components on the basis of the original model architecture. This design enables ChunkLLM to attain performance comparable\nto vanilla Transformer while utilizing a smaller KV cache, alongside achieving effective\ncontrol over computational scale. \n- We propose an attention distillation-based training approach for the QK Adapter, which\nleverages KL divergence to drive chunk attention toward approximating full attention, effectively enhancing the recall rate of key chunks. Furthermore, we introduce a novel ICAC\npattern, which yields notable improvements in inference efficiency for long-context scenarios. \n- Experimental evaluations show that ChunkLLM not only attains comparable performance\non short-text benchmarks but also maintains 98.64% of the performance on long-context\nbenchmarks while preserving a 48.58% key-value cache (kvcache) retention rate, relative\nto the vanilla Transformer. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7\nin comparison to the vanilla Transformer in the processing of 120K long texts. \n2 \n2 M ETHOD \nThe framework of ChunkLLM is shown in Figure 1. ChunkLLM can be built on any existing\ntransformer-based LLMs. Two extra lightweight and pluggable modules are designed to support\nchunk-related capability. One is Chunk Adapter, which is used to identify chunk boundaries. The\nother is the Q Adapter and K Adapter, which is tailored for efficient feature compression and chunk\nselection. This section elaborates on the details of the two modules. \n2.1 C HUNK A DAPTER \nThe Chunk Adapter is a one-layer forward\nneural network (FNN) classifier for chunk\nboundary prediction. Its input is the output\nof the first layer of the LLM, and the output is if or not the token is a chunk boundary, as depicted in the Figure 1. \nFor an input X = _{x_ 1 _, x_ 2 _, ..., x_ _n\u2212_ 1 _, x_ _n_ _}_\nwith _n_ tokens and their corresponding labels Y = _{y_ 1 _, y_ 2 _, ..., y_ _n\u2212_ 1 _, y_ _n_ _}_, _y_ _i_ _\u2208_\n_{_ 0 _,_ 1 _}_, where 1 indicates that the token\n_x_ _i_ is a chunk boundary, 0 indicates that it\ndoes not, **H** _[l]_ _i_ [1] [be the output of] _[ x]_ _[i]_ [ at first]\nlayer. The FNN based Chunk Adapter is\ngiven as in equation 1 \n\u02c6 1 _,_ Sigmoid(FFN( **H** _[l]_ _i_ [1] [))] _[ > \u03b1,]_\n_y_ _i_ =\n\ufffd0 _,_ _otherwise_\n(1) \noutput \nInput \nFigure 1: The framework of ChunkLLM. \nFor training the chunk adapter, we employ\nthe binary cross-entropy loss (BCE) (equation 2) as the objective function. Detailed\ninformation on the training dataset will be given in the experiment part. \n_L_ _CBP_ = _\u2212_ [1] \n_n_ \n_n_ \n\u02c6 \n\ufffd[ _y_ _i_ _\u00b7 log_ (\u02c6 _y_ _i_ ) + (1 _\u2212_ _y_ _i_ ) _\u00b7 log_ (1 _\u2212_ _y_ _i_ )] (2) \n_i_ =1 \n2.2 QK A DAPTER \nAt each layer of the LLM, we incorporate a Q-Adapter and a K-Adapter which used to compress the\nattention and select the chunks. \nFor each layer, let **Q** and **K** be the attention matrix respectively, _c_ be the chunk number of the input,\n_Index_ ~~_c_~~ = _{i_ 1 _, i_ 2 _, ...i_ _c_ _}_ is the index set of chunk boundary tokens. Let **K** [\u02c6] be the K matrix of these\ntokens. We then calculate chunk attention scores as follows:\n\n# Haojie Ouyang [1] **, Jianwei Lv** [2] **, Lei Ren** [2] **, Chen Wei** [2] **, Xiaojie Wang** [1] **, Fangxiang Feng** [1] \n1 School of Artificial Intelligence, Beijing University of Posts and Telecommunications\n2 Li Auto \nouyanghaojie@bupt.edu.cn \nA BSTRACT \nTransformer-based large models excel in natural language processing and computer vision, but face severe computational inefficiencies due to the self-attention\u2019s\nquadratic complexity with input tokens. Recently, researchers have proposed a\nseries of methods based on block selection and compression to alleviate this problem, but they either have issues with semantic incompleteness or poor traininginference efficiency. To comprehensively address these challenges, we propose\nChunkLLM, a lightweight and pluggable training framework. Specifically, we\nintroduce two components: QK Adapter (Q-Adapter and K-Adapter) and Chunk\nAdapter. The former is attached to each Transformer layer, serving dual purposes\nof feature compression and chunk attention acquisition. The latter operates at the\nbottommost layer of the model, functioning to detect chunk boundaries by leveraging contextual semantic information. During the training phase, the parameters\nof the backbone remain frozen, with only the QK Adapter and Chunk Adapter\nundergoing training. Notably, we design an attention distillation method for training the QK Adapter, which enhances the recall rate of key chunks. During the\ninference phase, chunk selection is triggered exclusively when the current token is\ndetected as a chunk boundary, thereby accelerating model inference. Experimental\nevaluations are conducted on a diverse set of long-text and short-text benchmark\ndatasets spanning multiple tasks. ChunkLLM not only attains comparable performance on short-text benchmarks but also maintains 98.64% of the performance\non long-context benchmarks while preserving a 48.58% key-value cache retention\nrate. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7 in comparison\nto the vanilla Transformer in the processing of 120K long texts. \n1 I NTRODUCTION \nTransformer-based large models (Vaswani et al., 2017) have demonstrated exceptional performance\nacross a diverse range of tasks, including natural language processing (Srivastava et al., 2025; Zhang\net al., 2024) and computer vision (Jiang et al., 2025). However, they have also faced significant challenges in terms of computational efficiency, particularly when scaling to larger structures and large\ncontext inputs. A core issue of efficiency limitations lies in the self-attention module, whose computational complexity is a quadratic relationship with the number of input tokens. Such deficiencies\nin computational efficiency exert a profound impact on both the training complexity and inference\nlatency of large models. \nEfficiency optimization of Transformer has emerged as a pivotal research domain, with efforts predominantly converging into three methodological paradigms. **Linear attention**, such as Mamba\n(Dao & Gu, 2024), RWKV (Peng et al., 2023b; 2024), and RetNet (Sun et al., 2023), seek to\napproximate and substitute the traditional softmax-based self-attention mechanism. However, the\nfundamental architectural disparities between linear attention and conventional attention mechanisms introduce non-trivial challenges: adapting pre-existing Transformer models to integrate linear attention often incurs prohibitive conversion costs(Mercat et al., 2024; Wang et al., 2024; Bick\net al., 2024), while alternative strategies necessitate end-to-end training of entirely new model from\nscratch(Li et al., 2025). Another optimization paradigm is **Sparse attention**, which leverages predefined structural constraints, such as sink-based attention mechanisms (Xiao et al., 2024) or sliding \n1 \nwindow attention mechanisms (Beltagy et al., 2020b), to exploit this sparsity. While these methods may yield certain effects, they often rely heavily on specific tasks, which can limit the overall\ngeneralization ability of the model. Dynamic sparse attention mechanisms (Tang et al., 2024; Jiang\net al., 2024; Liu et al., 2024) filter out subsets of tokens during the inference phase. Although such\nmethods can reduce the computational load of long sequences, they fail to significantly lower the\nhigh training costs of long-context models, making it difficult for large language models to efficiently scale to context-processing tasks with million-level token sizes. **Chunk Selective attention**,\na special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk\n(Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen\net al., 2024). Both approaches partition the input into discrete chunks: the former conducts partitioning with a fixed length, which gives rise to semantic incompleteness; the latter utilizes separators\nfor partitioning, yet ambiguities often emerge. For example, periods frequently occur in numerical\nvalues or abbreviations. Furthermore, during the inference phase, these methods necessitate chunk\nselection for each generated token, incurring additional computational overhead. It is thus evident\nthat existing efficient approaches still exhibit inherent limitations. \nTo address the aforementioned challenges, We propose ChunkLLM, which can be directly constructed by integrating two lightweight and trainable modules into existing LLMs: **QK Adapter**\nand **Chunk Adapter** . The Chunk Adapter connects to the output of the bottommost Transformer\nlayer and used for identify if a token is the last token of a chunk. The QK Adapter is in parallel with\nQ and K matrix at each Transformer layer. It maps full attention scores to chunk attention scores,\nand trained by a distillation approach. \nThe QK Adapter fulfills feature compression and the generation of chunk attention scores. To train\nthe QK Adapter, we propose an attention distillation approach designed to enhance the recall rate\nof key chunks. During training, LLM parameters are kept frozen, with the Kullback\u2013Leibler (KL)\ndivergence between chunk attention scores and full attention scores serving as a guidance signal for\noptimization. The Chunk Adapter determines whether a token corresponds to a chunk boundary by\nleveraging contextual semantic information. During the inference phase, we exploit the Intra-Chunk\nAttention Consistency (ICAC) pattern such that chunk selection is only updated when the current\ntoken is identified as a chunk boundary, which substantially enhances inference efficiency. Furthermore, ChunkLLM can achieve inference performance comparable to that of models optimized for\n120K context lengths, despite being trained solely on 4K context lengths, thereby substantially reducing the training overhead associated with 120K context scaling. Experimental results validate\nthat ChunkLLM yields a 4.48\u00d7 speedup relative to the vanilla Transformer when processing 120K\nlong texts. \nOur contributions are summarized as follows: \n- We introduce ChunkLLM that integrates two lightweight and pluggable components into\nexisting LLMs: the QK Adapter and the Chunk Adapter. The newly developed ChunkLLM only necessitates fine-tuning these lightweight components on the basis of the original model architecture. This design enables ChunkLLM to attain performance comparable\nto vanilla Transformer while utilizing a smaller KV cache, alongside achieving effective\ncontrol over computational scale. \n- We propose an attention distillation-based training approach for the QK Adapter, which\nleverages KL divergence to drive chunk attention toward approximating full attention, effectively enhancing the recall rate of key chunks. Furthermore, we introduce a novel ICAC\npattern, which yields notable improvements in inference efficiency for long-context scenarios. \n- Experimental evaluations show that ChunkLLM not only attains comparable performance\non short-text benchmarks but also maintains 98.64% of the performance on long-context\nbenchmarks while preserving a 48.58% key-value cache (kvcache) retention rate, relative\nto the vanilla Transformer. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7\nin comparison to the vanilla Transformer in the processing of 120K long texts. \n2 \n2 M ETHOD \nThe framework of ChunkLLM is shown in Figure 1. ChunkLLM can be built on any existing\ntransformer-based LLMs. Two extra lightweight and pluggable modules are designed to support\nchunk-related capability. One is Chunk Adapter, which is used to identify chunk boundaries. The\nother is the Q Adapter and K Adapter, which is tailored for efficient feature compression and chunk\nselection. This section elaborates on the details of the two modules. \n2.1 C HUNK A DAPTER \nThe Chunk Adapter is a one-layer forward\nneural network (FNN) classifier for chunk\nboundary prediction. Its input is the output\nof the first layer of the LLM, and the output is if or not the token is a chunk boundary, as depicted in the Figure 1. \nFor an input X = _{x_ 1 _, x_ 2 _, ..., x_ _n\u2212_ 1 _, x_ _n_ _}_\nwith _n_ tokens and their corresponding labels Y = _{y_ 1 _, y_ 2 _, ..., y_ _n\u2212_ 1 _, y_ _n_ _}_, _y_ _i_ _\u2208_\n_{_ 0 _,_ 1 _}_, where 1 indicates that the token\n_x_ _i_ is a chunk boundary, 0 indicates that it\ndoes not, **H** _[l]_ _i_ [1] [be the output of] _[ x]_ _[i]_ [ at first]\nlayer. The FNN based Chunk Adapter is\ngiven as in equation 1 \n\u02c6 1 _,_ Sigmoid(FFN( **H** _[l]_ _i_ [1] [))] _[ > \u03b1,]_\n_y_ _i_ =\n\ufffd0 _,_ _otherwise_\n(1) \noutput \nInput \nFigure 1: The framework of ChunkLLM. \nFor training the chunk adapter, we employ\nthe binary cross-entropy loss (BCE) (equation 2) as the objective function. Detailed\ninformation on the training dataset will be given in the experiment part. \n_L_ _CBP_ = _\u2212_ [1] \n_n_ \n_n_ \n\u02c6 \n\ufffd[ _y_ _i_ _\u00b7 log_ (\u02c6 _y_ _i_ ) + (1 _\u2212_ _y_ _i_ ) _\u00b7 log_ (1 _\u2212_ _y_ _i_ )] (2) \n_i_ =1 \n2.2 QK A DAPTER \nAt each layer of the LLM, we incorporate a Q-Adapter and a K-Adapter which used to compress the\nattention and select the chunks. \nFor each layer, let **Q** and **K** be the attention matrix respectively, _c_ be the chunk number of the input,\n_Index_ ~~_c_~~ = _{i_ 1 _, i_ 2 _, ...i_ _c_ _}_ is the index set of chunk boundary tokens. Let **K** [\u02c6] be the K matrix of these\ntokens. We then calculate chunk attention scores as follows:" + }, + "generated_answer": "Based on the provided notes, I can explain ChunkLLM's dynamic attention management mechanisms, how it derives chunk representations, and how it utilizes them for efficiency. However, the notes do not contain any information about Dynamic Hierarchical Sparse Attention (DHSA), so I cannot provide a contrast with it.\n\n---\n\n### ChunkLLM's Dynamic Attention Management\n\n**Answer:**\nChunkLLM addresses the computational inefficiency of Transformer models, particularly with long contexts, by introducing two lightweight and pluggable components: the **Chunk Adapter** and the **QK Adapter**. These components work together to manage attention dynamically, focusing computation on relevant parts of the input.\n\n1. **Dynamic Attention Management and Chunk Boundary Detection:**\n * The **Chunk Adapter** is responsible for dynamically identifying \"chunk boundaries\" within the input text. It operates at the bottommost layer of the LLM and is a one-layer Feedforward Neural Network (FNN) classifier. Its input is the output of the *first* layer of the LLM, and its job is to predict whether a given token is the last token of a chunk (a binary classification: 1 for boundary, 0 otherwise). This allows the model to adaptively segment the input based on contextual semantic information rather than fixed-length partitioning.\n * During the inference phase, ChunkLLM leverages an **Intra-Chunk Attention Consistency (ICAC)** pattern. This means that chunk selection\u2014which determines what parts of the input are attended to\u2014is *only updated when the current token is identified as a chunk boundary*. This dynamic updating mechanism avoids redundant computations for every single token, significantly enhancing inference efficiency.\n\n2. **Deriving and Utilizing Chunk Representations for Efficiency:**\n * The **QK Adapter** (consisting of Q-Adapter and K-Adapter) is attached to each Transformer layer, working in parallel with the standard Query (Q) and Key (K) matrices. Its primary roles are **feature compression** and **chunk attention acquisition**.\n * Instead of calculating full attention scores over all tokens, the QK Adapter maps these full attention scores to \"chunk attention scores.\" Crucially, it uses a subset of the Key matrix, specifically **K\u0302 (K-hat)**, which represents the K matrix of the *chunk boundary tokens* identified by the Chunk Adapter. This means that the \"chunk representation\" for attention calculation is effectively derived from the features of these semantically important boundary tokens.\n * By confining attention calculations to these selected chunk boundary tokens, ChunkLLM drastically reduces the computational load, as it no longer needs to compute attention over the entire sequence. This also leads to a more efficient use of the key-value (KV) cache.\n\n3. **Preserving Performance in Long-Context Models:**\n * To ensure that this efficiency doesn't come at the cost of performance, ChunkLLM employs an **attention distillation method** for training the QK Adapter. During training, the parameters of the main LLM backbone are frozen, and only the QK Adapter and Chunk Adapter are trained. The distillation process uses **Kullback\u2013Leibler (KL) divergence** to guide the chunk attention scores to approximate the full attention scores. This helps the QK Adapter \"learn\" to preserve the most critical information, thereby enhancing the recall rate of key chunks.\n * Experimental results show that ChunkLLM can achieve performance comparable to vanilla Transformers on short-text benchmarks and maintains 98.64% of the performance on long-context benchmarks, all while achieving a 4.48\u00d7 speedup and a 48.58% KV cache retention rate for 120K long texts. This demonstrates its ability to maintain high performance despite significant efficiency gains.\n\n**Regarding Dynamic Hierarchical Sparse Attention (DHSA):**\nI apologize, but the provided notes do not contain any information about Dynamic Hierarchical Sparse Attention (DHSA). Therefore, I cannot contrast ChunkLLM with DHSA based on the given context. If you have more notes that include DHSA, please provide them, and I'd be happy to help!\n\n---\n\n**Key Points/Summary:**\n* **Dynamic Chunking:** ChunkLLM uses a Chunk Adapter (FNN classifier) to dynamically detect chunk boundaries based on contextual semantics.\n* **Efficient Attention:** A QK Adapter performs feature compression and calculates chunk attention scores by attending to features of identified chunk boundary tokens (K\u0302).\n* **Inference Optimization:** Chunk selection is updated only when a chunk boundary is detected (ICAC pattern), significantly reducing computational overhead.\n* **Performance Preservation:** An attention distillation method (KL divergence) trains the QK Adapter to approximate full attention, ensuring high performance (e.g., 98.64% of vanilla Transformer on long contexts) while achieving substantial speedups and KV cache savings.\n\n**Extra Tip:**\nThink of the Chunk Adapter as a smart gatekeeper that decides when to \"re-evaluate\" the most important parts of a long conversation. Instead of constantly checking every single word, it only opens its gate and triggers a deeper look (via the QK Adapter's focused attention) when it senses a significant \"chunk boundary\" \u2013 like the end of a sentence, paragraph, or a topic shift. This selective processing is what makes ChunkLLM so efficient for very long texts!", + "correctness_score": 0.25, + "faithfulness_score": 1.0, + "model_tested": "Layout" + }, + { + "question_id": 7, + "document_path": "multiple files", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "retrieved_context": { + "context": "## 1.1 **Related Work** \nEntity-aware retrieval has gained increasing attention within biomedical NLP and question-answering \ndomains. Early retrieval-augmented methods such as RAG [2] demonstrated the potential of embedding \nbased chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches \nleveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., \n2016) provided partial improvements but often failed to maintain contextual continuity across long clin \nical narratives. \nCLEAR [1] represented a significant advancement by introducing entity-centered retrieval aligned \nwith clinical semantics. Our work extends this line of research by operationalizing CLEAR within an \nend-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale \ndocument sets. \nRecent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., \n2023; Xiong et al., 2024) emphasize that retrieval strategies often outperform naive long-context prompt \ning, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation \nframework within this paradigm, focusing on realistic EHR-scale clinical notes. \n3\n\n# 1. **INTRODUCTION** \nElectronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta \ntion within FHIR DocumentReference resources, typically encoded as base64 attachments containing \nunstructured clinical notes. These documents, ranging from brief progress notes to comprehensive dis \ncharge summaries, present substantial challenges for automated question-answering systems that require \nsemantically accurate information extraction rather than statistical correlation-based retrieval common \nin traditional vector database approaches. \nContemporary approaches to clinical document processing have largely focused on two paradigms: \nzero-shot inference with large context windows that process entire documents but face computational \n2 \nconstraints and the \u201dlost in the middle\u201d problem, and chunk-based retrieval-augmented generation (RAG) \nsystems that utilize vector databases for semantic similarity search but often fail to capture critical clin \nical entity relationships and contextual dependencies essential for accurate medical information extrac \ntion. \nA growing body of evidence shows that merely expanding context windows does not guarantee \neffective use of information: performance often drops when relevant spans occur in the middle of long \ninputs (\u201clost in the middle\u201d) [3]. This motivates entity-aware retrieval that selectively centers clinically \nrelevant spans rather than relying on statistically similar but potentially off-target chunks. \nThe CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in \n2025 [1], introduced a novel approach that addresses these limitations through entity-aware, entity \ncentered retrieval strategies. The original study demonstrated significant performance improvements \n(F1 score of 0.90 vs 0.86 for traditional RAG) with substantial efficiency gains (71% token reduction, \n72% faster inference time) on clinical information extraction tasks, positioning CLEAR as a potentially \ntransformative approach for production EHR processing systems. \nTo validate these claims in realistic healthcare scenarios and provide a robust evaluation framework \nfor clinical NLP approaches, we developed the Clinical Notes Q&A Evaluation Platform. This com \nprehensive validation study implements and compares three fundamental approaches: (1) wide context \nprocessing for zero-shot inference with large language models, (2) traditional vector database chunking \nwith embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu \nmentReference processing. Our contributions include: systematic validation of CLEAR\u2019s performance \nclaims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis \nof retrieval strategy performance across clinical documents of varying complexity and length.\n\n# Retrieval-Augmented Generation (RAG): Semantic chunking with embedding-based retrieval us \ning top-k chunk selection. This approach prioritizes efficiency with minimal token usage (average 544 \ntokens per query) but may miss critical clinical relationships.\n\n# 5. **CONCLUSION** \nThis study developed and deployed a comprehensive Clinical Notes Q&A Evaluation Platform and found \nresults consistent with the benefits reported for Clinical Entity Augmented Retrieval (CLEAR) in prior \nwork. In our synthetic EHR QA setting, entity-aware retrieval achieved stronger semantic similarity \nthan wide-context processing at markedly lower token budgets, echoing the efficiency\u2013quality trade-offs \nhighlighted by CLEAR [1]. \nThe validation results strongly confirm the original research findings, particularly demonstrating \nscalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe \nference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware \nretrieval becomes increasingly advantageous as document complexity increases, confirming the method \nology\u2019s suitability for enterprise healthcare environments. \n10 \nFigure 4: Interactive interface showing different analysis approaches and their corresponding performance scores, enabling experimentation to improve reasoning accuracy. \nThe Clinical Notes Q&A Evaluation Platform represents a significant contribution to clinical NLP \nresearch by providing a systematic framework for evaluating retrieval strategies in realistic EHR pro \ncessing scenarios. The platform\u2019s validation of CLEAR methodology demonstrates its viability as a \nproduction-ready approach for clinical information extraction systems requiring optimal balance be \ntween semantic accuracy and computational efficiency. \nThe demonstrated effectiveness of CLEAR through systematic platform-based validation provides \nevidence-based guidance for healthcare organizations implementing clinical question-answering sys \ntems. The evaluation platform framework enables continued research and development in clinical \nentity-aware retrieval methodologies while supporting reproducible evaluation of future clinical NLP \ninnovations in production-relevant contexts.\n\n# Background: Electronic Health Records (EHR) systems store clinical documentation in FHIR \nDocumentReference resources as base64-encoded attachments, presenting significant challenges \nfor semantic question-answering applications. Traditional approaches using statistical correlation \nthrough vector database chunking often fail to capture the nuanced clinical relationships required \nfor accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) \nmethodology, introduced by Lopez et al. (2025) [1], addresses these limitations through entity\naware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; \n_>_ 70% fewer tokens and faster inference).\n\n# mantic chunking, which aims to improve retrieval performance by dividing documents into\nsemantically coherent segments. Despite its\ngrowing adoption, the actual benefits over simpler **fixed-size chunking**, where documents are\nsplit into consecutive, fixed-size segments, remain unclear. This study systematically evaluates the effectiveness of semantic chunking using three common retrieval-related tasks: document retrieval, evidence retrieval, and retrievalbased answer generation. The results show that\nthe computational costs associated with semantic chunking are not justified by consistent performance gains. These findings challenge the\nprevious assumptions about semantic chunking and highlight the need for more efficient\nchunking strategies in RAG systems.\n\n# 4. **DISCUSSION** \n## 4.1 **Validation of Research Claims** \nOur enhanced implementation provides observations consistent with the direction of the original CLEAR \nfindings [1] on our synthetic clinical QA benchmark. While our task, retriever, and baselines differ from \nthe original study (which evaluated structured information extraction with model-based NER and on \ntology/LLM synonym augmentation), we still observe that entity-aware retrieval yields higher semantic \nsimilarity at substantially lower token budgets than wide-context processing. \nThe 75% win rate on large documents (65K+ tokens) supports the hypothesis that entity-aware \nretrieval advantages grow with document complexity, consistent with prior work emphasizing targeted \nretrieval over long context processing. This finding has significant implications for clinical applications \ninvolving comprehensive patient records and complex clinical assessments. \n8 \nFigure 2: CLEAR dominates at a 3% efficiency bonus, maintaining superior adjusted accuracy across\nall notes.\n\n# 5 **Conclusion** \nIn this paper, we evaluated semantic and fixed-size\nchunking strategies in RAG systems across document retrieval, evidence retrieval, and answer generation. Semantic chunking occasionally improved\nperformance, particularly on stitched datasets with\nhigh topic diversity. However, these benefits were\nhighly context-dependent and did not consistently\njustify the additional computational cost. On nonsynthetic datasets that better reflect real-world documents, fixed-size chunking often performed better.\nOverall, our results suggest that fixed-size chunking remains a more efficient and reliable choice for\npractical RAG applications. The impact of chunking strategy was often overshadowed by other factors, such as the quality of embeddings, especially\nwhen computational resources are limited or when\nworking with standard document structures.\n\n# Fig. 1. Comparison of early chunking (left) and late chunking (right) approaches for\nprocessing long documents. In early chunking, the document is divided into chunks\nbefore embedding, with each chunk processed independently by the embedding model\nand then pooled. In contrast, late chunking processes the entire document to generate\ntoken embeddings first, using boundary cues to create chunk embeddings, which are\nsubsequently pooled. \n_Rank Fusion._ In our methodology, we employ a rank fusion strategy that integrates dense embeddings with sparse embeddings of BM25 [19] to improve\nretrieval performance. Although embedding models adeptly capture semantic\nrelationships, they may overlook exact matches, which is particularly useful for\nunique identifiers or technical terms. BM25 uses a ranking function that builds\nupon Term Frequency-Inverse Document Frequency (TF-IDF), addressing this\nlimitation by emphasizing precise lexical matches. To combine the strengths of\nboth approaches, we conduct searches across both dense embedding vectors and\nBM25 sparse embedding vectors generated from both the chunk and its generated context. Initially, the assigned relative importance in the search for the two\nvector fields has been set to be of equal intensity, resulting in lowering the scoring\nresults in the retrieval evaluation. For this reason we use a weighting strategy\nassigning higher weights to dense vector fields, emphasizing them more in the\nfinal ranking. While different weight parameters have been tested, the final decision has been to define a ratio of importance 4:1 assigning weight 1 for the dense \n6 J. Singh and C. Merola\n\n# 1 **Introduction** \nIn Retrieval-Augmented Generation (RAG) systems, cutting documents into smaller units called\n\u201cchunks\u201d has a crucial effect on the quality of both\nretrieval and generation tasks (Chen et al., 2023;\nWadhwa et al., 2024; Shi et al., 2023; Yu et al., \n2023). By retrieving the most relevant chunks for\na given query and feeding them into a generative\nlanguage model, these systems aim to produce accurate and contextually appropriate answers. However, the effectiveness of chunking strategies remains a significant challenge in optimizing retrieval\nquality and computational efficiency (Lewis et al.,\n2020; Finardi et al., 2024). \nKnown as **fixed-size chunking**, the traditional\nway to chunk is to cut documents into chunks of a\nfixed length such as 200 tokens (Gao et al., 2023).\nWhile computationally simple, this approach can\nfragment semantically related content across multiple chunks, leading to suboptimal retrieval performance. Recently, there has been a surge of interest \nin **semantic chunking**, where documents are segmented based on semantic similarity, with some industry applications suggesting promising improvements in performance (LangChain, 2024; LlamaIndex, 2024; McCormick, 2024). However, there\nis no systematic evidence that semantic chunking\nyields a performance gain in downstream tasks,\nand if there is, the gain is significant enough to\njustify the computational overhead than fixed-size\nchunking.\nSuch a systematic evaluation is not trivial due to\nthe lack of data that can be directly used to compare chunking strategies. Therefore, we design an\nindirect evaluation using three proxy tasks: (1) document retrieval, measuring the ability to identify\nrelevant documents; (2) evidence retrieval, measuring the ability to locate ground-truth evidence;\nand (3) answer generation, testing the quality of\nanswers produced by a generative model using retrieved chunks. Our findings challenge prevailing\nassumptions about the benefits of semantic chunking, suggesting that its advantages are highly taskdependent and often insufficient to justify the added\ncomputational costs. This study lays the groundwork for future exploration of more efficient and\nadaptive chunking strategies in RAG systems.\nIn general, our contributions are: \n- We present a novel, large-scale evaluation\nframework comparing semantic and fixed-size\nchunking across diverse tasks. \n- We demonstrate that while semantic chunk\ning shows some benefits in certain scenarios,\nthese are inconsistent and often insufficient to\njustify the computational cost." + }, + "generated_answer": "Hello there! Let's break down the trade-offs between computational efficiency and semantic integrity for these different retrieval and chunking strategies, especially in the context of clinical documents.\n\n### Answer:\n\nUnderstanding the balance between computational efficiency (how fast and resource-intensive a method is) and semantic integrity (how well it captures the true meaning and relationships in the text) is crucial for effective information retrieval. Here's a look at the various strategies:\n\n1. **Fixed-size Chunking:**\n * **Computational Efficiency:** This approach is described as \"computationally simple\" and \"more efficient and reliable\" for practical RAG applications. It cuts documents into chunks of a fixed length, which is straightforward to implement.\n * **Semantic Integrity:** Its simplicity comes at a cost. Fixed-size chunking \"can fragment semantically related content across multiple chunks,\" which can lead to \"suboptimal retrieval performance\" because critical information might be split.\n\n2. **Semantic Chunking:**\n * **Computational Efficiency:** While it aims to segment documents based on semantic similarity, the notes state that \"the computational costs associated with semantic chunking are not justified by consistent performance gains.\" It introduces \"additional computational cost\" compared to fixed-size chunking.\n * **Semantic Integrity:** It is designed to create \"semantically coherent segments.\" However, its benefits are \"highly context-dependent\" and do not consistently outweigh the computational overhead. It \"occasionally improved performance, particularly on stitched datasets with high topic diversity,\" but not reliably on real-world documents.\n\n3. **Late Chunking:**\n * **Computational Efficiency & Semantic Integrity:** The notes describe late chunking as processing \"the entire document to generate token embeddings first, using boundary cues to create chunk embeddings.\" This contrasts with \"early chunking,\" where the document is divided *before* embedding. While the notes don't explicitly detail the efficiency vs. semantic integrity trade-offs for late chunking, processing the entire document initially might incur higher upfront computational costs but could potentially lead to a more holistic semantic understanding of the document before chunking. However, this is not directly elaborated upon in the provided text.\n\n4. **Contextual Retrieval (General RAG/Embedding-based Chunk Retrieval):**\n * **Computational Efficiency:** Traditional embedding-based chunk retrieval (like RAG) \"prioritizes efficiency with minimal token usage\" (e.g., an average of 544 tokens per query). Methods that use \"large context windows that process entire documents\" for zero-shot inference, while aiming for full context, \"face computational constraints.\"\n * **Semantic Integrity:** Early RAG methods \"lacked domain-specific entity modeling.\" Traditional approaches \"often fail to capture critical clinical entity relationships and contextual dependencies essential for accurate medical information extraction\" and may \"miss critical clinical relationships.\" Furthermore, expanding context windows for long documents often leads to the \"'lost in the middle' problem,\" where performance drops if relevant information is not at the beginning or end of the input, indicating a failure to effectively use all the provided context semantically.\n\n5. **Clinical Entity Augmented Retrieval (CLEAR):**\n * **Computational Efficiency:** CLEAR introduced \"substantial efficiency gains (71% token reduction, 72% faster inference time)\" compared to traditional RAG. It achieves \"stronger semantic similarity than wide-context processing at markedly lower token budgets,\" demonstrating an optimal balance.\n * **Semantic Integrity:** CLEAR is a \"novel approach that addresses these limitations through entity-aware, entity-centered retrieval strategies,\" meaning it's \"aligned with clinical semantics.\" It showed \"significant performance improvements (F1 score of 0.90 vs 0.86 for traditional RAG)\" by focusing on clinically relevant entities, thereby improving semantic accuracy.\n\n### Superior Scalability in High-Complexity Clinical Documents:\n\nThe **Clinical Entity Augmented Retrieval (CLEAR) methodology** demonstrated superior scalability advantages in high-complexity clinical documents.\n\nThe notes state: \"The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware retrieval becomes increasingly advantageous as document complexity increases, confirming the methodology\u2019s suitability for enterprise healthcare environments.\" This highlights CLEAR's ability to maintain performance and efficiency even with very long and complex clinical narratives, which is a significant advantage in real-world EHR settings.\n\n---\n\n### Key Points/Summary:\n\n* **Fixed-size chunking** is efficient but can fragment semantic content.\n* **Semantic chunking** aims for semantic coherence but often has high computational costs not consistently justified by performance gains.\n* **General RAG/Contextual retrieval** can be efficient in token usage but often misses nuanced semantic relationships and suffers from the \"lost in the middle\" problem in long contexts.\n* **CLEAR** achieves an optimal balance, providing both significant efficiency gains (reduced tokens, faster inference) and improved semantic accuracy through entity-aware retrieval.\n* **CLEAR** demonstrates superior scalability, performing robustly and advantageously on highly complex and long clinical documents (over 65,000 tokens).\n\n### Extra Tip:\n\nThink of fixed-size chunking like cutting a book into pages of equal thickness without regard for chapters or paragraphs. It's easy to do, but you might cut a sentence in half! Semantic chunking tries to cut at natural breaks (like chapter endings), which sounds better, but sometimes finding those breaks takes a lot of effort without a clear benefit. CLEAR, on the other hand, is like having an expert reader who knows exactly which *key concepts* in the book are important for your question and only retrieves those specific, relevant sections, saving you time and ensuring you get the most accurate information.", + "correctness_score": 0.85, + "faithfulness_score": 1.0, + "model_tested": "Layout" + } +] \ No newline at end of file diff --git a/evaluation/results/evaluation_results_Query Transformation.json b/evaluation/results/evaluation_results_Query Transformation.json new file mode 100644 index 0000000..88bf862 --- /dev/null +++ b/evaluation/results/evaluation_results_Query Transformation.json @@ -0,0 +1,244 @@ +[ + [ + [ + 0.95, + 1.0 + ], + [ + 0.95, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.95, + 1.0 + ], + [ + 0.15, + 1.0 + ], + [ + 0.95, + 1.0 + ] + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "retrieved_context": [ + "cluster loosens, increasing av- erage chunk size. As seen in Figure 3, this leads to a decrease in precision and an increase in recall for document and evidence retrieval, respectively, similar to the single-linkage case. Breakpoint-based Semantic Chunker As the dis- tance threshold between consecutive sentences in- creases, fewer breakpoints appear, resulting in larger chunks. Regardless of the threshold type, it ultimately determines chunk size. In Figure 4, we observe similar trends to Figure 2 and 3:", + "Is Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where", + "need for more efficient chunking strategies in RAG systems. 1 Introduction In Retrieval-Augmented Generation (RAG) sys- tems, cutting documents into smaller units called chunks has a crucial effect on the quality of both retrieval and generation tasks (Chen et al., 2023; Wadhwa et al., 2024; Shi et al., 2023; Yu et al., 2023). By retrieving the most relevant chunks for a given query and feeding them into a generative language model, these systems aim to produce ac- curate and contextually appropriate", + "However, these benefits were highly context-dependent and did not consistently justify the additional computational cost. On non- synthetic datasets that better reflect real-world doc- uments, fixed-size chunking often performed better. Overall, our results suggest that fixed-size chunk- ing remains a more efficient and reliable choice for practical RAG applications. The impact of chunk- ing strategy was often overshadowed by other fac- tors, such as the quality of embeddings, especially when computational", + "answers. How- ever, the effectiveness of chunking strategies re- mains a significant challenge in optimizing retrieval quality and computational efficiency (Lewis et al., 2020; Finardi et al., 2024). Known as fixed-size chunking, the traditional way to chunk is to cut documents into chunks of a fixed length such as 200 tokens (Gao et al., 2023). While computationally simple, this approach can fragment semantically related content across multi- ple chunks, leading to suboptimal retrieval perfor- mance.", + "documents are split into consecutive, fixed-size segments, re- main unclear. This study systematically evalu- ates the effectiveness of semantic chunking us- ing three common retrieval-related tasks: docu- ment retrieval, evidence retrieval, and retrieval- based answer generation. The results show that the computational costs associated with seman- tic chunking are not justified by consistent per- formance gains. These findings challenge the previous assumptions about semantic chunk- ing and highlight the", + "chunkers. Ini- tially, we experimented with datasets from BEIR (Thakur et al., 2021), but the short length of these documents showed no performance improvement with chunking. Short documents lack the complex- ity required to assess how chunkers manage context and semantic coherence across longer spans of text. To overcome this limitation, we created long documents by stitching shorter documents from ex- isting datasets. Each stitched document contains approximately 100 sentences, better reflecting real-", + "scenarios, these are inconsistent and often insufficient to justify the computational cost. 2 Chunking Strategies In this paper, a document is first split into sentences which are then grouped into chunks. We evaluate three chunking strategies, hereafter referred to as arXiv:2410.13070v1 [cs.CL] 16 Oct 2024", + "conditions. How- ever, simply splitting the document into structured sections before applying fixed-size chunking will solve this issue. In contrast, both semantic chunkers performed better on stitched documents, but still had issues. The Clustering-based Chunker made one error by grouping Sentence 16 (the last sentence of Doc- ument 4) into Chunk 2. This happened because, despite the large positional distance, the seman- tic similarity was high, causing the sentence to be incorrectly included. Without", + "minimal performance differ- ences. This suggests that adding semantic informa- tion did not significantly enhance performance, as the benefits of semantic grouping were often redun- dant when core evidence was already captured by sentence positions. These findings indicate that the performance of the chunkers largely depends on how effectively the embedding models capture the semantic richness of individual sentences, rather than the chunking strategy itself. Dataset Fixed-size Breakpoint Clustering", + "relying solely on semantic similarity. Sentences 8-9, though belong- ing to Chunk 3, were grouped into Chunk 2 due to high semantic similarity. This showed that even with added positional information, semantic-based chunking could misgroup content that shared con- text, as these sentences were clearly about the sales of Interact Home Computer. For the Breakpoint-based Chunker, errors seen in stitched documents were even more pronounced. Despite using the optimal configuration for each chunker (minimizing", + "Clustering-based Semantic Chunker groups semantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. chunkers. Fixed-size Chunker This is our baseline chunker that splits a document sequentially into fixed-size chunks, based on a predefined or user-specified number of sentences per chunk. Although this approach is simple and compu- tationally efficient, it may separate contextually related sentences, leading to potential degradation in retrieval quality (Lewis et", + "topic diversity. This diminished the advantage of Breakpoint-based Semantic Chunker, while Clustering-based Seman- tic Chunker improved. The gap between semantic and fixed-size chunkers narrowed, with Fixed-size Chunker benefiting from higher topic integrity. These results suggest that in real life, the topics in a document may not be as diverse as in our artificially noisy, stitched data, and hence semantic chunkers may not have an edge over fixed-size chunker there. 4.3 Evidence Retrieval As shown in", + "Recently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic", + "benefits of semantic chunk- ing, suggesting that its advantages are highly task- dependent and often insufficient to justify the added computational costs. This study lays the ground- work for future exploration of more efficient and adaptive chunking strategies in RAG systems. In general, our contributions are: We present a novel, large-scale evaluation framework comparing semantic and fixed-size chunking across diverse tasks. We demonstrate that while semantic chunk- ing shows some benefits in certain", + "embeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely", + "clusions. Dataset Fixed-size Breakpoint Clustering ExpertQA 0.65 0.65 0.65 DelucionQA 0.76 0.76 0.76 TechQA 0.68 0.68 0.68 ConditionalQA 0.42 0.43 0.43 Qasper 0.49 0.49 0.50 Table 3: BERTScore for Answer Generation. 5 Conclusion In this paper, we evaluated semantic and fixed-size chunking strategies in RAG systems across docu- ment retrieval, evidence retrieval, and answer gen- eration. Semantic chunking occasionally improved performance, particularly on stitched datasets with high topic diversity." + ], + "generated_answer": "Semantic chunking in Retrieval-Augmented Generation (RAG) systems is characterized by its approach to dividing documents into segments that are semantically coherent. This means that instead of simply cutting documents into fixed-size pieces, semantic chunking aims to group sentences or text based on their meaning or topic similarity.\n\nFor example, a \"Clustering-based Semantic Chunker\" specifically groups semantically similar sentences, even potentially combining non-consecutive text, to form chunks that are focused on a particular topic. The goal is to improve retrieval performance by ensuring that each chunk contains content that is related in meaning.\n\n**Key Points/Summary:**\n* **Semantic Coherence:** Documents are divided into segments that are meaningful and related in content.\n* **Semantic Similarity:** Segmentation is based on the semantic similarity of sentences or text.\n* **Topic-Based Grouping:** It aims to form chunks that are focused on specific topics.\n* **Improved Retrieval:** The goal is to enhance the quality of information retrieval in RAG systems.\n\n**Extra Tip:**\nThink of semantic chunking like organizing a book by chapters based on themes, rather than just cutting it into equal page counts. Each \"chapter\" (chunk) would then contain a complete thought or topic, making it easier to find relevant information quickly.", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "retrieved_context": [ + "similarity values are treated as 0, as they do not aid in retrieval or generation. Without losing generality, we employed single- linkage agglomerative clustering and DBSCAN (Ester et al., 1996) as representatives of clustering algorithms. Further details on these methods an their adjustments during experimentation are pro- vided in Appendix A. 3 Experiments In the absence of ground-truth chunk data, we de- signed three experiments to indirectly assess the quality of each chunker: document retrieval, evi-", + "Table 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the", + "evaluation is not trivial due to the lack of data that can be directly used to com- pare chunking strategies. Therefore, we design an indirect evaluation using three proxy tasks: (1) doc- ument retrieval, measuring the ability to identify relevant documents; (2) evidence retrieval, mea- suring the ability to locate ground-truth evidence; and (3) answer generation, testing the quality of answers produced by a generative model using re- trieved chunks. Our findings challenge prevailing assumptions about the", + "to identify chunker configurations that perform well across various datasets and k values, making it log- ical to average the results. The title of each plot row indicates the chunker and experiment being an- alyzed, while each subplot title specifies the fixed hyperparameter. The y-axis shows the metric score, and the x-axis represents the hyperparameter being analyzed. Blue lines denote recall, orange lines represent precision, and green lines indicate the F1 score. Clustering-based Semantic Chunker", + "scenarios, these are inconsistent and often insufficient to justify the computational cost. 2 Chunking Strategies In this paper, a document is first split into sentences which are then grouped into chunks. We evaluate three chunking strategies, hereafter referred to as arXiv:2410.13070v1 [cs.CL] 16 Oct 2024", + "minimal performance differ- ences. This suggests that adding semantic informa- tion did not significantly enhance performance, as the benefits of semantic grouping were often redun- dant when core evidence was already captured by sentence positions. These findings indicate that the performance of the chunkers largely depends on how effectively the embedding models capture the semantic richness of individual sentences, rather than the chunking strategy itself. Dataset Fixed-size Breakpoint Clustering", + "10]. Each retrieved chunk was mapped to its source document, and the retrieved documents were evaluated by comparing them to a set of relevant documents for each query. 3.2 Evidence Retrieval Here we evaluate chunkers at a finer granularity than the previous experiment by measuring their abilities to locate evidence sentences. We selected additional datasets from RAGBench (Friel et al., 2024), shown in Tables 2 and 5, because few datasets contain long documents with ground-truth evidence sentences. We", + "these biases, providing a better indicator of whether the chunker produces appropriately sized chunks that capture relevant ev- idence. Therefore, we focus on F1 scores in our analysis. Recall@k: Fraction of retrieved evidence sen- tences over all evidence sentences. Precision@k: Fraction of retrieved evidence sentences over all retrieved sentences. F1@k: The harmonic mean of precision and recall. Answer Generation Generated answers were as- sessed using BERTScore for semantic similarity between", + "experiment assessed the effectiveness of chun- kers in retrieving relevant documents for a given query. We used 10 datasets, shown in Tables 1 and 4. Most documents on the BEIR benchmark (Thakur et al., 2021) are too short for chunking to be effective. To address this, we synthesized longer documents by stitching short documents from six datasets where documents are too short (see Ap- pendix C for details). We randomly sampled 100 queries from each dataset and retrieved the top k chunks, where k [1, 3, 5,", + "Additionally, the answer sets in RAGBench (Friel et al., 2024) were generated by LLMs, which may not accu- rately assess chunk quality. A dataset containing all these elements is needed for a more thorough evaluation of chunking strategies. References Valeriia Bolotova-Baranova, Vladislav Blinov, Sofya Filippova, Falk Scholer, and Mark Sanderson. 2023. Wikihowqa: A comprehensive benchmark for multi- document non-factoid question answering. In Pro- ceedings of the 61st Annual Meeting of the Associa- tion", + "Figure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within the sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk breakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering semantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance thresholds between consecutive sentences to maintain coherence. (c)", + "thresholds are generally higher than those from percentiles or interquartile ranges. Fixed-size Chunker Figure 5 shows results for the Fixed-size Chunker. The trends mirror those seen in other chunkers. Adding one overlapping sentence between chunks does not notably improve performance, indicating that a single overlapping sentence is insufficient to significantly boost con- textual coherence. E.4 Chunk Inspection We examined the output chunks to (1) confirm that different chunkers were functioning as", + "embeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely", + "answers. 4 Results 4.1 Measuring and reporting performances As mentioned earlier, we used three proxy tasks the study chunking. We cannot directly assess the quality of retrieval at the chunk level due to the lack of ground-truth at the chunk level. Instead, each retrieved chunk is mapped back to either the source document or the included evidence sentences. Since the number of relevant documents or evi- dence sentences is not fixed (unlike the k value for retrieved chunks), traditional metrics such as Re-" + ], + "generated_answer": "The study designed three primary proxy tasks to indirectly evaluate the quality of different chunking strategies. These tasks were necessary because there was a lack of direct ground-truth data for comparing chunking methods.\n\nThe three proxy tasks are:\n1. **Document Retrieval:** This task measures how well a chunking strategy helps in identifying relevant documents for a given query.\n2. **Evidence Retrieval:** This task assesses the chunker's ability to locate specific ground-truth evidence sentences within documents.\n3. **Answer Generation:** This task evaluates the quality of answers produced by a generative model when it uses the chunks retrieved by a specific chunking strategy.\n\n### Key Points/Summary:\n* **Document Retrieval:** Evaluates the identification of relevant documents.\n* **Evidence Retrieval:** Assesses the location of ground-truth evidence sentences.\n* **Answer Generation:** Tests the quality of answers generated using retrieved chunks.\n\n### Extra Tip:\nThink of these proxy tasks as different levels of testing: document retrieval is broad (finding the right book), evidence retrieval is more specific (finding the right paragraph in the book), and answer generation is the ultimate application (using that information to answer a question).", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "retrieved_context": [ + "information split across chunks may not be effectively retrieved or integrated. To address these issues, we analyse and compare two recent techniques contextual retrieval1 and late chunking within a unified setup, evaluating their strengths and limitations in tackling challenges like context loss and incom- plete information retrieval. Contextual retrieval preserves coherence by prepend- ing LLM-generated context to chunks, while late chunking embeds entire docu- ments to retain global context before", + "advancements in scalability and embedding techniques, further establishing RAG as a foundational framework for knowledge-intensive applications. Document Segmentation.Document segmentation is essential for processing long texts in RAG workflows, with methods ranging fromfixed-size segmentation to more adaptive techniques likesemantic segmentation,3 which detect semantic breakpoints based on shifts in meaning. Recent advancements includesupervised segmentation models[14,12]and segment-then-predict", + "to preserve global context. Despite their potential, their comparative strengths and limitations remain unclear. This study presents a rigorous analysis of late chunking and contextual retrieval, evaluating their effectiveness and efficiency in optimizing RAG systems. Our results indicate that contextual retrieval preserves semantic coher- ence more effectively but requires greater computational resources. In contrast, late chunking offers higher efficiency but tends to sacrifice rel- evance and", + "RAG. A standard RAG workflow involves four main stages: document segmentation, chunk embedding, indexing, and retrieval. During segmentation, documents are divided into manageable chunks. These chunks are then trans- formed into vector representations using encoder models, often normalized to 1 2", + "models,weaimtoimprovetheoverallretrievalefficiencyandresponsegeneration within the RAG framework. Early Chunking. Documents are segmented into text chunks, and each chunk is processed by the embedding model. The model generates token-level embed- dings for each chunk, which are subsequently aggregated using mean pooling to produce a single embedding per chunk. Late Chunking. Late chunking defers the chunking process. As shown in Figure 3.1, instead of segmenting the document initially, the entire document", + "Contextual Chunking? In this workflow, traditional retrieval is compared to Contextual Retrieval with Rank Fusion technique. This has been introduced by Anthropic in September 2024.4 Three steps are added to the Traditional RAG process: Contextualization, Rank Fusion, Reraking. Contextualization. After document segmentation, each chunk is enriched with additional context from the entire document, ensuring that even when seg- mented, each piece retains a broader understanding of the content (Fig. 3.2). In", + "4 J. Singh and C. Merola to align with the early and late chunking strategies under evaluation. This ad- justment allows us to explore how different embedding techniques influence the retrieval quality and, subsequently, the overall performance of the RAG system. Additionally, we test dynamic segmenting models to further refine the chunk- ing process, providing an adaptive mechanism that adjusts chunk sizes based on content characteristics. By evaluating the impact of these dynamic segmenting", + "RAG:Managing extensive external documentsposessignificantissuesinRAGsystems.Despiteadvancements,many LLMs are limited to processing a few thousand tokens. Although some models have achieved context windows up to millions of tokens , these are exceptions rather than the norm. Moreover, research indicates that LLMs may exhibit posi- tional bias, performing better with information at the beginning of a document and struggling with content located in the middle or toward the end [11,16]. This issue is", + "completeness. Keywords: Contextual Retrieval Late Chunking Dynamic Chunking Rank Fusion. 1 Introduction Retrieval Augmented Generation (RAG) is a transformative approach that en- hances the capabilities of large language models (LLMs) by integrating external information retrieval directly into the text generation process. This method al- lows LLMs to dynamically access and utilize relevant external knowledge, sig- nificantly improving their ability to generate accurate, contextually grounded, and", + "question persists: how can vast volumes of external knowledge be man- aged effectively within the input constraints of LLMs? Traditional meth- ods address this by chunking external documents into smaller, fixed- size segments. While this approach alleviates input limitations, it often fragments context, resulting in incomplete retrieval and diminished co- herence in generation. To overcome these shortcomings, two advanced techniqueslate chunking and contextual retrievalhave been intro- duced, both aiming", + "retrieval, offering actionable insights into their relative strengths and trade-offs. 3 Methodology To guide our study, we define the following research questions (RQs), aimed at evaluating different strategies for chunking and retrieval in RAG systems: RQ#1: Compares the effectiveness ofearly versus late chunkingstrate- gies, utilizingdifferent text segmentersand embedding models to evalu- ate their impact on retrieval accuracy and downstream performance in RAG systems. RQ#2: Compares the effectiveness", + "models,trainedend-to-end without explicit labels to optimize chunking for downstream task performance . In 2024,late chunkingand contextual retrievalintroduced novel paradigms. Both techniques have proven effective in retrieval benchmarks but remain largely untested in integrated RAG workflows. Despite several RAG surveys [7,6,8], no prior work has compared these methods within a comprehensive evaluation framework. This study addresses this gap by holistically analyzing late chunking and contextual", + "ofcontextual retrieval versus tra- ditional early chunkingstrategies, utilizingdifferent text segmenters and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG systems. 3.1 RQ#1: Early or Late Chunking? In this workflow, the main architectural modification compared to the standard RAG lies in the document embedding process Figure 3.1. Specifically, we experi- ment with various embedding models to encode document chunks, tailoring them 3 chunking/", + "Reconstructing Context Evaluating Advanced Chunking Strategies for Retrieval-Augmented Generation Carlo Merola[0009-0000-1088-1495] and Jaspinder Singh[0009-0000-5147-1249] Department of Computer Science and Engineering, University of Bologna carlo.merola@studio.unibo.it, jaspinder.singh@studio.unibo.it Abstract. Retrieval-augmented generation (RAG) has become a trans- formative approach for enhancing large language models (LLMs) by grounding their outputs in external knowledge sources. Yet, a critical" + ], + "generated_answer": "Hello there! I can certainly help you with that.\n\nBased on your notes, two advanced chunking techniques, besides traditional early chunking, that are aimed at preserving global context within Retrieval-Augmented Generation (RAG) systems are **Contextual Retrieval** and **Late Chunking**.\n\nThese techniques were introduced to address issues like context fragmentation, incomplete retrieval, and diminished coherence that can arise from traditional methods of chunking documents into smaller, fixed-size segments.\n\n### How they work:\n\n1. **Contextual Retrieval**: This technique preserves semantic coherence by enriching each document chunk with additional context from the entire document. This means that even when a piece of content is segmented, it retains a broader understanding of its original context, often by prepending LLM-generated context to the chunks.\n2. **Late Chunking**: Instead of segmenting the document at the very beginning of the process (as in early chunking), late chunking defers the chunking process. It embeds entire documents to retain their global context before any segmentation occurs. This approach aims to maintain the overall meaning of the document throughout the embedding process.\n\n### Key Points/Summary:\n\n* **Contextual Retrieval**: Enriches individual chunks with broader document context to maintain semantic coherence.\n* **Late Chunking**: Defers the chunking process, embedding entire documents first to preserve global context.\n* Both techniques aim to overcome context fragmentation and improve retrieval in RAG systems.\n\n### Extra Tip:\nThink of it like this: If you're reading a book, traditional \"early chunking\" might be like tearing out individual pages and reading them in isolation. You might miss the plot! \"Contextual retrieval\" is like reading a page but having a little summary of the entire chapter or book at the top of each page. \"Late chunking\" is like reading the whole book first to understand the full story, and *then* going back to focus on specific pages when needed.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "retrieved_context": [ + "poses a limitation of low inference efficiency. As shown in Figure 6, after ICAC is removed, the inference latency is 2.12 times higher than that of ChunkLLM in the 110K120K token generation stage. Conversely, incorporating ICAC enables the model to main- tain near-lossless performance alongside improved inference efficiency, which provides additional validation of ICACs success. Appendix 6.4 shows a case study of the ICAC. 3.3.2 ANALYSIS OFFIXEDCHUNKS ANDSEMANTICCHUNKS We conduct an experimental analysis", + "perplexity (ppl) on the PG19 test set, and results are summarized in Table 2. Compared to the vanilla model, ChunkLLM yields a slight enhancement in ppl alongside a significant decrease in total infer- ence time. The underlying reason is that while the vanilla model maintains semantic integrity, it incurs linearly increasing inference time as gen- erated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up inference by leveraging its chunk selection and ICAC mechanisms. 3.2.4", + "demands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance", + "We set the top-k to 256 and the number of local chunks to 16 for ChunkLLM. As depicted in Figure 4, ChunkLLM outperforms SepLLM across all scenarios in the 64K-context NIAH evalu- ation conducted on Qwen2.5-7B and Llama3.1-8B, achieving superior performance. Notably, in scenarios where the context length exceeds 12K, SepLLM exhibits near-total loss of retrieval capa- bility (visualized in red), whereas ChunkLLM retains performance comparable to the vanilla model. This discrepancy is primarily attributed to", + "exclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison", + "ChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3", + "each layer, thereby deriving the global top-k chunks. These retrieved chunks are subsequently stored in the KV-cache. ICACWe find a phenomenon during the model inference, as illustrated in Figure 3. The chunks at- tended to by tokens within a generated chunk exhibit substantial consistency, whereas chunk updates 4", + "introduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9", + "comparison between our method v.s. VanillaLLM v.s. StreamingLLM Depth of needle Rate of kv usage Figure 8: Compare with StreamingLLM on Needle-in-a-Haystack. The last column represents the KV-cache utilization rate. 6.4 CASESTUDY To illustrate the reasoning process, we randomly sample one example from the LongBench summa- rization task, with its visualization results presented in Figure 9. As observed in the figure, during the generation phase, the chunks attended to by tokens within the same chunk", + "StreamingLLM across all scenarios in the 64K- context NIAH evaluation conducted on Qwen2.5-7B and Llama3.1-8B with lower KV-cache usage, achieving superior performance. SteamingLLM leverages initial tokens and local tokens as its core token selection strategy. However, this design inherently limits its ability to effectively capture critical information situated in the middle segment of the input sequence, thereby resulting in a notable deficiency in long-context retrieval performance. 14", + "inference phase. The sample is de- rived from the passkey retrieval task. The inference phase of ChunkLLM is depicted in Figure 2, encompassing two primary steps: top-k chunk selection and ICAC. In line with the ICAC paradigm, chunk updates are trig- gered exclusively when the current token func- tions as a chunk boundary. Top-k Chunk SelectionThis stage is primar- ily dedicated to selecting top-k chunks for each layer. To elaborate, we use the first layer as an illustrative example and defineeas the end", + "which gives rise to semantic incompleteness; the latter utilizes separators for partitioning, yet ambiguities often emerge. For example, periods frequently occur in numerical values or abbreviations. Furthermore, during the inference phase, these methods necessitate chunk selection for each generated token, incurring additional computational overhead. It is thus evident that existing efficient approaches still exhibit inherent limitations. To address the aforementioned challenges, We propose ChunkLLM,", + "exploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate", + "predominantly occur at chunk boundaries. We name this phenomenon the Intra-ChunkAttention Consistency (ICAC). ICAC makes it possible to save computational cost in chunk selection. We incorporate the chunk boundary prediction task into the inference phase. Only when the currently decoded token is a chunk boundary, do we update the chunk selection and integrate the complete chunk from the prediction phase intoKandV; otherwise, no update is executed. 3 EXPERIMENTS ANDRESULTS 3.1 EXPERIMENTALSETTINGS 3.1.1", + "4.48, which corroborates the efficacy of the proposed ICAC mechanism. During ChunkLLMs inference phase, chunk updates occur exclusively at chunk boundaries, minimizing the updates frequency and thereby boosting inference efficiency. We also conduct supplementary experiments using the FineWeb-Edu dataset, from which 1000 test corpora of 4k length are sampled. For the task of chunk boundary prediction, we evaluate its performance using three key metrics: precision, recall, and F1-score. The calculated", + "an 8.23 reduction (87.78 v.s. 79.55) in accuracy relative to ChunkLLM on the 64K NIAH task. This discrepancy stems from the semantic incompleteness of fixed chunks, which in turn compromises chunk selection during the inference phase. In contrast, ChunkLLM leverages contextual semantic information to identify chunk boundaries, preserving the semantic integrity of chunks. 4 RELATEDWORK KV Cache CompressionRecent research has primarily focused on overcoming the limitations of Large Language Models (LLMs) in", + "v.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high" + ], + "generated_answer": "The key phenomenon described in the context is called **Intra-Chunk Attention Consistency (ICAC)**.\n\nThis phenomenon refers to the observation that the chunks attended to by tokens within a generated chunk exhibit substantial consistency. ChunkLLM exploits this pattern to significantly enhance inference efficiency. Instead of updating chunk selection for every generated token, ChunkLLM only triggers chunk updates when the current token is identified as a chunk boundary.\n\n**Key Points/Summary:**\n* The phenomenon is named Intra-Chunk Attention Consistency (ICAC).\n* It describes that attention to chunks remains consistent within a generated chunk.\n* ChunkLLM leverages ICAC to reduce the frequency of chunk selection updates.\n* This leads to substantial improvements in inference efficiency, especially in long-context scenarios.\n\n**Extra Tip:**\nImagine you're reading a book. Instead of constantly deciding which chapter or section to focus on with every single word you read, you mostly stick to the current paragraph or section until you reach a natural break (like the end of a paragraph or a new heading). ICAC is similar; it recognizes that attention within a 'chunk' of text is stable, so ChunkLLM only re-evaluates its focus at these 'chunk boundaries,' saving computational effort.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "retrieved_context": [ + "complexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual", + "information extraction systems requiring optimal balance be- tween semantic accuracy and computational efficiency. The demonstrated effectiveness of CLEAR through systematic platform-based validation provides evidence-based guidance for healthcare organizations implementing clinical question-answering sys- tems. The evaluation platform framework enables continued research and development in clinical entity-aware retrieval methodologies while supporting reproducible evaluation of future clinical NLP", + "for semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes", + "statistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for", + "wide-context processing at markedly lower token budgets, echoing the efficiencyquality trade-offs highlighted by CLEAR . The validation results strongly confirm the original research findings, particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe- ference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware retrieval becomes increasingly advantageous as document complexity increases, confirming the", + "data processing. 4.4 Limitations and Future Work Several limitations should be acknowledged. The current implementation relies on keyword-based en- tity extraction, which, while effective, could benefit from advanced neural entity recognition models. Additionally, evaluation was limited to English clinical notes from specific domains, and the system lacks integration with standardized medical ontologies. Future research should focus on incorporating advanced clinical NER models, integrating standard- ized", + "fundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying", + "evaluation framework is designed for secure, offline analysis and can be integrated with institutional data governance processes to ensure regulatory compliance in healthcare NLP research. 5. CONCLUSION This study developed and deployed a comprehensive Clinical Notes Q&A Evaluation Platform and found results consistent with the benefits reported for Clinical Entity Augmented Retrieval (CLEAR) in prior work. In our synthetic EHR QA setting, entity-aware retrieval achieved stronger semantic similarity than", + "long inputs (lost in the middle) . This motivates entity-aware retrieval that selectively centers clinically relevant spans rather than relying on statistically similar but potentially off-target chunks. The CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in 2025 , introduced a novel approach that addresses these limitations through entity-aware, entity- centered retrieval strategies. The original study demonstrated significant performance improvements (F1 score of 0.90", + "context selection, and token optimization. 2.2.1 Enhanced Entity Extraction We developed a comprehensive medical entity recognition system using advanced keyword patterns, clinical value recognition for vital signs and laboratory values, and medical domain-specific entity types with confidence-based scoring. The system recognizes six primary entity categories: medications, symp- toms, diseases, procedures, laboratory values, and anatomical references. 2.2.2 Section-Aware Processing Clinical documents", + "clinical notes question-answering systems and validate CLEAR against established approachesincluding zero-shot large-context processing and traditional chunk-based retrieval-augmented generationin realistic EHR process- ing scenarios. Methods:We implemented a Clinical Notes Q&A Evaluation Platform with three retrieval strategies: (1) Wide Context processing for zero-shot inference with large context windows, (2) tra- ditional vector database chunking with semantic search, and (3) entity-aware CLEAR with", + "1 Beyond Long Context: When Semantics Matter More than Tokens Tarun Kumar Chawdhury and Jon D. Duke Georgia Institute of Technology tarunchawdhury@gatech.edu jon.duke@gatech.edu Keywords:Clinical NLP, Entity-Aware Retrieval, Evaluation Platform Abstract Background:Electronic Health Records (EHR) systems store clinical documentation in FHIR DocumentReference resources as base64-encoded attachments, presenting significant challenges for semantic question-answering applications. Traditional approaches using", + "strategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes." + ], + "generated_answer": "The methodology in the clinical domain that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes is called **Clinical Entity Augmented Retrieval (CLEAR)**.\n\nThis approach was introduced by Lopez et al. in 2025 and is designed to address the limitations of earlier retrieval methods. CLEAR utilizes entity-aware and entity-centered retrieval strategies, which have demonstrated significant performance improvements, including higher F1 scores and greater efficiency (e.g., >70% fewer tokens and faster inference compared to embedding RAG). Its effectiveness has been validated in systematic platform-based evaluations, particularly in EHR settings where semantic accuracy and computational efficiency are crucial.\n\n**Key Points/Summary:**\n* **Name:** Clinical Entity Augmented Retrieval (CLEAR).\n* **Strategy:** Employs entity-aware and entity-centered retrieval.\n* **Benefits:** Achieves improved semantic accuracy and computational efficiency.\n* **Application:** Specifically designed for processing Electronic Health Record (EHR) notes in clinical question-answering systems.\n\n**Extra Tip:**\nThink of CLEAR as a smart librarian for clinical notes. Instead of just looking for keywords in general sections (like traditional chunking), CLEAR understands that certain \"entities\" (like diseases, medications, symptoms) are critical. It focuses on retrieving information centered around these specific entities, ensuring that the most relevant clinical context is captured efficiently, even in very long and complex documents.", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "retrieved_context": [ + "demands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance", + "v.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high", + "experiment part. LCBP =- 1 n nX i=1 [yi log(yi) + (1-y i)log(1-y i)](2) 2.2 QK ADAPTER At each layer of the LLM, we incorporate a Q-Adapter and a K-Adapter which used to compress the attention and select the chunks. For each layer, letQandKbe the attention matrix respectively,cbe the chunk number of the input, Index c={i 1, i2, ...ic}is the index set of chunk boundary tokens. Let Kbe the K matrix of these tokens. We then calculate chunk attention scores as follows: As =Softmax( Mul( Q, K T )dk ) Q=FF N", + "costs of long-context models, making it difficult for large language models to effi- ciently scale to context-processing tasks with million-level token sizes.Chunk Selective attention, a special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk (Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen et al., 2024). Both approaches partition the input into discrete chunks: the former conducts parti- tioning with a fixed length,", + "ChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3", + "processing massive contextual inputs. SnapKV (Li et al., 2024) improves efficiency through KV cache compression, using attention scores to select and cluster important positional information; H2O (Zhang et al., 2023) implements a dynamic token retention policy that balances recent information and historically important information to optimize memory occupancy; StreamingLLM (Xiao et al., 2024) enables LLMs to handle sequences of infinite length without fine-tuning by retaining attention sinks and local", + "and chunk attention acquisition. The latter operates at the bottommost layer of the model, functioning to detect chunk boundaries by lever- aging contextual semantic information. During the training phase, the parameters of the backbone remain frozen, with only the QK Adapter and Chunk Adapter undergoing training. Notably, we design an attention distillation method for train- ing the QK Adapter, which enhances the recall rate of key chunks. During the inference phase, chunk selection is triggered", + "comparison between our method v.s. VanillaLLM v.s. StreamingLLM Depth of needle Rate of kv usage Figure 8: Compare with StreamingLLM on Needle-in-a-Haystack. The last column represents the KV-cache utilization rate. 6.4 CASESTUDY To illustrate the reasoning process, we randomly sample one example from the LongBench summa- rization task, with its visualization results presented in Figure 9. As observed in the figure, during the generation phase, the chunks attended to by tokens within the same chunk", + "exploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate", + "generation of chunk attention scores. To train the QK Adapter, we propose an attention distillation approach designed to enhance the recall rate of key chunks. During training, LLM parameters are kept frozen, with the KullbackLeibler (KL) divergence between chunk attention scores and full attention scores serving as a guidance signal for optimization. The Chunk Adapter determines whether a token corresponds to a chunk boundary by leveraging contextual semantic information. During the inference phase, we", + "introduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9", + "We set the top-k to 256 and the number of local chunks to 16 for ChunkLLM. As depicted in Figure 4, ChunkLLM outperforms SepLLM across all scenarios in the 64K-context NIAH evalu- ation conducted on Qwen2.5-7B and Llama3.1-8B, achieving superior performance. Notably, in scenarios where the context length exceeds 12K, SepLLM exhibits near-total loss of retrieval capa- bility (visualized in red), whereas ChunkLLM retains performance comparable to the vanilla model. This discrepancy is primarily attributed to", + "exclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison", + "#N-1 #N ChunkAdapter Input Student Attn Student Attn Chunk ids Vote [0,1,3,4,7] [0,2,3,5,7] [0,1,3,5,7] [0,1,3,5,7] Chunk ids Chunk ids Update [0,1,3,5,7]Chunk ids Yes Figure 2: The inference process of ChunkLLM. We employ the Kullback-Leibler (KL) diver- gence as the loss function for attention dis- tillation to guide the student modelA s in approximating the teacher models attention scoresA t : LN AD =KL(A t||As)(5) We average the KL divergence losses across the N layers to obtain the final attention", + "2023; Peng et al., 2023a; Sanh et al., 2019). 5 CONCLUSION We introduce ChunkLLM, a lightweight and pluggable framework, only necessitates fine-tuning lightweight components, QK Adapter and Chunk Adapter, on the basis of the original model archi- tecture. And then we propose an attention distillation-based training approach for the QK Adapter, which leverages KL divergence to drive chunk attention toward approximating full attention, ef- fectively enhancing the recall rate of key chunks. Furthermore, we" + ], + "generated_answer": "It looks like the provided notes contain detailed information about ChunkLLM, but they do not mention \"Dynamic Hierarchical Sparse Attention\" (DHSA). Therefore, I cannot directly contrast ChunkLLM with DHSA based on the given context.\n\nHowever, I can explain ChunkLLM's dynamic attention management mechanisms, how it derives chunk representations, and how it achieves efficiency gains while preserving performance in long-context models, as described in your notes.\n\n---\n\n### ChunkLLM's Dynamic Attention Management Mechanisms\n\nChunkLLM is a lightweight and pluggable framework designed to enhance the long-context understanding capabilities of large language models (LLMs) by efficiently managing attention. It focuses on improving the recall rate of critical chunks and reducing KV-Cache utilization.\n\nHere's how ChunkLLM handles dynamic attention and chunk representations:\n\n* **Chunk Representation Derivation:**\n * ChunkLLM enriches the representational capacity of chunks through an **attention distillation mechanism**. This mechanism is crucial for strengthening the feature representational capacity of individual chunks.\n * It incorporates a **QK Adapter** at each layer of the LLM, which is used to compress attention and select chunks.\n * A **Chunk Adapter** operates at the bottommost layer to detect chunk boundaries by leveraging contextual semantic information.\n * During training, an attention distillation method is used for the QK Adapter. This involves using the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores as a guidance signal. This approach drives chunk attention to approximate full attention, thereby enhancing the recall rate of key chunks.\n\n* **Utilization for Efficiency and Performance:**\n * **Selective Chunk Attention:** ChunkLLM employs a form of sparse attention where the input is partitioned into discrete chunks.\n * **Intra-Chunk Attention Consistency (ICAC):** ChunkLLM exploits an Intra-Chunk Attention Consistency pattern. This means chunk selection is only updated when the current token is identified as a chunk boundary. This significantly boosts inference efficiency.\n * **Improved Key Chunk Retrieval:** The enriched representational capacity and attention distillation allow ChunkLLM to effectively identify critical chunks with higher query relevance during chunk selection, leading to improved inference performance, especially in tasks like MDQA where critical information is dispersed.\n * **Reduced KV-Cache Utilization:** ChunkLLM exhibits a reduced KV-Cache utilization rate compared to methods like SepLLM, which further corroborates its effectiveness in key chunk retrieval and overall efficiency. It maintains a 48.58% key-value cache retention rate.\n * **Speedup and Performance Preservation:** ChunkLLM achieves a maximum speedup of 4.48 in processing 120K long texts compared to a vanilla Transformer. Despite being trained solely on 4K context lengths, it can achieve inference performance comparable to models optimized for 120K context lengths, significantly reducing training overhead. It retains 98.64% of the vanilla model's performance on long-context benchmarks.\n\n---\n\n**Key Points/Summary:**\n\n* ChunkLLM uses an **attention distillation mechanism** and **QK/Chunk Adapters** to enrich chunk representations and detect boundaries.\n* It improves **key chunk recall** by training the QK Adapter with KL divergence, guiding chunk attention to approximate full attention.\n* **Intra-Chunk Attention Consistency (ICAC)** enhances inference efficiency by updating chunk selection only at boundaries.\n* ChunkLLM achieves significant **speedup** and **reduced KV-Cache utilization** while preserving nearly 98.64% of the vanilla model's performance on long-context tasks.\n\n**Extra Tip:**\nThink of ChunkLLM's attention distillation like a student (chunk attention) learning from a master (full attention). By carefully mimicking the master's focus (through KL divergence), the student becomes very good at identifying the most important parts (key chunks) without needing to pay attention to everything all the time, making it much more efficient!", + "correctness_score": 0.15, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + }, + { + "question_id": 7, + "document_path": "static/Beyond Long Context.pdf", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "retrieved_context": [ + "complexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual", + "for semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes", + "long inputs (lost in the middle) . This motivates entity-aware retrieval that selectively centers clinically relevant spans rather than relying on statistically similar but potentially off-target chunks. The CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in 2025 , introduced a novel approach that addresses these limitations through entity-aware, entity- centered retrieval strategies. The original study demonstrated significant performance improvements (F1 score of 0.90", + "Contextual Chunking? In this workflow, traditional retrieval is compared to Contextual Retrieval with Rank Fusion technique. This has been introduced by Anthropic in September 2024.4 Three steps are added to the Traditional RAG process: Contextualization, Rank Fusion, Reraking. Contextualization. After document segmentation, each chunk is enriched with additional context from the entire document, ensuring that even when seg- mented, each piece retains a broader understanding of the content (Fig. 3.2). In", + "8 Figure 2: CLEAR dominates at a 3% efficiency bonus, maintaining superior adjusted accuracy across all notes. 4.2 Enhanced Implementation Benefits Our enhancements to the original CLEAR methodology provided measurable improvements. Section- aware processing contributed to better clinical reasoning preservation, while enhanced entity extraction improved medical concept recognition. The integration of medical domain knowledge through special- ized entity scoring and question-entity alignment resulted in more", + "scenarios, these are inconsistent and often insufficient to justify the computational cost. 2 Chunking Strategies In this paper, a document is first split into sentences which are then grouped into chunks. We evaluate three chunking strategies, hereafter referred to as arXiv:2410.13070v1 [cs.CL] 16 Oct 2024", + "topic diversity. This diminished the advantage of Breakpoint-based Semantic Chunker, while Clustering-based Seman- tic Chunker improved. The gap between semantic and fixed-size chunkers narrowed, with Fixed-size Chunker benefiting from higher topic integrity. These results suggest that in real life, the topics in a document may not be as diverse as in our artificially noisy, stitched data, and hence semantic chunkers may not have an edge over fixed-size chunker there. 4.3 Evidence Retrieval As shown in", + "maintained typical clinical section headings (e.g., HISTORY OF PRESENT ILLNESS, ASSESSMENT, PLAN) and included subtle contextual variations to challenge retrieval consistency. All three retrieval strategiesWide Context, RAG, and CLEARwere evaluated using these same ques- tions and gold-standard responses to ensure controlled, comparable measurement of semantic accuracy and token efficiency. 3. RESULTS 3.1 Overall Performance Comparison Table 1 presents the overall performance comparison across all three", + "ap- proach: semantic chunking and advanced filtering to refine retrieval results. Semantic Chunking Semantic chunking serves as the foundational step of our methodology, transforming the input docu- ment into semantically meaningful units to facil- itate effective retrieval. This stage involves three sub-processes:", + "4 Retrieval-Augmented Generation (RAG):Semantic chunking with embedding-based retrieval us- ing top-k chunk selection. This approach prioritizes efficiency with minimal token usage (average 544 tokens per query) but may miss critical clinical relationships. 2.4 Evaluation Framework Evaluation was conducted on a dataset of 12 clinical notes ranging from 10,000 to 65,000 tokens, repre- senting diverse clinical scenarios. Each note was accompanied by clinical questions requiring informa- tion extraction and", + "clinical notes question-answering systems and validate CLEAR against established approachesincluding zero-shot large-context processing and traditional chunk-based retrieval-augmented generationin realistic EHR process- ing scenarios. Methods:We implemented a Clinical Notes Q&A Evaluation Platform with three retrieval strategies: (1) Wide Context processing for zero-shot inference with large context windows, (2) tra- ditional vector database chunking with semantic search, and (3) entity-aware CLEAR with", + "medical domain knowledge. Evaluation encompassed 12 clinical documents (10K65K tokens) representing typical EHR DocumentReference content. Results:CLEAR showed a 58.3% win rate across test cases, achieving 0.878 average seman- tic similarity while requiring 78% fewer tokens than wide-context processing. Gains were most pronounced on large notes (75% win rate for 65K + tokens), consistent with published scalability claims. Conclusions:The Clinical Notes Q&A Evaluation Platform validates CLEARs advantages", + "continuity across long clin- ical narratives. CLEAR represented a significant advancement by introducing entity-centered retrieval aligned with clinical semantics. Our work extends this line of research by operationalizing CLEAR within an end-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale document sets. Recent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., 2023; Xiong et al., 2024) emphasize that retrieval", + "strategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes.", + "from datasets like Miracl and NQ, leading to high topic diversity. In such cases, Breakpoint-based Semantic Chunker outperformed others by better preserving topic integrity, splitting sentences based on semantic dissimilarity to form chunks similar to the original documents. In contrast, Fixed-size and Clustering-based Chunkers often mixed sen- tences from different documents, increasing noise and lowering retrieval quality. As document length increased, fewer documents were stitched together, reducing", + "2 constraints and the lost in the middle problem, and chunk-based retrieval-augmented generation (RAG) systems that utilize vector databases for semantic similarity search but often fail to capture critical clin- ical entity relationships and contextual dependencies essential for accurate medical information extrac- tion. A growing body of evidence shows that merely expanding context windows does not guarantee effective use of information: performance often drops when relevant spans occur in the middle of", + "information split across chunks may not be effectively retrieved or integrated. To address these issues, we analyse and compare two recent techniques contextual retrieval1 and late chunking within a unified setup, evaluating their strengths and limitations in tackling challenges like context loss and incom- plete information retrieval. Contextual retrieval preserves coherence by prepend- ing LLM-generated context to chunks, while late chunking embeds entire docu- ments to retain global context before", + "targeted information retrieval. 4.3 Clinical Applications and Impact The demonstrated performance characteristics show the potential of Enhanced CLEAR in real-world clinical applications. The optimal balance between accuracy and computational efficiency enables de- ployment in resource-constrained environments while maintaining clinical decision support quality. Po- tential applications include automated clinical documentation review, real-time decision support sys- tems, and large-scale clinical research", + "statistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for", + "data processing. 4.4 Limitations and Future Work Several limitations should be acknowledged. The current implementation relies on keyword-based en- tity extraction, which, while effective, could benefit from advanced neural entity recognition models. Additionally, evaluation was limited to English clinical notes from specific domains, and the system lacks integration with standardized medical ontologies. Future research should focus on incorporating advanced clinical NER models, integrating standard- ized", + "wide-context processing at markedly lower token budgets, echoing the efficiencyquality trade-offs highlighted by CLEAR . The validation results strongly confirm the original research findings, particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe- ference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware retrieval becomes increasingly advantageous as document complexity increases, confirming the", + "documents are split into consecutive, fixed-size segments, re- main unclear. This study systematically evalu- ates the effectiveness of semantic chunking us- ing three common retrieval-related tasks: docu- ment retrieval, evidence retrieval, and retrieval- based answer generation. The results show that the computational costs associated with seman- tic chunking are not justified by consistent per- formance gains. These findings challenge the previous assumptions about semantic chunk- ing and highlight the", + "local semantics. (CLEAR retrieves windows around entities but does not man- date a specific window size.) Our context selection algorithm incorporates questionentity semantic alignment and medical relationship scoring to prioritize clinically relevant spans. 2.3 Baseline Methods We compared our enhanced CLEAR implementation against two baseline approaches: Wide Context Processing:Complete clinical note processing using full document context. This approach provides comprehensive information access but", + "context selection, and token optimization. 2.2.1 Enhanced Entity Extraction We developed a comprehensive medical entity recognition system using advanced keyword patterns, clinical value recognition for vital signs and laboratory values, and medical domain-specific entity types with confidence-based scoring. The system recognizes six primary entity categories: medications, symp- toms, diseases, procedures, laboratory values, and anatomical references. 2.2.2 Section-Aware Processing Clinical documents", + "scores and token usage across all methods. Enhanced CLEAR demonstrates consistent performance across document sizes, with par- ticularly strong results on clinical notes 1, 2, 4, 5, 6, 9, 10, and 11. Table 2: Detailed Results by Clinical Note Note ID Size (tokens) Wide Sim. RAG Sim. CLEAR Sim. Best Strategy CLEAR Tokens clinical note1 10,025 0.847 0.8070.916CLEAR 8,446 clinical note2 10,142 0.880 0.8490.894CLEAR 8,493 clinical note3 10,2330.9290.835 0.909 Wide 8,318 clinical note4 10,098 0.857", + "Table 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the", + "exacerbated when retrieval fails to prioritize relevant information prop- erly. Thus, documents are often divided into smaller segments or \"chunks\" before embedding and retrieval. However, this chunking process can disrupt semantic coherence, leading to: Loss of Context:dividing documents without considering semantic bound- aries can result in chunks that lack sufficient context, impairing the models ability to generate accurate and coherent responses. Incomplete Information Retrieval:important", + "Is Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where", + "an 8.23 reduction (87.78 v.s. 79.55) in accuracy relative to ChunkLLM on the 64K NIAH task. This discrepancy stems from the semantic incompleteness of fixed chunks, which in turn compromises chunk selection during the inference phase. In contrast, ChunkLLM leverages contextual semantic information to identify chunk boundaries, preserving the semantic integrity of chunks. 4 RELATEDWORK KV Cache CompressionRecent research has primarily focused on overcoming the limitations of Large Language Models (LLMs) in", + "semantic chunking in improving the retrieval systems effectiveness. Table 3: Retriever Performance Comparison: Naive Retriever vs. ChunkRAG ( = 0.8). Retriever Type Average Relevance Score Naive Retriever 0.180 ChunkRAG ( = 0.8) 0.467 7 Discussion The ablation study highlights redundancy filterings key role in ChunkRAG, with dynamic chunk merg- ing and optimal similarity thresholds (validated at = 0.8) balancing chunk reduction and relevance while preventing over-filtering. Future work could investigate", + "answers. How- ever, the effectiveness of chunking strategies re- mains a significant challenge in optimizing retrieval quality and computational efficiency (Lewis et al., 2020; Finardi et al., 2024). Known as fixed-size chunking, the traditional way to chunk is to cut documents into chunks of a fixed length such as 200 tokens (Gao et al., 2023). While computationally simple, this approach can fragment semantically related content across multi- ple chunks, leading to suboptimal retrieval perfor- mance.", + "fundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying", + "Clustering-based Semantic Chunker groups semantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. chunkers. Fixed-size Chunker This is our baseline chunker that splits a document sequentially into fixed-size chunks, based on a predefined or user-specified number of sentences per chunk. Although this approach is simple and compu- tationally efficient, it may separate contextually related sentences, leading to potential degradation in retrieval quality (Lewis et", + "vs 0.86 for traditional RAG) with substantial efficiency gains (71% token reduction, 72% faster inference time) on clinical information extraction tasks, positioning CLEAR as a potentially transformative approach for production EHR processing systems. To validate these claims in realistic healthcare scenarios and provide a robust evaluation framework for clinical NLP approaches, we developed the Clinical Notes Q&A Evaluation Platform. This com- prehensive validation study implements and compares three", + "Recently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic", + "question persists: how can vast volumes of external knowledge be man- aged effectively within the input constraints of LLMs? Traditional meth- ods address this by chunking external documents into smaller, fixed- size segments. While this approach alleviates input limitations, it often fragments context, resulting in incomplete retrieval and diminished co- herence in generation. To overcome these shortcomings, two advanced techniqueslate chunking and contextual retrievalhave been intro- duced, both aiming", + "1 Beyond Long Context: When Semantics Matter More than Tokens Tarun Kumar Chawdhury and Jon D. Duke Georgia Institute of Technology tarunchawdhury@gatech.edu jon.duke@gatech.edu Keywords:Clinical NLP, Entity-Aware Retrieval, Evaluation Platform Abstract Background:Electronic Health Records (EHR) systems store clinical documentation in FHIR DocumentReference resources as base64-encoded attachments, presenting significant challenges for semantic question-answering applications. Traditional approaches using", + "to comprehensive dis- charge summaries, present substantial challenges for automated question-answering systems that require semantically accurate information extraction rather than statistical correlation-based retrieval common in traditional vector database approaches. Contemporary approaches to clinical document processing have largely focused on two paradigms: zero-shot inference with large context windows that process entire documents but face computational arXiv:2510.25816v1 [cs.CL] 29 Oct 2025" + ], + "generated_answer": "It's great you're exploring the nuances of chunking and retrieval strategies! Understanding these trade-offs is crucial for effective information retrieval, especially in complex domains like clinical notes.\n\nHere's an explanation of the observed trade-offs between computational efficiency and semantic integrity for the strategies you mentioned:\n\n### Explanation of Trade-offs\n\n**1. Fixed-size Chunking**\n* **Computational Efficiency:** This approach is highly efficient and computationally simple. Documents are cut into segments of a predefined length, such as 200 tokens, or a fixed number of sentences.\n* **Semantic Integrity:** The primary drawback is a significant risk to semantic integrity. Fixed-size chunking can fragment semantically related content across multiple chunks, leading to a \"loss of context\" and \"incomplete information retrieval.\" This can impair the model's ability to generate accurate and coherent responses because important information might be split or boundaries may not align with semantic meaning. While some studies noted it performed best on a few datasets, the overall consensus is that its simplicity often comes at the cost of semantic coherence.\n\n**2. Semantic Chunking (including Breakpoint-based and Clustering-based)**\n* **Computational Efficiency:** Semantic chunking aims to improve retrieval by dividing documents into semantically coherent segments. However, the notes indicate that the \"computational costs associated with semantic chunking are not justified by consistent performance gains.\" This suggests it often introduces overhead without a guaranteed significant improvement over simpler methods like fixed-size chunking, especially in real-life scenarios where topic diversity might not be as high as in artificially noisy data.\n* **Semantic Integrity:** The goal is to enhance semantic integrity by preserving topic integrity.\n * **Breakpoint-based Semantic Chunker:** This method preserves topic integrity by splitting sentences based on semantic dissimilarity, aiming to form chunks similar to the original document's structure. It showed an advantage in datasets with high topic diversity.\n * **Clustering-based Semantic Chunker:** This groups semantically similar sentences, potentially combining non-consecutive text to form topic-based chunks.\n Despite these aims, the notes suggest that semantic chunkers \"may not have an edge over fixed-size chunker\" in some real-life situations, and their benefits regarding performance gain are not always consistent enough to justify the computational overhead.\n\n**3. Late Chunking**\n* **Computational Efficiency:** While not explicitly detailed, late chunking \"embeds entire documents to retain global context.\" Embedding entire documents, especially long ones, can be computationally intensive, particularly for the initial processing step.\n* **Semantic Integrity:** This technique aims to overcome fragmentation and diminished coherence by embedding entire documents initially, thereby \"retaining global context.\" This approach prioritizes maintaining a broader understanding of the content before chunking, which should theoretically lead to better semantic integrity.\n\n**4. Contextual Retrieval**\n* **Computational Efficiency:** This strategy adds three steps to the traditional RAG process: Contextualization, Rank Fusion, and Reranking. \"Contextualization\" involves enriching each chunk with additional context from the entire document, ensuring a broader understanding. These additional steps, especially LLM-generated context and reranking, inherently introduce computational overhead compared to basic chunk retrieval.\n* **Semantic Integrity:** Contextual retrieval explicitly \"preserves coherence by prepending LLM-generated context to chunks.\" By enriching chunks, it ensures that even segmented pieces retain a broader understanding of the content, directly addressing context loss and incomplete information retrieval.\n\n**5. Clinical Entity Augmented Retrieval (CLEAR)**\n* **Computational Efficiency:** CLEAR demonstrates superior efficiency. It achieved \"78% fewer tokens than wide-context processing\" and \"71% token reduction, 72% faster inference time\" compared to traditional RAG. This makes it highly efficient for production environments.\n* **Semantic Integrity:** CLEAR excels in semantic integrity by using \"entity-aware, entity-centered retrieval strategies.\" It selectively centers clinically relevant spans, integrating medical domain knowledge through specialized entity scoring and question-entity alignment. This approach, along with section-aware processing and enhanced entity extraction, significantly improves medical concept recognition and clinical reasoning preservation. It addresses the limitation of traditional RAG, which often \"fail[s] to capture critical clinical entity relationships and contextual dependencies.\"\n\n### Superior Scalability Advantages\n\nThe **Clinical Entity Augmented Retrieval (CLEAR) methodology** demonstrated superior scalability advantages, particularly in high-complexity clinical documents.\n\n* **CLEAR's performance gains were \"most pronounced on large notes (75% win rate for 65K + tokens),\"** consistent with its published scalability claims.\n* The validation results confirmed that \"entity-aware retrieval becomes increasingly advantageous as document complexity increases.\"\n* It showed \"consistent performance across document sizes,\" with particularly strong results on clinical notes ranging from 10,025 to 65,000 tokens.\n\n### Key Points/Summary:\n\n* **Fixed-size Chunking:** High efficiency, but poor semantic integrity due to context fragmentation.\n* **Semantic Chunking:** Aims for semantic coherence but often has inconsistent performance gains that don't justify its computational cost.\n* **Late Chunking & Contextual Retrieval:** Improve semantic integrity by retaining global context or enriching chunks, but imply higher computational costs due to additional processing steps.\n* **CLEAR:** Achieves high semantic accuracy and significant computational efficiency, specifically designed to handle complex clinical relationships.\n* **Superior Scalability:** CLEAR demonstrates superior scalability, performing exceptionally well on large and complex clinical documents (e.g., 75% win rate on documents > 65K tokens).\n\n### Extra Tip:\nThink of it like this: Fixed-size chunking is like cutting a book into pages without reading it \u2013 fast, but you might cut a sentence in half. Semantic chunking tries to cut between chapters, which is better but takes more effort and might not always be perfectly aligned. Late chunking and contextual retrieval are like reading the whole book first, then summarizing each chapter with extra notes to ensure nothing's lost. CLEAR, however, is like having a medical expert read the clinical notes, identifying key medical entities and their relationships, ensuring both accuracy and efficiency, especially with very long patient histories!", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "Query Transformation" + } +] \ No newline at end of file diff --git a/evaluation/results/evaluation_results_base.json b/evaluation/results/evaluation_results_base.json new file mode 100644 index 0000000..d4794d7 --- /dev/null +++ b/evaluation/results/evaluation_results_base.json @@ -0,0 +1,123 @@ +[ + [ + [ + 1.0, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.5, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.95, + 1.0 + ], + [ + 0.1, + 1.0 + ], + [ + 0.75, + 1.0 + ] + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "retrieved_context": { + "context": "need for more efficient chunking strategies in RAG systems. 1 Introduction In Retrieval-Augmented Generation (RAG) sys- tems, cutting documents into smaller units called chunks has a crucial effect on the quality of both retrieval and generation tasks (Chen et al., 2023; Wadhwa et al., 2024; Shi et al., 2023; Yu et al., 2023). By retrieving the most relevant chunks for a given query and feeding them into a generative language model, these systems aim to produce ac- curate and contextually appropriate\n\nneed for more efficient chunking strategies in RAG systems. 1 Introduction In Retrieval-Augmented Generation (RAG) sys- tems, cutting documents into smaller units called chunks has a crucial effect on the quality of both retrieval and generation tasks (Chen et al., 2023; Wadhwa et al., 2024; Shi et al., 2023; Yu et al., 2023). By retrieving the most relevant chunks for a given query and feeding them into a generative language model, these systems aim to produce ac- curate and contextually appropriate\n\nbenefits of semantic chunk- ing, suggesting that its advantages are highly task- dependent and often insufficient to justify the added computational costs. This study lays the ground- work for future exploration of more efficient and adaptive chunking strategies in RAG systems. In general, our contributions are: We present a novel, large-scale evaluation framework comparing semantic and fixed-size chunking across diverse tasks. We demonstrate that while semantic chunk- ing shows some benefits in certain\n\nbenefits of semantic chunk- ing, suggesting that its advantages are highly task- dependent and often insufficient to justify the added computational costs. This study lays the ground- work for future exploration of more efficient and adaptive chunking strategies in RAG systems. In general, our contributions are: We present a novel, large-scale evaluation framework comparing semantic and fixed-size chunking across diverse tasks. We demonstrate that while semantic chunk- ing shows some benefits in certain\n\nIs Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\nIs Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\nRecently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic\n\nRecently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic\n\nclusions. Dataset Fixed-size Breakpoint Clustering ExpertQA 0.65 0.65 0.65 DelucionQA 0.76 0.76 0.76 TechQA 0.68 0.68 0.68 ConditionalQA 0.42 0.43 0.43 Qasper 0.49 0.49 0.50 Table 3: BERTScore for Answer Generation. 5 Conclusion In this paper, we evaluated semantic and fixed-size chunking strategies in RAG systems across docu- ment retrieval, evidence retrieval, and answer gen- eration. Semantic chunking occasionally improved performance, particularly on stitched datasets with high topic diversity.\n\nclusions. Dataset Fixed-size Breakpoint Clustering ExpertQA 0.65 0.65 0.65 DelucionQA 0.76 0.76 0.76 TechQA 0.68 0.68 0.68 ConditionalQA 0.42 0.43 0.43 Qasper 0.49 0.49 0.50 Table 3: BERTScore for Answer Generation. 5 Conclusion In this paper, we evaluated semantic and fixed-size chunking strategies in RAG systems across docu- ment retrieval, evidence retrieval, and answer gen- eration. Semantic chunking occasionally improved performance, particularly on stitched datasets with high topic diversity." + }, + "generated_answer": "**Answer:**\nIn Retrieval-Augmented Generation (RAG) systems, the defining characteristic of 'semantic chunking' is that it divides documents into **semantically coherent segments** or segments them **based on semantic similarity**. This approach aims to improve retrieval performance by ensuring that each chunk contains a complete and meaningful piece of information.\n\n**Key Points/Summary:**\n* Semantic chunking divides documents into smaller units.\n* These units are created based on their semantic coherence or similarity.\n* The goal is to improve the performance of information retrieval in RAG systems.\n\n**Extra Tip:**\nThink of it like organizing a book. Instead of just cutting the book into equal-sized pages (fixed-size chunking), semantic chunking would be like cutting it chapter by chapter, or even section by section within a chapter, ensuring each piece you get tells a complete story or discusses a single topic.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "retrieved_context": { + "context": "answers. 4 Results 4.1 Measuring and reporting performances As mentioned earlier, we used three proxy tasks the study chunking. We cannot directly assess the quality of retrieval at the chunk level due to the lack of ground-truth at the chunk level. Instead, each retrieved chunk is mapped back to either the source document or the included evidence sentences. Since the number of relevant documents or evi- dence sentences is not fixed (unlike the k value for retrieved chunks), traditional metrics such as Re-\n\nanswers. 4 Results 4.1 Measuring and reporting performances As mentioned earlier, we used three proxy tasks the study chunking. We cannot directly assess the quality of retrieval at the chunk level due to the lack of ground-truth at the chunk level. Instead, each retrieved chunk is mapped back to either the source document or the included evidence sentences. Since the number of relevant documents or evi- dence sentences is not fixed (unlike the k value for retrieved chunks), traditional metrics such as Re-\n\nevaluation is not trivial due to the lack of data that can be directly used to com- pare chunking strategies. Therefore, we design an indirect evaluation using three proxy tasks: (1) doc- ument retrieval, measuring the ability to identify relevant documents; (2) evidence retrieval, mea- suring the ability to locate ground-truth evidence; and (3) answer generation, testing the quality of answers produced by a generative model using re- trieved chunks. Our findings challenge prevailing assumptions about the\n\nevaluation is not trivial due to the lack of data that can be directly used to com- pare chunking strategies. Therefore, we design an indirect evaluation using three proxy tasks: (1) doc- ument retrieval, measuring the ability to identify relevant documents; (2) evidence retrieval, mea- suring the ability to locate ground-truth evidence; and (3) answer generation, testing the quality of answers produced by a generative model using re- trieved chunks. Our findings challenge prevailing assumptions about the\n\nembeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely\n\nembeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely\n\nTable 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the\n\nTable 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the\n\nthresholds are generally higher than those from percentiles or interquartile ranges. Fixed-size Chunker Figure 5 shows results for the Fixed-size Chunker. The trends mirror those seen in other chunkers. Adding one overlapping sentence between chunks does not notably improve performance, indicating that a single overlapping sentence is insufficient to significantly boost con- textual coherence. E.4 Chunk Inspection We examined the output chunks to (1) confirm that different chunkers were functioning as\n\nthresholds are generally higher than those from percentiles or interquartile ranges. Fixed-size Chunker Figure 5 shows results for the Fixed-size Chunker. The trends mirror those seen in other chunkers. Adding one overlapping sentence between chunks does not notably improve performance, indicating that a single overlapping sentence is insufficient to significantly boost con- textual coherence. E.4 Chunk Inspection We examined the output chunks to (1) confirm that different chunkers were functioning as" + }, + "generated_answer": "Hello there! I can certainly help you understand the evaluation methods used in this study.\n\nThe study designed an **indirect evaluation** using three primary proxy tasks to assess the quality of chunking strategies, because directly comparing chunking strategies was difficult due to a lack of suitable data.\n\nHere are the three proxy tasks:\n\n1. **Document Retrieval:** This task measured the ability of the system to identify and retrieve relevant source documents.\n2. **Evidence Retrieval:** This task focused on evaluating the system's ability to locate specific \"ground-truth evidence\" within the retrieved information.\n3. **Answer Generation:** This task tested the overall quality of the answers produced by a generative model, which used the retrieved chunks as its source material.\n\nThese tasks were used because the quality of retrieval at the chunk level couldn't be directly assessed due to the absence of \"ground-truth\" data specifically for chunks. Instead, retrieved chunks were mapped back to either their source documents or the evidence sentences they contained.\n\n### Key Points/Summary:\n* The study used an indirect evaluation method for chunking strategies.\n* Three proxy tasks were designed: Document Retrieval, Evidence Retrieval, and Answer Generation.\n* Direct chunk-level quality assessment was not possible due to a lack of ground-truth data.\n\n### Extra Tip:\nThink of \"proxy tasks\" as stand-ins or representatives. Since they couldn't directly measure how good the chunks themselves were, they measured related tasks (like finding documents, specific evidence, or generating answers from the chunks) to *indirectly* infer the effectiveness of different chunking strategies. It's like judging a chef's knife skills by how good the final meal tastes, rather than directly measuring their chopping speed!", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "retrieved_context": { + "context": "to preserve global context. Despite their potential, their comparative strengths and limitations remain unclear. This study presents a rigorous analysis of late chunking and contextual retrieval, evaluating their effectiveness and efficiency in optimizing RAG systems. Our results indicate that contextual retrieval preserves semantic coher- ence more effectively but requires greater computational resources. In contrast, late chunking offers higher efficiency but tends to sacrifice rel- evance and\n\nto preserve global context. Despite their potential, their comparative strengths and limitations remain unclear. This study presents a rigorous analysis of late chunking and contextual retrieval, evaluating their effectiveness and efficiency in optimizing RAG systems. Our results indicate that contextual retrieval preserves semantic coher- ence more effectively but requires greater computational resources. In contrast, late chunking offers higher efficiency but tends to sacrifice rel- evance and\n\n4 J. Singh and C. Merola to align with the early and late chunking strategies under evaluation. This ad- justment allows us to explore how different embedding techniques influence the retrieval quality and, subsequently, the overall performance of the RAG system. Additionally, we test dynamic segmenting models to further refine the chunk- ing process, providing an adaptive mechanism that adjusts chunk sizes based on content characteristics. By evaluating the impact of these dynamic segmenting\n\n4 J. Singh and C. Merola to align with the early and late chunking strategies under evaluation. This ad- justment allows us to explore how different embedding techniques influence the retrieval quality and, subsequently, the overall performance of the RAG system. Additionally, we test dynamic segmenting models to further refine the chunk- ing process, providing an adaptive mechanism that adjusts chunk sizes based on content characteristics. By evaluating the impact of these dynamic segmenting\n\nRAG. A standard RAG workflow involves four main stages: document segmentation, chunk embedding, indexing, and retrieval. During segmentation, documents are divided into manageable chunks. These chunks are then trans- formed into vector representations using encoder models, often normalized to 1 2\n\nRAG. A standard RAG workflow involves four main stages: document segmentation, chunk embedding, indexing, and retrieval. During segmentation, documents are divided into manageable chunks. These chunks are then trans- formed into vector representations using encoder models, often normalized to 1 2\n\nmodels,weaimtoimprovetheoverallretrievalefficiencyandresponsegeneration within the RAG framework. Early Chunking. Documents are segmented into text chunks, and each chunk is processed by the embedding model. The model generates token-level embed- dings for each chunk, which are subsequently aggregated using mean pooling to produce a single embedding per chunk. Late Chunking. Late chunking defers the chunking process. As shown in Figure 3.1, instead of segmenting the document initially, the entire document\n\nmodels,weaimtoimprovetheoverallretrievalefficiencyandresponsegeneration within the RAG framework. Early Chunking. Documents are segmented into text chunks, and each chunk is processed by the embedding model. The model generates token-level embed- dings for each chunk, which are subsequently aggregated using mean pooling to produce a single embedding per chunk. Late Chunking. Late chunking defers the chunking process. As shown in Figure 3.1, instead of segmenting the document initially, the entire document\n\nofcontextual retrieval versus tra- ditional early chunkingstrategies, utilizingdifferent text segmenters and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG systems. 3.1 RQ#1: Early or Late Chunking? In this workflow, the main architectural modification compared to the standard RAG lies in the document embedding process Figure 3.1. Specifically, we experi- ment with various embedding models to encode document chunks, tailoring them 3 chunking/\n\nofcontextual retrieval versus tra- ditional early chunkingstrategies, utilizingdifferent text segmenters and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG systems. 3.1 RQ#1: Early or Late Chunking? In this workflow, the main architectural modification compared to the standard RAG lies in the document embedding process Figure 3.1. Specifically, we experi- ment with various embedding models to encode document chunks, tailoring them 3 chunking/" + }, + "generated_answer": "The notes describe two advanced techniques that go beyond traditional early chunking to optimize RAG systems, with goals related to preserving context:\n\n**Answer:**\nBased on the provided notes, two advanced techniques, other than traditional early chunking, aimed at improving RAG systems and preserving context are **Late Chunking** and **Dynamic Segmenting Models**.\n\n1. **Late Chunking**: This technique defers the chunking process. Instead of segmenting the entire document initially, late chunking processes the document differently, aiming to preserve global context. While it offers higher efficiency, the notes indicate it may tend to sacrifice relevance.\n2. **Dynamic Segmenting Models**: These models are designed to refine the chunking process by providing an adaptive mechanism. This means they adjust chunk sizes based on the specific characteristics of the content, which helps improve overall retrieval efficiency and response generation within the RAG framework. By adapting chunk sizes, these models implicitly aim to better capture and preserve contextual information.\n\n**Key Points/Summary:**\n* **Late Chunking:** Delays the document segmentation process, aiming to preserve global context, though it might impact relevance.\n* **Dynamic Segmenting Models:** Adaptively adjust chunk sizes based on content characteristics to refine the chunking process, improving retrieval efficiency and response generation.\n\n**Extra Tip:**\nThink of \"dynamic segmenting\" like a smart editor who knows when to make paragraphs longer or shorter based on the flow of ideas, rather than just chopping text into fixed-size pieces. This helps ensure that important contextual information isn't split across different chunks unnecessarily.", + "correctness_score": 0.5, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "retrieved_context": { + "context": "exploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nexclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison\n\nexclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison\n\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high\n\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high\n\nperplexity (ppl) on the PG19 test set, and results are summarized in Table 2. Compared to the vanilla model, ChunkLLM yields a slight enhancement in ppl alongside a significant decrease in total infer- ence time. The underlying reason is that while the vanilla model maintains semantic integrity, it incurs linearly increasing inference time as gen- erated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up inference by leveraging its chunk selection and ICAC mechanisms. 3.2.4\n\nperplexity (ppl) on the PG19 test set, and results are summarized in Table 2. Compared to the vanilla model, ChunkLLM yields a slight enhancement in ppl alongside a significant decrease in total infer- ence time. The underlying reason is that while the vanilla model maintains semantic integrity, it incurs linearly increasing inference time as gen- erated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up inference by leveraging its chunk selection and ICAC mechanisms. 3.2.4" + }, + "generated_answer": "The key phenomenon described in the context, where chunks attended to by tokens within a generated chunk exhibit substantial consistency, is called the **Intra-Chunk Attention Consistency (ICAC) pattern**.\n\nChunkLLM exploits this pattern by updating chunk selection only when the current token is identified as a chunk boundary, which significantly enhances inference efficiency, especially in long-context scenarios.\n\n**Key Points/Summary:**\n* The phenomenon is named **Intra-Chunk Attention Consistency (ICAC)**.\n* It describes the consistency of chunks attended to by tokens within a generated chunk.\n* ChunkLLM leverages ICAC to improve inference efficiency by updating chunk selection only at chunk boundaries.\n* This mechanism helps reduce computational burden and speed up inference.\n\n**Extra Tip:**\nThink of it like reading a book. Instead of constantly deciding what to focus on with every single word, you might decide to shift your focus only when you reach the end of a paragraph or a new section (a \"chunk boundary\"). This makes your reading process much more efficient because you're not constantly re-evaluating your attention within a coherent thought unit.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "retrieved_context": { + "context": "strategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes.\n\nstrategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes.\n\nfor semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes\n\nfor semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes\n\ncomplexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\ncomplexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying" + }, + "generated_answer": "Hello there! I can help you with that.\n\n**Answer:**\nIn the clinical domain, the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes is called the **Clinical Entity Augmented Retrieval (CLEAR) methodology**.\n\nIntroduced by Lopez et al. (2025), CLEAR specifically addresses the limitations of previous methods, such as traditional embedding-based chunk retrieval (like RAG) and ontology-based retrieval, which often failed to capture nuanced clinical relationships or domain-specific entity modeling. The CLEAR methodology focuses on entity-aware retrieval strategies to enhance the accuracy of medical information extraction from clinical notes.\n\n**Key Points/Summary:**\n* The methodology is named **Clinical Entity Augmented Retrieval (CLEAR)**.\n* It was introduced by Lopez et al. (2025).\n* CLEAR uses **entity-aware retrieval strategies** to process EHR notes.\n* It demonstrates improved performance in terms of semantic accuracy (F1 0.90 vs. 0.86 for embedding RAG) and computational efficiency (over 70% fewer tokens and faster inference).\n\n**Extra Tip:**\nThink of \"entity-aware\" as a smart librarian who doesn't just look for keywords in books but understands what those keywords *mean* in the context of specific medical concepts (like diseases, medications, or procedures). This deeper understanding helps find more precise and relevant information, making the retrieval process much more accurate and efficient than just scanning for word matches.", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "retrieved_context": { + "context": "costs of long-context models, making it difficult for large language models to effi- ciently scale to context-processing tasks with million-level token sizes.Chunk Selective attention, a special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk (Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen et al., 2024). Both approaches partition the input into discrete chunks: the former conducts parti- tioning with a fixed length,\n\ncosts of long-context models, making it difficult for large language models to effi- ciently scale to context-processing tasks with million-level token sizes.Chunk Selective attention, a special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk (Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen et al., 2024). Both approaches partition the input into discrete chunks: the former conducts parti- tioning with a fixed length,\n\ndemands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance\n\ndemands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance\n\nChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3\n\nChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate" + }, + "generated_answer": "It looks like the provided notes discuss **ChunkLLM** in detail, but they do not contain information about **Dynamic Hierarchical Sparse Attention (DHSA)**. Therefore, I can't directly contrast ChunkLLM with DHSA based on the materials you've given me.\n\nHowever, I can explain ChunkLLM's dynamic attention management mechanisms, how it derives chunk representations, and how it achieves efficiency and performance in long-context models, based on your notes!\n\n### ChunkLLM's Dynamic Attention Management Mechanisms\n\nChunkLLM is designed to improve the efficiency and long-context understanding capability of large language models by using a special type of sparse attention called \"Chunk Selective attention.\" It addresses the challenges of processing million-level token sizes, which are costly for traditional long-context models.\n\nHere's how ChunkLLM works:\n\n1. **Chunk Representation Derivation:**\n * Unlike models that might use simpler methods like separators (e.g., SepLLM), ChunkLLM enriches the representational capacity of its chunks. It does this through an **attention distillation mechanism**. This mechanism strengthens the features of each chunk, making them more informative and robust.\n * By enhancing the representational capacity, ChunkLLM can better understand and identify the most relevant parts (critical chunks) of the input.\n\n2. **Utilizing Chunk Representations for Efficiency and Performance:**\n * **Improved Chunk Selection:** The enriched chunk representations allow ChunkLLM to effectively identify \"critical chunks\" with higher relevance to the query during the chunk selection process. This leads to improved inference performance because the model focuses its attention on the most important information.\n * **Reduced KV-Cache Utilization:** The efficient retrieval of key chunks results in a reduced KV-Cache utilization rate compared to other models like SepLLM. This is a significant efficiency gain, as the KV-Cache often consumes a lot of memory in long-context models.\n * **Intra-Chunk Attention Consistency (ICAC) Pattern:** ChunkLLM exploits a novel **Intra-Chunk Attention Consistency (ICAC) pattern**. This pattern ensures that chunk selection is only updated when the current token is identified as a chunk boundary. This substantially enhances inference efficiency, especially in long-context scenarios.\n * **Scalability with Reduced Training Overhead:** Remarkably, ChunkLLM can achieve inference performance comparable to models optimized for 120K context lengths, even when it has been trained solely on 4K context lengths. This significantly reduces the training overhead that would otherwise be associated with scaling models to such long contexts.\n * **Speedup:** Experimental results show that ChunkLLM can attain a maximum speedup of 4.48 times compared to a vanilla Transformer when processing 120K long texts. It also achieves 98.64% of the vanilla model's performance.\n\nIn summary, ChunkLLM uses sophisticated attention distillation to create rich chunk representations, which then enable more accurate and efficient selection of critical chunks. Combined with the ICAC pattern, this approach boosts inference efficiency, reduces resource usage (like KV-Cache), and allows for high performance on long contexts even with training on shorter contexts.\n\n---\n\n**Key Points/Summary:**\n\n* **No DHSA Info:** The provided notes do not contain information about Dynamic Hierarchical Sparse Attention (DHSA).\n* **Enriched Chunk Representations:** ChunkLLM uses an attention distillation mechanism to enhance the representational capacity of its chunks.\n* **Efficient Chunk Selection:** This enriched representation allows ChunkLLM to effectively identify \"critical chunks\" relevant to the query, improving inference.\n* **Efficiency Gains:** It achieves efficiency through reduced KV-Cache utilization and a novel Intra-Chunk Attention Consistency (ICAC) pattern, leading to significant speedups.\n* **Performance with Less Training:** ChunkLLM maintains high performance on 120K context lengths despite being trained only on 4K contexts, reducing training overhead.\n\n**Extra Tip:**\nThink of \"attention distillation\" like summarizing a long meeting. Instead of just picking out a few sentences (like using separators), ChunkLLM processes the whole \"chunk\" of the meeting to understand its core ideas and important points more deeply, making its summary much more effective for later recall and decision-making.", + "correctness_score": 0.1, + "faithfulness_score": 1.0, + "model_tested": "base" + }, + { + "question_id": 7, + "document_path": "static/Beyond Long Context.pdf", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "retrieved_context": { + "context": "4 Retrieval-Augmented Generation (RAG):Semantic chunking with embedding-based retrieval us- ing top-k chunk selection. This approach prioritizes efficiency with minimal token usage (average 544 tokens per query) but may miss critical clinical relationships. 2.4 Evaluation Framework Evaluation was conducted on a dataset of 12 clinical notes ranging from 10,000 to 65,000 tokens, repre- senting diverse clinical scenarios. Each note was accompanied by clinical questions requiring informa- tion extraction and\n\ncomplexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\nlong inputs (lost in the middle) . This motivates entity-aware retrieval that selectively centers clinically relevant spans rather than relying on statistically similar but potentially off-target chunks. The CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in 2025 , introduced a novel approach that addresses these limitations through entity-aware, entity- centered retrieval strategies. The original study demonstrated significant performance improvements (F1 score of 0.90\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\ndocuments are split into consecutive, fixed-size segments, re- main unclear. This study systematically evalu- ates the effectiveness of semantic chunking us- ing three common retrieval-related tasks: docu- ment retrieval, evidence retrieval, and retrieval- based answer generation. The results show that the computational costs associated with seman- tic chunking are not justified by consistent per- formance gains. These findings challenge the previous assumptions about semantic chunk- ing and highlight the\n\ncontinuity across long clin- ical narratives. CLEAR represented a significant advancement by introducing entity-centered retrieval aligned with clinical semantics. Our work extends this line of research by operationalizing CLEAR within an end-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale document sets. Recent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., 2023; Xiong et al., 2024) emphasize that retrieval\n\nanswers. How- ever, the effectiveness of chunking strategies re- mains a significant challenge in optimizing retrieval quality and computational efficiency (Lewis et al., 2020; Finardi et al., 2024). Known as fixed-size chunking, the traditional way to chunk is to cut documents into chunks of a fixed length such as 200 tokens (Gao et al., 2023). While computationally simple, this approach can fragment semantically related content across multi- ple chunks, leading to suboptimal retrieval perfor- mance.\n\nIs Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\nwide-context processing at markedly lower token budgets, echoing the efficiencyquality trade-offs highlighted by CLEAR . The validation results strongly confirm the original research findings, particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe- ference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware retrieval becomes increasingly advantageous as document complexity increases, confirming the" + }, + "generated_answer": "The notes describe several chunking and retrieval strategies, each with distinct trade-offs between computational efficiency and semantic integrity, particularly in the context of clinical documents.\n\nHere's a breakdown of the observed trade-offs:\n\n1. **Fixed-size Chunking:**\n * **Computational Efficiency:** This method is \"computationally simple\" as it involves cutting documents into segments of a fixed length (e.g., 200 tokens).\n * **Semantic Integrity:** Its simplicity comes at the cost of semantic integrity. It \"can fragment semantically related content across multiple chunks,\" leading to \"suboptimal retrieval performance\" because important information might be split.\n\n2. **Semantic Chunking:**\n * **Computational Efficiency:** This approach involves \"high computational costs.\" The notes state that \"the computational costs associated with semantic chunking are not justified by consistent performance gains.\"\n * **Semantic Integrity:** It \"aims to improve retrieval performance by dividing documents into semantically coherent segments.\" However, despite its growing adoption, \"the actual benefits over simpler fixed-size chunking... remain unclear\" and are not consistently justified by the increased costs.\n\n3. **Retrieval-Augmented Generation (RAG) with Embedding-based Retrieval (top-k chunk selection):**\n * **Computational Efficiency:** This approach \"prioritizes efficiency with minimal token usage (average 544 tokens per query).\"\n * **Semantic Integrity:** While efficient, it \"may miss critical clinical relationships,\" indicating a potential compromise in accurately capturing nuanced semantic information.\n\n4. **Clinical Entity Augmented Retrieval (CLEAR):**\n * **Computational Efficiency:** CLEAR demonstrates significant improvements in efficiency, reporting \">70% fewer tokens and faster inference\" compared to embedding-based RAG, and operating at \"markedly lower token budgets.\"\n * **Semantic Integrity:** It addresses the limitations of traditional methods by employing \"entity-aware, entity-centered retrieval strategies.\" This approach is \"aligned with clinical semantics\" and leads to \"improved performance (F1 score of 0.90 vs. 0.86 for embedding RAG),\" successfully capturing nuanced clinical relationships and maintaining contextual continuity.\n\n**Regarding Late Chunking and Contextual Retrieval:**\nThe provided notes do not explicitly define or discuss \"Late Chunking\" as a distinct strategy with its own set of trade-offs. Similarly, \"Contextual Retrieval\" is not presented as a specific methodology but rather a general goal or challenge (e.g., \"maintaining contextual continuity\" or \"wide context processing\") that various retrieval strategies aim to achieve or struggle with.\n\n**Superior Scalability Advantages in High-Complexity Clinical Documents:**\nThe **Clinical Entity Augmented Retrieval (CLEAR) methodology** demonstrated superior scalability advantages. The validation results \"strongly confirm the original research findings, particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentReference content.\" It achieved a \"75% win rate on documents exceeding 65,000 tokens,\" validating that \"entity-aware retrieval becomes increasingly advantageous as document complexity increases.\"\n\n---\n\n**Key Points/Summary:**\n* **Fixed-size Chunking:** Simple and efficient, but compromises semantic integrity by fragmenting content.\n* **Semantic Chunking:** Aims for semantic coherence but incurs high computational costs without consistent performance gains.\n* **Embedding-based RAG:** Efficient with low token usage, but risks missing critical clinical relationships.\n* **CLEAR:** Offers superior efficiency (fewer tokens, faster inference) and enhanced semantic integrity through entity-aware, entity-centered retrieval.\n* **Scalability:** CLEAR demonstrated superior scalability, becoming more advantageous for high-complexity clinical documents exceeding 65,000 tokens.\n\n**Extra Tip:**\nThink of the trade-off like packing a suitcase:\n* **Fixed-size chunking** is like cutting everything into equal squares, regardless of what it is. It's fast to cut, but you might cut a shirt in half!\n* **Semantic chunking** is like trying to fold clothes neatly, but it takes a lot more time and effort, and sometimes the neatness doesn't actually save space or make things easier to find.\n* **CLEAR** is like carefully organizing by item type (shirts with shirts, pants with pants) and then compressing them efficiently. It takes a smart approach, resulting in better organization (semantic integrity) and more efficient packing (computational efficiency), especially for a very large suitcase (complex document).", + "correctness_score": 0.75, + "faithfulness_score": 1.0, + "model_tested": "base" + } +] \ No newline at end of file diff --git a/evaluation/results/evaluation_results_everything.json b/evaluation/results/evaluation_results_everything.json new file mode 100644 index 0000000..f9e9e1e --- /dev/null +++ b/evaluation/results/evaluation_results_everything.json @@ -0,0 +1,109 @@ +[ + [ + [ + 0.97, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.98, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.15, + 1.0 + ], + [ + 0.9, + 1.0 + ] + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "retrieved_context": "# 5 **Conclusion** \nIn this paper, we evaluated semantic and fixed-size\nchunking strategies in RAG systems across document retrieval, evidence retrieval, and answer generation. Semantic chunking occasionally improved\nperformance, particularly on stitched datasets with\nhigh topic diversity. However, these benefits were\nhighly context-dependent and did not consistently\njustify the additional computational cost. On nonsynthetic datasets that better reflect real-world documents, fixed-size chunking often performed better.\nOverall, our results suggest that fixed-size chunking remains a more efficient and reliable choice for\npractical RAG applications. The impact of chunking strategy was often overshadowed by other factors, such as the quality of embeddings, especially\nwhen computational resources are limited or when\nworking with standard document structures.\n\n# mantic chunking, which aims to improve retrieval performance by dividing documents into\nsemantically coherent segments. Despite its\ngrowing adoption, the actual benefits over simpler **fixed-size chunking**, where documents are\nsplit into consecutive, fixed-size segments, remain unclear. This study systematically evaluates the effectiveness of semantic chunking using three common retrieval-related tasks: document retrieval, evidence retrieval, and retrievalbased answer generation. The results show that\nthe computational costs associated with semantic chunking are not justified by consistent performance gains. These findings challenge the\nprevious assumptions about semantic chunking and highlight the need for more efficient\nchunking strategies in RAG systems.\n\n# Breakpoint-based Semantic Chunker As the dis\ntance threshold between consecutive sentences in\ncreases, fewer breakpoints appear, resulting in\nlarger chunks. Regardless of the threshold type,\nit ultimately determines chunk size. In Figure 4,\nwe observe similar trends to Figure 2 and 3: as\nchunk size increases, precision decreases in both\nretrieval tasks, while recall increases sharply for\nevidence retrieval. The rise in standard deviation is \nexpected, as values from standard deviation-based \nthresholds are generally higher than those from\npercentiles or interquartile ranges.\n\n# Abstract \nRecent advances in Retrieval-Augmented Generation (RAG) systems have popularized **se-**\n\n# Clustering-based Semantic Chunker (Single-\n\n# Clustering-based Semantic Chunker (DBSCAN)\nAs EPS increases, the threshold for grouping samples into the same cluster loosens, increasing average chunk size. As seen in Figure 3, this leads\nto a decrease in precision and an increase in recall\nfor document and evidence retrieval, respectively,\nsimilar to the single-linkage case.\n\n# Sentence-level Chunking Our study focuses on\nsentence-level chunking, where documents are split\ninto individual sentences, and each sentence is\ntreated as a segment for grouping. This approach\nresults in sentence embeddings that lack contextual\ninformation. While we attempted to address this\nby overlapping sentences in Fixed-size Chunker\nand incorporating positional distance in Semantic Chunker (global), the embeddings themselves\nremained context-free. Further exploration of contextual embeddings is necessary before definitively\nconcluding the limitations of semantic chunking.\n\n## 4.2 **Document Retrieval** \nTable 1 shows varied chunker performance, with\nFixed-size Chunker excelling on non-stitched\ndatasets and Semantic Chunkers performing better\non stitched datasets. \nAs described in Appendix C, stitched documents,\naveraging 100 sentences, were formed by combining short documents (fewer than 10 sentences) \nDataset Fixed-size Breakpoint Clustering \nMiracl* 69.45 **81.89** 67.35 \nNQ* 43.79 **63.93** 41.01 \nScidocs* 16.82 17.60 **19.87** \nScifact* 35.27 **36.27** 35.70 \nBioASQ* 61.86 61.87 **62.49**\nNFCorpus* 21.36 21.07 **22.12**\nHotpotQA **90.59** 87.37 84.79\nMSMARCO **93.58** 92.23 93.18 \nConditionalQA **68.11** 64.44 65.94\nQasper **90.99** 89.27 90.77 \nTable 1: F1@5 for Document Retrieval ( % ). Datasets\nmarked with * are stitched. Rows are sorted by the average number of sentences per document (before stitching)\nin ascending order for easier comparison. \nfrom datasets like Miracl and NQ, leading to high\ntopic diversity. In such cases, Breakpoint-based\nSemantic Chunker outperformed others by better\npreserving topic integrity, splitting sentences based\non semantic dissimilarity to form chunks similar\nto the original documents. In contrast, Fixed-size\nand Clustering-based Chunkers often mixed sentences from different documents, increasing noise\nand lowering retrieval quality.\nAs document length increased, fewer documents\nwere stitched together, reducing topic diversity.\nThis diminished the advantage of Breakpoint-based\nSemantic Chunker, while Clustering-based Semantic Chunker improved. The gap between semantic\nand fixed-size chunkers narrowed, with Fixed-size\nChunker benefiting from higher topic integrity.\nThese results suggest that in real life, the topics\nin a document may not be as diverse as in our\nartificially noisy, stitched data, and hence semantic\nchunkers may not have an edge over fixed-size\nchunker there.\n\n# 2 **Chunking Strategies** \nIn this paper, a document is first split into sentences\nwhich are then grouped into chunks. We evaluate\nthree chunking strategies, hereafter referred to as \nFigure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within\nthe sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk\nbreakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering\nsemantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance\nthresholds between consecutive sentences to maintain coherence. (c) Clustering-based Semantic Chunker groups\nsemantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. \n\u201cchunkers.\u201d\n\n# Inspection on Stitched Documents In Figure 6,\nDocuments 1 and 3 have four sentences each, while\nDocuments 2 and 4 contain three and five sentences,\nrespectively. The Fixed-size Chunker, which ignores semantic relationships and document structure, frequently misassigned sentences, leading to\nerrors that propagated through subsequent chunks.\nFor instance, a sentence from Document 3 was appended to Document 2, illustrating the limitations\nof Fixed-size Chunking with stitched documents\ncontaining numerous short segments. This explains\nits poor performance under such conditions. However, simply splitting the document into structured\nsections before applying fixed-size chunking will\nsolve this issue. \nIn contrast, both semantic chunkers performed\nbetter on stitched documents, but still had issues.\nThe Clustering-based Chunker made one error by\ngrouping Sentence 16 (the last sentence of Document 4) into Chunk 2. This happened because,\ndespite the large positional distance, the semantic similarity was high, causing the sentence to be\nincorrectly included. Without considering positional structure like the Fixed-size and Breakpointbased Chunkers, the Clustering-based Chunker \noften mixed sentences from different documents. \nWhile this might be useful for multi-document\ntasks (Bolotova-Baranova et al., 2023; Zhu et al.,\n2024), it was problematic here, leading to worse\nperformance when many short documents were\nstitched together.\nThe Breakpoint-based Chunker also made errors. It could, like the Fixed-size Chunker, group a\nsentence with a different chunk due to low seman\ntic similarity with neighboring sentences, as seen\nwith Sentence 4 being moved to Chunk 2. This\nshows the advantage of the joint distance measure\nin Equation 1, which prevented this error for the\nClustering-based Chunker. Moreover, controlling\nchunk size was challenging; higher thresholds led\nto overly large chunks, while lower thresholds resulted in single-sentence chunks lacking contextual\ninformation, such as Chunk 4\u2019s meaningless \"Name\nbinding\" phrase.", + "generated_answer": "The defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems is its approach to dividing documents into **semantically coherent segments**.\n\nUnlike fixed-size chunking, which splits documents into consecutive, uniform segments without considering their meaning, semantic chunking aims to keep related information together. It achieves this by:\n* **Detecting semantic distance thresholds:** For example, a Breakpoint-based Semantic Chunker segments text by identifying points where the semantic distance between consecutive sentences exceeds a certain threshold, ensuring that sentences within a chunk are semantically related.\n* **Grouping semantically similar sentences:** A Clustering-based Semantic Chunker, for instance, groups sentences that are semantically similar, potentially combining non-consecutive text to form chunks focused on a particular topic.\n\nThis method intends to improve retrieval performance by ensuring that each chunk represents a cohesive idea or topic, which can be particularly useful in documents with high topic diversity.\n\n**Key Points/Summary:**\n* Semantic chunking divides documents into segments based on their meaning.\n* It aims to create \"semantically coherent\" chunks, keeping related information together.\n* This is achieved by detecting semantic distance between sentences or grouping semantically similar sentences.\n* It contrasts with fixed-size chunking, which splits text into uniform segments regardless of semantic content.\n\n**Extra Tip:** Imagine reading a book. Fixed-size chunking would be like tearing out pages in stacks of 5, regardless of whether a chapter ends halfway through a stack. Semantic chunking, however, would be like keeping each chapter or even each major section of a chapter intact, because they represent a complete idea.", + "correctness_score": 0.97, + "faithfulness_score": 1.0, + "model_tested": "everything" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "retrieved_context": "# 3 **Experiments** \nIn the absence of ground-truth chunk data, we designed three experiments to indirectly assess the\nquality of each chunker: document retrieval, evidence retrieval, and answer generation. Different\ndatasets and evaluation metrics were used for each \nexperiment to align with the specific task requirements. All documents were first split into sentences using SpaCy\u2019s en_core_web_sm model (Explosion, 2024) before being embedded and chunked. We tested three embedding models selected to\nrepresent a range of performances based on their\nrankings on the MTEB Leaderboard (Muennighoff\net al., 2022). See Appendix E.2 for details.\n\n# 4 **Results** \n## 4.1 **Measuring and reporting performances** \nAs mentioned earlier, we used three proxy tasks\nthe study chunking. We cannot directly assess the\nquality of retrieval at the chunk level due to the lack\nof ground-truth at the chunk level. Instead, each\nretrieved chunk is mapped back to either the source\ndocument or the included evidence sentences. \nSince the number of relevant documents or evi\ndence sentences is not fixed (unlike the _k_ value for\nretrieved chunks), traditional metrics such as Recall@k and NDCG@k are not suitable. F1 provides\na balanced measure that accounts for both precision\nand recall under these circumstances. Therefore,\nwe use **F1@5** as the metric. For further details, see\nAppendix D.\nFor each dataset, results are reported based on\nthe best hyperparameter configuration for each\nchunker, determined by the average F1 score across\nall _k_ values. All results to be reported below are\nobtained using dunzhang/stella_en_1.5B_v5 as the\nembedder for being the best among those tested.\nIn the following subsections, **Bold** values indicate the best performance on the respective dataset.\nThe results for Answer Generation closely matched\nthose of Evidence Retrieval and are discussed in \nAppendix E.1. Additional analysis of hyperparameters is provided in Appendix B. Inspection of the\noutputs of different chunkers is provided in Appendix E.4.\n\n# 2 **Chunking Strategies** \nIn this paper, a document is first split into sentences\nwhich are then grouped into chunks. We evaluate\nthree chunking strategies, hereafter referred to as \nFigure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within\nthe sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk\nbreakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering\nsemantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance\nthresholds between consecutive sentences to maintain coherence. (c) Clustering-based Semantic Chunker groups\nsemantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. \n\u201cchunkers.\u201d\n\n# mantic chunking, which aims to improve retrieval performance by dividing documents into\nsemantically coherent segments. Despite its\ngrowing adoption, the actual benefits over simpler **fixed-size chunking**, where documents are\nsplit into consecutive, fixed-size segments, remain unclear. This study systematically evaluates the effectiveness of semantic chunking using three common retrieval-related tasks: document retrieval, evidence retrieval, and retrievalbased answer generation. The results show that\nthe computational costs associated with semantic chunking are not justified by consistent performance gains. These findings challenge the\nprevious assumptions about semantic chunking and highlight the need for more efficient\nchunking strategies in RAG systems.\n\n## 3.2 **Evidence Retrieval** \nHere we evaluate chunkers at a finer granularity\nthan the previous experiment by measuring their\nabilities to locate evidence sentences. We selected \nadditional datasets from RAGBench (Friel et al.,\n2024), shown in Tables 2 and 5, because few\ndatasets contain long documents with ground-truth\nevidence sentences. We measured the number of \nground-truth evidence sentences present in the retrieved top-k chunks.\n\n## 3.1 **Document Retrieval** \nThis experiment assessed the effectiveness of chunkers in retrieving relevant documents for a given\nquery. We used 10 datasets, shown in Tables 1\nand 4. Most documents on the BEIR benchmark \n(Thakur et al., 2021) are too short for chunking to\nbe effective. To address this, we synthesized longer\ndocuments by stitching short documents from six\ndatasets where documents are too short (see Appendix C for details). We randomly sampled 100\nqueries from each dataset and retrieved the top _k_\nchunks, where _k \u2208_ [1 _,_ 3 _,_ 5 _,_ 10] . Each retrieved\nchunk was mapped to its source document, and the\nretrieved documents were evaluated by comparing\nthem to a set of relevant documents for each query.\n\n# Lack of Suitable Datasets Despite testing multiple datasets, our selection was constrained by\na lack of comprehensive datasets. An ideal\ndataset would include long documents representative of real-world use cases, diverse query types,\nhuman-generated answers, query-document relevance scores, and human-labeled evidence sentences. Our synthetic documents had artificially\nhigh topic diversity due to random stitching, potentially leading to unreliable results. Additionally,\nthe answer sets in RAGBench (Friel et al., 2024)\nwere generated by LLMs, which may not accurately assess chunk quality. A dataset containing\nall these elements is needed for a more thorough\nevaluation of chunking strategies.\n\n# E.4 **Chunk Inspection** \nWe examined the output chunks to (1) confirm that\ndifferent chunkers were functioning as intended,\nand (2) investigate the reasons behind performance\ndifferences. BEIR\u2019s HotpotQA dataset (Thakur\net al., 2021; Yang et al., 2018) was selected for its\nreasonably sized documents. We randomly sampled five documents, stitching the first four together\nto form a stitched document (Figure 6), and keeping the fifth as a normal document (Figure 7. The\ndocument IDs are: \n- Stitched: 44547136, 14115210, 5580754, \n54045118. \n- Normal: 30214079.\n\n# Lack of Chunk Quality Measures As noted in\nSection 4, while the output chunks differed between methods, retrieval and generation performances were similar across chunkers. In addition \nto the influence of embedding models, the absence\nof direct chunk quality metrics likely contributed\nto this issue. Having ground-truth query-chunk relevance scores would provide more accurate evaluations than relying solely on document or evidence\nmapping.\n\n# Fixed-size Chunker Figure 5 shows results for\nthe Fixed-size Chunker. The trends mirror those \nseen in other chunkers. Adding one overlapping\nsentence between chunks does not notably improve\nperformance, indicating that a single overlapping\nsentence is insufficient to significantly boost contextual coherence.", + "generated_answer": "The study presented in your notes designed three primary proxy tasks to indirectly evaluate the quality of different chunking strategies. Since there was no \"ground-truth chunk data\" available to directly assess chunk quality, these tasks were used to measure performance in related areas.\n\nThe three proxy tasks are:\n\n1. **Document Retrieval:** This experiment assessed how well chunkers could retrieve relevant documents for a given query. It involved mapping retrieved chunks back to their source documents and comparing them against a set of relevant documents.\n2. **Evidence Retrieval:** This task evaluated chunkers at a finer level of detail, measuring their ability to locate specific \"evidence sentences\" within the retrieved top-k chunks.\n3. **Answer Generation (or Retrieval-based Answer Generation):** This task focused on how well the chunking strategies supported the generation of answers based on the retrieved information. The notes mention that the results for Answer Generation closely matched those of Evidence Retrieval.\n\nThese tasks used different datasets and evaluation metrics to align with their specific requirements.\n\n---\n\n**Key Points/Summary:**\n\n* The study used three indirect proxy tasks to assess chunker quality due to the lack of ground-truth chunk data.\n* The tasks were Document Retrieval, Evidence Retrieval, and Answer Generation.\n* Each task evaluated different aspects of retrieval performance related to chunking.\n\n---\n\n**Extra Tip:**\nThink of these proxy tasks like testing how well a car engine works by measuring its fuel efficiency, acceleration, and emissions, rather than directly inspecting the engine's internal components. Each test gives an indirect but valuable measure of its overall performance!", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "everything" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "retrieved_context": "# Abstract. Retrieval-augmented generation (RAG) has become a transformative approach for enhancing large language models (LLMs) by\ngrounding their outputs in external knowledge sources. Yet, a critical\nquestion persists: how can vast volumes of external knowledge be managed effectively within the input constraints of LLMs? Traditional methods address this by chunking external documents into smaller, fixedsize segments. While this approach alleviates input limitations, it often\nfragments context, resulting in incomplete retrieval and diminished coherence in generation. To overcome these shortcomings, two advanced\ntechniques\u2014late chunking and contextual retrieval\u2014have been introduced, both aiming to preserve global context. Despite their potential,\ntheir comparative strengths and limitations remain unclear. This study\npresents a rigorous analysis of late chunking and contextual retrieval,\nevaluating their effectiveness and efficiency in optimizing RAG systems.\nOur results indicate that contextual retrieval preserves semantic coherence more effectively but requires greater computational resources. In\ncontrast, late chunking offers higher efficiency but tends to sacrifice relevance and completeness.\n\n# \u2013\n_Loss of Context:_ dividing documents without considering semantic boundaries can result in chunks that lack sufficient context, impairing the model\u2019s\nability to generate accurate and coherent responses. \n# \u2013\n_Incomplete Information Retrieval:_ important information split across chunks\nmay not be effectively retrieved or integrated. \nTo address these issues, we analyse and compare two recent techniques\u2014\ncontextual retrieval [1] and late chunking [9]\u2014within a unified setup, evaluating\ntheir strengths and limitations in tackling challenges like context loss and incomplete information retrieval. Contextual retrieval preserves coherence by prepending LLM-generated context to chunks, while late chunking embeds entire documents to retain global context before segmenting.\nOur study rigorously assesses their impact on generation performance in\nquestion-answering tasks, finding that neither technique offers a definitive solution. This work highlights the trade-offs between these methods and provides\npractical guidance for optimizing RAG systems.\nTo further support the community, we release all code, prompts, and data\nunder the permissive MIT license, enabling full reproducibility and empowering\npractitioners to adapt and extend our work. [2]\n\n## 3.1 **RQ#1: Early or Late Chunking?** \nIn this workflow, the main architectural modification compared to the standard\nRAG lies in the document embedding process Figure 3.1. Specifically, we experiment with various embedding models to encode document chunks, tailoring them \n3 `[https://docs.llamaindex.ai/en/stable/examples/node_parsers/semantic_](https://docs.llamaindex.ai/en/stable/examples/node_parsers/semantic_chunking/)`\n```\nchunking/\n\n``` \n4 J. Singh and C. Merola \nto align with the early and late chunking strategies under evaluation. This adjustment allows us to explore how different embedding techniques influence the\nretrieval quality and, subsequently, the overall performance of the RAG system.\nAdditionally, we test dynamic segmenting models to further refine the chunking process, providing an adaptive mechanism that adjusts chunk sizes based on\ncontent characteristics. By evaluating the impact of these dynamic segmenting\nmodels, we aim to improve the overall retrieval efficiency and response generation\nwithin the RAG framework. \n_Early Chunking._ Documents are segmented into text chunks, and each chunk\nis processed by the embedding model. The model generates token-level embeddings for each chunk, which are subsequently aggregated using mean pooling to\nproduce a single embedding per chunk. \n_Late Chunking._ Late chunking [9] defers the chunking process. As shown in\nFigure 3.1, instead of segmenting the document initially, the entire document\nis first embedded at the token level. The resulting token embeddings are then\nsegmented into chunks, and mean pooling is applied to each chunk to generate the\nfinal embeddings. This approach preserves the full contextual information within\nthe document, potentially leading to superior results across various retrieval\ntasks. It is adaptable to a wide range of long-context embedding models and\ncan be implemented without additional training. The two approaches are tested\nwith different embedding models.\n\n# 2 **Related Work** \n_Classic RAG._ A standard RAG workflow involves four main stages: document\nsegmentation, chunk embedding, indexing, and retrieval. During segmentation,\ndocuments are divided into manageable chunks. These chunks are then transformed into vector representations using encoder models, often normalized to \n1 `[https://www.anthropic.com/news/contextual-retrieval](https://www.anthropic.com/news/contextual-retrieval)`\n2 `[https://github.com/disi-unibo-nlp/rag-when-how-chunk](https://github.com/disi-unibo-nlp/rag-when-how-chunk)` \nReconstructing Context 3 \nensure unit magnitudes. The resulting embeddings are stored in indexed vector\ndatabases, enabling efficient approximate similarity searches. Retrieval involves\ncomparing query embeddings with the stored embeddings using metrics such as\ncosine similarity or Euclidean distance, which identify the most relevant chunks.\nSeminal works like [15] and [13] have demonstrated the effectiveness of RAG\nin tasks such as open-domain question answering. More recent studies, including [7], have introduced advancements in scalability and embedding techniques,\nfurther establishing RAG as a foundational framework for knowledge-intensive\napplications. \n_Document Segmentation._ Document segmentation is essential for processing long\ntexts in RAG workflows, with methods ranging from _fixed-size segmentation_ [7]\nto more adaptive techniques like _semantic segmentation_, [3] which detect semantic\nbreakpoints based on shifts in meaning. Recent advancements include _supervised_\n_segmentation models_ [14,12] and _segment-then-predict models_, trained end-to-end\nwithout explicit labels to optimize chunking for downstream task performance \n[17]. In 2024, _late chunking_ and _contextual retrieval_ introduced novel paradigms.\nBoth techniques have proven effective in retrieval benchmarks but remain largely\nuntested in integrated RAG workflows. Despite several RAG surveys [7,6,8],\nno prior work has compared these methods within a comprehensive evaluation\nframework. This study addresses this gap by holistically analyzing late chunking\nand contextual retrieval, offering actionable insights into their relative strengths\nand trade-offs.\n\n# Keywords: Contextual Retrieval \u00b7 Late Chunking \u00b7 Dynamic Chunking \n- Rank Fusion.\n\n## 3.2 **RQ#2: Early or Contextual Chunking?** \nIn this workflow, traditional retrieval is compared to Contextual Retrieval with\nRank Fusion technique. This has been introduced by Anthropic in September\n2024. [4] Three steps are added to the Traditional RAG process: Contextualization,\nRank Fusion, Reraking. \n_Contextualization._ After document segmentation, each chunk is enriched with\nadditional context from the entire document, ensuring that even when segmented, each piece retains a broader understanding of the content (Fig. 3.2).\nIn fact, when documents are split into smaller chunks, it might arise the problem where individual chunks lack sufficient context. For example, a chunk might\ncontain the text: \"The company\u2019s revenue grew by 3% over the previous quarter.\"\nHowever, this chunk on its own does not specify which company it is referring to\nor the relevant time period, making it difficult to retrieve the right information\nor use the information effectively. Contextualization improves the relevance and\naccuracy of retrieved information by maintaining contextual integrity. \n4 `[https://www.anthropic.com/news/contextual-retrieval](https://www.anthropic.com/news/contextual-retrieval)` \nReconstructing Context 5 \n|Embedding|....|Embed|Col4|\n|---|---|---|---|\n|Pooling
.....
Pooli
Pooling|Pooling
.....
Pooli
Pooling|Pooling
.....
Pooli
Pooling|Pooling
.....
Pooli
Pooling|\n|Pooling
.....
Pooli
Pooling|Pooling
.....
Pooli
Pooling|Pooling
.....
Pooli
Pooling|| \n|oken
emb|Token T
emb|oken
emb|Token To
emb e|ken
mb|Token
emb|\n|---|---|---|---|---|---|\n|Embedding
model|Embedding
model|Embedding
model|Embedding
model|Embedding
model|Embedding
model|\n|Long document|Long document|Long document|Long document|Long document|Long document|\n\n# ditional early chunking strategies, utilizing **different text segmenters**\nand embedding models to evaluate their impact on retrieval accuracy and\ndownstream performance in RAG systems.\n\n# \u2013 RQ#1 : Compares the effectiveness of **early versus late chunking** strategies, utilizing **different text segmenters** and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG\nsystems.\n\n# CM : Chunking Methods (FUC: Fixed-Window Uncontextualized Chunks, SUC:\nSemantic Uncontextualized Chunks, FCC: Fixed-Window Contextualized Chunks,\nSCC: Semantic Contextualized Chunks).\n\n# 1 **Introduction** \nRetrieval Augmented Generation (RAG) is a transformative approach that enhances the capabilities of large language models (LLMs) by integrating external\ninformation retrieval directly into the text generation process. This method allows LLMs to dynamically access and utilize relevant external knowledge, significantly improving their ability to generate accurate, contextually grounded,\nand informative responses. Unlike static LLMs that rely solely on pre-trained\ndata, RAG-enabled models can access up-to-date and domain-specific information. This dynamic integration ensures that the generated content remains both\nrelevant and accurate, even in rapidly evolving or specialized fields. \n_\u22c6_ Equal contribution. \n2 J. Singh and C. Merola \nRAG models combine two key components: a retrieval mechanism and a\ngenerative model. The retrieval mechanism fetches relevant documents or data\nfrom a large corpus, while the generative model synthesizes this information into\ncoherent, contextually enriched answers. This synergy enhances performance in\nknowledge-intensive natural language processing (NLP) tasks, enabling models\nto produce well-informed responses grounded in the retrieved data.\n_**The Context Dilemma in Classic RAG:**_ Managing extensive external\ndocuments poses significant issues in RAG systems. Despite advancements, many\nLLMs are limited to processing a few thousand tokens. Although some models\nhave achieved context windows up to millions of tokens [5], these are exceptions\nrather than the norm. Moreover, research indicates that LLMs may exhibit positional bias, performing better with information at the beginning of a document\nand struggling with content located in the middle or toward the end [11,16]. This\nissue is exacerbated when retrieval fails to prioritize relevant information properly. Thus, documents are often divided into smaller segments or \"chunks\" before\nembedding and retrieval. However, this chunking process can disrupt semantic\ncoherence, leading to:", + "generated_answer": "Two advanced chunking techniques, beyond traditional early chunking, that aim to preserve global context within Retrieval-Augmented Generation (RAG) systems are **late chunking** and **contextual retrieval**.\n\nThese techniques were introduced to address issues like fragmented context, incomplete retrieval, and diminished coherence that can arise from traditional fixed-size chunking methods.\n\nHere's a breakdown of each:\n\n1. **Late Chunking**:\n * **How it works**: Instead of segmenting a document into chunks at the very beginning (early chunking), late chunking first embeds the *entire document* at the token level. After the full document has been embedded, these token embeddings are then segmented into chunks. Mean pooling is applied to each of these resulting chunks to generate their final embeddings.\n * **Goal**: This approach is designed to preserve the full contextual information of the document during the initial embedding phase, potentially leading to better results in retrieval tasks.\n\n2. **Contextual Retrieval**:\n * **How it works**: In this method, after the initial document segmentation, each individual chunk is enriched with additional context derived from the *entire original document*. The notes also mention that contextual retrieval preserves coherence by \"prepending LLM-generated context to chunks.\"\n * **Goal**: The aim is to ensure that even when documents are split into smaller segments, each chunk still retains a broader understanding of the content. This helps to maintain contextual integrity, improving the relevance and accuracy of the retrieved information. For example, if a chunk says \"The company's revenue grew by 3%,\" contextualization would add information about *which* company or *when* to that specific chunk.\n\n**Key Points/Summary:**\n* **Late Chunking:** Embeds the entire document first, then segments the resulting token embeddings into chunks.\n* **Contextual Retrieval:** Enriches individual chunks with additional context from the whole document, often by prepending LLM-generated context.\n* **Shared Goal:** Both aim to overcome the context loss and incomplete information retrieval issues of traditional chunking by better preserving the global context.\n* **Trade-offs:** The notes indicate that contextual retrieval preserves semantic coherence more effectively but requires greater computational resources, while late chunking offers higher efficiency but may sacrifice relevance and completeness.\n\n**Extra Tip:** Think of it like this: if a book is chopped into chapters, traditional chunking might just give you a chapter. Late chunking would be like reading the whole book, *then* deciding where the chapters should be based on the overall flow, and then processing those \"chapters.\" Contextual retrieval would be like giving you a chapter, but also providing a short summary of the entire book or the preceding chapters along with it, so you don't lose the bigger picture.", + "correctness_score": 0.98, + "faithfulness_score": 1.0, + "model_tested": "everything" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "retrieved_context": "# ICAC We find a phenomenon during the model inference, as illustrated in Figure 3. The chunks attended to by tokens within a generated chunk exhibit substantial consistency, whereas chunk updates \nFigure 3: Attention visualization of chunk selection during the inference phase. The sample is derived from the passkey retrieval task. \n4 \npredominantly occur at chunk boundaries. We name this phenomenon the \u201d **I** ntra- **C** hunk **A** ttention\n\n# Haojie Ouyang [1] **, Jianwei Lv** [2] **, Lei Ren** [2] **, Chen Wei** [2] **, Xiaojie Wang** [1] **, Fangxiang Feng** [1] \n1 School of Artificial Intelligence, Beijing University of Posts and Telecommunications\n2 Li Auto \nouyanghaojie@bupt.edu.cn \nA BSTRACT \nTransformer-based large models excel in natural language processing and computer vision, but face severe computational inefficiencies due to the self-attention\u2019s\nquadratic complexity with input tokens. Recently, researchers have proposed a\nseries of methods based on block selection and compression to alleviate this problem, but they either have issues with semantic incompleteness or poor traininginference efficiency. To comprehensively address these challenges, we propose\nChunkLLM, a lightweight and pluggable training framework. Specifically, we\nintroduce two components: QK Adapter (Q-Adapter and K-Adapter) and Chunk\nAdapter. The former is attached to each Transformer layer, serving dual purposes\nof feature compression and chunk attention acquisition. The latter operates at the\nbottommost layer of the model, functioning to detect chunk boundaries by leveraging contextual semantic information. During the training phase, the parameters\nof the backbone remain frozen, with only the QK Adapter and Chunk Adapter\nundergoing training. Notably, we design an attention distillation method for training the QK Adapter, which enhances the recall rate of key chunks. During the\ninference phase, chunk selection is triggered exclusively when the current token is\ndetected as a chunk boundary, thereby accelerating model inference. Experimental\nevaluations are conducted on a diverse set of long-text and short-text benchmark\ndatasets spanning multiple tasks. ChunkLLM not only attains comparable performance on short-text benchmarks but also maintains 98.64% of the performance\non long-context benchmarks while preserving a 48.58% key-value cache retention\nrate. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7 in comparison\nto the vanilla Transformer in the processing of 120K long texts. \n1 I NTRODUCTION \nTransformer-based large models (Vaswani et al., 2017) have demonstrated exceptional performance\nacross a diverse range of tasks, including natural language processing (Srivastava et al., 2025; Zhang\net al., 2024) and computer vision (Jiang et al., 2025). However, they have also faced significant challenges in terms of computational efficiency, particularly when scaling to larger structures and large\ncontext inputs. A core issue of efficiency limitations lies in the self-attention module, whose computational complexity is a quadratic relationship with the number of input tokens. Such deficiencies\nin computational efficiency exert a profound impact on both the training complexity and inference\nlatency of large models. \nEfficiency optimization of Transformer has emerged as a pivotal research domain, with efforts predominantly converging into three methodological paradigms. **Linear attention**, such as Mamba\n(Dao & Gu, 2024), RWKV (Peng et al., 2023b; 2024), and RetNet (Sun et al., 2023), seek to\napproximate and substitute the traditional softmax-based self-attention mechanism. However, the\nfundamental architectural disparities between linear attention and conventional attention mechanisms introduce non-trivial challenges: adapting pre-existing Transformer models to integrate linear attention often incurs prohibitive conversion costs(Mercat et al., 2024; Wang et al., 2024; Bick\net al., 2024), while alternative strategies necessitate end-to-end training of entirely new model from\nscratch(Li et al., 2025). Another optimization paradigm is **Sparse attention**, which leverages predefined structural constraints, such as sink-based attention mechanisms (Xiao et al., 2024) or sliding \n1 \nwindow attention mechanisms (Beltagy et al., 2020b), to exploit this sparsity. While these methods may yield certain effects, they often rely heavily on specific tasks, which can limit the overall\ngeneralization ability of the model. Dynamic sparse attention mechanisms (Tang et al., 2024; Jiang\net al., 2024; Liu et al., 2024) filter out subsets of tokens during the inference phase. Although such\nmethods can reduce the computational load of long sequences, they fail to significantly lower the\nhigh training costs of long-context models, making it difficult for large language models to efficiently scale to context-processing tasks with million-level token sizes. **Chunk Selective attention**,\na special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk\n(Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen\net al., 2024). Both approaches partition the input into discrete chunks: the former conducts partitioning with a fixed length, which gives rise to semantic incompleteness; the latter utilizes separators\nfor partitioning, yet ambiguities often emerge. For example, periods frequently occur in numerical\nvalues or abbreviations. Furthermore, during the inference phase, these methods necessitate chunk\nselection for each generated token, incurring additional computational overhead. It is thus evident\nthat existing efficient approaches still exhibit inherent limitations. \nTo address the aforementioned challenges, We propose ChunkLLM, which can be directly constructed by integrating two lightweight and trainable modules into existing LLMs: **QK Adapter**\nand **Chunk Adapter** . The Chunk Adapter connects to the output of the bottommost Transformer\nlayer and used for identify if a token is the last token of a chunk. The QK Adapter is in parallel with\nQ and K matrix at each Transformer layer. It maps full attention scores to chunk attention scores,\nand trained by a distillation approach. \nThe QK Adapter fulfills feature compression and the generation of chunk attention scores. To train\nthe QK Adapter, we propose an attention distillation approach designed to enhance the recall rate\nof key chunks. During training, LLM parameters are kept frozen, with the Kullback\u2013Leibler (KL)\ndivergence between chunk attention scores and full attention scores serving as a guidance signal for\noptimization. The Chunk Adapter determines whether a token corresponds to a chunk boundary by\nleveraging contextual semantic information. During the inference phase, we exploit the Intra-Chunk\nAttention Consistency (ICAC) pattern such that chunk selection is only updated when the current\ntoken is identified as a chunk boundary, which substantially enhances inference efficiency. Furthermore, ChunkLLM can achieve inference performance comparable to that of models optimized for\n120K context lengths, despite being trained solely on 4K context lengths, thereby substantially reducing the training overhead associated with 120K context scaling. Experimental results validate\nthat ChunkLLM yields a 4.48\u00d7 speedup relative to the vanilla Transformer when processing 120K\nlong texts. \nOur contributions are summarized as follows: \n- We introduce ChunkLLM that integrates two lightweight and pluggable components into\nexisting LLMs: the QK Adapter and the Chunk Adapter. The newly developed ChunkLLM only necessitates fine-tuning these lightweight components on the basis of the original model architecture. This design enables ChunkLLM to attain performance comparable\nto vanilla Transformer while utilizing a smaller KV cache, alongside achieving effective\ncontrol over computational scale. \n- We propose an attention distillation-based training approach for the QK Adapter, which\nleverages KL divergence to drive chunk attention toward approximating full attention, effectively enhancing the recall rate of key chunks. Furthermore, we introduce a novel ICAC\npattern, which yields notable improvements in inference efficiency for long-context scenarios. \n- Experimental evaluations show that ChunkLLM not only attains comparable performance\non short-text benchmarks but also maintains 98.64% of the performance on long-context\nbenchmarks while preserving a 48.58% key-value cache (kvcache) retention rate, relative\nto the vanilla Transformer. Particularly, ChunkLLM attains a maximum speedup of 4.48\u00d7\nin comparison to the vanilla Transformer in the processing of 120K long texts. \n2 \n2 M ETHOD \nThe framework of ChunkLLM is shown in Figure 1. ChunkLLM can be built on any existing\ntransformer-based LLMs. Two extra lightweight and pluggable modules are designed to support\nchunk-related capability. One is Chunk Adapter, which is used to identify chunk boundaries. The\nother is the Q Adapter and K Adapter, which is tailored for efficient feature compression and chunk\nselection. This section elaborates on the details of the two modules. \n2.1 C HUNK A DAPTER \nThe Chunk Adapter is a one-layer forward\nneural network (FNN) classifier for chunk\nboundary prediction. Its input is the output\nof the first layer of the LLM, and the output is if or not the token is a chunk boundary, as depicted in the Figure 1. \nFor an input X = _{x_ 1 _, x_ 2 _, ..., x_ _n\u2212_ 1 _, x_ _n_ _}_\nwith _n_ tokens and their corresponding labels Y = _{y_ 1 _, y_ 2 _, ..., y_ _n\u2212_ 1 _, y_ _n_ _}_, _y_ _i_ _\u2208_\n_{_ 0 _,_ 1 _}_, where 1 indicates that the token\n_x_ _i_ is a chunk boundary, 0 indicates that it\ndoes not, **H** _[l]_ _i_ [1] [be the output of] _[ x]_ _[i]_ [ at first]\nlayer. The FNN based Chunk Adapter is\ngiven as in equation 1 \n\u02c6 1 _,_ Sigmoid(FFN( **H** _[l]_ _i_ [1] [))] _[ > \u03b1,]_\n_y_ _i_ =\n\ufffd0 _,_ _otherwise_\n(1) \noutput \nInput \nFigure 1: The framework of ChunkLLM. \nFor training the chunk adapter, we employ\nthe binary cross-entropy loss (BCE) (equation 2) as the objective function. Detailed\ninformation on the training dataset will be given in the experiment part. \n_L_ _CBP_ = _\u2212_ [1] \n_n_ \n_n_ \n\u02c6 \n\ufffd[ _y_ _i_ _\u00b7 log_ (\u02c6 _y_ _i_ ) + (1 _\u2212_ _y_ _i_ ) _\u00b7 log_ (1 _\u2212_ _y_ _i_ )] (2) \n_i_ =1 \n2.2 QK A DAPTER \nAt each layer of the LLM, we incorporate a Q-Adapter and a K-Adapter which used to compress the\nattention and select the chunks. \nFor each layer, let **Q** and **K** be the attention matrix respectively, _c_ be the chunk number of the input,\n_Index_ ~~_c_~~ = _{i_ 1 _, i_ 2 _, ...i_ _c_ _}_ is the index set of chunk boundary tokens. Let **K** [\u02c6] be the K matrix of these\ntokens. We then calculate chunk attention scores as follows:\n\n# A = _Softmax_ ( _[Mul]_ [(] **[Q]** _[,]_ **[ K]** _[T]_ ) **A** _\u2208_ R _[n][\u00d7][n]_\n~~_\u221a_~~ _d_ _k_ \nwhere **Q** _\u2208_ R _[n][\u00d7][d]_ _[k]_ and **K** _\u2208_ R _[n][\u00d7][d]_ _[k]_ are the matrices of query and key for one attention layer. For\nbrevity, the mask operation is omitted from the description. \n_Aggregate_ denotes the operation of summing the token scores within a single chunk. Assuming\nthat an input comprises _c_ chunks with _n_ tokens. Under this setting, _A_ _[t]_ _ij_ [denotes the attention score]\nof the current token _x_ _i_ relative to _j_ - _th_ chunk. For multi-head attention, we compute the average\nalong the head dimension, yielding matrix **A** _[t]_ . \nWe employ the Kullback-Leibler (KL) divergence as the loss function for attention distillation to guide the student model **A** _[s]_ in\napproximating the teacher model\u2019s attention\nscores **A** _[t]_ : \n_L_ _[N]_ _AD_ [=] _[ KL]_ [(] **[A]** _[t]_ _[||]_ **[A]** _[s]_ [)] (5) \nWe average the KL divergence losses across\nthe N layers to obtain the final attention distillation loss: \noutput \nVote \n[0,1,3,5,7] \nChunk ids \n[0,1,3,4,7] \nChunk ids \n[0,2,3,5,7] \nChunk ids \n[0,1,3,5,7] \n|Col1|Col2|\n|---|---|\n||[| \nInput \n_L_ _AD_ = [1] \n_N_ \n_N_\n\ufffd _L_ _[i]_ _AD_ (6) \n_i_ \nDuring the training phase, the parameters of\nthe backbone network are frozen, with only Figure 2: The inference process of ChunkLLM.\nthe Chunk Adapter and QK Adapter undergoing training, thereby achieving efficient training. \n2.3 I NFERENCE \nThe inference phase of ChunkLLM is depicted\nin Figure 2, encompassing two primary steps:\ntop-k chunk selection and ICAC. In line with\nthe ICAC paradigm, chunk updates are triggered exclusively when the current token functions as a chunk boundary.\n\n# C onsistency (ICAC)\u201d. \nICAC makes it possible to save computational cost in chunk selection. We incorporate the chunk\nboundary prediction task into the inference phase. Only when the currently decoded token is a chunk\nboundary, do we update the chunk selection and integrate the complete chunk from the prediction\nphase into **K** and **V** ; otherwise, no update is executed. \n3 E XPERIMENTS AND R ESULTS \n3.1 E XPERIMENTAL S ETTINGS \n3.1.1 M ODEL AND B ASELINES \nTwo representative open-source models, Qwen2.5-7B (Team, 2024) and Llama3.1-8B (Dubey et al.,\n2024), are chosen as the target models for evaluation. We select StreamingLLM (Xiao et al., 2024)\nand SepLLM (Chen et al., 2024) as the baselines to benchmark the proposed method. In detail,\nStreamingLLM retains both initial tokens and window tokens, whereas SepLLM, developed based\non StreamingLLM, treats separator features as chunk features and incorporates a specialized separator cache management mechanism in the inference stage. Detailed settings of the experimental\nparameters are provided in Appendix 6.1. \n3.1.2 T RAINING D ATASETS \nThe FineWeb-Edu dataset (Lozhkov et al., 2024) is employed as the training corpus in this study.\nDeveloped by the HuggingFaceFW team, this dataset undergoes filtering via an educational quality\nclassifier, which constructed based on annotations generated by Llama3-70B-Instruct (Meta, 2024). \nFor preprocessing the training data, the pySBD tool (Sadvilkar & Neumann, 2020), a rule-based sentence boundary detection module that works out-of-the-box, is utilized to annotate the end positions\nof chunks in sequences, serving as foundational input for training the chunk boundary prediction\nmodule. \n3.1.3 B ENCHMARKS\n\n# Sparse Attention The sparse attention mechanism constructs sparse attention matrices by confining\nattention to predefined patterns, such as local windows or fixed-stride block patterns. Beltagy et al.\n(2020a) combined dilated local window attention with task-specific global attention. MoBA (Lu\net al., 2025) proposes an innovative mixed block attention mechanism. ESA (Wang et al., 2025)\nreduces computational costs by selecting tokens most critical to the current generation for attention\ncalculation. NSA (Yuan et al., 2025) combines coarse-grained token compression and fine-grained\ntoken selection. SepLLM (Chen et al., 2024) finds that the segment information between separators\ncan be effectively compressed into the separators themselves without causing significant information\nloss.\n\n# KV Cache Compression Recent research has primarily focused on overcoming the limitations of\nLarge Language Models (LLMs) in processing massive contextual inputs. SnapKV (Li et al., 2024)\nimproves efficiency through KV cache compression, using attention scores to select and cluster\nimportant positional information; H2O (Zhang et al., 2023) implements a dynamic token retention\npolicy that balances recent information and historically important information to optimize memory\noccupancy; StreamingLLM (Xiao et al., 2024) enables LLMs to handle sequences of infinite length\nwithout fine-tuning by retaining attention sinks and local tokens; PyramidInfer (Yang et al., 2024)\nand PyramidKV (Cai et al., 2024) optimize performance by adjusting the KV cache capacity across\ndifferent layers However, most methods in this category cannot be applied to the training phase.\n\n# Long Context Benchmarks We select two long-context evaluation datasets, LongBench (Bai et al.,\n2024) and Needle In A Haystack (NIAH) (Kamradt, 2023), to assess the model\u2019s long-context ability\nThe average text length for most tasks ranges from 5k to 15k tokens in LongBench. We select 10 of\nits subtasks for evaluation. Comprehensive information regarding the characterization of subtasks,\nevaluation methodologies, and additional relevant details is available in Appendix 6.2. For NIAH,\nthe benchmark constructs prompts for LLMs by randomly inserting key information into long texts.\nThe primary objective of this test is to verify whether large models can successfully extract such\nembedded key information from long context, thereby gauging the models\u2019 proficiency in longcontext information extraction.\n\n# Top-k Chunk Selection This stage is primarily dedicated to selecting top-k chunks for each\nlayer. To elaborate, we use the first layer as an Figure 3: Attention visualization of chunk selecillustrative example and define _e_ as the end po- tion during the inference phase. The sample is desition of the input sequence. We then derive the rived from the passkey retrieval task.\nattention scores _A_ _[s]_ _e_ [=] **[ A]** _[s]_ [[] _[e,]_ [ :]] _[ \u2208]_ [R] [1] _[\u00d7][c]_ [ that cor-]\nrespond to the _c_ chunks associated with the end token. [ _,_ ] denotes the slicing operation. We select\nthe indices of the top-k chunks with the highest scores from _A_ _[s]_ _e_ [, where] _[ k][ \u226a]_ _[c]_ [, and retrieve the]\ncorresponding k chunks from **K** and **V**, which facilitates the selection of the top-k key chunks. **V** are\nthe value matrices for one attention layer. We propose a chunk voting mechanism that performs voting on the top-k chunks from each layer, thereby deriving the global top-k chunks. These retrieved\nchunks are subsequently stored in the KV-cache.\n\n# Short Context Benchmarks The selection of the evaluation datasets is primarily centered on the\nmodel\u2019s performance in three key dimensions: **General Knowledge**, which evaluates the model\u2019s\nbreadth of knowledge coverage and the accuracy of its knowledge, MMLU (Hendrycks et al., 2021)\n(5-shot), SciQ (Welbl et al., 2017) (5-shot), OpenBookQA (Mihaylov et al., 2018) (25-shot); **Ques-**", + "generated_answer": "The key phenomenon you're asking about is called **Intra-Chunk Attention Consistency (ICAC)**.\n\nThis phenomenon describes how, during model inference, the chunks that tokens within a generated chunk pay attention to show significant consistency. In simpler terms, tokens within the same chunk tend to focus on a similar set of other chunks.\n\nChunkLLM exploits this ICAC pattern to enhance inference efficiency by only updating the chunk selection process when the current token is identified as a chunk boundary. This means that instead of re-evaluating chunk selection for every single token, it only does so at specific, detected boundaries, which saves computational cost.\n\n**Key Points/Summary:**\n* The phenomenon is named **Intra-Chunk Attention Consistency (ICAC)**.\n* It refers to the observation that tokens within a generated chunk consistently attend to similar other chunks.\n* ChunkLLM uses ICAC to improve inference efficiency by only updating chunk selection at detected chunk boundaries.\n\n**Extra Tip:**\nImagine you're reading a book. Instead of constantly deciding which parts of the book are most important for every single word you read, you might only re-evaluate your focus on \"key sections\" or \"chapters\" as you finish one and start another. ICAC works similarly, allowing the model to maintain its focus on a set of chunks until a logical \"chunk boundary\" is reached, making the process much faster.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "everything" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "retrieved_context": "# Background: Electronic Health Records (EHR) systems store clinical documentation in FHIR \nDocumentReference resources as base64-encoded attachments, presenting significant challenges \nfor semantic question-answering applications. Traditional approaches using statistical correlation \nthrough vector database chunking often fail to capture the nuanced clinical relationships required \nfor accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) \nmethodology, introduced by Lopez et al. (2025) [1], addresses these limitations through entity\naware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; \n_>_ 70% fewer tokens and faster inference).\n\n# 5. **CONCLUSION** \nThis study developed and deployed a comprehensive Clinical Notes Q&A Evaluation Platform and found \nresults consistent with the benefits reported for Clinical Entity Augmented Retrieval (CLEAR) in prior \nwork. In our synthetic EHR QA setting, entity-aware retrieval achieved stronger semantic similarity \nthan wide-context processing at markedly lower token budgets, echoing the efficiency\u2013quality trade-offs \nhighlighted by CLEAR [1]. \nThe validation results strongly confirm the original research findings, particularly demonstrating \nscalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe \nference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware \nretrieval becomes increasingly advantageous as document complexity increases, confirming the method \nology\u2019s suitability for enterprise healthcare environments. \n10 \nFigure 4: Interactive interface showing different analysis approaches and their corresponding performance scores, enabling experimentation to improve reasoning accuracy. \nThe Clinical Notes Q&A Evaluation Platform represents a significant contribution to clinical NLP \nresearch by providing a systematic framework for evaluating retrieval strategies in realistic EHR pro \ncessing scenarios. The platform\u2019s validation of CLEAR methodology demonstrates its viability as a \nproduction-ready approach for clinical information extraction systems requiring optimal balance be \ntween semantic accuracy and computational efficiency. \nThe demonstrated effectiveness of CLEAR through systematic platform-based validation provides \nevidence-based guidance for healthcare organizations implementing clinical question-answering sys \ntems. The evaluation platform framework enables continued research and development in clinical \nentity-aware retrieval methodologies while supporting reproducible evaluation of future clinical NLP \ninnovations in production-relevant contexts.\n\n## 1.1 **Related Work** \nEntity-aware retrieval has gained increasing attention within biomedical NLP and question-answering \ndomains. Early retrieval-augmented methods such as RAG [2] demonstrated the potential of embedding \nbased chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches \nleveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., \n2016) provided partial improvements but often failed to maintain contextual continuity across long clin \nical narratives. \nCLEAR [1] represented a significant advancement by introducing entity-centered retrieval aligned \nwith clinical semantics. Our work extends this line of research by operationalizing CLEAR within an \nend-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale \ndocument sets. \nRecent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., \n2023; Xiong et al., 2024) emphasize that retrieval strategies often outperform naive long-context prompt \ning, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation \nframework within this paradigm, focusing on realistic EHR-scale clinical notes. \n3\n\n# Methods: We implemented a Clinical Notes Q&A Evaluation Platform with three retrieval \nstrategies: (1) Wide Context processing for zero-shot inference with large context windows, (2) tra\nditional vector database chunking with semantic search, and (3) entity-aware CLEAR with medical \ndomain knowledge. Evaluation encompassed 12 clinical documents (10K\u201365K tokens) representing \ntypical EHR DocumentReference content.\n\n# 4. **DISCUSSION** \n## 4.1 **Validation of Research Claims** \nOur enhanced implementation provides observations consistent with the direction of the original CLEAR \nfindings [1] on our synthetic clinical QA benchmark. While our task, retriever, and baselines differ from \nthe original study (which evaluated structured information extraction with model-based NER and on \ntology/LLM synonym augmentation), we still observe that entity-aware retrieval yields higher semantic \nsimilarity at substantially lower token budgets than wide-context processing. \nThe 75% win rate on large documents (65K+ tokens) supports the hypothesis that entity-aware \nretrieval advantages grow with document complexity, consistent with prior work emphasizing targeted \nretrieval over long context processing. This finding has significant implications for clinical applications \ninvolving comprehensive patient records and complex clinical assessments. \n8 \nFigure 2: CLEAR dominates at a 3% efficiency bonus, maintaining superior adjusted accuracy across\nall notes.\n\n# Conclusions: The Clinical Notes Q&A Evaluation Platform validates CLEAR\u2019s advantages for \nsemantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy \nare critical, and provides a reusable framework for evaluating clinical NLP approaches in production \nenvironments.\n\n# Keywords: Clinical NLP, Entity-Aware Retrieval, Evaluation Platform\n\n# Wide Context Processing: Complete clinical note processing using full document context. This \napproach provides comprehensive information access but requires significant computational resources \n(average 39,173 tokens per query). \n4\n\n## 4.2 **Enhanced Implementation Benefits** \nOur enhancements to the original CLEAR methodology provided measurable improvements. Section \naware processing contributed to better clinical reasoning preservation, while enhanced entity extraction \nimproved medical concept recognition. The integration of medical domain knowledge through special \nized entity scoring and question-entity alignment resulted in more targeted information retrieval.\n\n# 2.2.1 **Enhanced Entity Extraction** \nWe developed a comprehensive medical entity recognition system using advanced keyword patterns, \nclinical value recognition for vital signs and laboratory values, and medical domain-specific entity types \nwith confidence-based scoring. The system recognizes six primary entity categories: medications, symp \ntoms, diseases, procedures, laboratory values, and anatomical references.", + "generated_answer": "The methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes in the clinical domain is called **Clinical Entity Augmented Retrieval (CLEAR)**.\n\nIntroduced by Lopez et al. (2025), CLEAR specifically addresses the challenges of extracting nuanced clinical relationships from base64-encoded attachments within FHIR DocumentReference resources. Traditional methods often struggle with this, but CLEAR's entity-aware approach leads to better performance.\n\nStudies and evaluations, such as those conducted using the Clinical Notes Q&A Evaluation Platform, have consistently validated CLEAR's advantages. It has been shown to achieve stronger semantic similarity compared to wide-context processing, while requiring substantially lower token budgets. This makes it a production-ready approach for clinical information extraction systems that need an optimal balance between accuracy and computational efficiency, especially with large and complex clinical documents.\n\n**Key Points/Summary:**\n* **Name:** Clinical Entity Augmented Retrieval (CLEAR) methodology.\n* **Purpose:** Improves semantic accuracy and computational efficiency for question-answering on EHR clinical notes.\n* **Mechanism:** Utilizes entity-aware retrieval strategies.\n* **Benefits:** Achieves stronger semantic similarity with significantly lower token budgets, validated for scalability on large documents.\n\n**Extra Tip:**\nThink of CLEAR as a smart librarian for clinical notes. Instead of scanning every word in every book (wide-context processing) or just looking for keyword matches in random snippets (traditional chunking), CLEAR specifically identifies and focuses on the most important \"entities\" (like medications, symptoms, diseases) within the notes. This targeted approach helps it find the precise information you need more accurately and much faster, especially in very long and complex patient records.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "everything" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "retrieved_context": "# A = _Softmax_ ( _[Mul]_ [(] **[Q]** _[,]_ **[ K]** _[T]_ ) **A** _\u2208_ R _[n][\u00d7][n]_\n~~_\u221a_~~ _d_ _k_ \nwhere **Q** _\u2208_ R _[n][\u00d7][d]_ _[k]_ and **K** _\u2208_ R _[n][\u00d7][d]_ _[k]_ are the matrices of query and key for one attention layer. For\nbrevity, the mask operation is omitted from the description. \n_Aggregate_ denotes the operation of summing the token scores within a single chunk. Assuming\nthat an input comprises _c_ chunks with _n_ tokens. Under this setting, _A_ _[t]_ _ij_ [denotes the attention score]\nof the current token _x_ _i_ relative to _j_ - _th_ chunk. For multi-head attention, we compute the average\nalong the head dimension, yielding matrix **A** _[t]_ . \nWe employ the Kullback-Leibler (KL) divergence as the loss function for attention distillation to guide the student model **A** _[s]_ in\napproximating the teacher model\u2019s attention\nscores **A** _[t]_ : \n_L_ _[N]_ _AD_ [=] _[ KL]_ [(] **[A]** _[t]_ _[||]_ **[A]** _[s]_ [)] (5) \nWe average the KL divergence losses across\nthe N layers to obtain the final attention distillation loss: \noutput \nVote \n[0,1,3,5,7] \nChunk ids \n[0,1,3,4,7] \nChunk ids \n[0,2,3,5,7] \nChunk ids \n[0,1,3,5,7] \n|Col1|Col2|\n|---|---|\n||[| \nInput \n_L_ _AD_ = [1] \n_N_ \n_N_\n\ufffd _L_ _[i]_ _AD_ (6) \n_i_ \nDuring the training phase, the parameters of\nthe backbone network are frozen, with only Figure 2: The inference process of ChunkLLM.\nthe Chunk Adapter and QK Adapter undergoing training, thereby achieving efficient training. \n2.3 I NFERENCE \nThe inference phase of ChunkLLM is depicted\nin Figure 2, encompassing two primary steps:\ntop-k chunk selection and ICAC. In line with\nthe ICAC paradigm, chunk updates are triggered exclusively when the current token functions as a chunk boundary.\n\n# Sparse Attention The sparse attention mechanism constructs sparse attention matrices by confining\nattention to predefined patterns, such as local windows or fixed-stride block patterns. Beltagy et al.\n(2020a) combined dilated local window attention with task-specific global attention. MoBA (Lu\net al., 2025) proposes an innovative mixed block attention mechanism. ESA (Wang et al., 2025)\nreduces computational costs by selecting tokens most critical to the current generation for attention\ncalculation. NSA (Yuan et al., 2025) combines coarse-grained token compression and fine-grained\ntoken selection. SepLLM (Chen et al., 2024) finds that the segment information between separators\ncan be effectively compressed into the separators themselves without causing significant information\nloss.\n\n# ICAC We find a phenomenon during the model inference, as illustrated in Figure 3. The chunks attended to by tokens within a generated chunk exhibit substantial consistency, whereas chunk updates \nFigure 3: Attention visualization of chunk selection during the inference phase. The sample is derived from the passkey retrieval task. \n4 \npredominantly occur at chunk boundaries. We name this phenomenon the \u201d **I** ntra- **C** hunk **A** ttention", + "generated_answer": "It's a great question to understand how different models manage attention!\n\nBased on the provided notes, here's how ChunkLLM addresses dynamic attention management, and unfortunately, the notes do not contain information about Dynamic Hierarchical Sparse Attention (DHSA).\n\n### ChunkLLM's Dynamic Attention Management\n\nChunkLLM employs an **Intra-Chunk Attention Consistency (ICAC)** paradigm during inference to manage attention dynamically.\n\n1. **Deriving Chunk Representations:**\n * The notes define `Aggregate` as the \"operation of summing the token scores within a single chunk.\" This operation likely contributes to forming a representation for each chunk by combining the attention scores of its constituent tokens.\n * The attention score `A_t_ij` denotes the attention of the current token `x_i` relative to the `j`-th chunk. For multi-head attention, these scores are averaged along the head dimension to form a matrix `A_t`.\n\n2. **Utilizing Representations for Efficiency Gains:**\n * The inference process of ChunkLLM involves \"top-k chunk selection\" and ICAC.\n * The core idea of ICAC is based on the observation that \"chunks attended to by tokens within a generated chunk exhibit substantial consistency.\" This implies that once a token belongs to a chunk, its attention pattern largely aligns with that chunk.\n * \"Chunk updates are triggered exclusively when the current token functions as a chunk boundary.\" This suggests that full recalculations or changes to chunk-level attention are not needed for every token, but only at specific points (chunk boundaries), leading to computational efficiency.\n * During training, only the \"Chunk Adapter and QK Adapter\" are trained, while the backbone network is frozen, which contributes to efficient training.\n\n3. **Preserving Performance in Long-Context Models:**\n * ChunkLLM uses **attention distillation** as a loss function to guide the student model (`A_s`) in approximating the teacher model\u2019s attention scores (`A_t`).\n * The Kullback-Leibler (KL) divergence (`L_AD = KL(A_t || A_s)`) is used for this purpose, and these losses are averaged across multiple layers. This mechanism ensures that the student model, which likely incorporates the chunking strategy, learns to mimic the performance of a more robust teacher model, thereby preserving performance.\n\n### Information on Dynamic Hierarchical Sparse Attention (DHSA)\n\nThe provided notes **do not contain any information** regarding Dynamic Hierarchical Sparse Attention (DHSA). Therefore, I cannot contrast it with ChunkLLM based on this context. The \"Sparse Attention\" section lists other sparse attention mechanisms like MoBA, ESA, NSA, and SepLLM, but DHSA is not mentioned.\n\n---\n\n**Key Points/Summary:**\n\n* **Chunk Representation:** ChunkLLM derives chunk representations by \"summing the token scores within a single chunk\" (`Aggregate` operation).\n* **Efficiency:** Efficiency is gained through the ICAC paradigm, where chunk updates are limited to chunk boundaries, and \"top-k chunk selection\" during inference.\n* **Performance Preservation:** Performance is maintained using attention distillation with KL divergence, guiding a student model to approximate a teacher model's attention scores.\n* **DHSA Information:** The provided notes do not contain information about Dynamic Hierarchical Sparse Attention (DHSA).\n\n**Extra Tip:**\nThink of the ICAC paradigm like a \"smart update\" system for attention. Instead of re-evaluating everything all the time, ChunkLLM observes that attention patterns within a chunk are stable. So, it only makes significant updates (like selecting new chunks or updating existing ones) when it hits a \"chunk boundary,\" much like how you might only reorganize your notes when you finish a major topic, rather than after every sentence.", + "correctness_score": 0.15, + "faithfulness_score": 1.0, + "model_tested": "base_chunk_everything" + }, + { + "question_id": 7, + "document_path": "multiple files", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "retrieved_context": "# 5. **CONCLUSION** \nThis study developed and deployed a comprehensive Clinical Notes Q&A Evaluation Platform and found \nresults consistent with the benefits reported for Clinical Entity Augmented Retrieval (CLEAR) in prior \nwork. In our synthetic EHR QA setting, entity-aware retrieval achieved stronger semantic similarity \nthan wide-context processing at markedly lower token budgets, echoing the efficiency\u2013quality trade-offs \nhighlighted by CLEAR [1]. \nThe validation results strongly confirm the original research findings, particularly demonstrating \nscalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe \nference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware \nretrieval becomes increasingly advantageous as document complexity increases, confirming the method \nology\u2019s suitability for enterprise healthcare environments. \n10 \nFigure 4: Interactive interface showing different analysis approaches and their corresponding performance scores, enabling experimentation to improve reasoning accuracy. \nThe Clinical Notes Q&A Evaluation Platform represents a significant contribution to clinical NLP \nresearch by providing a systematic framework for evaluating retrieval strategies in realistic EHR pro \ncessing scenarios. The platform\u2019s validation of CLEAR methodology demonstrates its viability as a \nproduction-ready approach for clinical information extraction systems requiring optimal balance be \ntween semantic accuracy and computational efficiency. \nThe demonstrated effectiveness of CLEAR through systematic platform-based validation provides \nevidence-based guidance for healthcare organizations implementing clinical question-answering sys \ntems. The evaluation platform framework enables continued research and development in clinical \nentity-aware retrieval methodologies while supporting reproducible evaluation of future clinical NLP \ninnovations in production-relevant contexts.\n\n# Background: Electronic Health Records (EHR) systems store clinical documentation in FHIR \nDocumentReference resources as base64-encoded attachments, presenting significant challenges \nfor semantic question-answering applications. Traditional approaches using statistical correlation \nthrough vector database chunking often fail to capture the nuanced clinical relationships required \nfor accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) \nmethodology, introduced by Lopez et al. (2025) [1], addresses these limitations through entity\naware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; \n_>_ 70% fewer tokens and faster inference).\n\n# 4. **DISCUSSION** \n## 4.1 **Validation of Research Claims** \nOur enhanced implementation provides observations consistent with the direction of the original CLEAR \nfindings [1] on our synthetic clinical QA benchmark. While our task, retriever, and baselines differ from \nthe original study (which evaluated structured information extraction with model-based NER and on \ntology/LLM synonym augmentation), we still observe that entity-aware retrieval yields higher semantic \nsimilarity at substantially lower token budgets than wide-context processing. \nThe 75% win rate on large documents (65K+ tokens) supports the hypothesis that entity-aware \nretrieval advantages grow with document complexity, consistent with prior work emphasizing targeted \nretrieval over long context processing. This finding has significant implications for clinical applications \ninvolving comprehensive patient records and complex clinical assessments. \n8 \nFigure 2: CLEAR dominates at a 3% efficiency bonus, maintaining superior adjusted accuracy across\nall notes.\n\n# Conclusions: The Clinical Notes Q&A Evaluation Platform validates CLEAR\u2019s advantages for \nsemantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy \nare critical, and provides a reusable framework for evaluating clinical NLP approaches in production \nenvironments.\n\n## 4.2 **Document Retrieval** \nTable 1 shows varied chunker performance, with\nFixed-size Chunker excelling on non-stitched\ndatasets and Semantic Chunkers performing better\non stitched datasets. \nAs described in Appendix C, stitched documents,\naveraging 100 sentences, were formed by combining short documents (fewer than 10 sentences) \nDataset Fixed-size Breakpoint Clustering \nMiracl* 69.45 **81.89** 67.35 \nNQ* 43.79 **63.93** 41.01 \nScidocs* 16.82 17.60 **19.87** \nScifact* 35.27 **36.27** 35.70 \nBioASQ* 61.86 61.87 **62.49**\nNFCorpus* 21.36 21.07 **22.12**\nHotpotQA **90.59** 87.37 84.79\nMSMARCO **93.58** 92.23 93.18 \nConditionalQA **68.11** 64.44 65.94\nQasper **90.99** 89.27 90.77 \nTable 1: F1@5 for Document Retrieval ( % ). Datasets\nmarked with * are stitched. Rows are sorted by the average number of sentences per document (before stitching)\nin ascending order for easier comparison. \nfrom datasets like Miracl and NQ, leading to high\ntopic diversity. In such cases, Breakpoint-based\nSemantic Chunker outperformed others by better\npreserving topic integrity, splitting sentences based\non semantic dissimilarity to form chunks similar\nto the original documents. In contrast, Fixed-size\nand Clustering-based Chunkers often mixed sentences from different documents, increasing noise\nand lowering retrieval quality.\nAs document length increased, fewer documents\nwere stitched together, reducing topic diversity.\nThis diminished the advantage of Breakpoint-based\nSemantic Chunker, while Clustering-based Semantic Chunker improved. The gap between semantic\nand fixed-size chunkers narrowed, with Fixed-size\nChunker benefiting from higher topic integrity.\nThese results suggest that in real life, the topics\nin a document may not be as diverse as in our\nartificially noisy, stitched data, and hence semantic\nchunkers may not have an edge over fixed-size\nchunker there.\n\n# 5 **Conclusion** \nIn this paper, we evaluated semantic and fixed-size\nchunking strategies in RAG systems across document retrieval, evidence retrieval, and answer generation. Semantic chunking occasionally improved\nperformance, particularly on stitched datasets with\nhigh topic diversity. However, these benefits were\nhighly context-dependent and did not consistently\njustify the additional computational cost. On nonsynthetic datasets that better reflect real-world documents, fixed-size chunking often performed better.\nOverall, our results suggest that fixed-size chunking remains a more efficient and reliable choice for\npractical RAG applications. The impact of chunking strategy was often overshadowed by other factors, such as the quality of embeddings, especially\nwhen computational resources are limited or when\nworking with standard document structures.\n\n# \u2013\n_Loss of Context:_ dividing documents without considering semantic boundaries can result in chunks that lack sufficient context, impairing the model\u2019s\nability to generate accurate and coherent responses. \n# \u2013\n_Incomplete Information Retrieval:_ important information split across chunks\nmay not be effectively retrieved or integrated. \nTo address these issues, we analyse and compare two recent techniques\u2014\ncontextual retrieval [1] and late chunking [9]\u2014within a unified setup, evaluating\ntheir strengths and limitations in tackling challenges like context loss and incomplete information retrieval. Contextual retrieval preserves coherence by prepending LLM-generated context to chunks, while late chunking embeds entire documents to retain global context before segmenting.\nOur study rigorously assesses their impact on generation performance in\nquestion-answering tasks, finding that neither technique offers a definitive solution. This work highlights the trade-offs between these methods and provides\npractical guidance for optimizing RAG systems.\nTo further support the community, we release all code, prompts, and data\nunder the permissive MIT license, enabling full reproducibility and empowering\npractitioners to adapt and extend our work. [2]\n\n## 3.2 **Detailed Performance Analysis** \nTable 2 provides detailed results for each clinical note, showing accuracy scores and token usage across \nall methods. Enhanced CLEAR demonstrates consistent performance across document sizes, with par \nticularly strong results on clinical notes 1, 2, 4, 5, 6, 9, 10, and 11. \nTable 2: Detailed Results by Clinical Note \nNote ID Size (tokens) Wide Sim. RAG Sim. CLEAR Sim. Best Strategy CLEAR Tokens \nclinical ~~n~~ ote1 10,025 0.847 0.807 **0.916** CLEAR 8,446\nclinical ~~n~~ ote2 10,142 0.880 0.849 **0.894** CLEAR 8,493\nclinical ~~n~~ ote3 10,233 **0.929** 0.835 0.909 Wide 8,318\nclinical ~~n~~ ote4 10,098 0.857 0.805 **0.878** CLEAR 8,436\nclinical ~~n~~ ote5 42,011 0.843 0.836 **0.873** CLEAR 8,305\nclinical ~~n~~ ote6 42,181 0.869 0.860 **0.903** CLEAR 8,571\nclinical ~~n~~ ote7 42,072 **0.899** 0.871 0.891 Wide 8,489\nclinical ~~n~~ ote8 42,230 **0.910** 0.861 0.892 Wide 8,500\nclinical ~~n~~ ote9 65,186 0.859 0.870 **0.888** CLEAR 8,497\nclinical ~~n~~ ote10 65,233 0.842 0.791 **0.885** CLEAR 8,485\nclinical ~~n~~ ote11 65,141 0.829 0.830 **0.939** CLEAR 8,414\nclinical ~~n~~ ote12 65,310 0.730 **0.763** 0.742 RAG 8,525 \n6\n\n# 2 **Chunking Strategies** \nIn this paper, a document is first split into sentences\nwhich are then grouped into chunks. We evaluate\nthree chunking strategies, hereafter referred to as \nFigure 1: Illustration of the three chunkers tested in this study. Colored segments represent different topics within\nthe sample document: Purple for psychology, Green for programming, and Yellow for food. Red blocks mark chunk\nbreakpoints. (a) Fixed-size Chunker splits the document into consecutive, uniform chunks without considering\nsemantic content. (b) Breakpoint-based Semantic Chunker segments the text by detecting semantic distance\nthresholds between consecutive sentences to maintain coherence. (c) Clustering-based Semantic Chunker groups\nsemantically similar sentences, potentially combining non-consecutive text to form topic-based chunks. \n\u201cchunkers.\u201d\n\n# Methods: We implemented a Clinical Notes Q&A Evaluation Platform with three retrieval \nstrategies: (1) Wide Context processing for zero-shot inference with large context windows, (2) tra\nditional vector database chunking with semantic search, and (3) entity-aware CLEAR with medical \ndomain knowledge. Evaluation encompassed 12 clinical documents (10K\u201365K tokens) representing \ntypical EHR DocumentReference content.", + "generated_answer": "Hello there! I can certainly help you understand the trade-offs between computational efficiency and semantic integrity for different chunking and retrieval strategies mentioned in your notes.\n\n### Answer:\n\nThe notes discuss several strategies for processing clinical documents, each with its own balance of computational efficiency and semantic integrity.\n\n1. **Fixed-size Chunking:**\n * **Semantic Integrity Trade-offs:** This method splits documents into uniform chunks without considering semantic content. While simpler, it often leads to **loss of context** and **incomplete information retrieval** because important information might be split across different chunks, or sentences from different topics might be mixed, especially in documents with high topic diversity (like \"stitched\" datasets).\n * **Computational Efficiency Trade-offs:** Fixed-size chunking is generally described as a **more efficient and reliable choice** for practical RAG (Retrieval Augmented Generation) applications, particularly on non-synthetic datasets that better reflect real-world documents. It typically incurs **lower computational cost** compared to semantic chunking.\n\n2. **Semantic Chunking (Breakpoint-based and Clustering-based):**\n * **Semantic Integrity Trade-offs:** These methods aim to improve semantic integrity by considering the content.\n * **Breakpoint-based Semantic Chunker** segments text by detecting semantic distance thresholds between consecutive sentences, trying to maintain coherence. It excelled on \"stitched\" datasets with high topic diversity by preserving topic integrity.\n * **Clustering-based Semantic Chunker** groups semantically similar sentences, potentially combining non-consecutive text to form topic-based chunks.\n * However, the notes suggest that while semantic chunking occasionally improved performance, particularly on diverse stitched datasets, these benefits were **highly context-dependent** and did not consistently justify the additional computational cost.\n * **Computational Efficiency Trade-offs:** Semantic chunking often comes with **additional computational cost** compared to fixed-size chunking. As document length increased and topic diversity decreased, the advantage of Breakpoint-based Semantic Chunker diminished, and the gap between semantic and fixed-size chunkers narrowed.\n\n3. **Late Chunking & Contextual Retrieval:**\n * **Semantic Integrity Trade-offs:** These techniques were introduced to address issues like context loss and incomplete information retrieval.\n * **Contextual Retrieval** preserves coherence by prepending LLM-generated context to chunks.\n * **Late Chunking** embeds entire documents to retain global context before segmenting.\n * **Computational Efficiency Trade-offs:** The notes state that neither technique offers a definitive solution and highlights that there are **trade-offs** involved, without going into specific details about their computational efficiency.\n\n4. **Clinical Entity Augmented Retrieval (CLEAR):**\n * **Semantic Integrity Trade-offs:** CLEAR uses **entity-aware retrieval strategies** with medical domain knowledge, which allows it to achieve **stronger semantic similarity** and improved performance (F1 0.90 vs. 0.86 for embedding RAG). This method is designed to capture nuanced clinical relationships, which traditional statistical correlation often fails to do.\n * **Computational Efficiency Trade-offs:** A significant advantage of CLEAR is its efficiency. It operates at **markedly lower token budgets** (over 70% fewer tokens and faster inference) compared to wide-context processing, striking an **optimal balance between semantic accuracy and computational efficiency.**\n\n**Superior Scalability in High-Complexity Clinical Documents:**\n\nThe **Clinical Entity Augmented Retrieval (CLEAR) methodology** demonstrated superior scalability advantages in high-complexity clinical documents. The study found that entity-aware retrieval became increasingly advantageous as document complexity increased, particularly validated by a **75% win rate on documents exceeding 65,000 tokens**. This confirms CLEAR's suitability for enterprise healthcare environments involving comprehensive patient records.\n\n### Key Points/Summary:\n\n* **Fixed-size chunking** is computationally efficient but risks losing semantic context.\n* **Semantic chunking** improves context preservation but often at a higher computational cost, with benefits being context-dependent.\n* **Late Chunking and Contextual Retrieval** aim to enhance semantic integrity by preserving context but involve unstated trade-offs in efficiency.\n* **Clinical Entity Augmented Retrieval (CLEAR)** achieves both strong semantic accuracy and high computational efficiency (fewer tokens, faster inference).\n* **CLEAR** demonstrated superior scalability, especially with large, complex clinical documents (65K+ tokens), making it ideal for enterprise healthcare.\n\n### Extra Tip:\nThink of chunking like cutting a book into smaller sections. Fixed-size chunking is like cutting it every 10 pages, regardless of where the chapters end \u2013 you might cut a sentence in half! Semantic chunking tries to cut only at chapter or paragraph breaks, which is better for understanding but takes more effort to figure out where to cut. CLEAR is like having a smart assistant who knows the book's content and can quickly find exactly the relevant sentences based on specific medical terms, even in a very long book, without having to read everything.", + "correctness_score": 0.9, + "faithfulness_score": 1.0, + "model_tested": "everything" + } +] \ No newline at end of file diff --git a/evaluation/results/evaluation_results_plain.json b/evaluation/results/evaluation_results_plain.json new file mode 100644 index 0000000..62b165f --- /dev/null +++ b/evaluation/results/evaluation_results_plain.json @@ -0,0 +1,74 @@ +[ + [ + 0.95, + 0.0, + 0.0, + 0.2, + 0.0, + 0.1, + 0.95 + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "generated_answer": "Hello there! That's a great question about a really important aspect of RAG systems.\n\n### **Answer:**\n\nThe defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems is that **it groups text based on the coherence of its meaning or a complete idea, rather than just arbitrary size or fixed boundaries.**\n\nInstead of simply splitting a document into chunks of, say, 256 words, semantic chunking aims to keep related sentences and paragraphs together if they discuss the same concept, topic, or argument. The goal is for each chunk to be a self-contained unit of meaning, even if its length varies.\n\n### **Key Points/Summary:**\n\n* **Meaning-Based Grouping:** Chunks are formed around complete thoughts, topics, or arguments.\n* **Coherence:** Each chunk strives to be a self-contained, contextually rich unit.\n* **Variable Length:** Chunk sizes are not fixed but adapt to the natural breaks in meaning.\n* **Improved Relevance:** Leads to more relevant information being retrieved because chunks aren't cut off mid-idea.\n\n### **Extra Tip:**\n\nThink of it like organizing a bookshelf. Instead of just splitting books into piles of 200 pages each (which might cut a chapter in half!), semantic chunking is like keeping entire chapters or even whole books together if they deal with the same subject. This way, when you look for information on a specific topic, you're much more likely to pull out a complete and useful section!", + "correctness_score": 0.95, + "model_tested": "plain" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "generated_answer": "Hey there! It's a great question about how we can indirectly measure the effectiveness of chunking. When you can't directly measure \"chunk quality,\" you use proxy tasks that should improve if the chunking is good.\n\nHere are three primary proxy tasks commonly designed to indirectly evaluate the quality of chunking strategies:\n\n* **Memory Recall/Retention:** This task evaluates how well participants can remember the information that has been chunked. The assumption is that better chunking leads to more effective encoding and retrieval of information, resulting in higher recall scores or better retention over time.\n* **Comprehension/Understanding:** This task assesses how well participants grasp the meaning, relationships, and overall message of the content after it has been chunked. If information is chunked effectively, it should be easier to process and understand, which can be measured through comprehension questions or summary tasks.\n* **Search/Navigation Efficiency:** This task measures how quickly and accurately participants can find specific pieces of information or navigate through the chunked content to complete a particular goal. Good chunking should create logical and intuitive groups of information, making it easier and faster for users to locate what they need.\n\n---\n\n**Key Points/Summary:**\n\n* **Memory Recall:** Measures how much information is remembered from the chunks.\n* **Comprehension:** Assesses how well the chunked information is understood.\n* **Search Efficiency:** Evaluates how quickly and accurately information can be found within the chunks.\n\n---\n\n**Extra Tip:** Think of it like organizing your school notes! If you chunk them well (e.g., by topic, date, or concept), you'll likely remember more, understand the material better, and find specific details much faster when you're studying for an exam. These proxy tasks are trying to measure those exact benefits!", + "correctness_score": 0.0, + "model_tested": "plain" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "generated_answer": "It's a great question! Preserving global context is a key challenge in RAG systems, as traditional fixed-size chunking can often break up important information or isolate facts from their broader meaning.\n\nHere are two advanced chunking techniques aimed at preserving global context:\n\n1. **Parent-Child Chunking (or Hierarchical Chunking):**\n * **How it works:** This technique involves creating chunks at two different granularities. You create smaller, more precise chunks (the \"children\") that are optimized for retrieval (e.g., individual sentences, or small paragraphs). Alongside these, you also create larger, more comprehensive chunks (the \"parents\") that contain these smaller chunks (e.g., full paragraphs, sections, or even entire documents).\n * **Preserving Global Context:** During retrieval, the RAG system searches through the smaller, child chunks to find the most relevant pieces of information. Once a relevant child chunk is identified, its corresponding *larger parent chunk* (or a section of it) is passed to the Language Model (LLM). This ensures the LLM receives a broader context surrounding the specific retrieved information, helping it understand the full meaning and relationships.\n\n2. **Sentence Window Retrieval:**\n * **How it works:** With this method, the document is initially chunked into individual sentences. When a query is made, the RAG system first identifies the most relevant *individual sentences* (or very small, precise chunks) that best answer the query. Instead of just passing those exact sentences to the LLM, it then retrieves a \"window\" of surrounding sentences (e.g., 1-3 sentences before and after the relevant sentence). This expanded \"window\" of sentences is what is ultimately sent to the LLM.\n * **Preserving Global Context:** The initial retrieval is highly precise (at the sentence level), which helps in pinpointing exact answers and reducing noise. However, by adding the surrounding \"window\" of sentences, the technique ensures that the LLM receives enough local context to fully interpret the retrieved information, preventing key facts from being presented in isolation.\n\n---\n\n**Key Points/Summary:**\n\n* **Parent-Child Chunking:** Uses smaller chunks for retrieval and larger chunks for LLM input to provide broad context.\n* **Sentence Window Retrieval:** Retrieves precise relevant sentences, then expands to include a \"window\" of surrounding sentences for local context.\n* Both techniques aim to provide the LLM with more contextual information than just the direct match.\n\n**Extra Tip:** Think of Parent-Child Chunking like finding a specific page in a book (child chunk) but then reading the entire chapter (parent chunk) to understand it. Sentence Window Retrieval is like finding a specific sentence and then reading a few sentences before and after to get the immediate context.", + "correctness_score": 0.0, + "model_tested": "plain" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "generated_answer": "**Answer:**\n\nThe key phenomenon you're referring to is often called **attention locality** or **attention consistency within chunks**.\n\nIt describes the observation that when a Long Language Model (LLM) is generating a new segment (or \"chunk\") of text, the tokens within that newly generated chunk tend to consistently attend to a *specific and localized* set of previous chunks in the input context, rather than scattering their attention broadly across the entire long history.\n\n**Key Points/Summary:**\n\n* **Localized Attention:** Tokens within a *newly generated chunk* primarily focus their attention on a limited, consistent set of *earlier chunks*.\n* **Consistency:** This attention pattern is largely consistent across multiple tokens within the same generated chunk.\n* **ChunkLLM Exploitation:** ChunkLLM leverages this property to predict which past chunks will be relevant, allowing it to efficiently load only necessary parts of the context into memory, significantly improving inference speed for long contexts.\n\n**Extra Tip:**\nThink of it like reading a long book. When you're writing a summary for a specific chapter, you don't reread the *entire* book every time you write a sentence. Instead, you primarily focus on the current chapter and perhaps a few closely related preceding chapters, knowing that the most relevant information is usually nearby. This is analogous to attention locality!", + "correctness_score": 0.2, + "model_tested": "plain" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "generated_answer": "The methodology you're describing is often referred to as **Clinical Semantic Information Retrieval** or **Knowledge Graph-Enhanced Clinical Information Retrieval**.\n\nThis approach goes beyond simple keyword matching by understanding the specific clinical concepts (entities) within the Electronic Health Record (EHR) notes and using this understanding to guide the retrieval process.\n\nHere's how it works:\n1. **Entity Recognition and Linking:** It first employs Natural Language Processing (NLP) techniques, such as Named Entity Recognition (NER), to identify and extract relevant clinical entities from the unstructured text of EHR notes. These entities could be diseases, symptoms, medications, procedures, anatomical sites, etc. Crucially, these identified entities are then linked to standardized clinical ontologies and terminologies (like SNOMED CT, UMLS, RxNorm) to provide a consistent, unambiguous semantic meaning.\n2. **Knowledge Representation (e.g., Knowledge Graphs):** The extracted entities and their relationships are often used to build or enrich a structured representation, such as a clinical knowledge graph. This graph models the complex connections between different clinical concepts.\n3. **Entity-Aware Retrieval Strategies:** Instead of searching for literal strings, retrieval queries are formulated using these semantically linked entities. For example, a query might ask for all patients diagnosed with \"myocardial infarction\" (a specific entity) rather than just documents containing the phrase \"heart attack.\" The system understands that \"heart attack\" and \"myocardial infarction\" refer to the same concept. It can also retrieve information based on relationships between entities, like finding all medications prescribed for a specific condition.\n\nThis methodology leverages the deep understanding of clinical entities and their context to achieve its goals.\n\n---\n\n**Key Points/Summary:**\n* **Entity Identification:** Extracts specific clinical concepts (diseases, drugs, symptoms) from EHR notes using NLP.\n* **Semantic Linking:** Links identified entities to standardized clinical ontologies for consistent understanding.\n* **Knowledge-Based Retrieval:** Uses these semantically enriched entities to guide information retrieval, often from structured knowledge representations like graphs.\n* **Improved Outcomes:** Leads to more accurate and contextually relevant results, processed with greater efficiency.\n\n---\n\n**Extra Tip:**\nThink of it like searching for a specific book by its author, genre, and themes in a library catalog, rather than just looking for keywords on the spines of books on shelves. The \"entity-aware\" part means the system knows what an \"author\" is, what a \"genre\" is, and how they relate to a \"book,\" allowing for much more precise and relevant retrieval than a simple text search.", + "correctness_score": 0.0, + "model_tested": "plain" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "generated_answer": "It's great you're diving into the nuances of long-context models! Both ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) are clever ways to tackle the computational challenge of long sequences, but they approach dynamic attention management quite differently.\n\nLet's break them down:\n\n### ChunkLLM\n\n**Core Idea:** ChunkLLM operates by dividing the input sequence into fixed-size \"chunks\" and then introducing a set of \"global\" tokens (or summary tokens) that represent the information within each chunk.\n\n**How it derives chunk representations:**\n* **Explicit Summary Tokens:** For each fixed-size chunk of tokens, ChunkLLM explicitly computes one or more \"global\" or \"summary\" tokens. These global tokens are typically derived by having all local tokens within that chunk attend to a special global token, or by an aggregation mechanism (like averaging or a dedicated attention head) that condenses the chunk's information into this global representation. Think of it like taking the most important points from a paragraph and putting them into a single summary sentence.\n\n**How it utilizes them for efficiency and performance:**\n* **Efficiency:** The key is that local tokens (tokens within a chunk) attend only to other tokens within their own chunk *and* to all the global tokens from *all* other chunks. Global tokens, in turn, attend to all other global tokens. This significantly reduces the quadratic complexity of full self-attention. Instead of every token attending to every other token, tokens only attend to a limited local scope and a much smaller set of global summaries.\n* **Performance Preservation:** By having global tokens capture the essential information from each chunk, local tokens can still access a broad context. They don't need to attend to every single token in the entire sequence; they can infer the relevant context from the global summaries, which act as information bottlenecks, ensuring crucial information from distant parts of the sequence is still available.\n\n---\n\n### Dynamic Hierarchical Sparse Attention (DHSA)\n\n**Core Idea:** DHSA takes a more adaptive and dynamic approach. Instead of fixed chunks, it dynamically identifies important tokens (often called \"landmarks\" or \"centroids\") and forms clusters of related tokens around them. The attention patterns are then made sparse based on these dynamic clusters.\n\n**How it derives chunk representations:**\n* **Implicit & Dynamic Clustering:** DHSA doesn't necessarily *derive* a single, explicit summary token for a fixed chunk in the same way ChunkLLM does. Instead, it dynamically identifies \"landmark\" tokens or forms \"clusters\" of tokens based on their similarity (e.g., using k-means clustering on query/key embeddings) or importance. The \"representation\" of a dynamic group or cluster is often implicit through the landmark token itself, or through a centroid vector that represents the average of tokens within a cluster. It's less about a pre-defined summary token and more about *which* tokens are deemed important and *which* tokens are grouped together based on their content.\n\n**How it utilizes them for efficiency and performance:**\n* **Efficiency:** Tokens primarily attend to other tokens within their dynamically formed cluster and to a smaller set of landmark tokens (or cluster centroids) that represent broader context. This drastically reduces the number of attention computations because tokens are not attending to the entire sequence, but only to their relevant local cluster and a few critical global points. The attention graph is sparse and content-dependent.\n* **Performance Preservation:** By dynamically adapting the attention pattern to the content, DHSA ensures that critical information pathways are maintained. If two tokens are highly related, they are likely to be in the same cluster or have a landmark token that connects them, even if they are far apart in the original sequence. This allows the model to focus its attention on the most relevant parts of the context, preserving performance by adapting to the task and data.\n\n---\n\n### Contrast: ChunkLLM vs. DHSA\n\n| Feature | ChunkLLM | Dynamic Hierarchical Sparse Attention (DHSA) |\n| :---------------------------- | :------------------------------------------------ | :------------------------------------------------ |\n| **Chunk Formation** | **Fixed-size, pre-defined chunks.** | **Dynamic, content-dependent clusters/groups.** |\n| **Chunk Representation** | **Explicit \"global/summary\" tokens** computed for each fixed chunk. | **Implicit via landmark tokens/cluster centroids** derived from token similarity/importance. |\n| **Dynamism** | **Limited dynamism** in chunk formation; dynamism comes from global token interactions. | **Highly dynamic,** attention pattern adapts to the input content. |\n| **Efficiency Mechanism** | Local tokens attend to their chunk + all global tokens. Global tokens attend to all global tokens. | Tokens attend within their dynamic cluster + to landmark tokens/cluster representatives. |\n| **Performance Preservation** | Global tokens act as information bottlenecks, providing broad context. | Adapts attention to critical semantic connections, ensuring important pathways are maintained. |\n\n---\n\n**Key Points/Summary:**\n\n* **ChunkLLM** uses fixed-size chunks and explicit \"global tokens\" to summarize each chunk, allowing local tokens to access broad context via these summaries.\n* **DHSA** dynamically forms clusters of tokens based on content similarity or importance, with tokens primarily attending within their cluster and to a few \"landmark\" tokens.\n* The main difference lies in **how chunk representations are derived** (explicit summary vs. implicit dynamic clusters) and the **dynamism of the attention pattern** (fixed chunks vs. content-adaptive sparsity).\n\n**Extra Tip:**\nThink of ChunkLLM like a well-organized textbook where each chapter has a fixed-length summary. You read your section, and if you need broader context, you skim all the chapter summaries. DHSA, on the other hand, is like a highly intelligent research assistant who dynamically identifies the most important concepts and their related ideas across a vast document, linking them directly, and ignoring irrelevant details.", + "correctness_score": 0.1, + "model_tested": "plain" + }, + { + "question_id": 7, + "document_path": "multiple files", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "generated_answer": "Hey there! Let's break down these chunking and retrieval strategies to understand their trade-offs between getting answers quickly (computational efficiency) and making sure those answers are accurate and meaningful (semantic integrity).\n\nThe core tension is that simpler, faster methods often risk breaking apart important pieces of information, while methods that try to preserve meaning can be more complex and thus slower.\n\n---\n\n### **Answer:**\n\nThe observed trade-offs between computational efficiency and semantic integrity vary significantly across different chunking and retrieval strategies.\n\n1. **Fixed-size/Semantic Chunking:**\n * **Fixed-size Chunking:**\n * **Computational Efficiency:** High. It's very fast and straightforward to divide documents into chunks of a predetermined number of tokens or characters.\n * **Semantic Integrity:** Low. This method often cuts off sentences, paragraphs, or even entire sections mid-thought, destroying the semantic coherence and making the retrieved information hard to understand or potentially misleading.\n * **Semantic Chunking:**\n * **Computational Efficiency:** Moderate. It's more complex than fixed-size as it requires identifying natural breaks (e.g., paragraphs, headings, sentence boundaries) or even using NLP to detect topic shifts. This processing adds overhead.\n * **Semantic Integrity:** Moderate to High. By respecting natural document structures, it significantly improves semantic integrity compared to fixed-size chunks, ensuring that retrieved chunks are more self-contained and meaningful.\n\n2. **Late Chunking:**\n * **Concept:** Instead of chunking the entire document collection upfront, this strategy first retrieves larger, coarser-grained document units (e.g., entire documents or large sections) based on a query. Only *then* are these smaller, pre-selected documents/sections chunked more finely for detailed analysis.\n * **Computational Efficiency:** High. The most intensive chunking (fine-grained) is performed only on a small subset of potentially relevant data, rather than the entire corpus. This saves significant pre-processing time and storage.\n * **Semantic Integrity:** Moderate to High. If the initial coarse retrieval is effective at identifying relevant larger units, then the subsequent fine-grained chunking can preserve semantic integrity within that relevant scope. However, poor initial retrieval could miss relevant context entirely.\n\n3. **Contextual Retrieval:**\n * **Concept:** This strategy focuses not just on retrieving the most relevant chunk, but also on expanding that retrieval to include the surrounding context (e.g., preceding and succeeding sentences, paragraphs, or even a whole section) to provide a richer understanding.\n * **Computational Efficiency:** Moderate to Low. While the initial chunk retrieval might be fast, the subsequent step of identifying and extracting surrounding context adds computational overhead. Furthermore, providing larger contexts to downstream models (like LLMs) increases their input size and processing time.\n * **Semantic Integrity:** High. By explicitly including surrounding information, this method significantly enhances semantic integrity, reducing ambiguity and providing the necessary background for accurate interpretation of the retrieved information.\n\n4. **Clinical Entity Augmented Retrieval (CLEAR):**\n * **Concept:** This advanced strategy leverages specific domain knowledge by identifying and using clinical entities (e.g., diseases, medications, symptoms, procedures) within the documents. Retrieval is augmented by matching these entities in queries and documents, potentially using entity embeddings, semantic graphs, or custom indexing.\n * **Computational Efficiency:** Moderate to Low (initial setup), High (retrieval phase). The initial process of extracting, normalizing, and indexing clinical entities can be computationally very intensive. However, once this \"entity graph\" or augmented index is built, the *retrieval* phase can be exceptionally efficient and precise, as it directly targets semantically rich concepts.\n * **Semantic Integrity:** Very High. By focusing on the core clinical concepts and their relationships, CLEAR ensures that retrieved information is highly relevant, semantically accurate, and less prone to misinterpretation caused by ambiguous language or fragmented context, which is common in clinical text.\n\n---\n\n### **Superior Scalability Advantages in High-Complexity Clinical Documents:**\n\nIn high-complexity clinical documents, which are often long, dense with specialized terminology, and contain interconnected concepts, **Clinical Entity Augmented Retrieval (CLEAR)** demonstrated superior scalability advantages.\n\nWhile its initial setup (entity extraction, linking, and indexing) can be computationally intensive, this upfront investment pays off significantly during retrieval. CLEAR excels because:\n\n* **Precision:** It drastically improves the precision of retrieval by directly targeting semantically rich clinical concepts. This means it retrieves *less irrelevant information* and *more highly pertinent information*.\n* **Reduced Downstream Burden:** By providing highly focused and semantically coherent chunks, CLEAR reduces the amount of \"noise\" that downstream processing systems (like large language models) need to sift through. This leads to more efficient and accurate processing by these systems, effectively scaling better in terms of *getting the right answer quickly* from complex data, even if the raw number of operations for chunking/indexing is higher initially.\n* **Handles Domain Specificity:** It inherently understands the unique challenges and nuances of clinical language, which fixed-size or even general semantic chunking cannot effectively address.\n\nTherefore, while requiring more sophisticated initial processing, CLEAR ultimately provides a more scalable and effective solution for navigating and extracting insights from vast amounts of complex clinical data due to its high semantic integrity and precision during retrieval.\n\n---\n\n### **Key Points/Summary:**\n\n* **Efficiency vs. Integrity:** Simple chunking is efficient but loses meaning; complex chunking preserves meaning but adds computational cost.\n* **Late Chunking:** Efficient by delaying fine-grained chunking until after coarse retrieval.\n* **Contextual Retrieval:** Improves integrity by adding surrounding context, but increases downstream processing.\n* **CLEAR:** High initial cost for entity extraction but delivers superior semantic integrity and retrieval precision, especially in complex domains.\n* **Scalability Winner:** **Clinical Entity Augmented Retrieval (CLEAR)** offers superior scalability in high-complexity clinical documents due to its high precision and ability to reduce the burden of irrelevant information on downstream systems.\n\n---\n\n### **Extra Tip:**\n\nThink of it like building a specialized library. Fixed-size chunking is like cutting books into 10-page segments regardless of chapters. Semantic chunking is like cutting books by chapter. Late chunking is like first picking out relevant books, *then* cutting them by chapter. Contextual retrieval is like picking a chapter and also grabbing the previous and next chapters. CLEAR is like having a librarian who knows exactly what every medical term means and can instantly find precisely the paragraph you need, even if it took a lot of work to train that librarian initially!", + "correctness_score": 0.95, + "model_tested": "plain" + } +] \ No newline at end of file diff --git a/evaluation/results/evaluation_results_rerank.json b/evaluation/results/evaluation_results_rerank.json new file mode 100644 index 0000000..931d4d5 --- /dev/null +++ b/evaluation/results/evaluation_results_rerank.json @@ -0,0 +1,109 @@ +[ + [ + [ + 0.97, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 1.0, + 1.0 + ], + [ + 0.95, + 1.0 + ], + [ + 0.1, + 1.0 + ], + [ + 0.78, + 1.0 + ] + ], + { + "question_id": 1, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What is the defining characteristic of 'semantic chunking' in Retrieval-Augmented Generation (RAG) systems?", + "ground_truth": "Semantic chunking is a strategy that aims to improve retrieval performance by dividing documents into semantically coherent segments. This approach segments documents based on semantic similarity or detecting semantic distance thresholds between consecutive sentences to maintain coherence", + "retrieved_context": "Is Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\nIs Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\nneed for more efficient chunking strategies in RAG systems. 1 Introduction In Retrieval-Augmented Generation (RAG) sys- tems, cutting documents into smaller units called chunks has a crucial effect on the quality of both retrieval and generation tasks (Chen et al., 2023; Wadhwa et al., 2024; Shi et al., 2023; Yu et al., 2023). By retrieving the most relevant chunks for a given query and feeding them into a generative language model, these systems aim to produce ac- curate and contextually appropriate\n\nneed for more efficient chunking strategies in RAG systems. 1 Introduction In Retrieval-Augmented Generation (RAG) sys- tems, cutting documents into smaller units called chunks has a crucial effect on the quality of both retrieval and generation tasks (Chen et al., 2023; Wadhwa et al., 2024; Shi et al., 2023; Yu et al., 2023). By retrieving the most relevant chunks for a given query and feeding them into a generative language model, these systems aim to produce ac- curate and contextually appropriate\n\nclusions. Dataset Fixed-size Breakpoint Clustering ExpertQA 0.65 0.65 0.65 DelucionQA 0.76 0.76 0.76 TechQA 0.68 0.68 0.68 ConditionalQA 0.42 0.43 0.43 Qasper 0.49 0.49 0.50 Table 3: BERTScore for Answer Generation. 5 Conclusion In this paper, we evaluated semantic and fixed-size chunking strategies in RAG systems across docu- ment retrieval, evidence retrieval, and answer gen- eration. Semantic chunking occasionally improved performance, particularly on stitched datasets with high topic diversity.\n\nclusions. Dataset Fixed-size Breakpoint Clustering ExpertQA 0.65 0.65 0.65 DelucionQA 0.76 0.76 0.76 TechQA 0.68 0.68 0.68 ConditionalQA 0.42 0.43 0.43 Qasper 0.49 0.49 0.50 Table 3: BERTScore for Answer Generation. 5 Conclusion In this paper, we evaluated semantic and fixed-size chunking strategies in RAG systems across docu- ment retrieval, evidence retrieval, and answer gen- eration. Semantic chunking occasionally improved performance, particularly on stitched datasets with high topic diversity.\n\nbenefits of semantic chunk- ing, suggesting that its advantages are highly task- dependent and often insufficient to justify the added computational costs. This study lays the ground- work for future exploration of more efficient and adaptive chunking strategies in RAG systems. In general, our contributions are: We present a novel, large-scale evaluation framework comparing semantic and fixed-size chunking across diverse tasks. We demonstrate that while semantic chunk- ing shows some benefits in certain\n\nbenefits of semantic chunk- ing, suggesting that its advantages are highly task- dependent and often insufficient to justify the added computational costs. This study lays the ground- work for future exploration of more efficient and adaptive chunking strategies in RAG systems. In general, our contributions are: We present a novel, large-scale evaluation framework comparing semantic and fixed-size chunking across diverse tasks. We demonstrate that while semantic chunk- ing shows some benefits in certain\n\nRecently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic\n\nRecently, there has been a surge of interest in semantic chunking, where documents are seg- mented based on semantic similarity, with some in- dustry applications suggesting promising improve- ments in performance (LangChain, 2024; LlamaIn- dex, 2024; McCormick, 2024). However, there is no systematic evidence that semantic chunking yields a performance gain in downstream tasks, and if there is, the gain is significant enough to justify the computational overhead than fixed-size chunking. Such a systematic", + "generated_answer": "**Answer:**\nIn Retrieval-Augmented Generation (RAG) systems, the defining characteristic of 'semantic chunking' is its approach to dividing documents into segments that are **semantically coherent** or based on **semantic similarity**. This means that instead of just cutting documents into fixed-size pieces, semantic chunking aims to keep related ideas and topics together within each chunk, intending to improve retrieval performance.\n\n**Key Points/Summary:**\n* **Semantic Coherence:** Documents are divided into segments that are meaningful and self-contained in terms of their topic.\n* **Semantic Similarity:** Chunks are created by grouping text based on how similar their meanings are.\n* **Purpose:** The goal is to enhance the quality of information retrieval in RAG systems.\n\n**Extra Tip:**\nImagine a book where each chapter discusses a single, distinct topic. Semantic chunking tries to achieve a similar effect by ensuring that each \"chunk\" of a document focuses on one coherent idea, making it easier for a RAG system to find exactly what it needs when answering a query.", + "correctness_score": 0.97, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 2, + "document_path": "static/Computational cost of semantic chunking.pdf", + "question": "What three primary proxy tasks were designed to indirectly evaluate the quality of chunking strategies in the study presented in the sources?", + "ground_truth": "The study systematically evaluated the effectiveness of chunking strategies using three proxy tasks: document retrieval, evidence retrieval, and retrieval-based answer generation", + "retrieved_context": "evaluation is not trivial due to the lack of data that can be directly used to com- pare chunking strategies. Therefore, we design an indirect evaluation using three proxy tasks: (1) doc- ument retrieval, measuring the ability to identify relevant documents; (2) evidence retrieval, mea- suring the ability to locate ground-truth evidence; and (3) answer generation, testing the quality of answers produced by a generative model using re- trieved chunks. Our findings challenge prevailing assumptions about the\n\nevaluation is not trivial due to the lack of data that can be directly used to com- pare chunking strategies. Therefore, we design an indirect evaluation using three proxy tasks: (1) doc- ument retrieval, measuring the ability to identify relevant documents; (2) evidence retrieval, mea- suring the ability to locate ground-truth evidence; and (3) answer generation, testing the quality of answers produced by a generative model using re- trieved chunks. Our findings challenge prevailing assumptions about the\n\nanswers. 4 Results 4.1 Measuring and reporting performances As mentioned earlier, we used three proxy tasks the study chunking. We cannot directly assess the quality of retrieval at the chunk level due to the lack of ground-truth at the chunk level. Instead, each retrieved chunk is mapped back to either the source document or the included evidence sentences. Since the number of relevant documents or evi- dence sentences is not fixed (unlike the k value for retrieved chunks), traditional metrics such as Re-\n\nanswers. 4 Results 4.1 Measuring and reporting performances As mentioned earlier, we used three proxy tasks the study chunking. We cannot directly assess the quality of retrieval at the chunk level due to the lack of ground-truth at the chunk level. Instead, each retrieved chunk is mapped back to either the source document or the included evidence sentences. Since the number of relevant documents or evi- dence sentences is not fixed (unlike the k value for retrieved chunks), traditional metrics such as Re-\n\nembeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely\n\nembeddings is necessary before definitively concluding the limitations of semantic chunking. Lack of Chunk Quality Measures As noted in Section 4, while the output chunks differed be- tween methods, retrieval and generation perfor- mances were similar across chunkers. In addition to the influence of embedding models, the absence of direct chunk quality metrics likely contributed to this issue. Having ground-truth query-chunk rel- evance scores would provide more accurate evalua- tions than relying solely\n\nthresholds are generally higher than those from percentiles or interquartile ranges. Fixed-size Chunker Figure 5 shows results for the Fixed-size Chunker. The trends mirror those seen in other chunkers. Adding one overlapping sentence between chunks does not notably improve performance, indicating that a single overlapping sentence is insufficient to significantly boost con- textual coherence. E.4 Chunk Inspection We examined the output chunks to (1) confirm that different chunkers were functioning as\n\nthresholds are generally higher than those from percentiles or interquartile ranges. Fixed-size Chunker Figure 5 shows results for the Fixed-size Chunker. The trends mirror those seen in other chunkers. Adding one overlapping sentence between chunks does not notably improve performance, indicating that a single overlapping sentence is insufficient to significantly boost con- textual coherence. E.4 Chunk Inspection We examined the output chunks to (1) confirm that different chunkers were functioning as\n\nTable 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the\n\nTable 2, Fixed-size Chunker per- formed best on 3 out of 5 datasets, indicating a slight edge in capturing core evidence sentences. However, the performance differences between the Fixed-size Chunker and the two semantic chunkers were minimal, suggesting no clear advantage for any specific chunking strategy. See Appendix B for more details. Further inspection revealed that despite varia- tions in chunking methods, the top-k retrieved chunks frequently contained the same evidence sen- tences, explaining the", + "generated_answer": "Hello there! I can certainly help you with that.\n\nAccording to the notes, evaluating chunking strategies directly is challenging due to a lack of suitable data. Therefore, an **indirect evaluation** was designed using three primary proxy tasks. These tasks help to measure different aspects related to chunking performance:\n\n1. **Document Retrieval:** This task assesses the ability of the chunking strategy to help identify relevant documents from a collection.\n2. **Evidence Retrieval:** This task measures how well the strategy can locate the specific \"ground-truth evidence\" within the documents.\n3. **Answer Generation:** This task evaluates the quality of answers produced by a generative model when it uses the chunks retrieved by the chunking strategy.\n\nThese proxy tasks allow researchers to understand the effectiveness of different chunking methods even without direct chunk-level ground-truth data.\n\n### Key Points/Summary:\n* Evaluation of chunking strategies is indirect due to data limitations.\n* Three proxy tasks are used: Document Retrieval, Evidence Retrieval, and Answer Generation.\n* Document Retrieval measures identification of relevant documents.\n* Evidence Retrieval measures the ability to locate ground-truth evidence.\n* Answer Generation assesses the quality of model-generated answers using retrieved chunks.\n\n### Extra Tip:\nThink of these proxy tasks as a way to test the \"downstream\" impact of chunking. Instead of directly saying \"this chunk is good,\" we're saying \"if we use chunks created this way, does it lead to better document finding, better evidence spotting, or better answers?\" It's like evaluating a chef's knife by how well the meal tastes, rather than just how sharp the blade is in isolation!", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 3, + "document_path": "static/Reconstructing Context.pdf", + "question": "What are two advanced chunking techniques, besides traditional early chunking, aimed at preserving global context within RAG systems?", + "ground_truth": "Two advanced techniques introduced to preserve global context and mitigate context fragmentation are late chunking and contextual retrieval. Late chunking involves embedding the entire document first before segmentation to retain global context, potentially leading to superior results across various retrieval tasks.", + "retrieved_context": "to preserve global context. Despite their potential, their comparative strengths and limitations remain unclear. This study presents a rigorous analysis of late chunking and contextual retrieval, evaluating their effectiveness and efficiency in optimizing RAG systems. Our results indicate that contextual retrieval preserves semantic coher- ence more effectively but requires greater computational resources. In contrast, late chunking offers higher efficiency but tends to sacrifice rel- evance and\n\nto preserve global context. Despite their potential, their comparative strengths and limitations remain unclear. This study presents a rigorous analysis of late chunking and contextual retrieval, evaluating their effectiveness and efficiency in optimizing RAG systems. Our results indicate that contextual retrieval preserves semantic coher- ence more effectively but requires greater computational resources. In contrast, late chunking offers higher efficiency but tends to sacrifice rel- evance and\n\nmodels,weaimtoimprovetheoverallretrievalefficiencyandresponsegeneration within the RAG framework. Early Chunking. Documents are segmented into text chunks, and each chunk is processed by the embedding model. The model generates token-level embed- dings for each chunk, which are subsequently aggregated using mean pooling to produce a single embedding per chunk. Late Chunking. Late chunking defers the chunking process. As shown in Figure 3.1, instead of segmenting the document initially, the entire document\n\nmodels,weaimtoimprovetheoverallretrievalefficiencyandresponsegeneration within the RAG framework. Early Chunking. Documents are segmented into text chunks, and each chunk is processed by the embedding model. The model generates token-level embed- dings for each chunk, which are subsequently aggregated using mean pooling to produce a single embedding per chunk. Late Chunking. Late chunking defers the chunking process. As shown in Figure 3.1, instead of segmenting the document initially, the entire document\n\n4 J. Singh and C. Merola to align with the early and late chunking strategies under evaluation. This ad- justment allows us to explore how different embedding techniques influence the retrieval quality and, subsequently, the overall performance of the RAG system. Additionally, we test dynamic segmenting models to further refine the chunk- ing process, providing an adaptive mechanism that adjusts chunk sizes based on content characteristics. By evaluating the impact of these dynamic segmenting\n\n4 J. Singh and C. Merola to align with the early and late chunking strategies under evaluation. This ad- justment allows us to explore how different embedding techniques influence the retrieval quality and, subsequently, the overall performance of the RAG system. Additionally, we test dynamic segmenting models to further refine the chunk- ing process, providing an adaptive mechanism that adjusts chunk sizes based on content characteristics. By evaluating the impact of these dynamic segmenting\n\nofcontextual retrieval versus tra- ditional early chunkingstrategies, utilizingdifferent text segmenters and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG systems. 3.1 RQ#1: Early or Late Chunking? In this workflow, the main architectural modification compared to the standard RAG lies in the document embedding process Figure 3.1. Specifically, we experi- ment with various embedding models to encode document chunks, tailoring them 3 chunking/\n\nofcontextual retrieval versus tra- ditional early chunkingstrategies, utilizingdifferent text segmenters and embedding models to evaluate their impact on retrieval accuracy and downstream performance in RAG systems. 3.1 RQ#1: Early or Late Chunking? In this workflow, the main architectural modification compared to the standard RAG lies in the document embedding process Figure 3.1. Specifically, we experi- ment with various embedding models to encode document chunks, tailoring them 3 chunking/\n\nRAG. A standard RAG workflow involves four main stages: document segmentation, chunk embedding, indexing, and retrieval. During segmentation, documents are divided into manageable chunks. These chunks are then trans- formed into vector representations using encoder models, often normalized to 1 2\n\nRAG. A standard RAG workflow involves four main stages: document segmentation, chunk embedding, indexing, and retrieval. During segmentation, documents are divided into manageable chunks. These chunks are then trans- formed into vector representations using encoder models, often normalized to 1 2", + "generated_answer": "Hello there! I can certainly help you with that.\n\nBased on your notes, two advanced techniques, besides traditional early chunking, that are aimed at preserving global context within RAG systems are **Late Chunking** and **Contextual Retrieval**.\n\nLet's break them down:\n\n1. **Late Chunking:**\n * **How it works:** Unlike early chunking where the document is segmented right away, late chunking defers this process. Instead, the entire document is processed initially, and chunking happens later in the workflow.\n * **Goal regarding context:** The notes mention that both late chunking and contextual retrieval are studied \"to preserve global context.\"\n * **Characteristics:** It offers higher efficiency but, according to the notes, \"tends to sacrifice relevance\" compared to contextual retrieval.\n\n2. **Contextual Retrieval:**\n * **How it works:** Your notes present contextual retrieval as a technique evaluated alongside late chunking for optimizing RAG systems. It's compared against traditional early chunking strategies.\n * **Goal regarding context:** The notes explicitly state that \"contextual retrieval preserves semantic coherence more effectively.\" Semantic coherence is crucial for maintaining the overall meaning and context of a document.\n * **Characteristics:** While effective at preserving semantic coherence, it \"requires greater computational resources.\"\n\n### Key Points/Summary:\n\n* **Late Chunking:** Defers the chunking process, handling the entire document initially to help preserve global context, offering higher efficiency but potentially sacrificing some relevance.\n* **Contextual Retrieval:** Focuses on preserving semantic coherence more effectively, which directly contributes to maintaining global context, though it demands more computational resources.\n\n### Extra Tip:\n\nThink of \"global context\" like understanding the full story of a book.\n* **Early Chunking** might be like reading individual chapters without always remembering how they connect to the overarching plot.\n* **Late Chunking** tries to grasp the whole book first before diving into details, aiming to keep the main storyline in mind.\n* **Contextual Retrieval** specifically focuses on ensuring that even when you look at parts of the book, you understand their meaning within the entire narrative, maintaining the \"semantic coherence\" of the whole story.", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 4, + "document_path": "static/Accelerating LLM Inference.pdf", + "question": "In the context of long-context LLMs, what is the key phenomenon called where chunks attended to by tokens within a generated chunk exhibit substantial consistency, which ChunkLLM exploits to enhance inference efficiency?", + "ground_truth": "This phenomenon is called the Intra-Chunk Attention Consistency (ICAC) pattern. ChunkLLM exploits ICAC by updating chunk selection only when the currently decoded token is identified as a chunk boundary.", + "retrieved_context": "exploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nperplexity (ppl) on the PG19 test set, and results are summarized in Table 2. Compared to the vanilla model, ChunkLLM yields a slight enhancement in ppl alongside a significant decrease in total infer- ence time. The underlying reason is that while the vanilla model maintains semantic integrity, it incurs linearly increasing inference time as gen- erated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up inference by leveraging its chunk selection and ICAC mechanisms. 3.2.4\n\nperplexity (ppl) on the PG19 test set, and results are summarized in Table 2. Compared to the vanilla model, ChunkLLM yields a slight enhancement in ppl alongside a significant decrease in total infer- ence time. The underlying reason is that while the vanilla model maintains semantic integrity, it incurs linearly increasing inference time as gen- erated token count rises. Conversely, ChunkLLM reduces computational burden and speeds up inference by leveraging its chunk selection and ICAC mechanisms. 3.2.4\n\nexclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison\n\nexclusively when the current token is detected as a chunk boundary, thereby accelerating model inference. Experimental evaluations are conducted on a diverse set of long-text and short-text benchmark datasets spanning multiple tasks. ChunkLLM not only attains comparable perfor- mance on short-text benchmarks but also maintains 98.64% of the performance on long-context benchmarks while preserving a 48.58% key-value cache retention rate. Particularly, ChunkLLM attains a maximum speedup of 4.48 in comparison\n\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high\n\nv.s. 40.10 v.s. 41.51). In contrast to the short-text benchmark in Subsection 3.2.4, ChunkLLM demonstrates a remarkable improvement in long-context evaluation, which validates the advantage of ChunkLLM in retrieving key chunk information during long-context reasoning. (2) Notably, in the MDQA task, ChunkLLM yields a substantial improvement over SepLLM. We argue that the core challenge of MDQA lies in the dispersion of critical information across distinct positions within the context, which places high", + "generated_answer": "Hello there! I can certainly help you understand that.\n\n**Answer:**\nIn the context of long-context Large Language Models (LLMs), the key phenomenon where chunks attended to by tokens within a generated chunk exhibit substantial consistency is called the **Intra-Chunk Attention Consistency (ICAC) pattern**. ChunkLLM specifically exploits this pattern to significantly enhance its inference efficiency.\n\nThis means that instead of constantly re-evaluating attention for every new token, ChunkLLM only updates its selection of relevant chunks when a new \"chunk boundary\" is identified. This strategic updating process reduces the computational burden and speeds up how quickly the model can generate text, especially in long-context scenarios.\n\n**Key Points/Summary:**\n* **Intra-Chunk Attention Consistency (ICAC):** This pattern describes the consistent attention given to certain chunks by tokens within a generated chunk.\n* **Exploited by ChunkLLM:** ChunkLLM leverages ICAC to improve its performance.\n* **Efficiency Enhancement:** By only updating chunk selection at chunk boundaries, ChunkLLM substantially boosts inference efficiency.\n* **Reduced Overhead:** This mechanism helps ChunkLLM achieve high performance even when trained on smaller contexts (e.g., 4K) compared to models optimized for much larger contexts (e.g., 120K).\n\n**Extra Tip:**\nThink of it like reading a book. Instead of constantly shifting your focus to different paragraphs for every single word, you tend to focus on a particular section (a \"chunk\") for a while until you reach a natural break or a new topic. ChunkLLM applies a similar principle, maintaining focus on a set of relevant chunks until a \"chunk boundary\" signals a need to re-evaluate, making the reading (or inference) process much faster!", + "correctness_score": 1.0, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 5, + "document_path": "static/Beyond Long Context.pdf", + "question": "In the clinical domain, what is the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes?", + "ground_truth": "The methodology is Clinical Entity Augmented Retrieval (CLEAR). CLEAR addresses the limitations of traditional chunk-based RAG by employing entity-aware, entity-centered retrieval strategies and demonstrated a 78% reduction in token usage compared to wide-context processing in evaluations", + "retrieved_context": "complexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\ncomplexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying\n\nfor semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes\n\nfor semantic clinical retrieval in EHR settings where computational efficiency and semantic accuracy are critical, and provides a reusable framework for evaluating clinical NLP approaches in production environments. 1. INTRODUCTION Electronic Health Record (EHR) systems in modern healthcare infrastructure store clinical documenta- tion within FHIR DocumentReference resources, typically encoded as base64 attachments containing unstructured clinical notes. These documents, ranging from brief progress notes\n\nstrategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes.\n\nstrategies often outperform naive long-context prompt- ing, supporting the need for entity-aware retrieval. Our work contributes a reproducible evaluation framework within this paradigm, focusing on realistic EHR-scale clinical notes.", + "generated_answer": "In the clinical domain, the methodology that uses entity-aware retrieval strategies to achieve improved semantic accuracy and computational efficiency when processing Electronic Health Record (EHR) notes is called **Clinical Entity Augmented Retrieval (CLEAR)**.\n\nIntroduced by Lopez et al. (2025), CLEAR specifically addresses the limitations of earlier retrieval methods that struggled to capture nuanced clinical relationships. It's designed to work with EHR DocumentReference processing, which involves unstructured clinical notes stored within FHIR DocumentReference resources. CLEAR has been reported to show improved performance, achieving an F1 score of 0.90 compared to 0.86 for embedding RAG, with significantly fewer tokens (>70% fewer) and faster inference. This makes it particularly valuable for semantic clinical retrieval in EHR settings where both computational efficiency and semantic accuracy are crucial.\n\n### Key Points/Summary:\n* **Name:** Clinical Entity Augmented Retrieval (CLEAR) methodology.\n* **Purpose:** Improves semantic accuracy and computational efficiency in processing clinical information from EHR notes.\n* **Mechanism:** Utilizes entity-aware retrieval strategies to better understand nuanced clinical relationships.\n* **Benefits:** Reports improved performance (higher F1 score, fewer tokens, faster inference) compared to traditional embedding-based RAG.\n\n### Extra Tip:\nThink of \"entity-aware\" as a smart librarian who not only finds books related to your topic but also understands the specific medical terms (entities like diseases, treatments, medications) within those books, allowing for much more precise and relevant information retrieval than just matching keywords.", + "correctness_score": 0.95, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 6, + "document_path": "static/Accelerating_LLM_Inference.pdf", + "question": "Contrast ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) regarding their dynamic attention management mechanisms, specifically addressing how they derive chunk representations and utilize them to achieve efficiency gains while preserving performance in long-context models.", + "ground_truth": "ChunkLLM and Dynamic Hierarchical Sparse Attention (DHSA) both propose mechanisms for efficient long-context modeling by dynamically managing attention sparsity, but they employ different architectural additions and chunk representation strategies. ChunkLLM introduces two pluggable components: the QK Adapter (Q-Adapter and K-Adapter) and the Chunk Adapter. The Chunk Adapter is a one-layer feed-forward neural network (FNN) classifier that detects if a token is a chunk boundary using contextual semantic information. The QK Adapter fulfills feature compression and generates chunk attention scores, trained using an attention distillation approach where the Kullback-Leibler (KL) divergence between chunk attention scores and full attention scores guides optimization to enhance the recall rate of key chunks. ChunkLLM leverages the Intra-Chunk Attention Consistency (ICAC) pattern, triggering chunk selection updates exclusively when the current token is identified as a chunk boundary, substantially enhancing inference efficiency. ChunkLLM maintains 98.64% of the vanilla model's performance on long-context benchmarks and achieves a maximum speedup of 4.48x when processing 120K long texts. DHSA, conversely, is a plug-in module that dynamically predicts attention sparsity during prefill and decode stages without retraining the base model. DHSA employs a **Dynamic Hierarchical Sparsity Prediction approach. It first uses a boundary prediction function to adaptively segment input sequences into variable-length chunks. Chunk representations ($q_c$ and $k_c$) are derived by aggregating token queries and keys using a **length-normalized aggregation strategy, which involves scaling the sum of embeddings by the square root of the chunk size ($\\sqrt{|C|}$) to mitigate sensitivity to variable chunk lengths. It estimates chunk-level similarity ($S_c$) and then upsamples it to obtain the token-level similarity matrix ($S_t$), applying TOPK selection to generate the sparsity mask. DHSA reports matching dense attention in accuracy, while reducing prefill latency by 20-60% and peak memory usage by 35%.", + "retrieved_context": "demands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance\n\ndemands on the models context comprehension capability. SepLLM leverages separators as chunk features, which is plagued by constrained representational capacity and the problem of chunk semantic incompleteness. By contrast, ChunkLLM enriches the representational capacity of chunks via attention distillation, which enhances the recall rates of critical chunks. This, in turn, effectively boosts the models long-context understanding capability. (3) ChunkLLM attains 98.64% of the vanilla models performance\n\nChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3\n\nChunkLLMs attention distillation mechanism, which strengthens the feature representational capacity of chunks. Consequently, during chunk selection, the model effectively identifies critical chunks with higher query relevance, leading to improved in- ference performance. Additionally, ChunkLLM exhibits a reduced KV-Cache utilization rate relative to SepLLM, which further corroborates the effectiveness of key chunk retrieval. We also conduct experiments with StreamingLLM, as shown in Appendix 6.3. 3.2.3\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nexploit the Intra-Chunk Attention Consistency (ICAC) pattern such that chunk selection is only updated when the current token is identified as a chunk boundary, which substantially enhances inference efficiency. Further- more, ChunkLLM can achieve inference performance comparable to that of models optimized for 120K context lengths, despite being trained solely on 4K context lengths, thereby substantially re- ducing the training overhead associated with 120K context scaling. Experimental results validate\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\nintroduce a novel Intra-chunk Attention Consistency Pattern, which yields notable improvements in inference efficiency for long- context scenarios. Experimental results show that ChunkLLM attains a maximum speedup of 4.48 in comparison to the vanilla Transformer in the processing of 120K long texts. 9\n\ncosts of long-context models, making it difficult for large language models to effi- ciently scale to context-processing tasks with million-level token sizes.Chunk Selective attention, a special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk (Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen et al., 2024). Both approaches partition the input into discrete chunks: the former conducts parti- tioning with a fixed length,\n\ncosts of long-context models, making it difficult for large language models to effi- ciently scale to context-processing tasks with million-level token sizes.Chunk Selective attention, a special type of sparse attention, can be primarily categorized into two paradigms: fixed chunk (Lu et al., 2025; Yuan et al., 2025; Wang et al., 2025) and separators-based dynamic chunk (Chen et al., 2024). Both approaches partition the input into discrete chunks: the former conducts parti- tioning with a fixed length,", + "generated_answer": "It's a great question to understand the nuances of different attention mechanisms!\n\nBased on the provided notes, I can give you a detailed explanation of ChunkLLM's dynamic attention management. However, the notes **do not contain information about Dynamic Hierarchical Sparse Attention (DHSA)**, so I cannot directly contrast it with ChunkLLM using the given context. If you have notes on DHSA, please feel free to add them, and I'd be happy to help you understand!\n\nHere's how ChunkLLM manages dynamic attention, derives chunk representations, and uses them for efficiency and performance:\n\n### ChunkLLM's Dynamic Attention Management\n\nChunkLLM employs a sophisticated dynamic attention management mechanism designed to improve long-context understanding, efficiency, and performance.\n\n**1. Deriving Chunk Representations:**\nChunkLLM significantly enhances the way it understands and represents individual chunks of text. It does this through an **attention distillation mechanism**. This mechanism is crucial because it strengthens the \"feature representational capacity\" of each chunk. In simpler terms, it makes the model better at extracting and encoding meaningful information from each segment of the input text, going beyond just using simple separators as features (which is a limitation of methods like SepLLM).\n\n**2. Utilizing Representations for Efficiency Gains:**\nThe enriched chunk representations are then leveraged for several efficiency gains:\n* **Critical Chunk Identification:** With stronger feature representations, ChunkLLM can more effectively identify **critical chunks** that are highly relevant to the current query during the chunk selection process. This means the model doesn't waste computational resources on less important parts of the context.\n* **Reduced KV-Cache Utilization:** By accurately identifying and focusing on critical chunks, ChunkLLM achieves a reduced KV-Cache utilization rate compared to methods like SepLLM. The KV-Cache stores keys and values for past tokens, and reducing its size directly translates to lower memory consumption and faster inference.\n* **Intra-Chunk Attention Consistency (ICAC):** ChunkLLM exploits a novel pattern called Intra-Chunk Attention Consistency (ICAC). This pattern dictates that the chunk selection mechanism is only updated when the current token is recognized as a chunk boundary. This smart update strategy substantially enhances inference efficiency, as the model isn't constantly re-evaluating chunk relevance.\n* **Speedup:** This optimized approach allows ChunkLLM to achieve a maximum speedup of 4.48 times compared to a vanilla Transformer when processing very long texts (e.g., 120K tokens).\n\n**3. Preserving Performance in Long-Context Models:**\nDespite its efficiency optimizations, ChunkLLM is designed to maintain high performance in long-context scenarios:\n* **Enhanced Long-Context Understanding:** The improved recall rates of critical chunks, stemming from the enriched chunk representations, directly boost the model's overall long-context understanding capability. It ensures that the most important information is not overlooked.\n* **Comparable Performance with Less Training Overhead:** Remarkably, ChunkLLM can achieve inference performance comparable to models explicitly optimized for very long context lengths (like 120K tokens), even though it is trained solely on much shorter contexts (e.g., 4K tokens). This significantly reduces the training overhead and computational costs typically associated with scaling models to handle extremely long contexts.\n* **High Performance Relative to Vanilla Models:** The notes state that ChunkLLM attains 98.64% of the performance of a vanilla (standard) model, indicating that its efficiency gains do not come at a significant cost to overall performance.\n\nIn summary, ChunkLLM uses an attention distillation mechanism to create robust chunk representations, enabling it to intelligently select critical chunks, reduce memory usage, and optimize inference updates through ICAC, all while maintaining strong performance for long-context tasks with reduced training effort.\n\n---\n\n**Key Points/Summary:**\n* **Chunk Representation:** ChunkLLM enriches chunk representations via **attention distillation**, strengthening their feature capacity.\n* **Efficiency Gains:** It uses these representations to identify **critical chunks** for higher query relevance, reduces **KV-Cache utilization**, and employs **Intra-Chunk Attention Consistency (ICAC)** for efficient chunk selection updates, leading to significant inference speedups.\n* **Performance Preservation:** ChunkLLM boosts long-context understanding and achieves performance comparable to models trained on 120K contexts, despite being trained only on 4K contexts, thus reducing training overhead.\n\n**Extra Tip:**\nThink of \"attention distillation\" like creating a highly detailed summary for each chapter (chunk) of a very long book. Instead of just marking where chapters begin and end (like using separators), ChunkLLM reads each chapter deeply to understand its core essence. This way, when you ask a question about the book, it can quickly pinpoint the most relevant chapters because their \"summaries\" (representations) are so rich and informative. This saves a lot of time and effort compared to skimming every single word!", + "correctness_score": 0.1, + "faithfulness_score": 1.0, + "model_tested": "rerank" + }, + { + "question_id": 7, + "document_path": "multiple files", + "question": "Explain the observed trade-offs between computational efficiency and semantic integrity across various chunking and retrieval strategies\u2014including Fixed-size/Semantic Chunking, Late Chunking, Contextual Retrieval, and Clinical Entity Augmented Retrieval (CLEAR)\u2014and identify which approach demonstrated superior scalability advantages in high-complexity clinical documents.", + "ground_truth": "The sources reveal significant trade-offs among various chunking and retrieval strategies concerning computational cost and the preservation of semantic integrity. 1. Fixed-size vs. Semantic Chunking (RAG Baseline): Traditional fixed-size chunking is computationally simple and efficient. However, its simplicity risks fragmenting semantically related content, leading to suboptimal retrieval. Semantic chunking, which aims for semantically coherent segments, involves additional computational costs that the sources found were often not justified by consistent performance gains on standard document structures. Overall, fixed-size chunking was suggested as a more efficient and reliable choice for practical RAG applications on non-synthetic datasets. 2. Late Chunking vs. Contextual Retrieval: Late Chunking defers segmentation until after the entire document is embedded, preserving full contextual information for efficiency. Late Chunking offers higher efficiency but may sacrifice relevance and completeness. In contrast, Contextual Retrieval enhances chunks by prompting an LLM to generate additional context for each chunk, improving contextual integrity. This context preservation, particularly when combined with Rank Fusion (ContextualRankFusion), yields better overall results in retrieval evaluation than Late Chunking but incurs greater computational resources, potentially requiring up to 20GB of VRAM for chunk contextualization in long documents. 3. Clinical Entity Augmented Retrieval (CLEAR): This entity-aware method achieves a balance by selectively centering clinically relevant spans around identified entities, overcoming the positional bias ('lost in the middle' problem) associated with processing entire long documents. CLEAR achieved a 78.4% token savings compared to Wide Context processing while maintaining the highest average semantic similarity (0.878), demonstrating an optimal balance between accuracy and computational cost. The approach that demonstrated superior scalability advantages in high-complexity documents was CLEAR. In evaluations involving large clinical notes (exceeding 65,000 tokens), CLEAR achieved a 75% win rate, confirming that its entity-aware retrieval advantages grow as document complexity and document size increase, making it highly suitable for large EHR document processing", + "retrieved_context": "wide-context processing at markedly lower token budgets, echoing the efficiencyquality trade-offs highlighted by CLEAR . The validation results strongly confirm the original research findings, particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentRe- ference content. The 75% win rate on documents exceeding 65,000 tokens validates that entity-aware retrieval becomes increasingly advantageous as document complexity increases, confirming the\n\nanswers. How- ever, the effectiveness of chunking strategies re- mains a significant challenge in optimizing retrieval quality and computational efficiency (Lewis et al., 2020; Finardi et al., 2024). Known as fixed-size chunking, the traditional way to chunk is to cut documents into chunks of a fixed length such as 200 tokens (Gao et al., 2023). While computationally simple, this approach can fragment semantically related content across multi- ple chunks, leading to suboptimal retrieval perfor- mance.\n\nfundamental approaches: (1) wide context processing for zero-shot inference with large language models, (2) traditional vector database chunking with embedding-based retrieval, and (3) entity-aware CLEAR methodology adapted for EHR Docu- mentReference processing. Our contributions include: systematic validation of CLEARs performance claims, development of a reusable evaluation platform for clinical NLP research, and empirical analysis of retrieval strategy performance across clinical documents of varying\n\nlong inputs (lost in the middle) . This motivates entity-aware retrieval that selectively centers clinically relevant spans rather than relying on statistically similar but potentially off-target chunks. The CLinical Entity Augmented Retrieval (CLEAR) methodology, published by Lopez et al. in 2025 , introduced a novel approach that addresses these limitations through entity-aware, entity- centered retrieval strategies. The original study demonstrated significant performance improvements (F1 score of 0.90\n\nIs Semantic Chunking Worth the Computational Cost? Renyi Qu Vectara, Inc. renyi@vectara.com Forrest Bao Vectara, Inc. forrest@vectara.com Ruixuan Tu University of WisconsinMadison turx2003@gmail.com Abstract Recent advances in Retrieval-Augmented Gen- eration (RAG) systems have popularized se- mantic chunking, which aims to improve re- trieval performance by dividing documents into semantically coherent segments. Despite its growing adoption, the actual benefits over sim- pler fixed-size chunking, where\n\ncontinuity across long clin- ical narratives. CLEAR represented a significant advancement by introducing entity-centered retrieval aligned with clinical semantics. Our work extends this line of research by operationalizing CLEAR within an end-to-end evaluation platform, providing reproducible empirical validation across realistic EHR-scale document sets. Recent evaluations of retrieval-augmented models in long-context reasoning (e.g., Karpinska et al., 2023; Xiong et al., 2024) emphasize that retrieval\n\ndocuments are split into consecutive, fixed-size segments, re- main unclear. This study systematically evalu- ates the effectiveness of semantic chunking us- ing three common retrieval-related tasks: docu- ment retrieval, evidence retrieval, and retrieval- based answer generation. The results show that the computational costs associated with seman- tic chunking are not justified by consistent per- formance gains. These findings challenge the previous assumptions about semantic chunk- ing and highlight the\n\nstatistical correlation through vector database chunking often fail to capture the nuanced clinical relationships required for accurate medical information extraction. The Clinical Entity Augmented Retrieval (CLEAR) methodology, introduced by Lopez et al. (2025) , addresses these limitations through entity- aware retrieval strategies and reports improved performance (F1 0.90 vs. 0.86 for embedding RAG; >70% fewer tokens and faster inference). Objective:To develop a comprehensive evaluation platform for\n\ncomplexity and length. 1.1 Related Work Entity-aware retrieval has gained increasing attention within biomedical NLP and question-answering domains. Early retrieval-augmented methods such as RAG demonstrated the potential of embedding- based chunk retrieval but lacked domain-specific entity modeling. In the clinical domain, approaches leveraging UMLS concepts and ontology-based retrieval (e.g., Neumann et al., 2019; Johnson et al., 2016) provided partial improvements but often failed to maintain contextual\n\n4 Retrieval-Augmented Generation (RAG):Semantic chunking with embedding-based retrieval us- ing top-k chunk selection. This approach prioritizes efficiency with minimal token usage (average 544 tokens per query) but may miss critical clinical relationships. 2.4 Evaluation Framework Evaluation was conducted on a dataset of 12 clinical notes ranging from 10,000 to 65,000 tokens, repre- senting diverse clinical scenarios. Each note was accompanied by clinical questions requiring informa- tion extraction and", + "generated_answer": "It's a great question that gets to the heart of optimizing information retrieval! Balancing how computationally efficient a system is with how well it understands the meaning (semantic integrity) of the text is a significant challenge, especially with complex documents like clinical notes.\n\nHere's an explanation of the observed trade-offs across the strategies mentioned in your notes:\n\n### Fixed-size Chunking\n\n* **Approach:** This is the traditional and simplest method, where documents are cut into segments of a predetermined, fixed length (e.g., 200 tokens).\n* **Trade-offs:**\n * **Computational Efficiency:** It is \"computationally simple\" to implement, making it highly efficient from a processing standpoint.\n * **Semantic Integrity:** The major drawback is that this approach \"can fragment semantically related content across multiple chunks.\" This means that information that belongs together conceptually might be split, leading to \"suboptimal retrieval performance\" because the full context for a piece of information might not be available in a single chunk.\n\n### Semantic Chunking\n\n* **Approach:** This method aims to improve retrieval performance by dividing documents into \"semantically coherent segments,\" meaning it tries to keep related ideas together. It has become popular with Retrieval-Augmented Generation (RAG) systems.\n* **Trade-offs:**\n * **Computational Efficiency:** While it aims for better semantic integrity, the notes indicate that \"the computational costs associated with semantic chunking are not justified by consistent performance gains.\" This suggests that it requires more computational resources, but those extra resources don't reliably translate into better retrieval results compared to simpler methods.\n * **Semantic Integrity:** It *aims* to improve semantic coherence by grouping related content, but its effectiveness over fixed-size chunking \"remain unclear\" in terms of consistent performance gains.\n\n### Traditional Vector Database Chunking with Embedding-based Retrieval (Contextual Retrieval)\n\n* **Approach:** This involves splitting documents (often using semantic chunking) and then using embeddings to find chunks that are statistically similar to a query. This is a common approach in early RAG methods and \"wide context processing\" for zero-shot inference.\n* **Trade-offs:**\n * **Computational Efficiency:** For RAG with semantic chunking, it \"prioritizes efficiency with minimal token usage (average 544 tokens per query).\" However, \"wide-context processing\" generally implies managing larger contexts, which can have its own efficiency-quality trade-offs.\n * **Semantic Integrity:** While it uses embeddings to capture some context, it \"lacked domain-specific entity modeling\" and \"often fail to capture the nuanced clinical relationships required for accurate medical information extraction.\" This means it might retrieve statistically similar but \"potentially off-target chunks,\" missing critical clinical relationships. Compared to CLEAR, it might use \"more tokens and slower inference\" for a given level of performance.\n\n### Clinical Entity Augmented Retrieval (CLEAR)\n\n* **Approach:** This is a novel methodology that addresses the limitations of traditional methods by using \"entity-aware, entity-centered retrieval strategies.\" It \"selectively centers clinically relevant spans\" of text.\n* **Trade-offs:**\n * **Computational Efficiency:** CLEAR demonstrates significant improvements, achieving \"greater than 70% fewer tokens and faster inference\" compared to embedding-based RAG. It enables \"wide-context processing at markedly lower token budgets.\"\n * **Semantic Integrity:** It significantly enhances semantic integrity by being \"entity-aware\" and \"aligned with clinical semantics,\" which is crucial for accurate medical information extraction. This leads to \"improved performance\" (e.g., an F1 score of 0.90 vs. 0.86 for embedding RAG).\n\n### Late Chunking\n\nThe provided notes do not contain information about \"Late Chunking.\" It's possible this concept is covered in other materials not included here.\n\n### Superior Scalability Advantages in High-Complexity Clinical Documents\n\nThe **Clinical Entity Augmented Retrieval (CLEAR)** methodology demonstrated superior scalability advantages. The validation results \"strongly confirm... particularly demonstrating scalability advantages on large clinical documents characteristic of comprehensive EHR DocumentReference content.\" It achieved a \"75% win rate on documents exceeding 65,000 tokens,\" validating that \"entity-aware retrieval becomes increasingly advantageous as document complexity increases.\"\n\n---\n\n**Key Points/Summary:**\n\n* **Fixed-size Chunking:** High computational efficiency, but low semantic integrity due to fragmentation.\n* **Semantic Chunking:** Aims for semantic integrity, but its high computational cost is not consistently justified by performance gains.\n* **Traditional RAG/Embedding-based:** Moderate efficiency, but can miss nuanced, domain-specific relationships due to lack of entity awareness.\n* **CLEAR:** High semantic integrity (entity-aware) and high computational efficiency (fewer tokens, faster inference).\n* **Superior Scalability:** CLEAR demonstrated significant scalability advantages, especially with large and complex clinical documents.\n\n**Extra Tip:** Think of it like this: Fixed-size chunking is like cutting a book into pages without regard for chapters. Semantic chunking tries to cut by chapters but might spend too much time doing it for little benefit. Traditional RAG is like finding pages with similar words but might miss the specific plot points. CLEAR is like having a smart index that knows where all the key characters and events are, allowing you to quickly find exactly what you need with less effort.", + "correctness_score": 0.78, + "faithfulness_score": 1.0, + "model_tested": "rerank" + } +] \ No newline at end of file diff --git a/models/app_models.py b/models/app_models.py index f09b0a6..c4cc3df 100644 --- a/models/app_models.py +++ b/models/app_models.py @@ -5,4 +5,4 @@ class DocumentProcessRequest(BaseModel): document_path: Annotated[str, Field(min_length=1)] class QueryRequest(BaseModel): - query: Annotated[str, Field(min_length=1)] \ No newline at end of file + query: Annotated[str, Field(min_length=1)] diff --git a/pyproject.toml b/pyproject.toml index bacc6e4..5e1fcd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,4 +20,5 @@ dependencies = [ "sentence-transformers>=5.1.2", "streamlit>=1.51.0", "typing>=3.10.0.0", + "langchain-groq>=1.0.0", ] diff --git a/services/chroma_db_service.py b/services/chroma_db_service.py index f220ed4..fc534a6 100644 --- a/services/chroma_db_service.py +++ b/services/chroma_db_service.py @@ -3,12 +3,12 @@ import time from utils.logger import log -def connect_to_chroma_db() -> Chroma: +def connect_to_chroma_db(name:str = "document_store") -> Chroma: """ Establishes a connection with chroma database. """ vector_store = Chroma( - collection_name="document_store", + collection_name=name, embedding_function= get_embedding_model(), persist_directory="./data/chroma_langchain_db", # Where to save data locally, remove if not necessary ) @@ -33,7 +33,30 @@ def embed_and_add_document(documents: list, chroma_db: Chroma) -> None: end = time.perf_counter() log.info(f"INFO: Embedding process completed and has been stored into chroma database, took {end - start:.4f} seconds") - + +def retrieve(query: str, chroma_db: Chroma, k: int = 10) -> dict: + """ + Retrieves the top k most relevent documents based on the query. + """ + retrieved_docs = chroma_db.similarity_search(query, k = k) + + if not retrieved_docs: + print("no") + return {"context": "No relevant documents found"} + else: + return {"context": "\n\n".join(getattr(doc, "page_content", str(doc)) for doc in retrieved_docs)} + +def rerank_retrieve(query: str, chroma_db: Chroma, k: int = 10) -> list: + """ + Retrieves the top k most relevant documents and returns them as a + list of raw LangChain Document objects, suitable for reranking. + """ + # Perform the similarity search + retrieved_docs = chroma_db.similarity_search(query, k=k) + + # Return the raw list of document objects + return retrieved_docs + def multi_retrieve(queries: list, chroma_db: Chroma, k: int = 20) -> list: """ Retrieves the top k most relevent documents based on the query. diff --git a/services/query_service.py b/services/query_service.py index 96751a7..4d26e94 100644 --- a/services/query_service.py +++ b/services/query_service.py @@ -33,6 +33,19 @@ Friendly, approachable, and encouraging — like a helpful tutor. """ +TEST_PROMPT = """ + You are an intelligent and patient study assistant designed to help students understand their course notes. + IMPORTANT: do not use any RAG tools such as online search. Answer the given query with only your pre-trained data and knowledge. + + IMPORTANT, Response format: + - **Answer:** Give the main explanation clearly. + - **Key Points/Summary:** List 2–4 concise bullet points summarizing the answer. + - **Extra Tip (if relevant):** Add a short clarification, example, or analogy to help understanding. + + Tone: + Friendly, approachable, and encouraging — like a helpful tutor. +""" + query_transformation_prompt = ''' You are an AI language model assistant. Your task is to generate five different versions of the given user question to retrieve relevant documents from a vector @@ -77,6 +90,21 @@ def query_google_ai(query: str, google_ai: ChatGoogleGenerativeAI) -> dict: return {"query" : query, "response": response} +def query_google_ai_test(query: str, google_ai: ChatGoogleGenerativeAI) -> dict: + messages = [ + SystemMessage(content=TEST_PROMPT), + HumanMessage(content=query) + ] + log.info("Awaiting response from AI model") + start = time.perf_counter() + + response = google_ai.invoke(messages) + + end = time.perf_counter() + log.info(f"Response received, took {end - start:.4f} seconds") + + return {"query" : query, "response": response} + def query_transformation(query:str, google_ai: ChatGoogleGenerativeAI) -> list: messages = [ SystemMessage(content = query_transformation_prompt), @@ -90,7 +118,5 @@ def query_transformation(query:str, google_ai: ChatGoogleGenerativeAI) -> list: end = time.perf_counter() log.info(f"Response received, took {end - start:.4f} seconds") - - print(response.content.split('\n')) - + return response.content.split('\n') \ No newline at end of file diff --git a/static/Accelerating_LLM_Inference.pdf b/static/Accelerating_LLM_Inference.pdf new file mode 100644 index 0000000..4c8dade Binary files /dev/null and b/static/Accelerating_LLM_Inference.pdf differ diff --git a/static/Beyond_Long_Context.pdf b/static/Beyond_Long_Context.pdf new file mode 100644 index 0000000..f471c0b Binary files /dev/null and b/static/Beyond_Long_Context.pdf differ diff --git a/static/Computational_cost_of_semantic_chunking.pdf b/static/Computational_cost_of_semantic_chunking.pdf new file mode 100644 index 0000000..5cfe63e Binary files /dev/null and b/static/Computational_cost_of_semantic_chunking.pdf differ diff --git a/static/LLM_Chunk_Filtering_Method.pdf b/static/LLM_Chunk_Filtering_Method.pdf new file mode 100644 index 0000000..5ac2a77 Binary files /dev/null and b/static/LLM_Chunk_Filtering_Method.pdf differ diff --git a/static/Long_Context_Modelling.pdf b/static/Long_Context_Modelling.pdf new file mode 100644 index 0000000..4ee88a8 Binary files /dev/null and b/static/Long_Context_Modelling.pdf differ diff --git a/static/Reconstructing_Context.pdf b/static/Reconstructing_Context.pdf new file mode 100644 index 0000000..589faef Binary files /dev/null and b/static/Reconstructing_Context.pdf differ diff --git a/uv.lock b/uv.lock index cc68853..efc494a 100644 --- a/uv.lock +++ b/uv.lock @@ -1582,6 +1582,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/06/0e03587da37173c29a58bf17312793c2453df9ca2912e9adfe869c120437/langchain-1.0.2-py3-none-any.whl", hash = "sha256:e0c5647ea47cde7feb9534f56f4496c7f86a45084ad9bd152e7b19739f210ead", size = 107831, upload-time = "2025-10-21T21:08:25.009Z" }, ] +[package.optional-dependencies] +openai = [ + { name = "langchain-openai" }, +] + +[[package]] +name = "langchain-anthropic" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anthropic" }, + { name = "langchain-core" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/17/cac4e8dd225cfea43db167d062ea6108e179f42be5851a67f3c1d6e4aac5/langchain_anthropic-1.0.2.tar.gz", hash = "sha256:e88bd692e12d44d6368d336e2c9c650690f57291fa1c5060108f714c115a0c72", size = 669557, upload-time = "2025-11-07T21:51:07.239Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/b6/e3e00374124609ecf1157ad687e2d67cdc6e97e84177f8e70a936fd8c51f/langchain_anthropic-1.0.2-py3-none-any.whl", hash = "sha256:271740dacf08dc9a4fec3a282781b0a0359b2b081230785895afd27cc93c9c8f", size = 46423, upload-time = "2025-11-07T21:51:05.699Z" }, +] + [[package]] name = "langchain-chroma" version = "1.0.0" @@ -1639,7 +1658,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.0.1" +version = "1.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -1650,9 +1669,9 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/37/1bc4badb93eaa32406a7afdef011336b21719d21ce5ecebf35409a524f8c/langchain_core-1.0.1.tar.gz", hash = "sha256:d769e8d25854466abb672a721143a01bea11cc6ee2d7dae776aa092556db0a26", size = 764566, upload-time = "2025-10-24T16:39:35.495Z" } +sdist = { url = "https://files.pythonhosted.org/packages/93/35/147544d3422464d13a8ef88f9e25cff25e02c985eb44f8c106503f56ad50/langchain_core-1.0.4.tar.gz", hash = "sha256:086d408bcbeedecb0b152201e0163b85e7a6d9b26e11a75cc577b7371291df4e", size = 776329, upload-time = "2025-11-07T22:30:45.669Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/54/3cdbe9d151d06cd689b5aa937ac11403b64bbfe76486fda6431a24061721/langchain_core-1.0.1-py3-none-any.whl", hash = "sha256:c7ce58fc487359c44166e255cc0009ef30290da0b6307b75091152847919661e", size = 467122, upload-time = "2025-10-24T16:39:33.788Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ac/7032e5eb1c147a3d8e0a21a70e77d7efbd6295c8ce4833b90f6ff1750da9/langchain_core-1.0.4-py3-none-any.whl", hash = "sha256:53caa351d9d73b56f5d9628980f36851cfa725977508098869fdc2d246da43b3", size = 471198, upload-time = "2025-11-07T22:30:44.003Z" }, ] [[package]] @@ -1670,6 +1689,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/9d/bf0a4335f9df912e722fce34fb83aeb2c44e937dcb1d60afaae959b9abfc/langchain_google_genai-3.0.0-py3-none-any.whl", hash = "sha256:b26909a30c5c4cd8b5d634d8feafc1ec0c2583726b4cf747d732d7ad2c080ee7", size = 57783, upload-time = "2025-10-17T18:12:06.368Z" }, ] +[[package]] +name = "langchain-groq" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "groq" }, + { name = "langchain-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/9f/0a80b243f1b890cdc2d856f98d2243f49a58621de16fa80a73eba290bcb4/langchain_groq-1.0.0.tar.gz", hash = "sha256:46645591654c36db7ebc624503f26b38544b9aac0d8cf93e8db7105e049dab74", size = 196702, upload-time = "2025-10-17T15:24:30.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/0a/fd2b7545f63f2b05790c76260dc63b7cc9b0af88aa1c1e14fab9795805c2/langchain_groq-1.0.0-py3-none-any.whl", hash = "sha256:378124c2a5247df720093617afbd2dae28b06fc666372ffe7a279a0ce86b28d0", size = 16841, upload-time = "2025-10-17T15:24:29.348Z" }, +] + [[package]] name = "langchain-huggingface" version = "1.0.0" @@ -1684,6 +1716,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/30/7476a926e31498ebc4be3131e06650c5136ffb8ba12067f56212e187dd7c/langchain_huggingface-1.0.0-py3-none-any.whl", hash = "sha256:06d6ac57951c6a1c47d329c38f2b32472a839eab2fa14883be784916ea075da0", size = 27493, upload-time = "2025-10-17T15:30:34.023Z" }, ] +[[package]] +name = "langchain-openai" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "openai" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/3c/edb7ffca76fdcfd938ce8380bf8ec79a0a8be41ba7fdbf6f9fe1cb5fd1a8/langchain_openai-1.0.2.tar.gz", hash = "sha256:621e8295c52db9a1fc74806a0bd227ea215c132c6c5e421d2982c9ee78468769", size = 1025578, upload-time = "2025-11-03T14:08:32.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/9b/7af1d539a051d195c5ecc5990ebd483f208c40f75a8a9532846d16762704/langchain_openai-1.0.2-py3-none-any.whl", hash = "sha256:b3eb9b82752063b46452aa868d8c8bc1604e57631648c3bc325bba58d3aeb143", size = 81934, upload-time = "2025-11-03T14:08:30.655Z" }, +] + [[package]] name = "langchain-text-splitters" version = "1.0.0" @@ -4051,10 +4097,12 @@ source = { virtual = "." } dependencies = [ { name = "cohere" }, { name = "fastapi", extra = ["standard"] }, - { name = "langchain" }, + { name = "langchain", extra = ["openai"] }, + { name = "langchain-anthropic" }, { name = "langchain-chroma" }, { name = "langchain-community" }, { name = "langchain-google-genai" }, + { name = "langchain-groq" }, { name = "langchain-huggingface" }, { name = "langchain-text-splitters" }, { name = "pydantic" }, @@ -4070,10 +4118,12 @@ dependencies = [ requires-dist = [ { name = "cohere", specifier = ">=5.20.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.120.0" }, - { name = "langchain", specifier = ">=1.0.2" }, + { name = "langchain", extras = ["openai"], specifier = ">=1.0.2" }, + { name = "langchain-anthropic", specifier = ">=1.0.2" }, { name = "langchain-chroma", specifier = ">=1.0.0" }, { name = "langchain-community", specifier = ">=0.4" }, { name = "langchain-google-genai", specifier = ">=3.0.0" }, + { name = "langchain-groq", specifier = ">=1.0.0" }, { name = "langchain-huggingface", specifier = ">=1.0.0" }, { name = "langchain-text-splitters", specifier = ">=1.0.0" }, { name = "pydantic", specifier = ">=2.12.3" }, @@ -4134,6 +4184,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] +[[package]] +name = "tiktoken" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3", size = 1050802, upload-time = "2025-10-06T20:22:00.96Z" }, + { url = "https://files.pythonhosted.org/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160", size = 993995, upload-time = "2025-10-06T20:22:02.788Z" }, + { url = "https://files.pythonhosted.org/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa", size = 1128948, upload-time = "2025-10-06T20:22:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/0c/da/c028aa0babf77315e1cef357d4d768800c5f8a6de04d0eac0f377cb619fa/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be", size = 1151986, upload-time = "2025-10-06T20:22:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/a0/5a/886b108b766aa53e295f7216b509be95eb7d60b166049ce2c58416b25f2a/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a", size = 1194222, upload-time = "2025-10-06T20:22:06.265Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f8/4db272048397636ac7a078d22773dd2795b1becee7bc4922fe6207288d57/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3", size = 1255097, upload-time = "2025-10-06T20:22:07.403Z" }, + { url = "https://files.pythonhosted.org/packages/8e/32/45d02e2e0ea2be3a9ed22afc47d93741247e75018aac967b713b2941f8ea/tiktoken-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697", size = 879117, upload-time = "2025-10-06T20:22:08.418Z" }, + { url = "https://files.pythonhosted.org/packages/ce/76/994fc868f88e016e6d05b0da5ac24582a14c47893f4474c3e9744283f1d5/tiktoken-0.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16", size = 1050309, upload-time = "2025-10-06T20:22:10.939Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b8/57ef1456504c43a849821920d582a738a461b76a047f352f18c0b26c6516/tiktoken-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a", size = 993712, upload-time = "2025-10-06T20:22:12.115Z" }, + { url = "https://files.pythonhosted.org/packages/72/90/13da56f664286ffbae9dbcfadcc625439142675845baa62715e49b87b68b/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27", size = 1128725, upload-time = "2025-10-06T20:22:13.541Z" }, + { url = "https://files.pythonhosted.org/packages/05/df/4f80030d44682235bdaecd7346c90f67ae87ec8f3df4a3442cb53834f7e4/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb", size = 1151875, upload-time = "2025-10-06T20:22:14.559Z" }, + { url = "https://files.pythonhosted.org/packages/22/1f/ae535223a8c4ef4c0c1192e3f9b82da660be9eb66b9279e95c99288e9dab/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e", size = 1194451, upload-time = "2025-10-06T20:22:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/78/a7/f8ead382fce0243cb625c4f266e66c27f65ae65ee9e77f59ea1653b6d730/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25", size = 1253794, upload-time = "2025-10-06T20:22:16.624Z" }, + { url = "https://files.pythonhosted.org/packages/93/e0/6cc82a562bc6365785a3ff0af27a2a092d57c47d7a81d9e2295d8c36f011/tiktoken-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f", size = 878777, upload-time = "2025-10-06T20:22:18.036Z" }, + { url = "https://files.pythonhosted.org/packages/72/05/3abc1db5d2c9aadc4d2c76fa5640134e475e58d9fbb82b5c535dc0de9b01/tiktoken-0.12.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646", size = 1050188, upload-time = "2025-10-06T20:22:19.563Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7b/50c2f060412202d6c95f32b20755c7a6273543b125c0985d6fa9465105af/tiktoken-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88", size = 993978, upload-time = "2025-10-06T20:22:20.702Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/bf795595a2b897e271771cd31cb847d479073497344c637966bdf2853da1/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff", size = 1129271, upload-time = "2025-10-06T20:22:22.06Z" }, + { url = "https://files.pythonhosted.org/packages/f5/de/9341a6d7a8f1b448573bbf3425fa57669ac58258a667eb48a25dfe916d70/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830", size = 1151216, upload-time = "2025-10-06T20:22:23.085Z" }, + { url = "https://files.pythonhosted.org/packages/75/0d/881866647b8d1be4d67cb24e50d0c26f9f807f994aa1510cb9ba2fe5f612/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b", size = 1194860, upload-time = "2025-10-06T20:22:24.602Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1e/b651ec3059474dab649b8d5b69f5c65cd8fcd8918568c1935bd4136c9392/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b", size = 1254567, upload-time = "2025-10-06T20:22:25.671Z" }, + { url = "https://files.pythonhosted.org/packages/80/57/ce64fd16ac390fafde001268c364d559447ba09b509181b2808622420eec/tiktoken-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3", size = 921067, upload-time = "2025-10-06T20:22:26.753Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a4/72eed53e8976a099539cdd5eb36f241987212c29629d0a52c305173e0a68/tiktoken-0.12.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365", size = 1050473, upload-time = "2025-10-06T20:22:27.775Z" }, + { url = "https://files.pythonhosted.org/packages/e6/d7/0110b8f54c008466b19672c615f2168896b83706a6611ba6e47313dbc6e9/tiktoken-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e", size = 993855, upload-time = "2025-10-06T20:22:28.799Z" }, + { url = "https://files.pythonhosted.org/packages/5f/77/4f268c41a3957c418b084dd576ea2fad2e95da0d8e1ab705372892c2ca22/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63", size = 1129022, upload-time = "2025-10-06T20:22:29.981Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2b/fc46c90fe5028bd094cd6ee25a7db321cb91d45dc87531e2bdbb26b4867a/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0", size = 1150736, upload-time = "2025-10-06T20:22:30.996Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/3c7a39ff68022ddfd7d93f3337ad90389a342f761c4d71de99a3ccc57857/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a", size = 1194908, upload-time = "2025-10-06T20:22:32.073Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0d/c1ad6f4016a3968c048545f5d9b8ffebf577774b2ede3e2e352553b685fe/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0", size = 1253706, upload-time = "2025-10-06T20:22:33.385Z" }, + { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, +] + [[package]] name = "tokenizers" version = "0.22.1"