Issues with long documents

[diptanu]

Hey Gregory, great work! I played with this little and saw some issues - the output for a chat completion response from the content of a cloudflare 10K document was -

{answer.replace(chr(34), chr(39))}

It took about 100 seconds to get to this result.

here is the code -

import os
from typing import Optional
from pydantic import BaseModel, Field

from tensorlake.applications import Image, application, function


# Define the runtime image with required dependencies
image = (
    Image(base_image="python:3.11-slim", name="pdf-rlm-qa")
    .run("apt-get update && apt-get install -y git")
    .run("pip install tensorlake openai")
    .run("pip install git+https://github.com/ysz/recursive-llm.git litellm==1.74.9")
)


class PDFRequest(BaseModel):
    """Input for PDF question answering"""
    url: str = Field(description="URL to the PDF document")
    question: str = Field(description="Question to answer about the document")


class PDFContent(BaseModel):
    """Extracted PDF content"""
    url: str
    content: str
    question: str


class PDFAnalysis(BaseModel):
    """Complete analysis of a PDF document"""
    url: str
    question: str
    answer: str
    content_length: int


@application()
@function(image=image)
def pdf_qa_processor(request: PDFRequest) -> PDFAnalysis:
    """Main entry point for PDF question answering"""
    # Extract PDF content and answer question
    analysis = answer_with_rlm(extract_pdf_content(request))

    return analysis


@function(image=image, secrets=["TENSORLAKE_API_KEY"])
def extract_pdf_content(request: PDFRequest) -> PDFContent:
    """Extract OCR content from a PDF file using TensorLake DocumentAI"""
    from tensorlake.documentai import DocumentAI
    from tensorlake.documentai.models import ChunkingStrategy, ParsingOptions

    print(f"Extracting content from PDF: {request.url}")

    # Initialize DocumentAI
    doc_ai = DocumentAI()

    # Use the URL directly
    file_id = request.url
    print(f"Using remote file: {file_id}")

    # Configure parsing options for OCR
    parsing_options = ParsingOptions(
        chunking_strategy=ChunkingStrategy.PAGE,
        # skip_ocr=False ensures OCR is performed
    )

    # Parse the document
    print("Parsing document with OCR...")
    parse_id = doc_ai.parse(
        file_id,
        parsing_options=parsing_options,
    )

    # Wait for completion
    print("Waiting for OCR to complete...")
    result = doc_ai.wait_for_completion(parse_id=parse_id)

    print(f"Parse status: {result.status}")
    print(f"Extracted {len(result.chunks)} chunks")

    # Combine all chunks into a single text
    full_content = "\n\n".join([chunk.content for chunk in result.chunks])

    print(f"Total content length: {len(full_content)} characters")

    return PDFContent(
        url=request.url,
        content=full_content,
        question=request.question
    )


@function(image=image, secrets=["OPENAI_API_KEY"], timeout=600)
def answer_with_rlm(pdf_content: PDFContent) -> PDFAnalysis:
    """Answer questions using RLM (Recursive Language Models)"""
    from rlm import RLM

    print(f"Initializing RLM for question answering...")
    print(f"Question: {pdf_content.question}")
    print(f"Document length: {len(pdf_content.content)} characters")

    # Initialize RLM with GPT-4o-mini for cost-effective recursive processing
    # You can also use two models for optimization:
    # rlm = RLM(model="gpt-4o", recursive_model="gpt-4o-mini")
    rlm = RLM(
        model="gpt-5",
        max_depth=10,         # Maximum recursion depth
        max_iterations=50,   # Maximum REPL iterations
    )

    print("Processing with RLM (this may take a moment for long documents)...")

    # Use RLM to answer the question
    # RLM stores the context as a variable and recursively explores it
    result = rlm.completion(
        query=pdf_content.question,
        context=pdf_content.content
    )
    print(result)

    print(f"Answer generated successfully")

    return PDFAnalysis(
        url=pdf_content.url,
        question=pdf_content.question,
        answer=result,
        content_length=len(pdf_content.content)
    )


if __name__ == "__main__":
    from tensorlake.applications import run_local_application

    # Example usage with a sample PDF
    # Replace with your own PDF URL and question
    test_request = PDFRequest(
        url="https://d18rn0p25nwr6d.cloudfront.net/CIK-0001477333/2c5ce18c-bb02-4031-89cb-59ff64f22eaf.pdf",
        question="Give a very detailed summary like you are an analyst at a hedge fund about everything cloudflare is saying about AI and how it affects their business from this report. Include product details and other critical information. I want to be well informed. Return section number and page numbers as citation"
    )

    # Run the application locally
    response = run_local_application(
        pdf_qa_processor,
        test_request
    )

    result = response.output()

    print(result.answer)

You can run this locally in case you want to debug -

pip install tensorlake 

python code.py

This uses Tensorlake's OCR API and serverless application. You can get a key from our dashboard (cloud.tensorlake.ai) or DM me for a key. You could also just replace OCR with some hardcoded text or another API :) The application runs locally and on our platform as an API. You can run this locally for debugging using the steps above.

My observation is that it works well for shorter documents