From a9838ae2d0737378a87a3358b390e1514a12eafc Mon Sep 17 00:00:00 2001 From: AkshitMaheshwari Date: Sat, 4 Apr 2026 00:12:13 +0530 Subject: [PATCH 1/2] GLM-OCR --- Backend/.gitignore | 2 +- Backend/requirements.txt | 4 +- Backend/services/image_ocr.py | 100 ++++++++++++++++++++++------------ Frontend/.env | 2 +- Frontend/.env.example | 2 +- 5 files changed, 71 insertions(+), 39 deletions(-) diff --git a/Backend/.gitignore b/Backend/.gitignore index 6eea31a..68cc21a 100644 --- a/Backend/.gitignore +++ b/Backend/.gitignore @@ -67,7 +67,7 @@ instance/ # Scrapy stuff: .scrapy - +myenv/ # Sphinx documentation docs/_build/ diff --git a/Backend/requirements.txt b/Backend/requirements.txt index e317d2a..4fae5ef 100644 --- a/Backend/requirements.txt +++ b/Backend/requirements.txt @@ -3,10 +3,12 @@ groq PyJWT Pillow pandas -pytesseract python-dotenv python-multipart SQLAlchemy psycopg[binary] uvicorn Werkzeug +transformers +torch +accelerate diff --git a/Backend/services/image_ocr.py b/Backend/services/image_ocr.py index 7348568..1895fff 100644 --- a/Backend/services/image_ocr.py +++ b/Backend/services/image_ocr.py @@ -1,6 +1,8 @@ from __future__ import annotations import os +import tempfile +import uuid from io import BytesIO from typing import Any @@ -11,63 +13,91 @@ from services.groq_ai import extract_csv_from_ocr_text, groq_status from services.tax_constants import SUPPORTED_DOCUMENTS -try: - import pytesseract -except ImportError: # pragma: no cover - pytesseract = None - - IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".tiff", ".tif", ".bmp"} - class OcrConversionError(ValueError): pass +_processor = None +_model = None + +def _load_glm_ocr(): + global _processor, _model + if _processor is None or _model is None: + from transformers import AutoProcessor, AutoModelForImageTextToText + import torch + MODEL_PATH = "zai-org/GLM-OCR" + _processor = AutoProcessor.from_pretrained(MODEL_PATH) + _model = AutoModelForImageTextToText.from_pretrained( + pretrained_model_name_or_path=MODEL_PATH, + torch_dtype="auto", + device_map="auto", + ) + def ocr_status() -> dict[str, Any]: - config = get_runtime_config() - tesseract_cmd = str(getattr(config, "TESSERACT_CMD", "")).strip() return { - "pytesseract_installed": pytesseract is not None, - "tesseract_cmd_configured": bool(tesseract_cmd), + "glm_ocr_configured": True, "groq": groq_status(), } def _is_image_filename(filename: str) -> bool: return os.path.splitext(filename.lower())[1] in IMAGE_EXTENSIONS - def _extract_text_from_image(image_bytes: bytes) -> str: - if pytesseract is None: - raise OcrConversionError( - "pytesseract is not installed. Install it and ensure the Tesseract engine is available." - ) - - config = get_runtime_config() - tesseract_cmd = str(getattr(config, "TESSERACT_CMD", "")).strip() - if tesseract_cmd: - pytesseract.pytesseract.tesseract_cmd = tesseract_cmd + try: + _load_glm_ocr() + except Exception as exc: + raise OcrConversionError(f"Failed to load GLM-OCR model. Make sure transformers and torch are installed. Error: {exc}") + temp_filename = f"{uuid.uuid4().hex}.png" try: image = Image.open(BytesIO(image_bytes)) + image.save(temp_filename, format="PNG") except Exception as exc: + if os.path.exists(temp_filename): + os.remove(temp_filename) raise OcrConversionError(f"Uploaded file is not a readable image: {exc}") from exc - # Grayscale + auto-contrast usually improves OCR quality on scans. - processed = ImageOps.autocontrast(ImageOps.grayscale(image)) - try: - text = pytesseract.image_to_string(processed) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "url": temp_filename + }, + { + "type": "text", + "text": "Text Recognition:" + } + ], + } + ] + + inputs = _processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt" + ).to(_model.device) + + inputs.pop("token_type_ids", None) + generated_ids = _model.generate(**inputs, max_new_tokens=8192) + output_text = _processor.decode(generated_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=False) + + normalized = "\n".join(line.rstrip() for line in output_text.splitlines()) + if len(normalized.strip()) < 20: + raise OcrConversionError( + "OCR extracted too little text from the image. Upload a clearer image or use CSV." + ) + return normalized except Exception as exc: - raise OcrConversionError( - "OCR failed. Verify Tesseract is installed and TESSERACT_CMD is configured if needed." - ) from exc - - normalized = "\n".join(line.rstrip() for line in text.splitlines()) - if len(normalized.strip()) < 20: - raise OcrConversionError( - "OCR extracted too little text from the image. Upload a clearer image or use CSV." - ) - return normalized + raise OcrConversionError(f"OCR Inference failed: {exc}") from exc + finally: + if os.path.exists(temp_filename): + os.remove(temp_filename) def convert_image_to_csv_document( diff --git a/Frontend/.env b/Frontend/.env index b9ceeef..a165169 100644 --- a/Frontend/.env +++ b/Frontend/.env @@ -1,2 +1,2 @@ # Backend API URL -VITE_API_URL=https://taxai-77xc.onrender.com +VITE_API_URL=http://127.0.0.1:8000 diff --git a/Frontend/.env.example b/Frontend/.env.example index 04665c4..a165169 100644 --- a/Frontend/.env.example +++ b/Frontend/.env.example @@ -1,2 +1,2 @@ # Backend API URL -VITE_API_URL=http://127.0.0.1:5000 +VITE_API_URL=http://127.0.0.1:8000 From 75c0f36a8ebfef0244dd6cec0af93071440ff052 Mon Sep 17 00:00:00 2001 From: kaws26 Date: Sat, 4 Apr 2026 00:48:47 +0530 Subject: [PATCH 2/2] feat: Update API integration to use centralized API_BASE_URL and enhance OCR configuration --- Backend/config.py | 3 + Backend/requirements.txt | 4 +- Backend/services/image_ocr.py | 105 +++++++++++------------------ Frontend/src/pages/AIAssistant.jsx | 7 +- Frontend/src/pages/Dashboard.jsx | 5 +- Frontend/src/pages/Filing.jsx | 23 +++---- Frontend/src/pages/Upload.jsx | 13 ++-- Frontend/src/utils/api.js | 2 +- 8 files changed, 66 insertions(+), 96 deletions(-) diff --git a/Backend/config.py b/Backend/config.py index 073345d..5b43679 100644 --- a/Backend/config.py +++ b/Backend/config.py @@ -41,5 +41,8 @@ class Config: GROQ_BANK_ROWS_PER_CALL = int(os.getenv("GROQ_BANK_ROWS_PER_CALL", "20")) GROQ_TRANSACTION_ROWS_PER_CALL = int(os.getenv("GROQ_TRANSACTION_ROWS_PER_CALL", "25")) GROQ_OCR_MAX_INPUT_CHARS = int(os.getenv("GROQ_OCR_MAX_INPUT_CHARS", "12000")) + HF_TOKEN = os.getenv("HF_TOKEN", "") + HF_OCR_MODEL = os.getenv("HF_OCR_MODEL", "zai-org/GLM-OCR") + HF_OCR_PROVIDER = os.getenv("HF_OCR_PROVIDER", "zai-org") TESSERACT_CMD = os.getenv("TESSERACT_CMD", "") PDF_EXPORT_DIR = os.getenv("PDF_EXPORT_DIR", "generated_pdfs") diff --git a/Backend/requirements.txt b/Backend/requirements.txt index 4fae5ef..5040c31 100644 --- a/Backend/requirements.txt +++ b/Backend/requirements.txt @@ -9,6 +9,4 @@ SQLAlchemy psycopg[binary] uvicorn Werkzeug -transformers -torch -accelerate +huggingface_hub diff --git a/Backend/services/image_ocr.py b/Backend/services/image_ocr.py index 1895fff..a1f531f 100644 --- a/Backend/services/image_ocr.py +++ b/Backend/services/image_ocr.py @@ -2,102 +2,72 @@ import os import tempfile -import uuid -from io import BytesIO from typing import Any -from PIL import Image, ImageOps +from huggingface_hub import InferenceClient from runtime import get_runtime_config from services.document_ingestion import DocumentValidationError, parse_document from services.groq_ai import extract_csv_from_ocr_text, groq_status from services.tax_constants import SUPPORTED_DOCUMENTS + IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".tiff", ".tif", ".bmp"} + class OcrConversionError(ValueError): pass -_processor = None -_model = None - -def _load_glm_ocr(): - global _processor, _model - if _processor is None or _model is None: - from transformers import AutoProcessor, AutoModelForImageTextToText - import torch - MODEL_PATH = "zai-org/GLM-OCR" - _processor = AutoProcessor.from_pretrained(MODEL_PATH) - _model = AutoModelForImageTextToText.from_pretrained( - pretrained_model_name_or_path=MODEL_PATH, - torch_dtype="auto", - device_map="auto", - ) def ocr_status() -> dict[str, Any]: + config = get_runtime_config() + hf_token = str(getattr(config, "HF_TOKEN", "")).strip() return { - "glm_ocr_configured": True, + "provider": getattr(config, "HF_OCR_PROVIDER", "zai-org"), + "model": getattr(config, "HF_OCR_MODEL", "zai-org/GLM-OCR"), + "has_hf_token": bool(hf_token), "groq": groq_status(), } + def _is_image_filename(filename: str) -> bool: return os.path.splitext(filename.lower())[1] in IMAGE_EXTENSIONS -def _extract_text_from_image(image_bytes: bytes) -> str: - try: - _load_glm_ocr() - except Exception as exc: - raise OcrConversionError(f"Failed to load GLM-OCR model. Make sure transformers and torch are installed. Error: {exc}") - temp_filename = f"{uuid.uuid4().hex}.png" - try: - image = Image.open(BytesIO(image_bytes)) - image.save(temp_filename, format="PNG") - except Exception as exc: - if os.path.exists(temp_filename): - os.remove(temp_filename) - raise OcrConversionError(f"Uploaded file is not a readable image: {exc}") from exc +def _extract_text_from_image(image_bytes: bytes, source_name: str) -> str: + config = get_runtime_config() + hf_token = str(getattr(config, "HF_TOKEN", "")).strip() + if not hf_token: + raise OcrConversionError("HF_TOKEN is not configured. Set HF_TOKEN to use Hugging Face OCR.") + + client = InferenceClient( + provider=str(getattr(config, "HF_OCR_PROVIDER", "zai-org")).strip() or "zai-org", + api_key=hf_token, + ) + + suffix = os.path.splitext(source_name)[1].lower() or ".png" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file: + temp_file.write(image_bytes) + temp_path = temp_file.name try: - messages = [ - { - "role": "user", - "content": [ - { - "type": "image", - "url": temp_filename - }, - { - "type": "text", - "text": "Text Recognition:" - } - ], - } - ] - - inputs = _processor.apply_chat_template( - messages, - tokenize=True, - add_generation_prompt=True, - return_dict=True, - return_tensors="pt" - ).to(_model.device) - - inputs.pop("token_type_ids", None) - generated_ids = _model.generate(**inputs, max_new_tokens=8192) - output_text = _processor.decode(generated_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=False) - - normalized = "\n".join(line.rstrip() for line in output_text.splitlines()) + output = client.image_to_text( + temp_path, + model=str(getattr(config, "HF_OCR_MODEL", "zai-org/GLM-OCR")).strip() or "zai-org/GLM-OCR", + ) + normalized = "\n".join(line.rstrip() for line in str(output or "").splitlines()) if len(normalized.strip()) < 20: raise OcrConversionError( "OCR extracted too little text from the image. Upload a clearer image or use CSV." ) return normalized + except OcrConversionError: + raise except Exception as exc: - raise OcrConversionError(f"OCR Inference failed: {exc}") from exc + raise OcrConversionError(f"Hugging Face OCR inference failed: {exc}") from exc finally: - if os.path.exists(temp_filename): - os.remove(temp_filename) + if os.path.exists(temp_path): + os.remove(temp_path) def convert_image_to_csv_document( @@ -106,12 +76,13 @@ def convert_image_to_csv_document( source_name: str, image_bytes: bytes, ) -> dict[str, Any]: + config = get_runtime_config() if document_type not in SUPPORTED_DOCUMENTS: raise OcrConversionError(f"Unsupported document type: {document_type}") if not _is_image_filename(source_name): raise OcrConversionError(f"{source_name} is not a supported image format.") - ocr_text = _extract_text_from_image(image_bytes) + ocr_text = _extract_text_from_image(image_bytes, source_name) schema = SUPPORTED_DOCUMENTS[document_type] llm_result = extract_csv_from_ocr_text( document_type=document_type, @@ -152,7 +123,9 @@ def convert_image_to_csv_document( "metadata": { "source": "OCR", "conversion_meta": { - "origin": "image_ocr", + "origin": "huggingface_glm_ocr", + "provider": getattr(config, "HF_OCR_PROVIDER", "zai-org"), + "model": getattr(config, "HF_OCR_MODEL", "zai-org/GLM-OCR"), "original_source": source_name, "ocr_chars": len(ocr_text), "llm_meta": llm_result.get("meta", {}), diff --git a/Frontend/src/pages/AIAssistant.jsx b/Frontend/src/pages/AIAssistant.jsx index 59d48cd..2069276 100644 --- a/Frontend/src/pages/AIAssistant.jsx +++ b/Frontend/src/pages/AIAssistant.jsx @@ -1,7 +1,6 @@ import { useState } from 'react'; import AppLayout from '../components/AppLayout'; - -const API_URL = 'https://taxai-77xc.onrender.com'; +import { API_BASE_URL } from '../utils/api'; export default function AIAssistant() { const [activeTab, setActiveTab] = useState('tax'); @@ -120,7 +119,7 @@ export default function AIAssistant() { }); } - const response = await fetch(`${API_URL}/api/tax-assistant/analyze`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/analyze`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -161,7 +160,7 @@ export default function AIAssistant() { // For personal Q&A, call the API if (activeTab === 'personal' && taxAnalysis) { try { - const response = await fetch(`${API_URL}/api/tax-assistant/ask`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/ask`, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/Frontend/src/pages/Dashboard.jsx b/Frontend/src/pages/Dashboard.jsx index 9017e3b..ccde5c8 100644 --- a/Frontend/src/pages/Dashboard.jsx +++ b/Frontend/src/pages/Dashboard.jsx @@ -1,7 +1,6 @@ import { useState, useEffect } from 'react'; import AppLayout from '../components/AppLayout'; - -const API_URL = 'https://taxai-77xc.onrender.com'; +import { API_BASE_URL } from '../utils/api'; export default function Dashboard() { const [dashboardData, setDashboardData] = useState(null); @@ -12,7 +11,7 @@ export default function Dashboard() { const fetchDashboardData = async () => { try { const token = localStorage.getItem('access_token'); - const response = await fetch(`${API_URL}/api/tax-assistant/dashboard/financial-data`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/dashboard/financial-data`, { headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json', diff --git a/Frontend/src/pages/Filing.jsx b/Frontend/src/pages/Filing.jsx index 26abfc5..56ab814 100644 --- a/Frontend/src/pages/Filing.jsx +++ b/Frontend/src/pages/Filing.jsx @@ -1,8 +1,7 @@ import { useState, useEffect } from 'react'; import { useAuth } from '../context/AuthContext'; import AppLayout from '../components/AppLayout'; - -const API_URL = 'https://taxai-77xc.onrender.com'; +import { API_BASE_URL } from '../utils/api'; export default function Filing() { const { user } = useAuth(); @@ -29,7 +28,7 @@ export default function Filing() { useEffect(() => { const fetchOptions = async () => { try { - const response = await fetch(`${API_URL}/api/tax-assistant/options`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/options`, { headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}` }, }); const data = await response.json(); @@ -56,7 +55,7 @@ export default function Filing() { const fetchJobs = async () => { try { setLoading(true); - const response = await fetch(`${API_URL}/api/tax-assistant/jobs`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs`, { headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}` }, }); const data = await response.json(); @@ -82,7 +81,7 @@ export default function Filing() { if (formData.deduction_80d) payload.deduction_80d = parseFloat(formData.deduction_80d); if (formData.advance_tax) payload.advance_tax = parseFloat(formData.advance_tax); - const response = await fetch(`${API_URL}/api/tax-assistant/jobs`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -590,7 +589,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { const token = localStorage.getItem('access_token'); // Fetch options - const optResponse = await fetch(`${API_URL}/api/tax-assistant/options`, { + const optResponse = await fetch(`${API_BASE_URL}/api/tax-assistant/options`, { headers: { Authorization: `Bearer ${token}` }, }); if (!optResponse.ok) throw new Error('Failed to fetch options'); @@ -611,7 +610,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { // Fetch review state if status is review or approved if (job.status === 'review' || job.status === 'approved') { const revResponse = await fetch( - `${API_URL}/api/tax-assistant/jobs/${jobId}/review`, + `${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/review`, { headers: { Authorization: `Bearer ${token}` }, } @@ -651,7 +650,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { docFormData.append('files', file); }); - const uploadUrl = `${API_URL}/api/tax-assistant/jobs/${jobId}/documents`; + const uploadUrl = `${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/documents`; console.log('Uploading to:', uploadUrl, 'Document type:', docItem.documentType); const response = await fetch(uploadUrl, { @@ -706,7 +705,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { try { setProcessLoading(true); const jobId = currentJob.job_id || currentJob.id; - const response = await fetch(`${API_URL}/api/tax-assistant/jobs/${jobId}/process`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/process`, { method: 'POST', headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}`, @@ -721,7 +720,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { if (data.job?.status === 'review' || data.job?.status === 'approved') { try { const revResponse = await fetch( - `${API_URL}/api/tax-assistant/jobs/${jobId}/review`, + `${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/review`, { headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}` }, } @@ -754,7 +753,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { try { setApproveLoading(true); const jobId = currentJob.job_id || currentJob.id; - const response = await fetch(`${API_URL}/api/tax-assistant/jobs/${jobId}/approve`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/approve`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -786,7 +785,7 @@ function JobDetail({ job, onBack, onJobUpdate }) { try { setExportLoading(true); const jobId = currentJob.job_id || currentJob.id; - const response = await fetch(`${API_URL}/api/tax-assistant/jobs/${jobId}/export/itr-pdf`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/export/itr-pdf`, { headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}`, }, diff --git a/Frontend/src/pages/Upload.jsx b/Frontend/src/pages/Upload.jsx index 32a58d6..70cef90 100644 --- a/Frontend/src/pages/Upload.jsx +++ b/Frontend/src/pages/Upload.jsx @@ -1,7 +1,6 @@ import { useState, useEffect } from 'react'; import AppLayout from '../components/AppLayout'; - -const API_URL = 'https://taxai-77xc.onrender.com'; +import { API_BASE_URL } from '../utils/api'; export default function Upload() { const [mode, setMode] = useState('analyze'); // 'analyze' or 'filing' @@ -35,7 +34,7 @@ export default function Upload() { const token = localStorage.getItem('access_token'); // Fetch options - const optResponse = await fetch(`${API_URL}/api/tax-assistant/options`, { + const optResponse = await fetch(`${API_BASE_URL}/api/tax-assistant/options`, { headers: { Authorization: `Bearer ${token}` }, }); const optData = await optResponse.json(); @@ -43,7 +42,7 @@ export default function Upload() { setAnalyzeOptions(optData); // Fetch jobs - const jobResponse = await fetch(`${API_URL}/api/tax-assistant/jobs`, { + const jobResponse = await fetch(`${API_BASE_URL}/api/tax-assistant/jobs`, { headers: { Authorization: `Bearer ${token}` }, }); const jobData = await jobResponse.json(); @@ -63,7 +62,7 @@ export default function Upload() { try { const jobId = selectedJob.job_id || selectedJob.id; const response = await fetch( - `${API_URL}/api/tax-assistant/jobs/${jobId}/documents`, + `${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/documents`, { headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}` }, } @@ -142,7 +141,7 @@ export default function Upload() { }); const response = await fetch( - `${API_URL}/api/tax-assistant/jobs/${jobId}/documents`, + `${API_BASE_URL}/api/tax-assistant/jobs/${jobId}/documents`, { method: 'POST', headers: { @@ -218,7 +217,7 @@ export default function Upload() { formData.append('files', file); }); - const response = await fetch(`${API_URL}/api/tax-assistant/analyze-files`, { + const response = await fetch(`${API_BASE_URL}/api/tax-assistant/analyze-files`, { method: 'POST', headers: { Authorization: `Bearer ${localStorage.getItem('access_token')}`, diff --git a/Frontend/src/utils/api.js b/Frontend/src/utils/api.js index 6b4c3ac..e8ba015 100644 --- a/Frontend/src/utils/api.js +++ b/Frontend/src/utils/api.js @@ -1,4 +1,4 @@ -const rawBaseUrl = import.meta.env.VITE_API_URL || 'https://taxai-77xc.onrender.com'; +const rawBaseUrl = import.meta.env.VITE_API_URL || 'http://127.0.0.1:8000'; export const API_BASE_URL = rawBaseUrl.replace(/\/+$/, ''); export const API_PREFIX = `${API_BASE_URL}/api`;