diff --git a/README.md b/README.md index c64736c..aaa1cd5 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # Budget Tracker -**Budget Tracker** is a Python-based tool for processing, categorizing, and summarizing monthly financial transactions from Excel files. +**Budget Tracker** is a Python-based tool for processing, categorizing, and summarizing monthly financial transactions from Excel files and text-based PDF credit card statements. It supports Hebrew-language inputs and outputs a categorized summary into a pre-designed Excel dashboard. ## Features ### Core Features - **Modern GUI Interface**: Easy-to-use graphical interface with Hebrew/English support -- **Automated Processing**: Load and normalize Excel transaction files (`.xlsx`, `.xls`) +- **Automated Processing**: Load and normalize transaction files (`.xlsx`, `.xls`, text-based `.pdf`) - **Smart Categorization**: Map merchants to user-defined budget categories and subcategories - **Dashboard Integration**: Output categorized monthly summaries into an existing formatted dashboard - **Backups**: Automatic backups of your dashboard, archived transaction files - **Hebrew Support**: Built-in RTL support for Hebrew text and date formats ### File Management -- **Drag & Drop**: Import files by dragging them into the window +- **Drag & Drop**: Import Excel and text-based PDF files by dragging them into the window - **Duplicate Detection**: SHA256 hash-based detection prevents duplicate imports - **File Preview**: See transaction count, date range, and total when selecting a file - **Quick Delete**: Remove files with confirmation dialog @@ -72,7 +72,7 @@ The Budget Tracker uses a two-tier category system: *(GUI starts by default)* 2. **Import Files**: - - Drag & drop Excel files into the window, OR + - Drag & drop Excel or text-based PDF files into the window, OR - Click "Import Files" button to browse 3. **Preview Files**: Click on a file to see transaction count, dates, and total diff --git a/gui_app.py b/gui_app.py index 6ba88bc..d3bbe02 100644 --- a/gui_app.py +++ b/gui_app.py @@ -37,7 +37,7 @@ from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas # noqa: E402 from matplotlib.figure import Figure # noqa: E402 -from src.config import TRANSACTIONS_DIR, CATEGORIES_FILE_PATH, DASHBOARD_FILE_PATH, APPDATA_DIR, LOG_FILE_NAME, ARCHIVE_DIR # noqa: E402 +from src.config import TRANSACTIONS_DIR, CATEGORIES_FILE_PATH, DASHBOARD_FILE_PATH, APPDATA_DIR, LOG_FILE_NAME, ARCHIVE_DIR, SUPPORTED_EXTENSIONS # noqa: E402 from src.file_manager import ensure_dirs, load_transaction_files, _load_transaction_file # noqa: E402 from src.logger import setup_logging # noqa: E402 from src.normalizer import Normalizer # noqa: E402 @@ -665,29 +665,54 @@ def on_log_level_changed(self, level_name: str): handler.setLevel(LOG_LEVEL_MAP[level_name]) -class LogViewerHandler(logging.Handler, QObject): - """ - Logging handler that forwards log records to the GUI log viewer. - """ +class _LogSignalEmitter(QObject): + """Qt signal bridge for forwarding log records to the viewer.""" log_signal = pyqtSignal(str, str) + +class LogViewerHandler(logging.Handler): + """Logging handler that forwards log records to the GUI log viewer.""" + def __init__(self, viewer: LogViewerWidget): - QObject.__init__(self) - logging.Handler.__init__(self) - self.log_signal.connect(viewer.add_log) + super().__init__() + self._emitter = _LogSignalEmitter() + self._viewer = viewer + self._is_closed = False + self._emitter.log_signal.connect(viewer.add_log) def emit(self, record: logging.LogRecord): + if self._is_closed or self._emitter is None: + return + try: msg = self.format(record) - # Check if the widget still exists before emitting - if hasattr(self, 'log_signal'): - self.log_signal.emit(record.levelname, msg) + self._emitter.log_signal.emit(record.levelname, msg) except RuntimeError: # Widget has been deleted, ignore pass except Exception: self.handleError(record) + def close(self): + """Disconnect Qt resources before logging shutdown runs.""" + self._is_closed = True + + if self._emitter is not None: + try: + self._emitter.log_signal.disconnect() + except (TypeError, RuntimeError): + pass + + if QApplication.instance() is not None: + try: + self._emitter.deleteLater() + except RuntimeError: + pass + + self._emitter = None + self._viewer = None + super().close() + class QuickStatsWidget(QWidget): """Widget displaying quick statistics cards.""" @@ -1086,7 +1111,7 @@ def dropEvent(self, event: QDropEvent): """ Handle file drop event. - Extracts Excel file paths from dropped URLs and emits files_dropped signal. + Extracts supported transaction file paths from dropped URLs and emits files_dropped signal. Args: event: Drop event @@ -1094,7 +1119,7 @@ def dropEvent(self, event: QDropEvent): files = [] for url in event.mimeData().urls(): file_path = url.toLocalFile() - if file_path.endswith(('.xlsx', '.xls')): + if file_path.lower().endswith(tuple(SUPPORTED_EXTENSIONS)): files.append(file_path) if files: @@ -1656,13 +1681,13 @@ def perform_startup_validation(self): def import_dropped_files(self, files: List[str]): """Import files dropped via drag and drop with duplicate detection.""" - from src.validators import validate_excel_file, ValidationError + from src.validators import validate_transaction_file, ValidationError count, errors, duplicates = 0, [], [] for file_path in files: try: src = Path(file_path) - validate_excel_file(src) + validate_transaction_file(src) # Calculate hash src_hash = calculate_file_hash(src) @@ -2020,7 +2045,7 @@ def refresh_files(self): """ Refresh the transaction files list. - Scans the transactions directory for Excel files and updates the file list + Scans the transactions directory for supported transaction files and updates the file list widget. Displays file count and total size information. """ self.file_list.clear() @@ -2029,7 +2054,10 @@ def refresh_files(self): ensure_dirs([TRANSACTIONS_DIR]) return - files = list(TRANSACTIONS_DIR.glob('*.xls*')) + files = [ + file_path for file_path in TRANSACTIONS_DIR.glob('*') + if file_path.is_file() and file_path.suffix.lower() in SUPPORTED_EXTENSIONS + ] for file_path in files: self.file_list.addItem(file_path.name) @@ -2093,7 +2121,7 @@ def refresh_archive(self): """ Refresh the archive files list. - Scans the archive directory for Excel files and updates the archive list widget. + Scans the archive directory for supported transaction files and updates the archive list widget. Displays archive file count. """ self.archive_list.clear() @@ -2102,7 +2130,10 @@ def refresh_archive(self): if not archive_path.exists(): return - files = list(archive_path.glob('*.xls*')) + files = [ + file_path for file_path in archive_path.glob('*') + if file_path.is_file() and file_path.suffix.lower() in SUPPORTED_EXTENSIONS + ] for file_path in files: self.archive_list.addItem(file_path.name) @@ -2125,9 +2156,9 @@ def refresh_all(self): def import_files(self): """ - Import Excel files via file dialog with duplicate detection. + Import transaction files via file dialog with duplicate detection. - Opens a file selection dialog allowing user to choose one or more Excel files. + Opens a file selection dialog allowing user to choose one or more supported transaction files. Copies selected files to the transactions directory and refreshes the file list. Shows success message with count of imported files. Validates files before importing and provides user-friendly error messages. @@ -2137,7 +2168,7 @@ def import_files(self): self, self.translations.get('select_files'), "", - "Excel Files (*.xlsx *.xls)" + "Transaction Files (*.xlsx *.xls *.pdf)" ) if not files: @@ -2170,7 +2201,7 @@ def import_files(self): # Check if source file is locked if is_file_locked(src): - errors.append(f"{src.name}: File is locked (may be open in Excel)") + errors.append(f"{src.name}: File is locked (may be open in another application)") continue dst = TRANSACTIONS_DIR / src.name @@ -2274,7 +2305,7 @@ def process_transactions(self): from src.file_manager import _compute_file_hash, _load_processed_hashes transaction_files = [ f for f in Path(TRANSACTIONS_DIR).glob('*') - if f.suffix in ('.xlsx', '.xls') and f.is_file() + if f.suffix.lower() in SUPPORTED_EXTENSIONS and f.is_file() ] file_hashes = {} @@ -2737,8 +2768,9 @@ def clear_archive(self): if reply == QMessageBox.StandardButton.Yes: archive_path = ARCHIVE_DIR if archive_path.exists(): - for file_path in archive_path.glob('*.xls*'): - file_path.unlink() + for file_path in archive_path.glob('*'): + if file_path.is_file() and file_path.suffix.lower() in SUPPORTED_EXTENSIONS: + file_path.unlink() self.refresh_archive() QMessageBox.information( @@ -2793,6 +2825,7 @@ def closeEvent(self, event): try: logging.getLogger().removeHandler(self._log_viewer_handler) self._log_viewer_handler.close() + self._log_viewer_handler = None except Exception: pass # Ignore errors during cleanup diff --git a/requirements.txt b/requirements.txt index 84a02f5..97c302f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ # Dependencies pandas # Data manipulation and transaction processing openpyxl # Excel file reading and writing +pypdf # Text-based PDF extraction for transaction imports +pdfplumber # Layout-aware PDF parsing for credit card statements PyQt5>=5.15.0 # Modern GUI framework matplotlib>=3.7.0 # Charts and graphs generation diff --git a/roadmap.md b/roadmap.md index 7914dd5..b894832 100644 --- a/roadmap.md +++ b/roadmap.md @@ -8,6 +8,8 @@ - [x] **File Preview**: Show transaction count, date range, and total when selecting file - [x] **Interactive Chart Filtering**: Click category to show subcategory breakdown with RTL support - [x] **Robust File Parsing**: Flexible header detection (handles quote variations, extra spaces, simplified names) +- [x] **PDF Transaction Import**: Import text-based credit card statement PDFs through the same GUI flow as Excel +- [x] **PDF Statement Parsing Hardening**: Added layout-aware parsing, merchant cleanup, foreign-currency support, and focused parser tests - [x] **Pre-loaded Common Mappings**: Default merchant-to-category mappings for 211 common Israeli merchants --- @@ -44,7 +46,7 @@ _(Currently none)_ - [ ] Column position hints (amount typically in rightmost columns) - [ ] **File Format Support** - [ ] CSV import support - - [ ] PDF bank statement parsing + - [x] PDF bank statement parsing - [ ] Direct bank API integration --- diff --git a/src/config.py b/src/config.py index 2feada5..34138e7 100644 --- a/src/config.py +++ b/src/config.py @@ -28,7 +28,7 @@ LOG_FILE_NAME = 'budget.log' TEMPLATE_SHEET_NAME = "Template" -SUPPORTED_EXTENSIONS = ['.xlsx', '.xls'] +SUPPORTED_EXTENSIONS = ['.xlsx', '.xls', '.pdf'] # Default log level LOG_SEVERITY = logging.DEBUG @@ -93,9 +93,9 @@ def get_log_level_name() -> str: } # Security and Validation Constants -MAX_FILE_SIZE_MB = 50 # Maximum file size for Excel files +MAX_FILE_SIZE_MB = 50 # Maximum file size for supported transaction files MAX_CATEGORIES = 10000 # Maximum number of categories to prevent unbounded growth MAX_MERCHANT_NAME_LENGTH = 200 # Maximum length for merchant names -ALLOWED_FILE_EXTENSIONS = ['.xlsx', '.xls'] +ALLOWED_FILE_EXTENSIONS = ['.xlsx', '.xls', '.pdf'] BACKUP_SUFFIX = '.backup' PROCESSING_TIMEOUT_SECONDS = 300 # 5 minutes max processing time \ No newline at end of file diff --git a/src/file_manager.py b/src/file_manager.py index 9e0280f..e7da457 100644 --- a/src/file_manager.py +++ b/src/file_manager.py @@ -1,6 +1,7 @@ import os import json import hashlib +import re from pathlib import Path from zipfile import BadZipFile from datetime import datetime @@ -8,9 +9,26 @@ import pandas as pd import logging from typing import List, Optional, Dict +from pypdf import PdfReader from src.config import SUPPORTED_EXTENSIONS, ARCHIVE_DIR, TRANSACTIONS_DIR, FILE_HEADER_KEYWORDS, PROCESSED_HASHES_PATH +from src.pdf_statement_rules import ( + MERCHANT_STOP_TOKENS, + PDF_DETAIL_NOISE_TOKENS, + PDF_DETAIL_PHRASES, + PDF_HEADER_KEYWORD_TOKENS, + PDF_NOISE_LINE_MARKERS, + PDF_SECTOR_SUFFIXES, +) logger = logging.getLogger(__name__) +DATE_PATTERN = re.compile(r"\b\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?\b") +PDF_TRANSACTION_LINE_PATTERN = re.compile( + r"^\s*(?P[₪$]\s*-?[\d,.]+)\s+" + r"(?:(?P\d{2}/\d{2}/\d{2})\s+)?" + r"(?:(?P[₪$]\s*-?[\d,.]+)\s+)?" + r"(?P.+?)\s+" + r"(?P\d{2}/\d{2}/\d{4})\s*$" +) def _normalize_for_matching(text: str) -> str: """ @@ -56,22 +74,510 @@ def _detect_header_row(raw: pd.DataFrame) -> Optional[int]: return None + +def _build_invalid_file_error(file_path: Path) -> ValueError: + return ValueError( + f"Invalid Transaction File: Could not find the header row in '{file_path.name}'.\n" + f"\nThe file must contain column headers with at least:\n" + f" - תאריך (Date)\n" + f" - שם בית העסק (Merchant)\n" + f" - סכום (Amount)\n" + f"\nPlease verify this is a valid credit card transaction export file." + ) + + +def _contains_hebrew(text: str) -> bool: + return any('\u0590' <= char <= '\u05FF' for char in text) + + +def _normalize_pdf_inline_token(token: str) -> str: + token = token.strip() + if not token: + return token + if _contains_hebrew(token): + return token[::-1] + return token + + +def _normalize_pdf_token_group(tokens: List[str]) -> List[str]: + normalized: List[str] = [] + idx = 0 + + while idx < len(tokens): + token = tokens[idx] + if _contains_hebrew(token): + normalized.append(token) + idx += 1 + continue + + group: List[str] = [] + while idx < len(tokens) and not _contains_hebrew(tokens[idx]): + group.append(tokens[idx]) + idx += 1 + + numeric_suffix: List[str] = [] + while group and re.fullmatch(r"\d{4}", group[-1]): + numeric_suffix.insert(0, group.pop()) + + if len(numeric_suffix) == 1 and numeric_suffix[0].startswith('0'): + numeric_suffix[0] = numeric_suffix[0][::-1] + + if group: + group = list(reversed(group)) + + normalized.extend(group + numeric_suffix) + + return normalized + + +def _normalize_pdf_body_text(text: str) -> str: + tokens = [part for part in re.split(r"\s+", text.strip()) if part] + tokens.reverse() + tokens = [_normalize_pdf_inline_token(token) for token in tokens] + tokens = _normalize_pdf_token_group(tokens) + return ' '.join(tokens).strip() + + +def _normalize_compact_hebrew(text: str) -> str: + return _normalize_for_matching(text).replace(' ', '') + + +def _is_pdf_noise_line(line: str) -> bool: + normalized = _normalize_for_matching(line) + if not normalized: + return True + + normalized_body = _normalize_for_matching(_normalize_pdf_body_text(line)) + if any(marker in normalized for marker in PDF_NOISE_LINE_MARKERS): + return True + if any(marker in normalized_body for marker in PDF_NOISE_LINE_MARKERS): + return True + + body_tokens = { + token.strip('.,:;()[]{}') + for token in _normalize_pdf_body_text(line).split() + if token.strip('.,:;()[]{}') + } + keyword_hits = sum(1 for token in body_tokens if token in PDF_HEADER_KEYWORD_TOKENS) + return keyword_hits >= 3 + + +def _is_pdf_detail_token(token: str) -> bool: + cleaned = token.strip('.,:;()[]{}') + if cleaned in MERCHANT_STOP_TOKENS: + return True + if re.fullmatch(r"\d{4}", cleaned): + return True + if re.fullmatch(r"[\d,.]+", cleaned): + return True + return False + + +def _split_pdf_sector_phrase(merchant: str) -> tuple[str, str]: + tokens = merchant.split() + if not any(re.search(r"[A-Za-z]", token) for token in tokens): + return merchant, '' + + for suffix in PDF_SECTOR_SUFFIXES: + suffix_tokens = suffix.split() + suffix_len = len(suffix_tokens) + for idx in range(len(tokens) - suffix_len + 1): + if tokens[idx:idx + suffix_len] != suffix_tokens: + continue + if not any(re.search(r"[A-Za-z]", token) for token in tokens[:idx]): + continue + + merchant_part = ' '.join(tokens[:idx]).strip() + detail_part = ' '.join(tokens[idx:]).strip() + if merchant_part and detail_part: + return merchant_part, detail_part + + return merchant, '' + + +def _strip_pdf_sector_suffix(merchant: str) -> tuple[str, str]: + merchant = merchant.strip() + compact_merchant = _normalize_compact_hebrew(merchant) + + for suffix in PDF_SECTOR_SUFFIXES: + compact_suffix = _normalize_compact_hebrew(suffix) + if compact_merchant.endswith(compact_suffix): + cut_index = len(merchant) - len(suffix) + stripped = merchant[:cut_index].strip() + if stripped: + return stripped, merchant[cut_index:].strip() + + return merchant, '' + + +def _extract_pdf_detail_phrase(merchant: str) -> tuple[str, str]: + normalized_merchant = _normalize_for_matching(merchant) + for phrase in PDF_DETAIL_PHRASES: + phrase_idx = normalized_merchant.find(_normalize_for_matching(phrase)) + if phrase_idx == -1: + continue + + words = merchant.split() + rebuilt_prefix: List[str] = [] + rebuilt_suffix: List[str] = [] + normalized_progress = '' + + for word in words: + candidate = (normalized_progress + ' ' + _normalize_for_matching(word)).strip() + if phrase_idx >= len(candidate): + rebuilt_prefix.append(word) + normalized_progress = candidate + else: + rebuilt_suffix.append(word) + + prefix = ' '.join(rebuilt_prefix).strip() + suffix = ' '.join(rebuilt_suffix).strip() + if prefix and suffix: + return prefix, suffix + + return merchant, '' + + +def _clean_pdf_details(details: str) -> str: + if not details: + return '' + + tokens = [token for token in details.split() if token] + filtered_tokens = [ + token for token in tokens + if _normalize_for_matching(token.strip('.,:;()[]{}')) not in PDF_DETAIL_NOISE_TOKENS + ] + cleaned = ' '.join(filtered_tokens).strip() + cleaned = re.sub(r"\s+", " ", cleaned) + return cleaned + + +def _is_short_noise_continuation(continuation: str) -> bool: + tokens = continuation.split() + if len(tokens) != 1: + return False + token = tokens[0] + if re.search(r"[A-Za-z0-9]", token): + return False + return len(token) <= 4 + + +def _clean_pdf_merchant_and_details(merchant: str, details: str) -> tuple[str, str]: + merchant = merchant.strip() + details = details.strip() + + merchant, detail_from_phrase = _extract_pdf_detail_phrase(merchant) + merchant, detail_from_sector_phrase = _split_pdf_sector_phrase(merchant) + merchant, sector_suffix = _strip_pdf_sector_suffix(merchant) + merchant_tokens = merchant.split() + + if len(merchant_tokens) >= 2 and re.fullmatch(r"\d{1,3}", merchant_tokens[0]): + detail_from_phrase = f"{detail_from_phrase} {merchant_tokens[0]}".strip() + merchant = ' '.join(merchant_tokens[1:]).strip() + merchant_tokens = merchant.split() + + if len(merchant_tokens) >= 2 and re.fullmatch(r"\d{1,4}", merchant_tokens[-1]): + details = f"{merchant_tokens[-1]} {details}".strip() + merchant = ' '.join(merchant_tokens[:-1]).strip() + + extra_details = [ + part for part in [ + sector_suffix, + detail_from_sector_phrase, + details, + detail_from_phrase, + ] + if part + ] + merged_details = _clean_pdf_details(' '.join(extra_details).strip()) + return merchant, merged_details + + +def _split_pdf_merchant_and_details(body_text: str) -> tuple[str, str]: + tokens = [token for token in body_text.split() if token] + if not tokens: + return '', '' + + merchant_tokens: List[str] = [] + details_tokens: List[str] = [] + stop_found = False + + for token in tokens: + if not stop_found and not _is_pdf_detail_token(token): + merchant_tokens.append(token) + else: + stop_found = True + details_tokens.append(token) + + if not merchant_tokens: + merchant_tokens = tokens[:3] + details_tokens = tokens[3:] + + merchant = ' '.join(merchant_tokens).strip() + details = ' '.join(details_tokens).strip() + return _clean_pdf_merchant_and_details(merchant, details) + + +def _append_pdf_continuation_to_record(record: Dict[str, str], line: str) -> None: + continuation = _normalize_pdf_body_text(line) + if not continuation: + return + if _is_short_noise_continuation(continuation): + return + + card_match = re.search(r"\b\d{4}\b", continuation) + if card_match and not record.get('כרטיס'): + record['כרטיס'] = card_match.group(0) + + existing_text = ' '.join([ + record.get('שם בית העסק', ''), + record.get('הערות', ''), + ]).strip() + if continuation and continuation in existing_text: + return + + continuation = _clean_pdf_details(continuation) + if not continuation: + return + + if record.get('הערות'): + record['הערות'] = f"{record['הערות']} {continuation}".strip() + else: + record['הערות'] = continuation + + +def _build_pdf_transaction_record(match: re.Match) -> Dict[str, str]: + body_text = _normalize_pdf_body_text(match.group('body')) + merchant, details = _split_pdf_merchant_and_details(body_text) + record: Dict[str, str] = { + 'תאריך': match.group('transaction_date'), + 'שם בית העסק': merchant or body_text, + 'סכום': match.group('amount'), + 'הערות': details or body_text, + } + + purchase_amount = match.group('purchase_amount') + if purchase_amount: + record['סכום קנייה'] = purchase_amount + + charge_due_date = match.group('charge_due_date') + if charge_due_date: + record['חיוב לתאריך'] = charge_due_date + + card_match = re.search(r"\b\d{4}\b", body_text) + if card_match: + record['כרטיס'] = card_match.group(0) + + return record + + +def _rows_to_dataframe(rows: List[List[str]]) -> pd.DataFrame: + if not rows: + return pd.DataFrame() + + max_width = max(len(row) for row in rows) + normalized_rows = [ + row + [None] * (max_width - len(row)) + for row in rows + ] + return pd.DataFrame(normalized_rows) + + +def _is_header_like_line(cells: List[str]) -> bool: + line_normalized = _normalize_for_matching(' '.join(cell for cell in cells if cell)) + has_date = any( + _normalize_for_matching(keyword) in line_normalized + for keyword in FILE_HEADER_KEYWORDS['mandatory']['transaction_date'] + ) + has_merchant = any( + _normalize_for_matching(keyword) in line_normalized + for keyword in FILE_HEADER_KEYWORDS['mandatory']['merchant'] + ) + has_amount = any( + _normalize_for_matching(keyword) in line_normalized + for keyword in FILE_HEADER_KEYWORDS['mandatory']['amount'] + ) + return has_date and has_merchant and has_amount + + +def _find_matching_column(headers: List[str], aliases: List[str]) -> Optional[int]: + normalized_aliases = {_normalize_for_matching(alias) for alias in aliases} + for idx, header in enumerate(headers): + if _normalize_for_matching(header) in normalized_aliases: + return idx + return None + + +def _append_pdf_continuation(previous_row: List[str], headers: List[str], extra_cells: List[str]) -> None: + extra_text = ' '.join(cell for cell in extra_cells if cell).strip() + if not extra_text: + return + + target_idx = _find_matching_column(headers, FILE_HEADER_KEYWORDS['optional']['misc']) + if target_idx is None: + target_idx = _find_matching_column(headers, FILE_HEADER_KEYWORDS['mandatory']['merchant']) + if target_idx is None: + target_idx = len(previous_row) - 1 + + previous_value = previous_row[target_idx].strip() + previous_row[target_idx] = f"{previous_value} {extra_text}".strip() + + +def _normalize_pdf_table(rows: List[List[str]], file_path: Path) -> pd.DataFrame: + raw = _rows_to_dataframe(rows) + if raw.empty: + raise ValueError(f"Invalid Transaction File: '{file_path.name}' does not contain readable text.") + + header_idx = _detect_header_row(raw) + if header_idx is None: + raise _build_invalid_file_error(file_path) + + headers = [ + '' if pd.isna(cell) else str(cell).strip() + for cell in raw.iloc[header_idx].tolist() + ] + expected_width = len(headers) + normalized_rows: List[List[str]] = [] + + for row_values in raw.iloc[header_idx + 1:].values.tolist(): + cells = [ + '' if pd.isna(cell) else str(cell).strip() + for cell in row_values + ] + + while cells and not cells[-1]: + cells.pop() + + if not any(cells): + continue + + if _is_header_like_line(cells): + continue + + if len(cells) > expected_width: + cells = cells[:expected_width - 1] + [' '.join(cells[expected_width - 1:]).strip()] + elif len(cells) < expected_width: + has_date = any(DATE_PATTERN.search(cell) for cell in cells if cell) + if normalized_rows and not has_date: + _append_pdf_continuation(normalized_rows[-1], headers, cells) + continue + cells = cells + [''] * (expected_width - len(cells)) + + normalized_rows.append(cells) + + df = pd.DataFrame(normalized_rows, columns=headers) + df['source_file'] = file_path.name + return df + + +def _split_pdf_line(line: str) -> List[str]: + cleaned = line.replace('\u00a0', ' ').replace('\uf0b7', ' ').strip() + if not cleaned: + return [] + + cells = [part.strip() for part in re.split(r"\s{2,}|\t+", cleaned) if part.strip()] + return cells or [cleaned] + + +def _extract_pdf_lines(file_path: Path) -> List[str]: + import pdfplumber + + lines: List[str] = [] + with pdfplumber.open(str(file_path)) as pdf: + for page in pdf.pages: + for line in page.extract_text_lines(layout=True): + text = (line.get('text') or '').strip() + if text: + lines.append(' '.join(text.split())) + return lines + + +def _extract_pdf_rows(file_path: Path) -> List[List[str]]: + reader = PdfReader(str(file_path)) + rows: List[List[str]] = [] + + for page in reader.pages: + try: + text = page.extract_text(extraction_mode="layout") + except TypeError: + text = page.extract_text() + + if not text: + text = page.extract_text() + if not text: + continue + + for line in text.splitlines(): + cells = _split_pdf_line(line) + if cells: + rows.append(cells) + + return rows + + +def _load_layout_aware_pdf_transaction_file(file_path: Path) -> Optional[pd.DataFrame]: + records: List[Dict[str, str]] = [] + current_record: Optional[Dict[str, str]] = None + + for line in _extract_pdf_lines(file_path): + if _is_pdf_noise_line(line): + continue + + match = PDF_TRANSACTION_LINE_PATTERN.match(line) + if match: + if current_record and current_record.get('שם בית העסק'): + records.append(current_record) + + current_record = _build_pdf_transaction_record(match) + continue + + if ( + current_record + and not DATE_PATTERN.search(line) + and not line.lstrip().startswith(('₪', '$')) + ): + _append_pdf_continuation_to_record(current_record, line) + + if current_record and current_record.get('שם בית העסק'): + records.append(current_record) + + if not records: + return None + + df = pd.DataFrame(records) + df['source_file'] = file_path.name + return df + + +def _load_excel_transaction_file(file_path: Path) -> pd.DataFrame: + raw = pd.read_excel(file_path, header=None, engine='openpyxl') + header_idx = _detect_header_row(raw) + if header_idx is None: + raise _build_invalid_file_error(file_path) + df = pd.read_excel(file_path, header=header_idx, engine='openpyxl') + df['source_file'] = file_path.name + return df + + +def _load_pdf_transaction_file(file_path: Path) -> pd.DataFrame: + try: + layout_df = _load_layout_aware_pdf_transaction_file(file_path) + if layout_df is not None and not layout_df.empty: + return layout_df + except Exception as exc: + logger.warning(f"Layout-aware PDF parsing failed for {file_path.name}: {exc}") + + rows = _extract_pdf_rows(file_path) + return _normalize_pdf_table(rows, file_path) + def _load_transaction_file(file_path: Path) -> Optional[pd.DataFrame]: try: - raw = pd.read_excel(file_path, header=None, engine='openpyxl') - header_idx = _detect_header_row(raw) - if header_idx is None: - raise ValueError( - f"Invalid Transaction File: Could not find the header row in '{file_path.name}'.\n" - f"\nThe file must contain column headers with at least:\n" - f" - תאריך (Date)\n" - f" - שם בית העסק (Merchant)\n" - f" - סכום (Amount)\n" - f"\nPlease verify this is a valid credit card transaction export file." - ) - df = pd.read_excel(file_path, header=header_idx, engine='openpyxl') - df['source_file'] = file_path.name - logger.info(f"Loaded file: {file_path.name} (rows: {len(df)}, header row at: {header_idx})") + suffix = file_path.suffix.lower() + if suffix == '.pdf': + df = _load_pdf_transaction_file(file_path) + else: + df = _load_excel_transaction_file(file_path) + logger.info(f"Loaded file: {file_path.name} (rows: {len(df)})") return df except Exception as e: logger.error(f"Failed to load {file_path.name}: {e}") @@ -80,12 +586,12 @@ def _load_transaction_file(file_path: Path) -> Optional[pd.DataFrame]: def load_transaction_files(transactions_dir: str | Path) -> List[pd.DataFrame]: """ - Load all Excel files from the transactions directory that have a recognizable header row. + Load all supported transaction files with a recognizable header row. """ files = list(Path(transactions_dir).glob('*')) dataframes = [] for file_path in files: - if file_path.suffix not in SUPPORTED_EXTENSIONS: + if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS: continue df = _load_transaction_file(file_path) if df is not None: diff --git a/src/logger.py b/src/logger.py index 2354f2a..0279594 100644 --- a/src/logger.py +++ b/src/logger.py @@ -10,6 +10,19 @@ # Default log rotation settings MAX_LOG_SIZE_MB = 10 BACKUP_COUNT = 5 +NOISY_LIBRARY_LOGGERS = ( + 'matplotlib', + 'PIL', + 'pdfminer', + 'pdfminer.converter', + 'pdfminer.layout', + 'pdfminer.pdfdocument', + 'pdfminer.pdfinterp', + 'pdfminer.pdfpage', + 'pdfminer.pdfparser', + 'pdfminer.psparser', + 'pdfplumber', +) class StructuredFormatter(logging.Formatter): @@ -102,6 +115,10 @@ def setup_logging(log_dir: str, log_file_name: str, console_handler.setFormatter(formatter) logger.addHandler(console_handler) + # Keep third-party internals from flooding the app logs in DEBUG mode. + for logger_name in NOISY_LIBRARY_LOGGERS: + logging.getLogger(logger_name).setLevel(logging.WARNING) + logger.debug("Logging initialized", extra={ 'log_file': log_file, 'max_size_mb': max_bytes_mb, diff --git a/src/normalizer.py b/src/normalizer.py index aa16ce5..dca7194 100644 --- a/src/normalizer.py +++ b/src/normalizer.py @@ -70,7 +70,7 @@ def _parse_date(series: pd.Series) -> pd.Series: @staticmethod def _parse_amount(series: pd.Series) -> pd.Series: - s = series.astype(str).str.replace(r"[,\s₪]", "", regex=True) + s = series.astype(str).str.replace(r"[,\s₪$]", "", regex=True) return pd.to_numeric(s, errors='coerce') @staticmethod diff --git a/src/pdf_statement_rules.py b/src/pdf_statement_rules.py new file mode 100644 index 0000000..a5f05ec --- /dev/null +++ b/src/pdf_statement_rules.py @@ -0,0 +1,56 @@ +""" +Rules and token lists for text-based PDF statement parsing. +""" + +MERCHANT_STOP_TOKENS = { + 'מזהה', 'כרטיס', 'לא', 'תשלום', 'מבצע', 'הוראת', 'קבע', 'זיכוי', + 'ש"ח', 'ח"ש', 'MC', 'Credit', 'Debit' +} + +PDF_NOISE_LINE_MARKERS = ( + 'סה"כ לתאריך', + 'פירוט עסק', + 'דף חיוב', + 'עמוד', + 'prd-', + 'www.', + 'ריביות', + 'תנאי השירות', + 'הודעה', +) + +PDF_DETAIL_PHRASES = ( + 'עיגול לטובה', + 'עיגלת לטובה', + 'הוראת קבע', + 'מזהה כרטיס', + 'תשלום', + 'זיכוי', +) + +PDF_SECTOR_SUFFIXES = ( + 'פנאי בילוי', + 'מזון ומשקא', + 'מזון ומשקאות', + 'ריהוט ובית', + 'ביטוח ופינ', + 'ביטוח ופיננסים', + 'מסעדות', + 'מוסדות', + 'תקשורת', + 'גז', + 'רכב ותחבורה', + 'רכבותחבור', + 'רכבותחבורה', + 'תיירות', +) + +PDF_HEADER_KEYWORD_TOKENS = { + 'תאריך', 'חיוב', 'סכום', 'כרטיס', 'פירוט', 'ענף', + 'שם', 'בית', 'העסק', 'עסק', 'עסקה', 'הוצג', +} + +PDF_DETAIL_NOISE_TOKENS = { + 'תאריך', 'חיוב', 'סכום', 'פירוט', 'ענף', 'הוצג', + 'עסקה', 'העסקה', 'עסק', 'העסק', +} diff --git a/src/translations.py b/src/translations.py index 95d9a45..1d1bf01 100644 --- a/src/translations.py +++ b/src/translations.py @@ -22,7 +22,7 @@ class Translations: 'refresh_all': 'רענן דשבורד', 'files_count': 'קבצים: {count}', 'total_size': 'גודל כולל: {size}', - 'drag_drop_hint': 'גרור קבצי Excel לכאן', + 'drag_drop_hint': 'גרור קבצי Excel או PDF לכאן', # Process section 'process_transactions': 'עבד עסקאות', @@ -105,7 +105,7 @@ class Translations: # Messages 'no_files': 'לא נמצאו קבצי עסקאות', - 'select_files': 'בחר קבצי Excel', + 'select_files': 'בחר קבצי Excel או PDF', 'file_imported': 'קובץ יובא בהצלחה', 'files_imported': '{count} קבצים יובאו בהצלחה', 'dashboard_opened': 'הדשבורד נפתח באקסל', @@ -153,7 +153,7 @@ class Translations: 'refresh_all': 'Refresh Dashboard', 'files_count': 'Files: {count}', 'total_size': 'Total Size: {size}', - 'drag_drop_hint': 'Drag Excel files here', + 'drag_drop_hint': 'Drag Excel or PDF files here', # Process section 'process_transactions': 'Process Transactions', @@ -248,7 +248,7 @@ class Translations: # Messages 'no_files': 'No transaction files found', - 'select_files': 'Select Excel Files', + 'select_files': 'Select Excel or PDF Files', 'file_imported': 'File imported successfully', 'files_imported': '{count} files imported successfully', 'dashboard_opened': 'Dashboard opened in Excel', diff --git a/src/validators.py b/src/validators.py index 0f0acd5..294c2cf 100644 --- a/src/validators.py +++ b/src/validators.py @@ -253,12 +253,12 @@ def validate_user_input(input_str: str, input_type: str) -> tuple[bool, Optional return True, None -def validate_excel_file(file_path: Path) -> bool: +def validate_transaction_file(file_path: Path) -> bool: """ - Comprehensive validation for Excel files. + Comprehensive validation for supported transaction files. Args: - file_path: Path to Excel file + file_path: Path to transaction file Returns: True if all validations pass @@ -273,7 +273,12 @@ def validate_excel_file(file_path: Path) -> bool: raise ValidationError( f"Invalid File: '{file_path}' is not a valid file.\n" f"It may be a directory or a special system item.\n" - f"Please select a regular Excel file (.xlsx or .xls)." + f"Please select a regular transaction file (.xlsx, .xls, or .pdf)." ) return True + + +def validate_excel_file(file_path: Path) -> bool: + """Backward-compatible wrapper for transaction file validation.""" + return validate_transaction_file(file_path) diff --git a/tests/unit/test_file_manager.py b/tests/unit/test_file_manager.py index ce668ab..91141be 100644 --- a/tests/unit/test_file_manager.py +++ b/tests/unit/test_file_manager.py @@ -4,7 +4,7 @@ """ import pytest import pandas as pd -from pathlib import Path +from src import file_manager from src.file_manager import ensure_dirs @@ -35,3 +35,162 @@ def test_ensure_dirs_handles_existing(temp_dir): ensure_dirs([existing_dir]) assert existing_dir.exists() + + +def test_load_transaction_file_excel_with_header_detection(tmp_path): + """Test loading a real Excel transaction file with header detection.""" + file_path = tmp_path / "transactions.xlsx" + + raw_df = pd.DataFrame([ + ["Statement period", "", "", ""], + ["Generated automatically", "", "", ""], + ["תאריך", "שם בית העסק", "סכום", "שם כרטיס"], + ["01/01/2025", "Supermarket", 100.50, "1234"], + ["02/01/2025", "Coffee Shop", 24.90, "1234"], + ]) + raw_df.to_excel(file_path, index=False, header=False) + + result = file_manager._load_transaction_file(file_path) + + assert result is not None + assert len(result) == 2 + assert list(result["שם בית העסק"]) == ["Supermarket", "Coffee Shop"] + assert list(result["סכום"]) == [100.50, 24.90] + assert set(result["source_file"]) == {"transactions.xlsx"} + + +def test_load_transaction_file_pdf_from_extracted_rows(monkeypatch, tmp_path): + """Test loading a statement-style PDF transaction file from extracted lines.""" + file_path = tmp_path / "transactions.pdf" + file_path.write_bytes(b"%PDF-1.4\n") + + extracted_lines = [ + "תוקסע טוריפי", + "₪ 100.00 ₪ 100.00 4321 Demo Wallet סיטרכ ההזמ יוליב יאנפ המגודל תונח 01/02/2026", + "₪ 991.20 ₪ 11,900.00 אל 12 - מ 12 םולשת תיבו טוהיר המגודל תיב תונח 02/03/2025", + "₪ -36.00 ₪ -36.00 אל יוכיז תודעסמ SHOPONLINE 13/02/2026", + '₪ 3,971.63 02/03/26 ךיראתל כ"הס', + ] + + monkeypatch.setattr(file_manager, "_extract_pdf_lines", lambda _: extracted_lines) + + result = file_manager._load_transaction_file(file_path) + + assert result is not None + assert len(result) == 3 + assert list(result["שם בית העסק"]) == [ + "חנות לדוגמה", + "חנות בית לדוגמה", + "SHOPONLINE", + ] + assert result.iloc[0]["כרטיס"] == "4321" + assert result.iloc[1]["סכום קנייה"] == "₪ 11,900.00" + assert "פנאי בילוי" in result.iloc[0]["הערות"] + assert "מסעדות" in result.iloc[2]["הערות"] + assert set(result["source_file"]) == {"transactions.pdf"} + + from src.normalizer import Normalizer + normalized = Normalizer().normalize(result) + assert len(normalized) == 3 + assert "merchant" in normalized.columns + + +def test_load_transaction_file_pdf_with_due_date_and_continuation(monkeypatch, tmp_path): + """Test PDF parsing with optional due date and continuation lines.""" + file_path = tmp_path / "transactions_2.pdf" + file_path.write_bytes(b"%PDF-1.4\n") + + extracted_lines = [ + "תוקסע טוריפי", + "₪ 58.00 15/02/26 ₪ 57.38 אל םיילטיגיד םיתוריש EXAMPLENET 10/02/2026", + "₪ 44.00 ₪ 44.00 הבוטל תלגיע STOREONLINE 11/02/2026", + "4321 Digital Wallet", + '₪ 102.00 15/02/26 ךיראתל כ"הס', + "PRD-123456 footer line", + ] + + monkeypatch.setattr(file_manager, "_extract_pdf_lines", lambda _: extracted_lines) + + result = file_manager._load_transaction_file(file_path) + + assert result is not None + assert len(result) == 2 + assert list(result["שם בית העסק"]) == [ + "EXAMPLENET שירותים דיגיטליים", + "STOREONLINE", + ] + assert result.iloc[0]["חיוב לתאריך"] == "15/02/26" + assert result.iloc[0]["סכום קנייה"] == "₪ 57.38" + assert result.iloc[1]["כרטיס"] == "4321" + assert "עיגלת לטובה" in result.iloc[1]["הערות"] + assert "Digital Wallet" in result.iloc[1]["הערות"] + + from src.normalizer import Normalizer + normalized = Normalizer().normalize(result) + assert len(normalized) == 2 + assert list(normalized["merchant"]) == [ + "EXAMPLENET שירותים דיגיטליים", + "STOREONLINE", + ] + + +def test_load_transaction_file_pdf_strips_branch_codes_and_keeps_foreign_currency(monkeypatch, tmp_path): + """Test PDF parsing for short numeric merchant codes and dollar amounts.""" + file_path = tmp_path / "transactions_3.pdf" + file_path.write_bytes(b"%PDF-1.4\n") + + extracted_lines = [ + "תוקסע טוריפי", + "$ 23.79 $ 23.79 אל םייללכ םיתוריש STOREUSD 25/02/2026", + "₪ 172.83 ₪ 2,074.00 אל 12 - מ 5 םולשת ניפו חוטיב הבוח חוטיב 9 10/10/2025", + "םוכס ךיראת םוכס סיטרכ טוריפ ףנע קסעה תיב םש ךיראת", + '₪ 196.62 15/02/26 ךיראתל כ"הס', + ] + + monkeypatch.setattr(file_manager, "_extract_pdf_lines", lambda _: extracted_lines) + + result = file_manager._load_transaction_file(file_path) + + assert result is not None + assert len(result) == 2 + assert list(result["שם בית העסק"]) == [ + "STOREUSD שירותים כלליים", + "ביטוח חובה", + ] + assert result.iloc[1]["סכום קנייה"] == "₪ 2,074.00" + assert result.iloc[1]["הערות"].startswith("ביטוח ופינ") + assert "9" in result.iloc[1]["הערות"] + + from src.normalizer import Normalizer + normalized = Normalizer().normalize(result) + assert len(normalized) == 2 + assert list(normalized["amount"]) == [23.79, 172.83] + + +def test_load_transaction_file_pdf_splits_foreign_merchant_suffixes(monkeypatch, tmp_path): + """Test foreign merchants keep the merchant name and move sector/country to details.""" + file_path = tmp_path / "transactions_4.pdf" + file_path.write_bytes(b"%PDF-1.4\n") + + extracted_lines = [ + "תוקסע טוריפי", + "$ 7.92 $ 7.92 אל 3968 טנרטניא סיטרכ ההזמ .הינטירב יוליב יאנפ PAYPAL *GOOGLE YOUTUBE 07/02/2026", + "חמו", + '₪ 7.92 15/02/26 ךיראתל כ"הס', + ] + + monkeypatch.setattr(file_manager, "_extract_pdf_lines", lambda _: extracted_lines) + + result = file_manager._load_transaction_file(file_path) + + assert result is not None + assert len(result) == 1 + assert result.iloc[0]["שם בית העסק"] == "PAYPAL *GOOGLE YOUTUBE" + assert "פנאי בילוי" in result.iloc[0]["הערות"] + assert "בריטניה" in result.iloc[0]["הערות"] + assert "חמו" not in result.iloc[0]["הערות"] + + from src.normalizer import Normalizer + normalized = Normalizer().normalize(result) + assert len(normalized) == 1 + assert normalized.iloc[0]["merchant"] == "PAYPAL *GOOGLE YOUTUBE" diff --git a/tests/unit/test_gui_main_window.py b/tests/unit/test_gui_main_window.py index f80c3ec..86c2af5 100644 --- a/tests/unit/test_gui_main_window.py +++ b/tests/unit/test_gui_main_window.py @@ -2,7 +2,8 @@ Tests for main window GUI components. """ import pytest -from PyQt5.QtWidgets import QApplication +import gui_app as gui_module +from PyQt5.QtWidgets import QApplication, QMessageBox from PyQt5.QtCore import Qt from gui_app import BudgetTrackerGUI from src.config import TRANSACTIONS_DIR, DASHBOARD_FILE_PATH @@ -15,17 +16,30 @@ @pytest.fixture def gui_window(qapp, tmp_path, monkeypatch): """Create GUI window with temporary directories.""" + transactions_dir = tmp_path / 'transactions' + archive_dir = tmp_path / 'archive' + dashboard_path = tmp_path / 'dashboard.xlsx' + appdata_dir = tmp_path / 'appdata' + categories_path = tmp_path / 'categories.json' + # Patch config paths to use temp directory - monkeypatch.setattr('src.config.TRANSACTIONS_DIR', tmp_path / 'transactions') - monkeypatch.setattr('src.config.DASHBOARD_FILE_PATH', tmp_path / 'dashboard.xlsx') - monkeypatch.setattr('src.config.APPDATA_DIR', tmp_path / 'appdata') - monkeypatch.setattr('src.config.ARCHIVE_DIR', tmp_path / 'archive') - monkeypatch.setattr('src.config.CATEGORIES_FILE_PATH', tmp_path / 'categories.json') + monkeypatch.setattr('src.config.TRANSACTIONS_DIR', transactions_dir) + monkeypatch.setattr('src.config.DASHBOARD_FILE_PATH', dashboard_path) + monkeypatch.setattr('src.config.APPDATA_DIR', appdata_dir) + monkeypatch.setattr('src.config.ARCHIVE_DIR', archive_dir) + monkeypatch.setattr('src.config.CATEGORIES_FILE_PATH', categories_path) + + # Patch already-imported module globals used by the GUI + monkeypatch.setattr(gui_module, 'TRANSACTIONS_DIR', transactions_dir) + monkeypatch.setattr(gui_module, 'DASHBOARD_FILE_PATH', dashboard_path) + monkeypatch.setattr(gui_module, 'APPDATA_DIR', appdata_dir) + monkeypatch.setattr(gui_module, 'ARCHIVE_DIR', archive_dir) + monkeypatch.setattr(gui_module, 'CATEGORIES_FILE_PATH', categories_path) # Create directories - (tmp_path / 'transactions').mkdir() + transactions_dir.mkdir() (tmp_path / 'output').mkdir() - (tmp_path / 'archive').mkdir() + archive_dir.mkdir() window = BudgetTrackerGUI('en') return window @@ -75,3 +89,56 @@ def test_clear_archive(self, gui_window): # Method exists and handles confirmation dialog assert hasattr(gui_window, 'clear_archive') + def test_import_files_accepts_pdf(self, gui_window, tmp_path, monkeypatch): + """Test importing a PDF through the file dialog flow.""" + source_pdf = tmp_path / "statement.pdf" + source_pdf.write_bytes(b"%PDF-1.4\n") + + monkeypatch.setattr( + gui_module.QFileDialog, + 'getOpenFileNames', + lambda *args, **kwargs: ([str(source_pdf)], "Transaction Files (*.xlsx *.xls *.pdf)") + ) + monkeypatch.setattr(gui_module, 'check_file_permissions', lambda path, mode: (True, None)) + monkeypatch.setattr(gui_module, 'is_file_locked', lambda path: False) + monkeypatch.setattr( + gui_module.QMessageBox, + 'information', + lambda *args, **kwargs: QMessageBox.StandardButton.Ok + ) + monkeypatch.setattr( + gui_module.QMessageBox, + 'warning', + lambda *args, **kwargs: QMessageBox.StandardButton.Ok + ) + + gui_window.import_files() + + imported_path = gui_module.TRANSACTIONS_DIR / source_pdf.name + assert imported_path.exists() + assert gui_window.file_list.count() == 1 + assert gui_window.file_list.item(0).text() == source_pdf.name + + def test_import_dropped_files_accepts_pdf(self, gui_window, tmp_path, monkeypatch): + """Test importing a PDF through the dropped files flow.""" + source_pdf = tmp_path / "dropped_statement.pdf" + source_pdf.write_bytes(b"%PDF-1.4\n") + + monkeypatch.setattr( + gui_module.QMessageBox, + 'information', + lambda *args, **kwargs: QMessageBox.StandardButton.Ok + ) + monkeypatch.setattr( + gui_module.QMessageBox, + 'warning', + lambda *args, **kwargs: QMessageBox.StandardButton.Ok + ) + + gui_window.import_dropped_files([str(source_pdf)]) + + imported_path = gui_module.TRANSACTIONS_DIR / source_pdf.name + assert imported_path.exists() + assert gui_window.file_list.count() == 1 + assert gui_window.file_list.item(0).text() == source_pdf.name +