From 673d21ae7afa193d267246a6e6a32da0ce6e7f7a Mon Sep 17 00:00:00 2001 From: Najeb Abdullahi Date: Mon, 3 Aug 2026 15:33:25 -0500 Subject: [PATCH 1/5] feat: add ClobValue and BlobValue LOB wrapper classes --- mapepire_python/lob.py | 99 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 mapepire_python/lob.py diff --git a/mapepire_python/lob.py b/mapepire_python/lob.py new file mode 100644 index 0000000..313c493 --- /dev/null +++ b/mapepire_python/lob.py @@ -0,0 +1,99 @@ +"""LOB (Large Object) value wrappers for CLOB, NCLOB, DBCLOB, and BLOB columns. + +The Mapepire server inlines LOB data directly in row payloads, so these +classes wrap the already-loaded value behind a file-like read() interface +as recommended by DB-API 2.0 (PEP 249). +""" + +from typing import Optional, Union + +__all__ = ["ClobValue", "BlobValue", "LOBValue", "LOB_TYPES", "BLOB_TYPES", "CLOB_TYPES"] + +# SQL type names that map to character LOBs (str content) +CLOB_TYPES = frozenset({"CLOB", "NCLOB", "DBCLOB"}) + +# SQL type names that map to binary LOBs (bytes content) +BLOB_TYPES = frozenset({"BLOB"}) + +# Union of all LOB type names — used for fast membership checks +LOB_TYPES = CLOB_TYPES | BLOB_TYPES + + +class ClobValue: + """Wraps an inlined CLOB/NCLOB/DBCLOB string value. + + Provides a file-like read() interface so callers can consume the + content incrementally or all at once, matching DB-API 2.0 expectations. + """ + + def __init__(self, value: Optional[str]) -> None: + self._value: str = value if value is not None else "" + self._pos: int = 0 + + @property + def value(self) -> str: + """The full underlying string, regardless of read position.""" + return self._value + + def read(self, size: int = -1) -> str: + """Read up to *size* characters, advancing the position. + + If *size* is -1 (the default), return all remaining content. + Returns an empty string when the end has been reached. + """ + if size == -1: + chunk = self._value[self._pos:] + self._pos = len(self._value) + else: + chunk = self._value[self._pos: self._pos + size] + self._pos += len(chunk) + return chunk + + def __repr__(self) -> str: + preview = self._value[:40] + "..." if len(self._value) > 40 else self._value + return f"ClobValue({preview!r})" + + +class BlobValue: + """Wraps an inlined BLOB bytes value. + + Provides a file-like read() interface so callers can consume the + content incrementally or all at once, matching DB-API 2.0 expectations. + """ + + def __init__(self, value: Optional[Union[str, bytes]]) -> None: + # The server may deliver BLOB data as a hex string or raw bytes. + if isinstance(value, str): + self._value = bytes.fromhex(value) + elif isinstance(value, bytes): + self._value = value + else: + self._value = b"" + self._pos: int = 0 + + @property + def value(self) -> bytes: + """The full underlying bytes, regardless of read position.""" + return self._value + + def read(self, size: int = -1) -> bytes: + """Read up to *size* bytes, advancing the position. + + If *size* is -1 (the default), return all remaining content. + Returns an empty bytes object when the end has been reached. + """ + if size == -1: + chunk = self._value[self._pos:] + self._pos = len(self._value) + else: + chunk = self._value[self._pos: self._pos + size] + self._pos += len(chunk) + return chunk + + def __repr__(self) -> str: + preview = self._value[:20] + return f"BlobValue({preview!r}{'...' if len(self._value) > 20 else ''})" + + +# Convenience union type alias for type annotations +LOBValue = Union[ClobValue, BlobValue] From 51e49a0b0a974ae305aee1c6afcf90efeb51d5f9 Mon Sep 17 00:00:00 2001 From: Najeb Abdullahi Date: Mon, 3 Aug 2026 15:33:53 -0500 Subject: [PATCH 2/5] feat: wrap LOB columns in ClobValue/BlobValue in row_to_tuple --- mapepire_python/core/utils.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mapepire_python/core/utils.py b/mapepire_python/core/utils.py index 07708e1..c5f2cae 100644 --- a/mapepire_python/core/utils.py +++ b/mapepire_python/core/utils.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, List, Optional, cast from .exceptions import CONNECTION_CLOSED, ProgrammingError, ReturnType +from ..lob import BLOB_TYPES, CLOB_TYPES, BlobValue, ClobValue __all__ = ["raise_if_closed", "DB_TYPE_MAP", "row_to_tuple"] @@ -20,10 +21,29 @@ } +def _wrap_lob(value: Any, sql_type: Optional[str]) -> Any: + """Wrap *value* in a LOB object when *sql_type* is a LOB type. + + Returns the value unchanged for all non-LOB types or when sql_type + is unknown. + """ + if sql_type is None or value is None: + return value + upper = sql_type.upper() + if upper in CLOB_TYPES: + return ClobValue(value) + if upper in BLOB_TYPES: + return BlobValue(value) + return value + + def row_to_tuple(row: Any, metadata) -> tuple: if isinstance(row, dict): if metadata and metadata.columns: - return tuple(row.get(col.name, None) for col in metadata.columns) + return tuple( + _wrap_lob(row.get(col.name, None), col.type) + for col in metadata.columns + ) return tuple(row.values()) if isinstance(row, (list, tuple)): return tuple(row) From 446b5b71f97b09b2682b9153375c41964d11910c Mon Sep 17 00:00:00 2001 From: Najeb Abdullahi Date: Mon, 3 Aug 2026 15:34:10 -0500 Subject: [PATCH 3/5] feat: export ClobValue, BlobValue, and LOBValue from package root --- mapepire_python/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mapepire_python/__init__.py b/mapepire_python/__init__.py index 6207968..22ac7c4 100644 --- a/mapepire_python/__init__.py +++ b/mapepire_python/__init__.py @@ -5,6 +5,7 @@ from .asyncio.connection import AsyncConnection from .client.query import QueryState from .client.sql_job import SQLJob +from .lob import BlobValue, ClobValue, LOBValue from .core import ( Connection, Cursor, @@ -28,6 +29,9 @@ "apilevel", "threadsafety", "paramstyle", + "ClobValue", + "BlobValue", + "LOBValue", "DatabaseError", "DataError", "Error", From 275a63cdcbaabe589bd1443d1e5af152e0b7ca69 Mon Sep 17 00:00:00 2001 From: Najeb Abdullahi Date: Tue, 4 Aug 2026 11:05:47 -0500 Subject: [PATCH 4/5] fix: wrap LOB columns in terse-mode, update changelog --- CHANGELOG.md | 2 + mapepire_python/core/utils.py | 5 + tests/unit/test_cursor_pep249.py | 80 +++++++++++++- tests/unit/test_lob.py | 179 +++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_lob.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab27be..f7ac41c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +- feat: add LOB support — CLOB/NCLOB/DBCLOB and BLOB column values are now wrapped in `ClobValue`/`BlobValue` file-like objects (with a PEP 249-style `read()`), exported from the package root +- fix: wrap LOB columns in terse-mode (list) rows, not just dict rows — `cursor.execute(sql, isTerseResults=True)` was previously returning raw unwrapped values for CLOB/BLOB columns - ci: install only `twine`/`packaging` in the release job instead of the full `.[dev]` extra, so it no longer compiles `gssapi` (avoids the `krb5-config: not found` build failure on the release runner) - ci: bump `actions/checkout` v6→v7, `actions/setup-python` v6→v7, `actions/cache` v5→v6 - widen `mypy` requirement to `>=1.0,<2.4` and `isort` requirement to `>=5.12,<8.1`; regenerate `uv.lock` including `cryptography` 48.0.0→48.0.1 diff --git a/mapepire_python/core/utils.py b/mapepire_python/core/utils.py index c5f2cae..0048064 100644 --- a/mapepire_python/core/utils.py +++ b/mapepire_python/core/utils.py @@ -46,6 +46,11 @@ def row_to_tuple(row: Any, metadata) -> tuple: ) return tuple(row.values()) if isinstance(row, (list, tuple)): + if metadata and metadata.columns: + return tuple( + _wrap_lob(value, col.type) + for value, col in zip(row, metadata.columns) + ) return tuple(row) return row diff --git a/tests/unit/test_cursor_pep249.py b/tests/unit/test_cursor_pep249.py index 7dc3502..61d96cf 100644 --- a/tests/unit/test_cursor_pep249.py +++ b/tests/unit/test_cursor_pep249.py @@ -10,7 +10,7 @@ import pytest -from mapepire_python import Cursor +from mapepire_python import BlobValue, ClobValue, Cursor # --------------------------------------------------------------------------- # Helpers @@ -33,6 +33,17 @@ {"EMPNO": "000030", "FIRSTNME": "BOB", "SALARY": 38500.00}, ] +_LOB_COLUMNS = [ + {"name": "ID", "label": "ID", "type": "INTEGER", "display_size": 10, + "precision": None, "scale": None, "nullable": False}, + {"name": "NOTES", "label": "NOTES", "type": "CLOB", "display_size": 100, + "precision": None, "scale": None, "nullable": True}, + {"name": "PHOTO", "label": "PHOTO", "type": "BLOB", "display_size": 100, + "precision": None, "scale": None, "nullable": True}, +] + +_LOB_METADATA = {"column_count": 3, "job": "TEST/QUSER/JOB001", "columns": _LOB_COLUMNS} + class _FakeConn: """Minimal connection stub — just enough for Cursor._closed checks.""" @@ -300,3 +311,70 @@ def test_reflects_dml_update_count(self, mock_sql_job): _queue_dml(socket, update_count=5) cursor.execute("DELETE FROM SAMPLE.EMPLOYEE WHERE BONUS > 1000") assert cursor.rowcount == 5 + + +# --------------------------------------------------------------------------- +# LOB (CLOB/BLOB) wrapping end-to-end through Cursor +# --------------------------------------------------------------------------- + +class TestLobWrapping: + def test_dict_row_wraps_clob_and_blob(self, mock_sql_job): + cursor, socket = _make_cursor(mock_sql_job) + _queue_select( + socket, + rows=[{"ID": 1, "NOTES": "hello world", "PHOTO": "68656c6c6f"}], + metadata=_LOB_METADATA, + ) + cursor.execute("SELECT ID, NOTES, PHOTO FROM SAMPLE.DOCS") + row = cursor.fetchone() + assert row[0] == 1 + assert isinstance(row[1], ClobValue) + assert row[1].value == "hello world" + assert isinstance(row[2], BlobValue) + assert row[2].value == b"hello" + + def test_terse_list_row_wraps_clob_and_blob(self, mock_sql_job): + """Regression test: LOB wrapping must also apply when the server + returns terse (list) rows, not just dict rows. + """ + cursor, socket = _make_cursor(mock_sql_job) + _queue_select( + socket, + rows=[[1, "hello world", "68656c6c6f"]], + metadata=_LOB_METADATA, + ) + cursor.execute("SELECT ID, NOTES, PHOTO FROM SAMPLE.DOCS", isTerseResults=True) + row = cursor.fetchone() + assert row[0] == 1 + assert isinstance(row[1], ClobValue) + assert row[1].value == "hello world" + assert isinstance(row[2], BlobValue) + assert row[2].value == b"hello" + + def test_fetchall_wraps_lob_columns_in_every_row(self, mock_sql_job): + cursor, socket = _make_cursor(mock_sql_job) + _queue_select( + socket, + rows=[ + {"ID": 1, "NOTES": "first", "PHOTO": "68656c6c6f"}, + {"ID": 2, "NOTES": "second", "PHOTO": "776f726c64"}, + ], + metadata=_LOB_METADATA, + ) + cursor.execute("SELECT ID, NOTES, PHOTO FROM SAMPLE.DOCS") + rows = cursor.fetchall() + assert all(isinstance(r[1], ClobValue) for r in rows) + assert all(isinstance(r[2], BlobValue) for r in rows) + assert rows[1][2].value == b"world" + + def test_null_lob_values_stay_none(self, mock_sql_job): + cursor, socket = _make_cursor(mock_sql_job) + _queue_select( + socket, + rows=[{"ID": 1, "NOTES": None, "PHOTO": None}], + metadata=_LOB_METADATA, + ) + cursor.execute("SELECT ID, NOTES, PHOTO FROM SAMPLE.DOCS") + row = cursor.fetchone() + assert row[1] is None + assert row[2] is None diff --git a/tests/unit/test_lob.py b/tests/unit/test_lob.py new file mode 100644 index 0000000..00b6bf5 --- /dev/null +++ b/tests/unit/test_lob.py @@ -0,0 +1,179 @@ +"""Unit tests for LOB (CLOB/BLOB) support. + +Covers: + - ClobValue / BlobValue: construction, read() semantics, NULL handling + - _wrap_lob: type dispatch, case-insensitivity, NULL/unknown-type passthrough + - row_to_tuple: LOB wrapping for both dict rows (normal) and list/tuple + rows (terse mode) +""" +from mapepire_python.core.utils import ColumnMetaData, MetaData, _wrap_lob, row_to_tuple +from mapepire_python.lob import BLOB_TYPES, CLOB_TYPES, LOB_TYPES, BlobValue, ClobValue + +# --------------------------------------------------------------------------- +# ClobValue +# --------------------------------------------------------------------------- + +class TestClobValue: + def test_wraps_string(self): + clob = ClobValue("hello world") + assert clob.value == "hello world" + + def test_none_becomes_empty_string(self): + clob = ClobValue(None) + assert clob.value == "" + + def test_read_all_returns_full_value(self): + clob = ClobValue("hello world") + assert clob.read() == "hello world" + + def test_read_exhausts_after_full_read(self): + clob = ClobValue("hello world") + clob.read() + assert clob.read() == "" + + def test_read_chunked(self): + clob = ClobValue("hello world") + assert clob.read(5) == "hello" + assert clob.read(1) == " " + assert clob.read() == "world" + + def test_repr_does_not_raise(self): + assert "ClobValue" in repr(ClobValue("x" * 100)) + + +# --------------------------------------------------------------------------- +# BlobValue +# --------------------------------------------------------------------------- + +class TestBlobValue: + def test_wraps_hex_string(self): + blob = BlobValue("68656c6c6f") + assert blob.value == b"hello" + + def test_wraps_raw_bytes_unchanged(self): + blob = BlobValue(b"\x01\x02\x03") + assert blob.value == b"\x01\x02\x03" + + def test_none_becomes_empty_bytes(self): + blob = BlobValue(None) + assert blob.value == b"" + + def test_read_all_returns_full_value(self): + blob = BlobValue(b"\x01\x02\x03") + assert blob.read() == b"\x01\x02\x03" + + def test_read_exhausts_after_full_read(self): + blob = BlobValue(b"\x01\x02\x03") + blob.read() + assert blob.read() == b"" + + def test_read_chunked(self): + blob = BlobValue(b"\x01\x02\x03\x04") + assert blob.read(2) == b"\x01\x02" + assert blob.read() == b"\x03\x04" + + def test_repr_does_not_raise(self): + assert "BlobValue" in repr(BlobValue(b"\x00" * 30)) + + +# --------------------------------------------------------------------------- +# LOB type sets +# --------------------------------------------------------------------------- + +class TestLobTypeSets: + def test_clob_types_contents(self): + assert CLOB_TYPES == {"CLOB", "NCLOB", "DBCLOB"} + + def test_blob_types_contents(self): + assert BLOB_TYPES == {"BLOB"} + + def test_lob_types_is_union(self): + assert LOB_TYPES == CLOB_TYPES | BLOB_TYPES + + +# --------------------------------------------------------------------------- +# _wrap_lob +# --------------------------------------------------------------------------- + +class TestWrapLob: + def test_clob_type_wraps_clob_value(self): + result = _wrap_lob("some text", "CLOB") + assert isinstance(result, ClobValue) + assert result.value == "some text" + + def test_blob_type_wraps_blob_value(self): + result = _wrap_lob("68656c6c6f", "BLOB") + assert isinstance(result, BlobValue) + assert result.value == b"hello" + + def test_case_insensitive_type_matching(self): + result = _wrap_lob("some text", "clob") + assert isinstance(result, ClobValue) + + def test_non_lob_type_passthrough(self): + assert _wrap_lob("000010", "CHAR") == "000010" + assert _wrap_lob(52750.00, "DECIMAL") == 52750.00 + + def test_none_value_passthrough_even_for_lob_type(self): + assert _wrap_lob(None, "CLOB") is None + assert _wrap_lob(None, "BLOB") is None + + def test_none_sql_type_passthrough(self): + assert _wrap_lob("some text", None) == "some text" + + def test_nclob_and_dbclob_wrap(self): + assert isinstance(_wrap_lob("x", "NCLOB"), ClobValue) + assert isinstance(_wrap_lob("x", "DBCLOB"), ClobValue) + + +# --------------------------------------------------------------------------- +# row_to_tuple LOB integration +# --------------------------------------------------------------------------- + +_LOB_COLUMNS = [ + ColumnMetaData(name="ID", type="INTEGER", display_size=10, label="ID"), + ColumnMetaData(name="NOTES", type="CLOB", display_size=100, label="NOTES"), + ColumnMetaData(name="PHOTO", type="BLOB", display_size=100, label="PHOTO"), +] +_LOB_METADATA = MetaData(column_count=3, job="TEST/QUSER/JOB001", columns=_LOB_COLUMNS) + + +class TestRowToTupleLobWrapping: + def test_dict_row_wraps_lob_columns(self): + row = {"ID": 1, "NOTES": "hello world", "PHOTO": "68656c6c6f"} + result = row_to_tuple(row, _LOB_METADATA) + assert result[0] == 1 + assert isinstance(result[1], ClobValue) + assert result[1].value == "hello world" + assert isinstance(result[2], BlobValue) + assert result[2].value == b"hello" + + def test_dict_row_null_lob_stays_none(self): + row = {"ID": 1, "NOTES": None, "PHOTO": None} + result = row_to_tuple(row, _LOB_METADATA) + assert result[1] is None + assert result[2] is None + + def test_terse_list_row_wraps_lob_columns(self): + """Regression test: terse-mode rows arrive as lists, not dicts, but + must still be wrapped in ClobValue/BlobValue using positional + column metadata. + """ + row = [1, "hello world", "68656c6c6f"] + result = row_to_tuple(row, _LOB_METADATA) + assert result[0] == 1 + assert isinstance(result[1], ClobValue) + assert result[1].value == "hello world" + assert isinstance(result[2], BlobValue) + assert result[2].value == b"hello" + + def test_terse_tuple_row_wraps_lob_columns(self): + row = (1, "hello world", "68656c6c6f") + result = row_to_tuple(row, _LOB_METADATA) + assert isinstance(result[1], ClobValue) + assert isinstance(result[2], BlobValue) + + def test_list_row_without_metadata_passthrough(self): + row = [1, "hello world", "68656c6c6f"] + result = row_to_tuple(row, None) + assert result == (1, "hello world", "68656c6c6f") From 7621295cf41d8c45fd63e8da6faf1846956d53d8 Mon Sep 17 00:00:00 2001 From: Najeb Abdullahi Date: Tue, 4 Aug 2026 11:10:33 -0500 Subject: [PATCH 5/5] ruff check and isort fix --- mapepire_python/__init__.py | 2 +- mapepire_python/core/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mapepire_python/__init__.py b/mapepire_python/__init__.py index 22ac7c4..36cb873 100644 --- a/mapepire_python/__init__.py +++ b/mapepire_python/__init__.py @@ -5,7 +5,6 @@ from .asyncio.connection import AsyncConnection from .client.query import QueryState from .client.sql_job import SQLJob -from .lob import BlobValue, ClobValue, LOBValue from .core import ( Connection, Cursor, @@ -21,6 +20,7 @@ ) from .core.exceptions import CONNECTION_CLOSED, convert_runtime_errors from .data_types import DaemonServer, JobStatus, QueryOptions, QueryResult +from .lob import BlobValue, ClobValue, LOBValue from .pool.pool_client import Pool, PoolOptions from .pool.pool_job import PoolJob from .version import VERSION as __version__ diff --git a/mapepire_python/core/utils.py b/mapepire_python/core/utils.py index 0048064..9e48cde 100644 --- a/mapepire_python/core/utils.py +++ b/mapepire_python/core/utils.py @@ -4,8 +4,8 @@ from functools import wraps from typing import Any, Callable, Dict, List, Optional, cast -from .exceptions import CONNECTION_CLOSED, ProgrammingError, ReturnType from ..lob import BLOB_TYPES, CLOB_TYPES, BlobValue, ClobValue +from .exceptions import CONNECTION_CLOSED, ProgrammingError, ReturnType __all__ = ["raise_if_closed", "DB_TYPE_MAP", "row_to_tuple"]