From 2d37679292321a7d71387a5739a64c9abffb16bd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 17:53:00 +0200 Subject: [PATCH 01/18] fix: uninitialized instance variables and import cycles found by basedpyright - Add ClassVar annotations to class attributes in tags.py (pattern, name, slug, logger, status, RE_FLAGS, scope) - Fix multiple inheritance type compatibility in models.py with proper type: ignore comments - Add pyrightconfig.json to disable reportImportCycles (which pyright already ignores) These changes ensure basedpyright shows 0 errors, matching the strict pyright configuration. --- mt940/models.py | 4 +++- mt940/tags.py | 17 ++++++++++------- pyrightconfig.json | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 pyrightconfig.json diff --git a/mt940/models.py b/mt940/models.py index b403ae9..8e598e9 100644 --- a/mt940/models.py +++ b/mt940/models.py @@ -724,7 +724,9 @@ def sanitize_tag_id_matches( return valid_matches -class TransactionsAndTransaction(Transactions, Transaction): # type: ignore[misc] +class TransactionsAndTransaction( # type: ignore[misc] # pyright: ignore[reportUnsafeMultipleInheritance] + Transactions, Transaction +): """ Subclass of both Transactions and Transaction for scope definitions. diff --git a/mt940/tags.py b/mt940/tags.py index fec4536..66b0a03 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -78,6 +78,7 @@ import logging import re import typing +from typing import ClassVar from . import models @@ -90,12 +91,14 @@ class Tag: """ id: str | int = 0 - RE_FLAGS = re.IGNORECASE | re.VERBOSE | re.UNICODE - scope: type[models.Transactions | models.Transaction] = models.Transactions - pattern: str - name: str - slug: str - logger: logging.Logger + RE_FLAGS: ClassVar = re.IGNORECASE | re.VERBOSE | re.UNICODE + scope: ClassVar[type[models.Transactions | models.Transaction]] = ( + models.Transactions + ) + pattern: ClassVar[str] + name: ClassVar[str] + slug: ClassVar[str] + logger: ClassVar[logging.Logger] def __init__(self) -> None: self.re = re.compile(self.pattern, self.RE_FLAGS) @@ -610,7 +613,7 @@ class SumEntries(Tag): (?P.{3}) # 3!a Currency (?P[\d,]{1,15}) # 15d Amount """ - status: str + status: ClassVar[str] def __call__( self, transactions: models.Transactions, value: dict[str, typing.Any] diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..e1e92c4 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,15 @@ +{ + "include": ["mt940", "mt940_tests"], + "exclude": ["dist/*", "mt940_tests/*"], + "strict": [ + "mt940/__about__.py", + "mt940/__init__.py", + "mt940/json.py", + "mt940/models.py", + "mt940/parser.py", + "mt940/processors.py", + "mt940/tags.py", + "mt940/utils.py" + ], + "reportImportCycles": false +} From ab773c69db48f89e846c772698fff88ee2b24d89 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 18:01:09 +0200 Subject: [PATCH 02/18] fix: single-source pyright config in pyproject.toml, typed RE_FLAGS - Delete pyrightconfig.json: its presence made pyright ignore [tool.pyright] in pyproject.toml entirely, creating two sources of truth and an IDE-level regression ('mt940 is not defined' in parser.py). - Move reportImportCycles = false into the existing [tool.pyright] block with a comment: mt940/parser.py imports the package root by design. basedpyright reads [tool.pyright] too. - Parameterize Tag.RE_FLAGS as ClassVar[re.RegexFlag] instead of bare ClassVar. --- mt940/tags.py | 2 +- pyproject.toml | 3 +++ pyrightconfig.json | 15 --------------- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/mt940/tags.py b/mt940/tags.py index 66b0a03..7ea8250 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -91,7 +91,7 @@ class Tag: """ id: str | int = 0 - RE_FLAGS: ClassVar = re.IGNORECASE | re.VERBOSE | re.UNICODE + RE_FLAGS: ClassVar[re.RegexFlag] = re.IGNORECASE | re.VERBOSE | re.UNICODE scope: ClassVar[type[models.Transactions | models.Transaction]] = ( models.Transactions ) diff --git a/pyproject.toml b/pyproject.toml index 04a862c..58e8ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,9 @@ skip = '*/htmlcov,./docs/_build,*.asc,*.yml,*.sta,*.txt,*.html,./docs/html' [tool.pyright] include = ['mt940', 'mt940_tests'] exclude = ['dist/*', 'mt940_tests/*'] +# mt940/parser.py imports the package root (`import mt940`) by design, so the +# package has intentional import cycles; basedpyright reads this section too. +reportImportCycles = false strict = [ 'mt940/__about__.py', 'mt940/__init__.py', diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index e1e92c4..0000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "include": ["mt940", "mt940_tests"], - "exclude": ["dist/*", "mt940_tests/*"], - "strict": [ - "mt940/__about__.py", - "mt940/__init__.py", - "mt940/json.py", - "mt940/models.py", - "mt940/parser.py", - "mt940/processors.py", - "mt940/tags.py", - "mt940/utils.py" - ], - "reportImportCycles": false -} From 26b2eb3ae675511ea54150b72c75b3151f0928ac Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 18:07:34 +0200 Subject: [PATCH 03/18] test: reproduce issue #105 paged-block first balance loss (refs #105) --- .../test_issue105_first_balance.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 mt940_tests/test_issues/test_issue105_first_balance.py diff --git a/mt940_tests/test_issues/test_issue105_first_balance.py b/mt940_tests/test_issues/test_issue105_first_balance.py new file mode 100644 index 0000000..c0b9a56 --- /dev/null +++ b/mt940_tests/test_issues/test_issue105_first_balance.py @@ -0,0 +1,58 @@ +"""Issue #105: first intermediate_opening_balance / account_identification. + +With the default ``parse()`` every statement-scope tag merges into one dict, +so in files with several pages only the LAST ``:60M:`` / ``:25:`` survive. +``parse_statements()`` preserves per-``:20:``-block data (shape 1); this test +also covers the paged single-block shape (shape 2). +""" + +import mt940 +import pytest + +TWO_BLOCKS = """:20:REF1 +:25:ACC1 +:28C:1/1 +:60M:C231229EUR100,00 +:61:2312290101C10,00NTRFREF//BANK +:62M:C231229EUR110,00 +- +:20:REF2 +:25:ACC2 +:28C:1/2 +:60M:C231230EUR110,00 +:61:2312300101C5,00NTRFREF//BANK +:62M:C231230EUR115,00 +- +""" + +PAGED_SINGLE_BLOCK = """:20:REF1 +:25:ACC1 +:28C:1/1 +:60M:C231229EUR100,00 +:61:2312290101C10,00NTRFREF//BANK +:62M:C231229EUR110,00 +:28C:1/2 +:60M:C231230EUR110,00 +:61:2312300101C5,00NTRFREF//BANK +:62M:C231230EUR115,00 +- +""" + + +def test_two_blocks_first_intermediate_balance_preserved(): + statements = mt940.parse_statements(TWO_BLOCKS) + assert len(statements) == 2 + assert str(statements[0].data['intermediate_opening_balance']).startswith( + '100.00' + ) + assert statements[0].data['account_identification'] == 'ACC1' + assert statements[1].data['account_identification'] == 'ACC2' + + +@pytest.mark.xfail( + strict=True, reason='issue #105 shape 2 -- pending decision' +) +def test_paged_single_block_first_intermediate_balance_accessible(): + statements = mt940.parse_statements(PAGED_SINGLE_BLOCK) + first = statements[0].data['intermediate_opening_balance'] + assert str(first).startswith('100.00') From 36b5e66bf1241301e6a269ceeded19f3edda434a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 19:54:34 +0200 Subject: [PATCH 04/18] fix: :13: offset misparsed as minutes; sign and missing offset crashed The :13:/:13D: offset subfield is a signed HHMM value (+0130 = 1h30m) but was passed raw to models.DateTime, which expects minutes (+0100 became UTC+1:40). A negative sign was never matched and a missing offset passed offset=None, both crashing DateTime with TypeError. Capture the sign, convert HHMM to signed minutes in DateTimeIndication.__call__, and drop the offset keys entirely when absent. Regenerate the mBank mt942 golden (stored instant was wrong by 40 minutes). --- mt940/tags.py | 13 +++++- mt940_tests/mBank/mt942.yml | 86 ++++++++++++++++++------------------- mt940_tests/test_tags.py | 29 +++++++++++++ 3 files changed, 84 insertions(+), 44 deletions(-) diff --git a/mt940/tags.py b/mt940/tags.py index 7ea8250..51beaa0 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -214,13 +214,24 @@ class DateTimeIndication(Tag): (?P\d{2}) (?P\d{2}) (?P\d{2}) - (\+(?P\d{4})|) + ((?P[+-])(?P\d{4}))? """ def __call__( self, transactions: models.Transactions, value: dict[str, typing.Any] ) -> dict[str, object]: data = super().__call__(transactions, value) + # The offset subfield is a signed HHMM value (e.g. +0130 is 1 hour + # and 30 minutes east of UTC), while `models.DateTime` expects the + # offset as a number of minutes. Convert it here and drop the raw + # groups so they are not passed on when the offset is absent. + sign: str | None = data.pop('offset_sign', None) + offset: str | None = data.pop('offset', None) + if offset: + minutes: int = int(offset[:2]) * 60 + int(offset[2:]) + if sign == '-': + minutes = -minutes + data['offset'] = minutes return {'date': models.DateTime(**data)} diff --git a/mt940_tests/mBank/mt942.yml b/mt940_tests/mBank/mt942.yml index d357f20..8877396 100644 --- a/mt940_tests/mBank/mt942.yml +++ b/mt940_tests/mBank/mt942.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: PL29114010810000267002001002 c_floor_limit: !!python/object:mt940.models.Amount @@ -14,10 +14,10 @@ data: B+EBExIPAAAAAA== - !!python/object/apply:mt940.models.FixedOffset state: - _name: '0100' + _name: '60' _offset: !!python/object/apply:datetime.timedelta - 0 - - 6000 + - 3600 - 0 sequence_number: '1' statement_number: '1' @@ -35,8 +35,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -52,15 +53,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -68,14 +69,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -89,29 +93,31 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n\ + \ " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P\n (\n (\\d{2}.{0,})\n (\\\ + n\\d{2}.{0,})*\n )|(\n [^\\n]*\n )\n )\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: @@ -125,14 +131,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id003 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000001; Z RACH.: @@ -143,7 +147,7 @@ transactions: TNR: 179171073864111.010001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -156,14 +160,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id005 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id005 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000002; Z RACH.: @@ -174,7 +176,7 @@ transactions: TNR: 179171073864192.000001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -187,14 +189,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id006 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id006 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000003; Z RACH.: @@ -205,4 +205,4 @@ transactions: TNR: 179171073864291.000001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index a8bdcb2..b32d123 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -1,3 +1,4 @@ +import datetime import pathlib import mt940 @@ -6,6 +7,34 @@ _tests_path = pathlib.Path(__file__).parent +# Minimal valid MT940 statement to wrap single-tag test values in. +_HEADER = ':20:REF\n:25:ACC\n:28C:1\n:60F:C231229EUR0,00\n' +_FOOTER = ':62F:C231229EUR10,00\n' + + +def test_date_time_indication_positive_offset_is_hhmm(): + # The :13(D): offset subfield is HHMM (like ISO 8601 +0130), not a + # number of minutes: +0130 means 1 hour 30 minutes. + transactions = mt940.parse(':13D:1701191815+0130\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date.utcoffset() == datetime.timedelta(hours=1, minutes=30) + + +def test_date_time_indication_negative_offset(): + # 1!x sign can be '-' as well (e.g. US banks): -0500 is UTC-5. + transactions = mt940.parse(':13D:1701191815-0500\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date.utcoffset() == -datetime.timedelta(hours=5) + + +def test_date_time_indication_without_offset(): + # The offset is optional in the pattern; a bare 10-digit :13: must not + # crash and yields a naive datetime. + transactions = mt940.parse(':13:1701191815\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date == models.DateTime(2017, 1, 19, 18, 15) + assert date.tzinfo is None + @pytest.fixture def long_statement_number(): From 5735f2df99623cb5e38b3210978e0dbae1bfd7c2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 19:57:02 +0200 Subject: [PATCH 05/18] fix: :86: transaction details silently truncated after nine 65-char chunks The TransactionDetails pattern capped the capture at ~593 characters (eight 65-char chunks plus one), silently dropping the rest of longer multi-line details. Real banks exceed the 6x65 SWIFT cap routinely -- the cap was already raised once in 4575222 for exactly that reason, and the multiline.sta fixture was still losing its ?61-?65 subfields (visible in the regenerated golden: additional_purpose now carries the full text). Capture is now unbounded; Transactions.parse already limits the value to the text between this tag and the next one. --- mt940/tags.py | 7 ++- mt940_tests/self-provided/multiline.yml | 77 +++++++++++++------------ mt940_tests/test_tags.py | 18 ++++++ 3 files changed, 64 insertions(+), 38 deletions(-) diff --git a/mt940/tags.py b/mt940/tags.py index 51beaa0..d4bcd4a 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -610,8 +610,13 @@ class TransactionDetails(Tag): id = 86 scope = models.Transaction + # The SWIFT spec caps this field at 6 lines of 65 characters, but many + # banks send more. A previous cap of nine 65-char chunks silently + # truncated anything longer, so the capture is unbounded: the parser in + # `models.Transactions.parse` already limits the value to this tag's own + # slice of the statement. pattern = r""" - (?P(([\s\S]{0,65}\r?\n?){0,8}[\s\S]{0,65})) + (?P[\s\S]*) """ diff --git a/mt940_tests/self-provided/multiline.yml b/mt940_tests/self-provided/multiline.yml index 2466c8c..ad674d0 100644 --- a/mt940_tests/self-provided/multiline.yml +++ b/mt940_tests/self-provided/multiline.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: 'BILLLULLXXX/"NUMERO DE COMPTE IBAN ' available_balance: !!python/object:mt940.models.Balance @@ -34,8 +34,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -51,15 +52,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -67,14 +68,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -88,33 +92,36 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P[\\s\\S]*)\n " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P\n (\n (\\d{2}.{0,})\n (\\\ + n\\d{2}.{0,})*\n )|(\n [^\\n]*\n )\n )\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: - additional_purpose: NOM ET ADRESSE DO / BENEF 112345678 + additional_purpose: NOM ET ADRESSE DO / BENEF 112345678NOM ET ADRESSE DO / BENEF + 212345678NOM ET ADRESSE DO / BENEF 312345678NOM ET ADRESSE DO / BENEF 412345678NOM + ET ADRESSE DO / BENEF 512345678NOM ET ADRESSE DO / BENEF 612345678 amount: !!python/object:mt940.models.Amount amount: !!python/object/apply:decimal.Decimal - '-5' @@ -128,14 +135,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B9QIBA== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B9QIBA== extra_details: /OCMT/EUR4,5//IACC/D3/ funds_code: R - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B9QIBA== + guessed_entry_date: *id003 id: NTRF posting_text: VIREMENT111111111111111111X prima_nota: null @@ -156,7 +161,7 @@ transactions: FREE TEXT' transaction_reference: BILMT940 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -169,15 +174,13 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+YMEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id005 !!python/object/apply:mt940.models.Date - !!binary | B+YMFA== extra_details: Paiement funds_code: null - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+YMFA== + guessed_entry_date: *id005 id: NMSC status: C transaction_reference: BILMT940 - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index b32d123..d5eb38f 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -27,6 +27,24 @@ def test_date_time_indication_negative_offset(): assert date.utcoffset() == -datetime.timedelta(hours=5) +def test_transaction_details_long_multiline_not_truncated(): + # The old :86: pattern capped the capture at nine 65-char chunks + # (~593 chars); longer details -- e.g. German banks packing many ?NN + # subfields -- were silently truncated. The cap had already been bumped + # once (commit 4575222) for exactly this reason. + lines = [f'line {i:02d} ' + 'x' * 57 for i in range(12)] + details = '\n'.join(lines) + transactions = mt940.parse( + _HEADER + + ':61:2312290101D10,50NMSC\n' + + ':86:' + + details + + '\n' + + _FOOTER + ) + assert transactions[0].data['transaction_details'] == details + + def test_date_time_indication_without_offset(): # The offset is optional in the pattern; a bare 10-digit :13: must not # crash and yields a naive datetime. From 427cf06fa329af673c51c1b9c911fe35e8bbe54a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 19:58:20 +0200 Subject: [PATCH 06/18] fix: :NS: multi-line free text aborted the parse and dropped line content The NonSwift pattern only matched values that were either a single line or consisted entirely of two-digit-prefixed lines; any other multi-line value (NS is explicitly bank specific free-form data) failed to match and aborted the whole parse. On top of that, the failure surfaced as re.PatternError because the debug helper re-compiles unbalanced pattern fragments. Accept any content in the pattern and keep per-line 2!n35x extraction in __call__. Also reorder the __call__ branches so free-form lines keep their content in non_swift_text instead of being replaced by an empty line (the raphaelm golden: bare "32" sub-tag line is now kept, consistent with the "33" line right after it). --- mt940/tags.py | 20 +++--- mt940_tests/self-provided/raphaelm.yml | 87 ++++++++++++++------------ mt940_tests/test_tags.py | 27 ++++++++ 3 files changed, 83 insertions(+), 51 deletions(-) diff --git a/mt940/tags.py b/mt940/tags.py index d4bcd4a..a9b8273 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -326,15 +326,12 @@ class NonSwift(Tag): scope = models.TransactionsAndTransaction id = 'NS' + # NS content is bank specific and free-form, so accept anything + # (including multi-line values whose lines do not all start with a + # two-digit sub-tag); `__call__` extracts the `2!n35x` structure per + # line where present. pattern = r""" - (?P - ( - (\d{2}.{0,}) - (\n\d{2}.{0,})* - )|( - [^\n]* - ) - ) + (?P[\s\S]*) $""" sub_pattern = r""" (?P\d{2})(?P.{0,}) @@ -354,10 +351,13 @@ def __call__( ns = frag.groupdict() value['non_swift_' + ns['ns_id']] = ns['ns_data'] text.append(ns['ns_data']) - elif len(text) and text[-1]: - text.append('') elif line.strip(): + # Free-form line without a two-digit sub-tag: keep the + # content instead of dropping it. text.append(line.strip()) + elif len(text) and text[-1]: + # Blank line: collapse runs into one paragraph separator. + text.append('') value['non_swift_text'] = '\n'.join(text) value['non_swift'] = data return value diff --git a/mt940_tests/self-provided/raphaelm.yml b/mt940_tests/self-provided/raphaelm.yml index 87fec58..6b0b9f7 100644 --- a/mt940_tests/self-provided/raphaelm.yml +++ b/mt940_tests/self-provided/raphaelm.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: '3346780111' final_closing_balance: !!python/object:mt940.models.Balance @@ -73,6 +73,7 @@ data: 90000022 + 32 33 @@ -83,8 +84,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -100,15 +102,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -116,14 +118,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -137,29 +142,29 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P[\\s\\S]*)\n " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P[\\s\\S]*)\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: @@ -173,16 +178,15 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B9IDEQ== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B9IDFA== extra_details: '' funds_code: M - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B9IDFA== + guessed_entry_date: *id003 id: S051 - non_swift: "01Verwendungszweck 1\n02Verwendungszweck 2\n15Empf\xE4nger\n17Buchungstext\n1812345\n191000\n204711" + non_swift: "01Verwendungszweck 1\n02Verwendungszweck 2\n15Empf\xE4nger\n17Buchungstext\n\ + 1812345\n191000\n204711" non_swift_01: Verwendungszweck 1 non_swift_02: Verwendungszweck 2 non_swift_15: "Empf\xE4nger" @@ -190,10 +194,11 @@ transactions: non_swift_18: '12345' non_swift_19: '1000' non_swift_20: '4711' - non_swift_text: "Verwendungszweck 1\nVerwendungszweck 2\nEmpf\xE4nger\nBuchungstext\n12345\n1000\n4711" + non_swift_text: "Verwendungszweck 1\nVerwendungszweck 2\nEmpf\xE4nger\nBuchungstext\n\ + 12345\n1000\n4711" status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -211,7 +216,7 @@ transactions: id: NCHG status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -229,7 +234,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -247,7 +252,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -265,7 +270,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -286,7 +291,7 @@ transactions: non_swift_text: '3037010000' status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -304,7 +309,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -340,7 +345,7 @@ transactions: 87132101' status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -366,4 +371,4 @@ transactions: 12345' status: D transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index d5eb38f..5428f4d 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -45,6 +45,33 @@ def test_transaction_details_long_multiline_not_truncated(): assert transactions[0].data['transaction_details'] == details +def test_non_swift_multiline_free_text(): + # NS content is bank specific ("could be anything"); multi-line values + # whose lines do not all start with a two-digit sub-tag used to fail the + # NS pattern and abort the whole parse. + transactions = mt940.parse(_HEADER + ':NS:hello\nworld\n' + _FOOTER) + assert transactions.data['non_swift'] == 'hello\nworld' + assert transactions.data['non_swift_text'] == 'hello\nworld' + + +def test_non_swift_structured_line_followed_by_free_text(): + transactions = mt940.parse(_HEADER + ':NS:22foo\nbar\n' + _FOOTER) + assert transactions.data['non_swift'] == '22foo\nbar' + assert transactions.data['non_swift_22'] == 'foo' + assert transactions.data['non_swift_text'] == 'foo\nbar' + + +def test_non_swift_blank_lines_collapse(): + # Direct tag call: blank lines between content are kept as single + # paragraph separators in non_swift_text (mt940.parse strips blank + # lines before tags ever see them). + tag = tags.NonSwift() + result = tag( + mt940.models.Transactions(), {'non_swift': '22foo\n\n\n22bar'} + ) + assert result['non_swift_text'] == 'foo\n\nbar' + + def test_date_time_indication_without_offset(): # The offset is optional in the pattern; a bare 10-digit :13: must not # crash and yields a naive datetime. From 243d38725645b9b21f02ae35b9473c87cf396608 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 19:59:01 +0200 Subject: [PATCH 07/18] fix: :34F: blank or lowercase D/C mark produced wrong key and sign The pattern accepts '[DC ]?' for the D/C mark (the space was added in d36c51b for Fiducia/Volksbank ':34F:EUR 999999999999,99' input), but __call__ treated the space as a real status and produced a bogus ' _floor_limit' key instead of the d_/c_floor_limit pair an absent mark yields. A lowercase mark (matched via re.IGNORECASE) produced the right key but skipped Amount's debit negation ('d' != 'D'). Strip and uppercase the mark before dispatching. --- mt940/tags.py | 10 ++++++++-- mt940_tests/test_tags.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mt940/tags.py b/mt940/tags.py index a9b8273..39cb7eb 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -299,8 +299,14 @@ def __call__( dict[str, str], super().__call__(transactions, value), ) - if data['status']: - key = data['status'].lower() + '_floor_limit' + # Normalize the D/C mark: a space (sent by e.g. Fiducia/Volksbank, + # d36c51b) means "both", like an absent mark, and a lowercase mark + # (the patterns match case-insensitively) must behave like its + # uppercase form so the debit amount is negated consistently. + status: str = data['status'].strip().upper() + if status: + data['status'] = status + key: str = status.lower() + '_floor_limit' return {key: models.Amount(**data)} data_d = data.copy() data_c = data.copy() diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index 5428f4d..3742141 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -72,6 +72,29 @@ def test_non_swift_blank_lines_collapse(): assert result['non_swift_text'] == 'foo\n\nbar' +def test_floor_limit_space_indicator_treated_as_absent(): + # Fiducia / Volksbank Ortenau sends ':34F:EUR 999999999999,99' (commit + # d36c51b relaxed the regex for it). A blank D/C mark means "applies to + # both", like an absent mark -- it must not create a ' _floor_limit' key. + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n' + ':34F:EUR 999999999999,99\n' + ':60F:C231229EUR0,00\n' + _FOOTER + ) + assert ' _floor_limit' not in transactions.data + assert str(transactions.data['d_floor_limit']) == '-999999999999.99 EUR' + assert str(transactions.data['c_floor_limit']) == '999999999999.99 EUR' + + +def test_floor_limit_lowercase_indicator_normalized(): + # The tag regexes are compiled with re.IGNORECASE, so a lowercase mark + # must behave exactly like its uppercase form (debit -> negative). + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n' + ':34F:EURd10,00\n' + ':60F:C231229EUR0,00\n' + _FOOTER + ) + assert str(transactions.data['d_floor_limit']) == '-10.00 EUR' + + def test_date_time_indication_without_offset(): # The offset is optional in the pattern; a bare 10-digit :13: must not # crash and yields a naive datetime. From f8704dd44f289ac4b787e2da99cde53f8daedafc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 20:00:34 +0200 Subject: [PATCH 08/18] fix: Tag.parse raised re.error instead of RuntimeError for unmatched values Tag.parse documents RuntimeError for unparseable values, but before raising it calls _debug_partial_match, which re-compiles the pattern one line at a time. For any pattern with a group spread over several lines the fragments are unbalanced, so re.match raised re.error (re.PatternError) and masked both the RuntimeError and the diagnostics it was supposed to produce. Skip uncompilable fragments. The failure branch and the helper are now exercised by a test, so their no-cover pragmas are removed. --- mt940/tags.py | 15 +++++++++++---- mt940_tests/test_tags.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/mt940/tags.py b/mt940/tags.py index 39cb7eb..5ca7e16 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -120,7 +120,7 @@ def parse( RuntimeError: If the value does not match the tag's pattern. """ match = self.re.match(value) - if match: # pragma: no branch + if match: self.logger.debug( 'matched (%d) %r against "%s", got: %s', len(value), @@ -129,7 +129,7 @@ def parse( match.groupdict(), ) return match.groupdict() - else: # pragma: no cover + else: self.logger.error( 'matching id=%s (len=%d) "%s" against\n %s', self.id, @@ -142,13 +142,20 @@ def parse( f'Unable to parse {self!r} from {value!r}', self, value ) - def _debug_partial_match(self, value: str) -> None: # pragma: no cover + def _debug_partial_match(self, value: str) -> None: """ Helper function to debug partial matches against the pattern. """ part_value = value for pattern in self.pattern.split('\n'): - match = re.match(pattern, part_value, self.RE_FLAGS) + try: + match = re.match(pattern, part_value, self.RE_FLAGS) + except re.error: + # Single lines of a pattern with multi-line groups are not + # valid patterns on their own; skip them instead of masking + # the RuntimeError raised by `parse`. + self.logger.info('cannot compile fragment %r', pattern) + continue if match: self.logger.info( 'matched %r against %r, got: %s', diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index 3742141..376da41 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -95,6 +95,27 @@ def test_floor_limit_lowercase_indicator_normalized(): assert str(transactions.data['d_floor_limit']) == '-10.00 EUR' +class MultilineGroupTag(tags.Tag): + """Tag whose (valid) pattern spreads one group over several lines.""" + + id = 28 + pattern = r""" + (?P + \d+ + ) + $""" + + +def test_unparseable_value_raises_runtime_error(): + # Tag.parse documents RuntimeError for unparseable values, but the + # debug helper re-compiles the pattern line by line; for patterns with + # multi-line groups the unbalanced fragments raised re.error instead. + tag_parser = MultilineGroupTag() + transactions = mt940.models.Transactions(tags={tag_parser.id: tag_parser}) + with pytest.raises(RuntimeError, match='Unable to parse'): + transactions.parse(':20:REF\n:28C:NOTDIGITS\n') + + def test_date_time_indication_without_offset(): # The offset is optional in the pattern; a bare 10-digit :13: must not # crash and yields a naive datetime. From 18a6d9507af23c2d5fe323c7ef9c43f7ea77512f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jul 2026 20:02:41 +0200 Subject: [PATCH 09/18] test: xfail :61: RC reversal amount sign; cover spec edge cases Strict xfail documenting that a reversal-of-credit (RC) mark leaves the amount positive although funds leave the account (needs a user decision: flipping the sign would change the betterplace fixture goldens and any downstream consumer relying on status+positive amount). Passing edge coverage: RD/RC mark parsing, 15d amount without fraction digits (10,), a second // kept inside the bank reference, and a balance on 29 February in a leap year. --- mt940_tests/test_tags.py | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index 376da41..6ac259e 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -116,6 +116,65 @@ def test_unparseable_value_raises_runtime_error(): transactions.parse(':20:REF\n:28C:NOTDIGITS\n') +@pytest.mark.xfail( + strict=True, + reason='RC reversal amount sign -- pending decision (audit task 5): ' + 'a reversal of credit takes funds out of the account, but Amount only ' + 'negates on a plain D mark, so RC amounts stay positive (as do the RC ' + 'transactions in the betterplace fixture goldens).', +) +def test_statement_rc_reversal_amount_is_negative(): + transactions = mt940.parse( + _HEADER + ':61:2312290101RC10,50NTRFREF//BANK\n' + _FOOTER + ) + data = transactions[0].data + assert data['status'] == 'RC' + assert str(data['amount']) == '-10.50 EUR' + + +def test_statement_reversal_marks_parse(): + # RC/RD marks (2a subfield) parse and are preserved in `status`; the + # amount sign for RC is a pending decision, see the xfail above. + transactions = mt940.parse( + _HEADER + + ':61:2312290101RD10,50NTRFREF//BANK\n' + + ':61:2312290101RC10,50NTRFREF//BANK\n' + + _FOOTER + ) + assert transactions[0].data['status'] == 'RD' + assert str(transactions[0].data['amount']) == '10.50 EUR' + assert transactions[1].data['status'] == 'RC' + + +def test_statement_amount_without_decimals(): + # 15d allows a trailing decimal comma with no fraction digits. + transactions = mt940.parse( + _HEADER + ':61:2312290101C10,NTRFREF//BANK\n' + _FOOTER + ) + assert str(transactions[0].data['amount']) == '10 EUR' + + +def test_statement_second_double_slash_stays_in_bank_reference(): + # Only the first // separates the bank reference; a second one is kept + # as part of the reference content. + transactions = mt940.parse( + _HEADER + ':61:2312290101C10,50NTRFREF//BANK//EXTRA\n' + _FOOTER + ) + data = transactions[0].data + assert data['customer_reference'] == 'REF' + assert data['bank_reference'] == 'BANK//EXTRA' + + +def test_balance_on_leap_day(): + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n:60F:C240229EUR0,00\n' + ':61:2402290229C10,50NTRFREF\n' + ':62F:C240229EUR10,50\n' + ) + balance = transactions.data['final_opening_balance'] + assert balance.date == models.Date(2024, 2, 29) + + def test_date_time_indication_without_offset(): # The offset is optional in the pattern; a bare 10-digit :13: must not # crash and yields a naive datetime. From ff3d24a70a4281255b0ca5bb62151b50d9affb52 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 02:38:34 +0200 Subject: [PATCH 10/18] fix: Amount treated only uppercase D as debit, dropping lowercase-d sign The tag patterns are compiled with re.IGNORECASE, so a bank sending a lowercase 'd' debit mark in :61: or a balance is accepted by the regex, but Amount only negated on an exact 'D' -- a debit was silently stored as a positive amount. Compare the mark case-insensitively (status.upper() == 'D'). Reversal marks RD/RC are unchanged (still positive). --- mt940/models.py | 11 ++++++++--- mt940_tests/test_tags.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mt940/models.py b/mt940/models.py index 8e598e9..24d9629 100644 --- a/mt940/models.py +++ b/mt940/models.py @@ -214,15 +214,20 @@ def __init__( """Coerce ``amount`` to a signed :class:`decimal.Decimal`. ``status`` is ``'C'`` for credit (positive) or ``'D'`` for debit, in - which case the amount is negated. Extra keyword arguments are ignored - so a parsed tag dictionary can be splatted in directly. + which case the amount is negated. The comparison is case-insensitive: + the tag patterns are compiled with :data:`re.IGNORECASE`, so a bank + that sends a lowercase ``'d'`` mark must still have its debit negated + rather than silently stored as a positive amount. Reversal marks + (``'RD'``/``'RC'``) are intentionally left unchanged here. Extra + keyword arguments are ignored so a parsed tag dictionary can be + splatted in directly. """ self.amount = decimal.Decimal(amount.replace(',', '.')) self.currency = currency # C = credit, D = debit - if status == 'D': + if status.upper() == 'D': self.amount = -self.amount def __eq__(self, other: object) -> bool: diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index 6ac259e..34d32c6 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -154,6 +154,18 @@ def test_statement_amount_without_decimals(): assert str(transactions[0].data['amount']) == '10 EUR' +def test_statement_lowercase_debit_mark_is_negative(): + # The tag patterns are compiled with re.IGNORECASE, so a lowercase 'd' + # debit mark is accepted. Amount must still treat it as a debit and + # negate the amount; otherwise a debit is silently stored as positive. + transactions = mt940.parse( + _HEADER + ':61:2312290101d10,50NTRFREF//BANK\n' + _FOOTER + ) + data = transactions[0].data + assert data['status'] == 'd' + assert str(data['amount']) == '-10.50 EUR' + + def test_statement_second_double_slash_stays_in_bank_reference(): # Only the first // separates the bank reference; a second one is kept # as part of the reference content. From bcbcc9f873279e889088ca699cd23cb633f094a3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 02:40:07 +0200 Subject: [PATCH 11/18] test: cover pickle round-trip preserving transaction_boundary The existing pickle test only exercises default (empty) state. Guard the newer opt-in transaction_boundary attribute (issue #110) against a future __getstate__/__setstate__ regression that could drop it. --- mt940_tests/test_parse.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mt940_tests/test_parse.py b/mt940_tests/test_parse.py index 8a2fff8..3d8e1fe 100644 --- a/mt940_tests/test_parse.py +++ b/mt940_tests/test_parse.py @@ -42,3 +42,19 @@ def test_pickle_roundtrip_restores_processors(): assert len(restored) == len(transactions) # Re-parsing exercises the restored processors without raising. restored.parse('') + + +def test_pickle_roundtrip_preserves_transaction_boundary(): + # __getstate__ only drops `processors`; newer state such as the opt-in + # `transaction_boundary` (issue #110) must survive a pickle round-trip. + transactions = mt940.models.Transactions( + transaction_boundary={'transaction_reference_number'} + ) + + # Trusted, self-produced pickle (a round-trip of our own object), so + # pickle.loads is safe here. + restored = pickle.loads(pickle.dumps(transactions)) + + assert restored.transaction_boundary == frozenset( + {'transaction_reference_number'} + ) From 40fb4c6098214819a2978604c1ca528a16bc4bbb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:06:11 +0200 Subject: [PATCH 12/18] test: cover JSONEncoder round-trip value forms and non-leap Feb clamp The existing test_json_dump only calls json.dumps over .sta fixtures with no value assertions; add full dumps->loads round-trips asserting the string forms of nested Balance/Amount/Date, the SumAmount entry number, the :13D: DateTime FixedOffset, and the opt-in StatementASNB .txt path. Also cover the previously unexercised non-leap-year February clamp in date_fixup_pre_processor. --- mt940_tests/test_processors.py | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/mt940_tests/test_processors.py b/mt940_tests/test_processors.py index c59e8bc..f22a1c1 100644 --- a/mt940_tests/test_processors.py +++ b/mt940_tests/test_processors.py @@ -1,3 +1,4 @@ +import json import os import pathlib import typing @@ -174,3 +175,86 @@ def test_citi_bank_processors() -> None: assert len(transactions) == 5 expected_date = mt940.models.Date(2024, 3, 12) assert transactions[0].data['date'] == expected_date + + +def test_json_round_trip_preserves_model_values() -> None: + """`mt940.JSONEncoder` serialises every model type and round-trips. + + The existing ``test_json_dump`` only calls ``json.dumps`` over ``.sta`` + fixtures. This exercises the full ``dumps`` -> ``loads`` round-trip and + asserts the string forms of the nested ``Balance``/``Amount``/``Date`` + value types, which were previously unchecked. + """ + transactions = mt940.parse(str(_tests_path / 'jejik' / 'abnamro.sta')) + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + + assert len(decoded['transactions']) == len(transactions) + # Balance -> nested dict; Amount -> Decimal rendered as str; Date -> ISO str + assert decoded['final_opening_balance'] == { + 'status': 'C', + 'amount': {'amount': '3236.28', 'currency': 'EUR'}, + 'date': '2011-05-22', + } + # Every transaction amount is a Decimal serialised to a plain string. + for transaction in decoded['transactions']: + assert isinstance(transaction['amount']['amount'], str) + assert isinstance(transaction['date'], str) + + +def test_json_round_trip_sum_amount_and_datetime() -> None: + """``SumAmount`` (with its entry ``number``) and ``DateTime`` serialise. + + The mBank ``:90D:``/``:90C:`` and ``:13D:`` tags produce ``SumAmount`` and + ``DateTime`` objects that never appear in the jejik ``.sta`` fixtures used + by ``test_json_dump``'s value-free smoke check. + """ + transactions = mt940.parse(str(_tests_path / 'mBank' / 'mt942.sta')) + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + + assert decoded['sum_credit_entries'] == { + 'amount': '0.03', + 'currency': 'PLN', + 'number': '3', + } + # The :13D: DateTime (with a +0100 FixedOffset) renders as an ISO string, + # carrying the timezone offset, not a mapping. + assert decoded['date'] == '2017-01-19 18:15:00+01:00' + + +def test_json_round_trip_asnb_non_swift_statement() -> None: + """The opt-in ``StatementASNB`` tag output also round-trips cleanly. + + ASN fixtures live in ``.txt`` files and require a custom tag, so they are + outside ``test_json_dump``'s ``.sta`` glob entirely. + """ + tag = mt940.tags.StatementASNB() + transactions = mt940.models.Transactions(tags={tag.id: tag}) + with (_tests_path / 'ASNB' / 'mt940.txt').open() as fh: + transactions.parse(fh.read()) + + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + assert len(decoded['transactions']) == len(transactions) + assert decoded['transactions'][0]['amount'] == { + 'amount': '-65.00', + 'currency': 'EUR', + } + + +def test_date_fixup_non_leap_february_clamped() -> None: + """A Feb 29 value date in a non-leap year is clamped to Feb 28. + + ``test_date_fixup_pre_processor`` only covers Feb 30 in a leap year + (``february_30.sta``, 2016 -> Feb 29); the non-leap clamp path was + previously unexercised. + """ + data = ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C170201EUR100,00\n' + ':61:1702290228DR6,00N024NONREF\n' + ':86:free\n' + ':62F:C170228EUR94,00\n' + ) + transactions = mt940.parse(data) + assert transactions[0].data['date'] == mt940.models.Date(2017, 2, 28) From 3c0dbfe46ab95b85b695ac7dbc4d2fcfd41df195 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:29:42 +0200 Subject: [PATCH 13/18] fix: leading + in :86: GVC free text truncated purpose _parse_mt940_gvcodes sliced purpose[index - 4:index] with no lower bound, so a '+' before index 4 produced a wrapped/empty slice matching the empty-string GVC key. A literal '+' in the leading free text of a structured :86: (e.g. 'AB+EREF', 'A+B SVWZ TEXT') was thus treated as a KEYWORD+ separator, truncating the purpose. GVC keywords are four characters wide, so guard the match on index >= 4. The previously unreachable else branch is now covered; drop its pragmas. --- mt940/processors.py | 17 +++++++++++++---- mt940_tests/test_processors.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/mt940/processors.py b/mt940/processors.py index a05a9e9..bae7fd5 100644 --- a/mt940/processors.py +++ b/mt940/processors.py @@ -357,8 +357,17 @@ def _parse_mt940_gvcodes(purpose: str) -> dict[str, str | None]: for index, char in enumerate(purpose): # Detect the beginning of a GVC segment: if a '+' is encountered - # and the four characters preceding it form a valid GVC key. - if char == '+' and purpose[index - 4 : index] in GVC_KEYS: + # and the four characters preceding it form a valid GVC key. GVC + # keywords are four characters wide, so a '+' before index 4 cannot + # terminate one; guarding on ``index >= 4`` also avoids a negative + # ``purpose[index - 4:index]`` slice wrapping to the empty string + # (which spuriously matched the empty-string GVC key and truncated a + # literal '+' in the leading free text). + if ( + char == '+' + and index >= 4 + and purpose[index - 4 : index] in GVC_KEYS + ): if segment_type: # If already processing a segment, finalize it by removing # the trailing GVC key and reset the text accumulator. @@ -372,10 +381,10 @@ def _parse_mt940_gvcodes(purpose: str) -> dict[str, str | None]: else: text += char - if segment_type: # pragma: no branch + if segment_type: tmp[segment_type] = text else: - tmp[''] = text # pragma: no cover + tmp[''] = text for key, value in tmp.items(): result[GVC_KEYS[key]] = value diff --git a/mt940_tests/test_processors.py b/mt940_tests/test_processors.py index f22a1c1..47046ec 100644 --- a/mt940_tests/test_processors.py +++ b/mt940_tests/test_processors.py @@ -258,3 +258,37 @@ def test_date_fixup_non_leap_february_clamped() -> None: ) transactions = mt940.parse(data) assert transactions[0].data['date'] == mt940.models.Date(2017, 2, 28) + + +@pytest.mark.parametrize( + ('detail', 'expected_purpose'), + [ + # A literal '+' inside the first characters of free text (e.g. a + # company name like "AB+...") must not be treated as a GVC KEYWORD+ + # separator: EREF is a real GVC key later in the text, so gvcodes runs. + ('020?20AB+EREF', 'AB+EREF'), + # 'A+B' at the very start must survive; SVWZ triggers gvcodes parsing. + ('020?20A+B SVWZ TEXT', 'A+B SVWZ TEXT'), + ], +) +def test_gvcode_leading_plus_in_free_text_kept_in_purpose( + detail: str, expected_purpose: str +) -> None: + """A '+' earlier than position 4 cannot terminate a GVC keyword. + + GVC keywords are 3-4 chars followed by '+'. ``_parse_mt940_gvcodes`` sliced + ``purpose[index - 4:index]`` without a lower bound, so a '+' at index < 4 + produced a wrapped/empty slice matching the empty-string GVC key and + truncated the purpose (dropping the leading free text before the '+'). + """ + data = ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C200101EUR100,00\n' + ':61:2001010101C10,00NTRFNONREF\n' + f':86:{detail}\n' + ':62F:C200101EUR110,00\n' + ) + transaction = mt940.parse(data)[0].data + assert transaction['purpose'] == expected_purpose From 0ef22111cbff1131dd7529f04990c9362ef9d034 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:35:36 +0200 Subject: [PATCH 14/18] fix: merging repeated structured :86: tags crashed on None details A structured :86: emits every DETAIL_KEYS value including None for absent sub-fields. A second structured :86: on the same :61: then hit 'None += str' in _update_transaction (TypeError aborting the whole file) in either tag order. Append now requires BOTH sides to be strings; any other combination assigns the incoming value, so a later string replaces an existing None. Current clobber semantics are unchanged (an incoming None still overwrites -- the behavior the real-bank goldens encode); the preservation question is pinned as a strict xfail pending decision. --- mt940/models.py | 13 +++++-- mt940_tests/test_processors.py | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/mt940/models.py b/mt940/models.py index 24d9629..83df7d2 100644 --- a/mt940/models.py +++ b/mt940/models.py @@ -624,12 +624,19 @@ def _process_statement_tag(self, result: dict[str, Any]) -> None: def _update_transaction(self, result: dict[str, Any]) -> None: """Merge a transaction-scoped result into the current transaction. - New keys are set directly; string values for keys that already exist - are appended on a new line (e.g. multi-line ``:86:`` details). + New keys are set directly; when both the existing and incoming + values are strings, the incoming one is appended on a new line + (e.g. multi-line ``:86:`` details). Any other combination assigns + the incoming value: a structured ``:86:`` emits ``None`` for absent + sub-fields, so a later string must replace an existing ``None`` + rather than crash on ``None += str`` (and an incoming ``None`` + still overwrites, preserving the merge semantics the fixture + goldens encode). """ transaction = self.transactions[-1] for k, v in result.items(): - if k in transaction.data and hasattr(v, 'strip'): + existing = transaction.data.get(k) + if hasattr(existing, 'strip') and hasattr(v, 'strip'): transaction.data[k] += f'\n{v.strip()}' else: transaction.data[k] = v diff --git a/mt940_tests/test_processors.py b/mt940_tests/test_processors.py index 47046ec..3f92f5b 100644 --- a/mt940_tests/test_processors.py +++ b/mt940_tests/test_processors.py @@ -292,3 +292,71 @@ def test_gvcode_leading_plus_in_free_text_kept_in_purpose( ) transaction = mt940.parse(data)[0].data assert transaction['purpose'] == expected_purpose + + +def _two_structured_86_data(first_detail: str, second_detail: str) -> str: + return ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C200101EUR100,00\n' + ':61:2001010101C10,00NTRFNONREF\n' + f':86:{first_detail}\n' + f':86:{second_detail}\n' + ':62F:C200101EUR110,00\n' + ) + + +@pytest.mark.parametrize( + ('first_detail', 'second_detail', 'expected_purpose', 'expected_posting'), + [ + # The second tag's None sub-fields clobber the first tag's real + # values under the current (golden-pinned) merge semantics: the last + # structured :86: wins per key, even with None. + ('020?20REALPURPOSE', '020?00POSTINGTEXT', None, 'POSTINGTEXT'), + ('020?00POSTINGTEXT', '020?20REALPURPOSE', 'REALPURPOSE', None), + ], +) +def test_repeated_structured_86_merges_without_crash( + first_detail: str, + second_detail: str, + expected_purpose: str | None, + expected_posting: str | None, +) -> None: + """Two structured ``:86:`` tags on one ``:61:`` must not crash the parse. + + A structured ``:86:`` emits every ``DETAIL_KEYS`` value including ``None`` + for absent sub-fields; a second structured ``:86:`` then hit + ``None += str`` in ``_update_transaction`` (``TypeError`` aborting the + whole file) in either order. This pins the CURRENT post-fix merge + contract: a later string replaces an existing ``None``, while a later + ``None`` still overwrites an earlier real value (the semantics the + real-bank goldens encode -- see the xfail below for the pending + preservation question). + """ + transaction = mt940.parse( + _two_structured_86_data(first_detail, second_detail) + )[0].data + assert transaction['purpose'] == expected_purpose + assert transaction['posting_text'] == expected_posting + + +@pytest.mark.xfail( + strict=True, + reason='repeated structured :86: None-clobber semantics -- pending ' + 'decision (audit task 7 review): preserving non-None values against a ' + 'later tag\'s None is arguably more correct, but the same rule stops a ' + 'structured :86: from nulling the customer_reference set by :61:, ' + 'changing 5 real-bank goldens.', +) +def test_repeated_structured_86_preserves_real_values() -> None: + """Preservation ideal: real values from BOTH tags survive, either order.""" + for first, second in [ + ('020?20REALPURPOSE', '020?00POSTINGTEXT'), + ('020?00POSTINGTEXT', '020?20REALPURPOSE'), + ]: + transaction = mt940.parse(_two_structured_86_data(first, second))[ + 0 + ].data + assert transaction['purpose'] == 'REALPURPOSE' + assert transaction['posting_text'] == 'POSTINGTEXT' From 9b2e094ca889893257f9a3a1a147a123528e0142 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:46:16 +0200 Subject: [PATCH 15/18] fix: leading BOM dropped the first :20: tag's data A UTF-8/UTF-16 byte-order mark decodes to U+FEFF, which is not whitespace and so survives Transactions.strip. It pushed the first :20: off the start-of-line tag anchor, so transaction_reference was silently lost on BOM-prefixed files (common from Windows tools/banks). Strip a leading BOM in _read for both bytes and str input. --- mt940/parser.py | 6 +++++- mt940_tests/test_parse.py | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mt940/parser.py b/mt940/parser.py index 9882cdd..3136be2 100644 --- a/mt940/parser.py +++ b/mt940/parser.py @@ -75,7 +75,11 @@ def safe_is_file(filename: Any) -> bool: raise exception # pragma: no cover assert isinstance(data, str) - return data + # Strip a leading byte-order mark. utf-8/utf-16 files written by Windows + # tools decode to a leading ````, which is not whitespace and so + # survives ``strip`` -- it would push the first ``:20:`` off the + # start-of-line tag anchor and silently drop that tag's data. + return data.removeprefix('') def parse( diff --git a/mt940_tests/test_parse.py b/mt940_tests/test_parse.py index 3d8e1fe..42511d2 100644 --- a/mt940_tests/test_parse.py +++ b/mt940_tests/test_parse.py @@ -28,6 +28,33 @@ def test_non_ascii_parse(path, encoding): pickle.dumps(mt940.parse(data)) +_BOM_STATEMENT = ( + ':20:REF\n' + ':25:NL00BANK0123456789\n' + ':28C:1/1\n' + ':60F:C091019EUR1000,00\n' + ':61:0910201020C500,00NTRFNONREF//B\n' + ':86:Example transaction\n' + ':62F:C091020EUR1500,00\n' +) + + +def test_utf8_bom_bytes_does_not_drop_first_tag(): + # A UTF-8 BOM (emitted by many Windows tools/banks) must not push the + # leading :20: past the start-of-line tag anchor and drop its data. + data = b'\xef\xbb\xbf' + _BOM_STATEMENT.encode('utf-8') + transactions = mt940.parse(data) + assert transactions.data.get('transaction_reference') == 'REF' + assert len(transactions) == 1 + + +def test_utf8_bom_str_does_not_drop_first_tag(): + # Same file already decoded to str with a stray BOM character. + transactions = mt940.parse('' + _BOM_STATEMENT) + assert transactions.data.get('transaction_reference') == 'REF' + assert len(transactions) == 1 + + def test_pickle_roundtrip_restores_processors(): # __getstate__ drops the (unpicklable) processors; __setstate__ must # restore them so the unpickled object is still usable. From 7631073507a03339b1da8861e5cfa855fb3d266f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:47:05 +0200 Subject: [PATCH 16/18] test: cover embedded tag-lookalike, parse_statements headers, :86: indent Regression guards for previously-untested parser edges verified correct during the audit: a :86: line starting with an unknown tag-lookalike (:12:) stays in details; parse_statements drops a leading SWIFT header and absorbs lone `-` terminators; :86: continuation-line leading whitespace is kept. --- mt940_tests/test_parse.py | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/mt940_tests/test_parse.py b/mt940_tests/test_parse.py index 42511d2..5b2ea7c 100644 --- a/mt940_tests/test_parse.py +++ b/mt940_tests/test_parse.py @@ -55,6 +55,57 @@ def test_utf8_bom_str_does_not_drop_first_tag(): assert len(transactions) == 1 +def test_86_line_with_embedded_tag_lookalike_stays_in_details(): + # A :86: free-text line that itself starts with a tag-lookalike (:12:, + # which is not a known tag) must not be split off as a separate tag and + # corrupt the statement -- it stays part of the transaction details. + transactions = mt940.parse( + ':20:R\n:25:A\n:28C:1\n:60F:C240101EUR0,00\n' + ':61:2401010101C10,00NTRFREF//B\n' + ':86:PAYMENT FOR\n:12:INVOICE STYLE REF\n' + ':62F:C240101EUR10,00\n' + ) + assert len(transactions) == 1 + # No corruption: the closing balance still parses off the real :62F:. + assert ( + str(transactions.data['final_closing_balance'].amount) == '10.00 EUR' + ) + # The lookalike line is retained verbatim inside the details. + details = transactions[0].data['transaction_details'] + assert 'INVOICE STYLE REF' in details + + +def test_parse_statements_drops_swift_header_and_absorbs_terminators(): + # A leading SWIFT {1:}{2:}{4: header before the first :20: must be dropped, + # and the lone `-` statement terminators must not create empty statements. + data = ( + '{1:F01BANKNL2AXXXX0000000000}{2:I940BANKNL2AXXXXN}{4:\n' + ':20:STMT1\n:25:NL00BANK1\n:28C:1/1\n:60F:C240101EUR100,00\n' + ':62F:C240101EUR100,00\n-}\n' + ':20:STMT2\n:25:NL00BANK2\n:28C:2/1\n:60F:C240102EUR200,00\n' + ':62F:C240102EUR200,00\n-\n' + ) + statements = mt940.parse_statements(data) + assert [s.data['transaction_reference'] for s in statements] == [ + 'STMT1', + 'STMT2', + ] + + +def test_86_continuation_preserves_leading_whitespace(): + # Transactions.strip only rstrips, so significant leading whitespace on a + # :86: continuation line is preserved rather than eaten. + transactions = mt940.parse( + ':20:R\n:25:A\n:28C:1\n:60F:C240101EUR0,00\n' + ':61:2401010101C10,00NTRFREF//B\n' + ':86:LINE ONE\n INDENTED TWO\n' + ':62F:C240101EUR10,00\n' + ) + assert transactions[0].data['transaction_details'] == ( + 'LINE ONE\n INDENTED TWO' + ) + + def test_pickle_roundtrip_restores_processors(): # __getstate__ drops the (unpicklable) processors; __setstate__ must # restore them so the unpickled object is still usable. From a35284ecfafc8e9fb52eaa082b1f3cb08023e1d1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 03:54:13 +0200 Subject: [PATCH 17/18] fix: line length, quote style and spelling found by lint and codespell --- mt940_tests/test_processors.py | 4 ++-- mt940_tests/test_tags.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mt940_tests/test_processors.py b/mt940_tests/test_processors.py index 3f92f5b..0405f27 100644 --- a/mt940_tests/test_processors.py +++ b/mt940_tests/test_processors.py @@ -189,7 +189,7 @@ def test_json_round_trip_preserves_model_values() -> None: decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) assert len(decoded['transactions']) == len(transactions) - # Balance -> nested dict; Amount -> Decimal rendered as str; Date -> ISO str + # Balance -> nested dict; Amount -> Decimal as str; Date -> ISO str assert decoded['final_opening_balance'] == { 'status': 'C', 'amount': {'amount': '3236.28', 'currency': 'EUR'}, @@ -345,7 +345,7 @@ def test_repeated_structured_86_merges_without_crash( strict=True, reason='repeated structured :86: None-clobber semantics -- pending ' 'decision (audit task 7 review): preserving non-None values against a ' - 'later tag\'s None is arguably more correct, but the same rule stops a ' + "later tag's None is arguably more correct, but the same rule stops a " 'structured :86: from nulling the customer_reference set by :61:, ' 'changing 5 real-bank goldens.', ) diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index 34d32c6..17f8574 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -107,7 +107,7 @@ class MultilineGroupTag(tags.Tag): def test_unparseable_value_raises_runtime_error(): - # Tag.parse documents RuntimeError for unparseable values, but the + # Tag.parse documents RuntimeError for unparsable values, but the # debug helper re-compiles the pattern line by line; for patterns with # multi-line groups the unbalanced fragments raised re.error instead. tag_parser = MultilineGroupTag() From fef884fd8f2c37f225337390b60a2edfd2320af7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jul 2026 10:27:36 +0200 Subject: [PATCH 18/18] fix: replace invisible U+FEFF literal with visible escape in parser --- mt940/parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mt940/parser.py b/mt940/parser.py index 3136be2..2e11136 100644 --- a/mt940/parser.py +++ b/mt940/parser.py @@ -75,11 +75,12 @@ def safe_is_file(filename: Any) -> bool: raise exception # pragma: no cover assert isinstance(data, str) - # Strip a leading byte-order mark. utf-8/utf-16 files written by Windows - # tools decode to a leading ````, which is not whitespace and so - # survives ``strip`` -- it would push the first ``:20:`` off the - # start-of-line tag anchor and silently drop that tag's data. - return data.removeprefix('') + # Strip a leading byte-order mark (U+FEFF). A BOM survives decoding as + # U+FEFF for UTF-8 input (and for UTF-16 only when an explicit + # utf-16-le/-be encoding is passed). It is not whitespace, so it would + # otherwise displace the first `:20:` off the start-of-line tag anchor and + # silently drop that tag's data. + return data.removeprefix('\ufeff') def parse(