diff --git a/mt940/models.py b/mt940/models.py index b403ae9..83df7d2 100644 --- a/mt940/models.py +++ b/mt940/models.py @@ -214,15 +214,20 @@ def __init__( """Coerce ``amount`` to a signed :class:`decimal.Decimal`. ``status`` is ``'C'`` for credit (positive) or ``'D'`` for debit, in - which case the amount is negated. Extra keyword arguments are ignored - so a parsed tag dictionary can be splatted in directly. + which case the amount is negated. The comparison is case-insensitive: + the tag patterns are compiled with :data:`re.IGNORECASE`, so a bank + that sends a lowercase ``'d'`` mark must still have its debit negated + rather than silently stored as a positive amount. Reversal marks + (``'RD'``/``'RC'``) are intentionally left unchanged here. Extra + keyword arguments are ignored so a parsed tag dictionary can be + splatted in directly. """ self.amount = decimal.Decimal(amount.replace(',', '.')) self.currency = currency # C = credit, D = debit - if status == 'D': + if status.upper() == 'D': self.amount = -self.amount def __eq__(self, other: object) -> bool: @@ -619,12 +624,19 @@ def _process_statement_tag(self, result: dict[str, Any]) -> None: def _update_transaction(self, result: dict[str, Any]) -> None: """Merge a transaction-scoped result into the current transaction. - New keys are set directly; string values for keys that already exist - are appended on a new line (e.g. multi-line ``:86:`` details). + New keys are set directly; when both the existing and incoming + values are strings, the incoming one is appended on a new line + (e.g. multi-line ``:86:`` details). Any other combination assigns + the incoming value: a structured ``:86:`` emits ``None`` for absent + sub-fields, so a later string must replace an existing ``None`` + rather than crash on ``None += str`` (and an incoming ``None`` + still overwrites, preserving the merge semantics the fixture + goldens encode). """ transaction = self.transactions[-1] for k, v in result.items(): - if k in transaction.data and hasattr(v, 'strip'): + existing = transaction.data.get(k) + if hasattr(existing, 'strip') and hasattr(v, 'strip'): transaction.data[k] += f'\n{v.strip()}' else: transaction.data[k] = v @@ -724,7 +736,9 @@ def sanitize_tag_id_matches( return valid_matches -class TransactionsAndTransaction(Transactions, Transaction): # type: ignore[misc] +class TransactionsAndTransaction( # type: ignore[misc] # pyright: ignore[reportUnsafeMultipleInheritance] + Transactions, Transaction +): """ Subclass of both Transactions and Transaction for scope definitions. diff --git a/mt940/parser.py b/mt940/parser.py index 9882cdd..2e11136 100644 --- a/mt940/parser.py +++ b/mt940/parser.py @@ -75,7 +75,12 @@ def safe_is_file(filename: Any) -> bool: raise exception # pragma: no cover assert isinstance(data, str) - return data + # Strip a leading byte-order mark (U+FEFF). A BOM survives decoding as + # U+FEFF for UTF-8 input (and for UTF-16 only when an explicit + # utf-16-le/-be encoding is passed). It is not whitespace, so it would + # otherwise displace the first `:20:` off the start-of-line tag anchor and + # silently drop that tag's data. + return data.removeprefix('\ufeff') def parse( diff --git a/mt940/processors.py b/mt940/processors.py index a05a9e9..bae7fd5 100644 --- a/mt940/processors.py +++ b/mt940/processors.py @@ -357,8 +357,17 @@ def _parse_mt940_gvcodes(purpose: str) -> dict[str, str | None]: for index, char in enumerate(purpose): # Detect the beginning of a GVC segment: if a '+' is encountered - # and the four characters preceding it form a valid GVC key. - if char == '+' and purpose[index - 4 : index] in GVC_KEYS: + # and the four characters preceding it form a valid GVC key. GVC + # keywords are four characters wide, so a '+' before index 4 cannot + # terminate one; guarding on ``index >= 4`` also avoids a negative + # ``purpose[index - 4:index]`` slice wrapping to the empty string + # (which spuriously matched the empty-string GVC key and truncated a + # literal '+' in the leading free text). + if ( + char == '+' + and index >= 4 + and purpose[index - 4 : index] in GVC_KEYS + ): if segment_type: # If already processing a segment, finalize it by removing # the trailing GVC key and reset the text accumulator. @@ -372,10 +381,10 @@ def _parse_mt940_gvcodes(purpose: str) -> dict[str, str | None]: else: text += char - if segment_type: # pragma: no branch + if segment_type: tmp[segment_type] = text else: - tmp[''] = text # pragma: no cover + tmp[''] = text for key, value in tmp.items(): result[GVC_KEYS[key]] = value diff --git a/mt940/tags.py b/mt940/tags.py index fec4536..5ca7e16 100644 --- a/mt940/tags.py +++ b/mt940/tags.py @@ -78,6 +78,7 @@ import logging import re import typing +from typing import ClassVar from . import models @@ -90,12 +91,14 @@ class Tag: """ id: str | int = 0 - RE_FLAGS = re.IGNORECASE | re.VERBOSE | re.UNICODE - scope: type[models.Transactions | models.Transaction] = models.Transactions - pattern: str - name: str - slug: str - logger: logging.Logger + RE_FLAGS: ClassVar[re.RegexFlag] = re.IGNORECASE | re.VERBOSE | re.UNICODE + scope: ClassVar[type[models.Transactions | models.Transaction]] = ( + models.Transactions + ) + pattern: ClassVar[str] + name: ClassVar[str] + slug: ClassVar[str] + logger: ClassVar[logging.Logger] def __init__(self) -> None: self.re = re.compile(self.pattern, self.RE_FLAGS) @@ -117,7 +120,7 @@ def parse( RuntimeError: If the value does not match the tag's pattern. """ match = self.re.match(value) - if match: # pragma: no branch + if match: self.logger.debug( 'matched (%d) %r against "%s", got: %s', len(value), @@ -126,7 +129,7 @@ def parse( match.groupdict(), ) return match.groupdict() - else: # pragma: no cover + else: self.logger.error( 'matching id=%s (len=%d) "%s" against\n %s', self.id, @@ -139,13 +142,20 @@ def parse( f'Unable to parse {self!r} from {value!r}', self, value ) - def _debug_partial_match(self, value: str) -> None: # pragma: no cover + def _debug_partial_match(self, value: str) -> None: """ Helper function to debug partial matches against the pattern. """ part_value = value for pattern in self.pattern.split('\n'): - match = re.match(pattern, part_value, self.RE_FLAGS) + try: + match = re.match(pattern, part_value, self.RE_FLAGS) + except re.error: + # Single lines of a pattern with multi-line groups are not + # valid patterns on their own; skip them instead of masking + # the RuntimeError raised by `parse`. + self.logger.info('cannot compile fragment %r', pattern) + continue if match: self.logger.info( 'matched %r against %r, got: %s', @@ -211,13 +221,24 @@ class DateTimeIndication(Tag): (?P\d{2}) (?P\d{2}) (?P\d{2}) - (\+(?P\d{4})|) + ((?P[+-])(?P\d{4}))? """ def __call__( self, transactions: models.Transactions, value: dict[str, typing.Any] ) -> dict[str, object]: data = super().__call__(transactions, value) + # The offset subfield is a signed HHMM value (e.g. +0130 is 1 hour + # and 30 minutes east of UTC), while `models.DateTime` expects the + # offset as a number of minutes. Convert it here and drop the raw + # groups so they are not passed on when the offset is absent. + sign: str | None = data.pop('offset_sign', None) + offset: str | None = data.pop('offset', None) + if offset: + minutes: int = int(offset[:2]) * 60 + int(offset[2:]) + if sign == '-': + minutes = -minutes + data['offset'] = minutes return {'date': models.DateTime(**data)} @@ -285,8 +306,14 @@ def __call__( dict[str, str], super().__call__(transactions, value), ) - if data['status']: - key = data['status'].lower() + '_floor_limit' + # Normalize the D/C mark: a space (sent by e.g. Fiducia/Volksbank, + # d36c51b) means "both", like an absent mark, and a lowercase mark + # (the patterns match case-insensitively) must behave like its + # uppercase form so the debit amount is negated consistently. + status: str = data['status'].strip().upper() + if status: + data['status'] = status + key: str = status.lower() + '_floor_limit' return {key: models.Amount(**data)} data_d = data.copy() data_c = data.copy() @@ -312,15 +339,12 @@ class NonSwift(Tag): scope = models.TransactionsAndTransaction id = 'NS' + # NS content is bank specific and free-form, so accept anything + # (including multi-line values whose lines do not all start with a + # two-digit sub-tag); `__call__` extracts the `2!n35x` structure per + # line where present. pattern = r""" - (?P - ( - (\d{2}.{0,}) - (\n\d{2}.{0,})* - )|( - [^\n]* - ) - ) + (?P[\s\S]*) $""" sub_pattern = r""" (?P\d{2})(?P.{0,}) @@ -340,10 +364,13 @@ def __call__( ns = frag.groupdict() value['non_swift_' + ns['ns_id']] = ns['ns_data'] text.append(ns['ns_data']) - elif len(text) and text[-1]: - text.append('') elif line.strip(): + # Free-form line without a two-digit sub-tag: keep the + # content instead of dropping it. text.append(line.strip()) + elif len(text) and text[-1]: + # Blank line: collapse runs into one paragraph separator. + text.append('') value['non_swift_text'] = '\n'.join(text) value['non_swift'] = data return value @@ -596,8 +623,13 @@ class TransactionDetails(Tag): id = 86 scope = models.Transaction + # The SWIFT spec caps this field at 6 lines of 65 characters, but many + # banks send more. A previous cap of nine 65-char chunks silently + # truncated anything longer, so the capture is unbounded: the parser in + # `models.Transactions.parse` already limits the value to this tag's own + # slice of the statement. pattern = r""" - (?P(([\s\S]{0,65}\r?\n?){0,8}[\s\S]{0,65})) + (?P[\s\S]*) """ @@ -610,7 +642,7 @@ class SumEntries(Tag): (?P.{3}) # 3!a Currency (?P[\d,]{1,15}) # 15d Amount """ - status: str + status: ClassVar[str] def __call__( self, transactions: models.Transactions, value: dict[str, typing.Any] diff --git a/mt940_tests/mBank/mt942.yml b/mt940_tests/mBank/mt942.yml index d357f20..8877396 100644 --- a/mt940_tests/mBank/mt942.yml +++ b/mt940_tests/mBank/mt942.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: PL29114010810000267002001002 c_floor_limit: !!python/object:mt940.models.Amount @@ -14,10 +14,10 @@ data: B+EBExIPAAAAAA== - !!python/object/apply:mt940.models.FixedOffset state: - _name: '0100' + _name: '60' _offset: !!python/object/apply:datetime.timedelta - 0 - - 6000 + - 3600 - 0 sequence_number: '1' statement_number: '1' @@ -35,8 +35,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -52,15 +53,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -68,14 +69,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -89,29 +93,31 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n\ + \ " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P\n (\n (\\d{2}.{0,})\n (\\\ + n\\d{2}.{0,})*\n )|(\n [^\\n]*\n )\n )\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: @@ -125,14 +131,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id003 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000001; Z RACH.: @@ -143,7 +147,7 @@ transactions: TNR: 179171073864111.010001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -156,14 +160,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id005 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id005 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000002; Z RACH.: @@ -174,7 +176,7 @@ transactions: TNR: 179171073864192.000001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -187,14 +189,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id006 !!python/object/apply:mt940.models.Date - !!binary | B+EBEw== extra_details: 911-TRANSAKCJA IPH funds_code: N - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+EBEw== + guessed_entry_date: *id006 id: NTRF status: C transaction_details: '911 TRANSAKCJA COLLECT; ID IPH: XX000000000003; Z RACH.: @@ -205,4 +205,4 @@ transactions: TNR: 179171073864291.000001' transaction_reference: ST170119CYC/0001 - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/self-provided/multiline.yml b/mt940_tests/self-provided/multiline.yml index 2466c8c..ad674d0 100644 --- a/mt940_tests/self-provided/multiline.yml +++ b/mt940_tests/self-provided/multiline.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: 'BILLLULLXXX/"NUMERO DE COMPTE IBAN ' available_balance: !!python/object:mt940.models.Balance @@ -34,8 +34,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -51,15 +52,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -67,14 +68,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -88,33 +92,36 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P[\\s\\S]*)\n " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P\n (\n (\\d{2}.{0,})\n (\\\ + n\\d{2}.{0,})*\n )|(\n [^\\n]*\n )\n )\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: - additional_purpose: NOM ET ADRESSE DO / BENEF 112345678 + additional_purpose: NOM ET ADRESSE DO / BENEF 112345678NOM ET ADRESSE DO / BENEF + 212345678NOM ET ADRESSE DO / BENEF 312345678NOM ET ADRESSE DO / BENEF 412345678NOM + ET ADRESSE DO / BENEF 512345678NOM ET ADRESSE DO / BENEF 612345678 amount: !!python/object:mt940.models.Amount amount: !!python/object/apply:decimal.Decimal - '-5' @@ -128,14 +135,12 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B9QIBA== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B9QIBA== extra_details: /OCMT/EUR4,5//IACC/D3/ funds_code: R - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B9QIBA== + guessed_entry_date: *id003 id: NTRF posting_text: VIREMENT111111111111111111X prima_nota: null @@ -156,7 +161,7 @@ transactions: FREE TEXT' transaction_reference: BILMT940 - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -169,15 +174,13 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B+YMEw== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id005 !!python/object/apply:mt940.models.Date - !!binary | B+YMFA== extra_details: Paiement funds_code: null - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B+YMFA== + guessed_entry_date: *id005 id: NMSC status: C transaction_reference: BILMT940 - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/self-provided/raphaelm.yml b/mt940_tests/self-provided/raphaelm.yml index 87fec58..6b0b9f7 100644 --- a/mt940_tests/self-provided/raphaelm.yml +++ b/mt940_tests/self-provided/raphaelm.yml @@ -1,4 +1,4 @@ -&id003 !!python/object:mt940.models.Transactions +&id004 !!python/object:mt940.models.Transactions data: account_identification: '3346780111' final_closing_balance: !!python/object:mt940.models.Balance @@ -73,6 +73,7 @@ data: 90000022 + 32 33 @@ -83,8 +84,9 @@ data: tags: 13: !!python/object:mt940.tags.DateTimeIndication re: !!python/object/apply:re._compile - - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n - \ (?P\\d{2})\n (\\+(?P\\d{4})|)\n " + - "^\n (?P\\d{2})\n (?P\\d{2})\n (?P\\d{2})\n (?P\\\ + d{2})\n (?P\\d{2})\n ((?P[+-])(?P\\d{4}))?\n\ + \ " - 98 20: !!python/object:mt940.tags.TransactionReferenceNumber re: !!python/object/apply:re._compile @@ -100,15 +102,15 @@ tags: - 98 28: !!python/object:mt940.tags.StatementNumber re: !!python/object/apply:re._compile - - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\d{1,5}))? - \ # [/5n]\n $" + - "\n (?P\\d{1,5}) # 5n\n (?:/?(?P\\\ + d{1,5}))? # [/5n]\n $" - 98 60: !!python/object:mt940.tags.OpeningBalance re: &id001 !!python/object/apply:re._compile - - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value - Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3}) - \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n " + - "^\n (?P[DC]) # 1!a Debit/Credit\n (?P\\d{2}) # 6!n Value\ + \ Date (YYMMDD)\n (?P\\d{2})\n (?P\\d{2})\n (?P.{3})\ + \ # 3!a Currency\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n " - 98 60M: !!python/object:mt940.tags.IntermediateOpeningBalance re: *id001 @@ -116,14 +118,17 @@ tags: re: *id001 61: !!python/object:mt940.tags.Statement re: !!python/object/apply:re._compile - - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n - \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date - (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit - Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the currency\n - \ # code, if needed)\n [\\n ]?\n (?P[\\d,]{1,15}) - \ # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n (?P((?!//)[^\\n]){0,16})\n - \ (//(?P.{0,23}))?\n (\\n?(?P.{0,34}))?\n - \ $" + - "^\n (?P\\d{2}) # 6!n Value Date (YYMMDD)\n (?P\\d{2})\n\ + \ (?P\\d{2})\n (?P\\d{2}|\\s{2})? # [4!n] Entry Date\ + \ (MMDD)\n (?P\\d{2}|\\s{2})?\n (?PR?[DC]) # 2a Debit/Credit\ + \ Mark\n (?P[A-Z])? # [1!a] Funds Code (3rd character of the\ + \ currency\n # code, if needed)\n [\\n ]?\n \ + \ (?P[\\d,]{1,15}) # 15d Amount\n (?P[A-Z][A-Z0-9 ]{3})?\n\ + \ (?P((?!//)[^\\n]){0,16})\n (//(?P.{0,23}))?\n\ + \ # Supplementary details: the SWIFT spec caps this at 34x, but some banks\n\ + \ # (e.g. Wise, issue #117) send more, so the length limit is relaxed. This\n\ + \ # only ever turns a previous parse error into a successful parse.\n \ + \ (\\n?(?P.*))?\n $" - 98 62: !!python/object:mt940.tags.ClosingBalance re: *id001 @@ -137,29 +142,29 @@ tags: re: *id001 86: !!python/object:mt940.tags.TransactionDetails re: !!python/object/apply:re._compile - - "\n (?P(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n - \ " + - "\n (?P[\\s\\S]*)\n " - 98 34: !!python/object:mt940.tags.FloorLimitIndicator re: !!python/object/apply:re._compile - - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a - Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal - sign, so 16)\n $" + - "^\n (?P[A-Z]{3}) # 3!a Currency\n (?P[DC ]?) # 2a\ + \ Debit/Credit Mark\n (?P[0-9,]{0,16}) # 15d Amount (includes decimal\ + \ sign, so 16)\n $" - 98 NS: !!python/object:mt940.tags.NonSwift re: !!python/object/apply:re._compile - - "\n (?P\n (\n (\\d{2}.{0,})\n (\\n\\d{2}.{0,})*\n - \ )|(\n [^\\n]*\n )\n )\n $" + - "\n (?P[\\s\\S]*)\n $" - 98 90: !!python/object:mt940.tags.SumEntries re: &id002 !!python/object/apply:re._compile - - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\d,]{1,15}) - \ # 15d Amount\n " + - "^\n (?P\\d*)\n (?P.{3}) # 3!a Currency\n (?P[\\\ + d,]{1,15}) # 15d Amount\n " - 98 90D: !!python/object:mt940.tags.SumDebitEntries re: *id002 90C: !!python/object:mt940.tags.SumCreditEntries re: *id002 +transaction_boundary: !!python/object/apply:builtins.frozenset +- [] transactions: - !!python/object:mt940.models.Transaction data: @@ -173,16 +178,15 @@ transactions: date: !!python/object/apply:mt940.models.Date - !!binary | B9IDEQ== - entry_date: !!python/object/apply:mt940.models.Date + entry_date: &id003 !!python/object/apply:mt940.models.Date - !!binary | B9IDFA== extra_details: '' funds_code: M - guessed_entry_date: !!python/object/apply:mt940.models.Date - - !!binary | - B9IDFA== + guessed_entry_date: *id003 id: S051 - non_swift: "01Verwendungszweck 1\n02Verwendungszweck 2\n15Empf\xE4nger\n17Buchungstext\n1812345\n191000\n204711" + non_swift: "01Verwendungszweck 1\n02Verwendungszweck 2\n15Empf\xE4nger\n17Buchungstext\n\ + 1812345\n191000\n204711" non_swift_01: Verwendungszweck 1 non_swift_02: Verwendungszweck 2 non_swift_15: "Empf\xE4nger" @@ -190,10 +194,11 @@ transactions: non_swift_18: '12345' non_swift_19: '1000' non_swift_20: '4711' - non_swift_text: "Verwendungszweck 1\nVerwendungszweck 2\nEmpf\xE4nger\nBuchungstext\n12345\n1000\n4711" + non_swift_text: "Verwendungszweck 1\nVerwendungszweck 2\nEmpf\xE4nger\nBuchungstext\n\ + 12345\n1000\n4711" status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -211,7 +216,7 @@ transactions: id: NCHG status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -229,7 +234,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -247,7 +252,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -265,7 +270,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -286,7 +291,7 @@ transactions: non_swift_text: '3037010000' status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -304,7 +309,7 @@ transactions: id: S051 status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -340,7 +345,7 @@ transactions: 87132101' status: C transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 - !!python/object:mt940.models.Transaction data: amount: !!python/object:mt940.models.Amount @@ -366,4 +371,4 @@ transactions: 12345' status: D transaction_reference: STARTUMS - transactions: *id003 + transactions: *id004 diff --git a/mt940_tests/test_issues/test_issue105_first_balance.py b/mt940_tests/test_issues/test_issue105_first_balance.py new file mode 100644 index 0000000..c0b9a56 --- /dev/null +++ b/mt940_tests/test_issues/test_issue105_first_balance.py @@ -0,0 +1,58 @@ +"""Issue #105: first intermediate_opening_balance / account_identification. + +With the default ``parse()`` every statement-scope tag merges into one dict, +so in files with several pages only the LAST ``:60M:`` / ``:25:`` survive. +``parse_statements()`` preserves per-``:20:``-block data (shape 1); this test +also covers the paged single-block shape (shape 2). +""" + +import mt940 +import pytest + +TWO_BLOCKS = """:20:REF1 +:25:ACC1 +:28C:1/1 +:60M:C231229EUR100,00 +:61:2312290101C10,00NTRFREF//BANK +:62M:C231229EUR110,00 +- +:20:REF2 +:25:ACC2 +:28C:1/2 +:60M:C231230EUR110,00 +:61:2312300101C5,00NTRFREF//BANK +:62M:C231230EUR115,00 +- +""" + +PAGED_SINGLE_BLOCK = """:20:REF1 +:25:ACC1 +:28C:1/1 +:60M:C231229EUR100,00 +:61:2312290101C10,00NTRFREF//BANK +:62M:C231229EUR110,00 +:28C:1/2 +:60M:C231230EUR110,00 +:61:2312300101C5,00NTRFREF//BANK +:62M:C231230EUR115,00 +- +""" + + +def test_two_blocks_first_intermediate_balance_preserved(): + statements = mt940.parse_statements(TWO_BLOCKS) + assert len(statements) == 2 + assert str(statements[0].data['intermediate_opening_balance']).startswith( + '100.00' + ) + assert statements[0].data['account_identification'] == 'ACC1' + assert statements[1].data['account_identification'] == 'ACC2' + + +@pytest.mark.xfail( + strict=True, reason='issue #105 shape 2 -- pending decision' +) +def test_paged_single_block_first_intermediate_balance_accessible(): + statements = mt940.parse_statements(PAGED_SINGLE_BLOCK) + first = statements[0].data['intermediate_opening_balance'] + assert str(first).startswith('100.00') diff --git a/mt940_tests/test_parse.py b/mt940_tests/test_parse.py index 8a2fff8..5b2ea7c 100644 --- a/mt940_tests/test_parse.py +++ b/mt940_tests/test_parse.py @@ -28,6 +28,84 @@ def test_non_ascii_parse(path, encoding): pickle.dumps(mt940.parse(data)) +_BOM_STATEMENT = ( + ':20:REF\n' + ':25:NL00BANK0123456789\n' + ':28C:1/1\n' + ':60F:C091019EUR1000,00\n' + ':61:0910201020C500,00NTRFNONREF//B\n' + ':86:Example transaction\n' + ':62F:C091020EUR1500,00\n' +) + + +def test_utf8_bom_bytes_does_not_drop_first_tag(): + # A UTF-8 BOM (emitted by many Windows tools/banks) must not push the + # leading :20: past the start-of-line tag anchor and drop its data. + data = b'\xef\xbb\xbf' + _BOM_STATEMENT.encode('utf-8') + transactions = mt940.parse(data) + assert transactions.data.get('transaction_reference') == 'REF' + assert len(transactions) == 1 + + +def test_utf8_bom_str_does_not_drop_first_tag(): + # Same file already decoded to str with a stray BOM character. + transactions = mt940.parse('' + _BOM_STATEMENT) + assert transactions.data.get('transaction_reference') == 'REF' + assert len(transactions) == 1 + + +def test_86_line_with_embedded_tag_lookalike_stays_in_details(): + # A :86: free-text line that itself starts with a tag-lookalike (:12:, + # which is not a known tag) must not be split off as a separate tag and + # corrupt the statement -- it stays part of the transaction details. + transactions = mt940.parse( + ':20:R\n:25:A\n:28C:1\n:60F:C240101EUR0,00\n' + ':61:2401010101C10,00NTRFREF//B\n' + ':86:PAYMENT FOR\n:12:INVOICE STYLE REF\n' + ':62F:C240101EUR10,00\n' + ) + assert len(transactions) == 1 + # No corruption: the closing balance still parses off the real :62F:. + assert ( + str(transactions.data['final_closing_balance'].amount) == '10.00 EUR' + ) + # The lookalike line is retained verbatim inside the details. + details = transactions[0].data['transaction_details'] + assert 'INVOICE STYLE REF' in details + + +def test_parse_statements_drops_swift_header_and_absorbs_terminators(): + # A leading SWIFT {1:}{2:}{4: header before the first :20: must be dropped, + # and the lone `-` statement terminators must not create empty statements. + data = ( + '{1:F01BANKNL2AXXXX0000000000}{2:I940BANKNL2AXXXXN}{4:\n' + ':20:STMT1\n:25:NL00BANK1\n:28C:1/1\n:60F:C240101EUR100,00\n' + ':62F:C240101EUR100,00\n-}\n' + ':20:STMT2\n:25:NL00BANK2\n:28C:2/1\n:60F:C240102EUR200,00\n' + ':62F:C240102EUR200,00\n-\n' + ) + statements = mt940.parse_statements(data) + assert [s.data['transaction_reference'] for s in statements] == [ + 'STMT1', + 'STMT2', + ] + + +def test_86_continuation_preserves_leading_whitespace(): + # Transactions.strip only rstrips, so significant leading whitespace on a + # :86: continuation line is preserved rather than eaten. + transactions = mt940.parse( + ':20:R\n:25:A\n:28C:1\n:60F:C240101EUR0,00\n' + ':61:2401010101C10,00NTRFREF//B\n' + ':86:LINE ONE\n INDENTED TWO\n' + ':62F:C240101EUR10,00\n' + ) + assert transactions[0].data['transaction_details'] == ( + 'LINE ONE\n INDENTED TWO' + ) + + def test_pickle_roundtrip_restores_processors(): # __getstate__ drops the (unpicklable) processors; __setstate__ must # restore them so the unpickled object is still usable. @@ -42,3 +120,19 @@ def test_pickle_roundtrip_restores_processors(): assert len(restored) == len(transactions) # Re-parsing exercises the restored processors without raising. restored.parse('') + + +def test_pickle_roundtrip_preserves_transaction_boundary(): + # __getstate__ only drops `processors`; newer state such as the opt-in + # `transaction_boundary` (issue #110) must survive a pickle round-trip. + transactions = mt940.models.Transactions( + transaction_boundary={'transaction_reference_number'} + ) + + # Trusted, self-produced pickle (a round-trip of our own object), so + # pickle.loads is safe here. + restored = pickle.loads(pickle.dumps(transactions)) + + assert restored.transaction_boundary == frozenset( + {'transaction_reference_number'} + ) diff --git a/mt940_tests/test_processors.py b/mt940_tests/test_processors.py index c59e8bc..0405f27 100644 --- a/mt940_tests/test_processors.py +++ b/mt940_tests/test_processors.py @@ -1,3 +1,4 @@ +import json import os import pathlib import typing @@ -174,3 +175,188 @@ def test_citi_bank_processors() -> None: assert len(transactions) == 5 expected_date = mt940.models.Date(2024, 3, 12) assert transactions[0].data['date'] == expected_date + + +def test_json_round_trip_preserves_model_values() -> None: + """`mt940.JSONEncoder` serialises every model type and round-trips. + + The existing ``test_json_dump`` only calls ``json.dumps`` over ``.sta`` + fixtures. This exercises the full ``dumps`` -> ``loads`` round-trip and + asserts the string forms of the nested ``Balance``/``Amount``/``Date`` + value types, which were previously unchecked. + """ + transactions = mt940.parse(str(_tests_path / 'jejik' / 'abnamro.sta')) + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + + assert len(decoded['transactions']) == len(transactions) + # Balance -> nested dict; Amount -> Decimal as str; Date -> ISO str + assert decoded['final_opening_balance'] == { + 'status': 'C', + 'amount': {'amount': '3236.28', 'currency': 'EUR'}, + 'date': '2011-05-22', + } + # Every transaction amount is a Decimal serialised to a plain string. + for transaction in decoded['transactions']: + assert isinstance(transaction['amount']['amount'], str) + assert isinstance(transaction['date'], str) + + +def test_json_round_trip_sum_amount_and_datetime() -> None: + """``SumAmount`` (with its entry ``number``) and ``DateTime`` serialise. + + The mBank ``:90D:``/``:90C:`` and ``:13D:`` tags produce ``SumAmount`` and + ``DateTime`` objects that never appear in the jejik ``.sta`` fixtures used + by ``test_json_dump``'s value-free smoke check. + """ + transactions = mt940.parse(str(_tests_path / 'mBank' / 'mt942.sta')) + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + + assert decoded['sum_credit_entries'] == { + 'amount': '0.03', + 'currency': 'PLN', + 'number': '3', + } + # The :13D: DateTime (with a +0100 FixedOffset) renders as an ISO string, + # carrying the timezone offset, not a mapping. + assert decoded['date'] == '2017-01-19 18:15:00+01:00' + + +def test_json_round_trip_asnb_non_swift_statement() -> None: + """The opt-in ``StatementASNB`` tag output also round-trips cleanly. + + ASN fixtures live in ``.txt`` files and require a custom tag, so they are + outside ``test_json_dump``'s ``.sta`` glob entirely. + """ + tag = mt940.tags.StatementASNB() + transactions = mt940.models.Transactions(tags={tag.id: tag}) + with (_tests_path / 'ASNB' / 'mt940.txt').open() as fh: + transactions.parse(fh.read()) + + decoded = json.loads(json.dumps(transactions, cls=mt940.JSONEncoder)) + assert len(decoded['transactions']) == len(transactions) + assert decoded['transactions'][0]['amount'] == { + 'amount': '-65.00', + 'currency': 'EUR', + } + + +def test_date_fixup_non_leap_february_clamped() -> None: + """A Feb 29 value date in a non-leap year is clamped to Feb 28. + + ``test_date_fixup_pre_processor`` only covers Feb 30 in a leap year + (``february_30.sta``, 2016 -> Feb 29); the non-leap clamp path was + previously unexercised. + """ + data = ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C170201EUR100,00\n' + ':61:1702290228DR6,00N024NONREF\n' + ':86:free\n' + ':62F:C170228EUR94,00\n' + ) + transactions = mt940.parse(data) + assert transactions[0].data['date'] == mt940.models.Date(2017, 2, 28) + + +@pytest.mark.parametrize( + ('detail', 'expected_purpose'), + [ + # A literal '+' inside the first characters of free text (e.g. a + # company name like "AB+...") must not be treated as a GVC KEYWORD+ + # separator: EREF is a real GVC key later in the text, so gvcodes runs. + ('020?20AB+EREF', 'AB+EREF'), + # 'A+B' at the very start must survive; SVWZ triggers gvcodes parsing. + ('020?20A+B SVWZ TEXT', 'A+B SVWZ TEXT'), + ], +) +def test_gvcode_leading_plus_in_free_text_kept_in_purpose( + detail: str, expected_purpose: str +) -> None: + """A '+' earlier than position 4 cannot terminate a GVC keyword. + + GVC keywords are 3-4 chars followed by '+'. ``_parse_mt940_gvcodes`` sliced + ``purpose[index - 4:index]`` without a lower bound, so a '+' at index < 4 + produced a wrapped/empty slice matching the empty-string GVC key and + truncated the purpose (dropping the leading free text before the '+'). + """ + data = ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C200101EUR100,00\n' + ':61:2001010101C10,00NTRFNONREF\n' + f':86:{detail}\n' + ':62F:C200101EUR110,00\n' + ) + transaction = mt940.parse(data)[0].data + assert transaction['purpose'] == expected_purpose + + +def _two_structured_86_data(first_detail: str, second_detail: str) -> str: + return ( + ':20:REF\n' + ':25:123456789\n' + ':28C:0\n' + ':60F:C200101EUR100,00\n' + ':61:2001010101C10,00NTRFNONREF\n' + f':86:{first_detail}\n' + f':86:{second_detail}\n' + ':62F:C200101EUR110,00\n' + ) + + +@pytest.mark.parametrize( + ('first_detail', 'second_detail', 'expected_purpose', 'expected_posting'), + [ + # The second tag's None sub-fields clobber the first tag's real + # values under the current (golden-pinned) merge semantics: the last + # structured :86: wins per key, even with None. + ('020?20REALPURPOSE', '020?00POSTINGTEXT', None, 'POSTINGTEXT'), + ('020?00POSTINGTEXT', '020?20REALPURPOSE', 'REALPURPOSE', None), + ], +) +def test_repeated_structured_86_merges_without_crash( + first_detail: str, + second_detail: str, + expected_purpose: str | None, + expected_posting: str | None, +) -> None: + """Two structured ``:86:`` tags on one ``:61:`` must not crash the parse. + + A structured ``:86:`` emits every ``DETAIL_KEYS`` value including ``None`` + for absent sub-fields; a second structured ``:86:`` then hit + ``None += str`` in ``_update_transaction`` (``TypeError`` aborting the + whole file) in either order. This pins the CURRENT post-fix merge + contract: a later string replaces an existing ``None``, while a later + ``None`` still overwrites an earlier real value (the semantics the + real-bank goldens encode -- see the xfail below for the pending + preservation question). + """ + transaction = mt940.parse( + _two_structured_86_data(first_detail, second_detail) + )[0].data + assert transaction['purpose'] == expected_purpose + assert transaction['posting_text'] == expected_posting + + +@pytest.mark.xfail( + strict=True, + reason='repeated structured :86: None-clobber semantics -- pending ' + 'decision (audit task 7 review): preserving non-None values against a ' + "later tag's None is arguably more correct, but the same rule stops a " + 'structured :86: from nulling the customer_reference set by :61:, ' + 'changing 5 real-bank goldens.', +) +def test_repeated_structured_86_preserves_real_values() -> None: + """Preservation ideal: real values from BOTH tags survive, either order.""" + for first, second in [ + ('020?20REALPURPOSE', '020?00POSTINGTEXT'), + ('020?00POSTINGTEXT', '020?20REALPURPOSE'), + ]: + transaction = mt940.parse(_two_structured_86_data(first, second))[ + 0 + ].data + assert transaction['purpose'] == 'REALPURPOSE' + assert transaction['posting_text'] == 'POSTINGTEXT' diff --git a/mt940_tests/test_tags.py b/mt940_tests/test_tags.py index a8bdcb2..17f8574 100644 --- a/mt940_tests/test_tags.py +++ b/mt940_tests/test_tags.py @@ -1,3 +1,4 @@ +import datetime import pathlib import mt940 @@ -6,6 +7,194 @@ _tests_path = pathlib.Path(__file__).parent +# Minimal valid MT940 statement to wrap single-tag test values in. +_HEADER = ':20:REF\n:25:ACC\n:28C:1\n:60F:C231229EUR0,00\n' +_FOOTER = ':62F:C231229EUR10,00\n' + + +def test_date_time_indication_positive_offset_is_hhmm(): + # The :13(D): offset subfield is HHMM (like ISO 8601 +0130), not a + # number of minutes: +0130 means 1 hour 30 minutes. + transactions = mt940.parse(':13D:1701191815+0130\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date.utcoffset() == datetime.timedelta(hours=1, minutes=30) + + +def test_date_time_indication_negative_offset(): + # 1!x sign can be '-' as well (e.g. US banks): -0500 is UTC-5. + transactions = mt940.parse(':13D:1701191815-0500\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date.utcoffset() == -datetime.timedelta(hours=5) + + +def test_transaction_details_long_multiline_not_truncated(): + # The old :86: pattern capped the capture at nine 65-char chunks + # (~593 chars); longer details -- e.g. German banks packing many ?NN + # subfields -- were silently truncated. The cap had already been bumped + # once (commit 4575222) for exactly this reason. + lines = [f'line {i:02d} ' + 'x' * 57 for i in range(12)] + details = '\n'.join(lines) + transactions = mt940.parse( + _HEADER + + ':61:2312290101D10,50NMSC\n' + + ':86:' + + details + + '\n' + + _FOOTER + ) + assert transactions[0].data['transaction_details'] == details + + +def test_non_swift_multiline_free_text(): + # NS content is bank specific ("could be anything"); multi-line values + # whose lines do not all start with a two-digit sub-tag used to fail the + # NS pattern and abort the whole parse. + transactions = mt940.parse(_HEADER + ':NS:hello\nworld\n' + _FOOTER) + assert transactions.data['non_swift'] == 'hello\nworld' + assert transactions.data['non_swift_text'] == 'hello\nworld' + + +def test_non_swift_structured_line_followed_by_free_text(): + transactions = mt940.parse(_HEADER + ':NS:22foo\nbar\n' + _FOOTER) + assert transactions.data['non_swift'] == '22foo\nbar' + assert transactions.data['non_swift_22'] == 'foo' + assert transactions.data['non_swift_text'] == 'foo\nbar' + + +def test_non_swift_blank_lines_collapse(): + # Direct tag call: blank lines between content are kept as single + # paragraph separators in non_swift_text (mt940.parse strips blank + # lines before tags ever see them). + tag = tags.NonSwift() + result = tag( + mt940.models.Transactions(), {'non_swift': '22foo\n\n\n22bar'} + ) + assert result['non_swift_text'] == 'foo\n\nbar' + + +def test_floor_limit_space_indicator_treated_as_absent(): + # Fiducia / Volksbank Ortenau sends ':34F:EUR 999999999999,99' (commit + # d36c51b relaxed the regex for it). A blank D/C mark means "applies to + # both", like an absent mark -- it must not create a ' _floor_limit' key. + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n' + ':34F:EUR 999999999999,99\n' + ':60F:C231229EUR0,00\n' + _FOOTER + ) + assert ' _floor_limit' not in transactions.data + assert str(transactions.data['d_floor_limit']) == '-999999999999.99 EUR' + assert str(transactions.data['c_floor_limit']) == '999999999999.99 EUR' + + +def test_floor_limit_lowercase_indicator_normalized(): + # The tag regexes are compiled with re.IGNORECASE, so a lowercase mark + # must behave exactly like its uppercase form (debit -> negative). + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n' + ':34F:EURd10,00\n' + ':60F:C231229EUR0,00\n' + _FOOTER + ) + assert str(transactions.data['d_floor_limit']) == '-10.00 EUR' + + +class MultilineGroupTag(tags.Tag): + """Tag whose (valid) pattern spreads one group over several lines.""" + + id = 28 + pattern = r""" + (?P + \d+ + ) + $""" + + +def test_unparseable_value_raises_runtime_error(): + # Tag.parse documents RuntimeError for unparsable values, but the + # debug helper re-compiles the pattern line by line; for patterns with + # multi-line groups the unbalanced fragments raised re.error instead. + tag_parser = MultilineGroupTag() + transactions = mt940.models.Transactions(tags={tag_parser.id: tag_parser}) + with pytest.raises(RuntimeError, match='Unable to parse'): + transactions.parse(':20:REF\n:28C:NOTDIGITS\n') + + +@pytest.mark.xfail( + strict=True, + reason='RC reversal amount sign -- pending decision (audit task 5): ' + 'a reversal of credit takes funds out of the account, but Amount only ' + 'negates on a plain D mark, so RC amounts stay positive (as do the RC ' + 'transactions in the betterplace fixture goldens).', +) +def test_statement_rc_reversal_amount_is_negative(): + transactions = mt940.parse( + _HEADER + ':61:2312290101RC10,50NTRFREF//BANK\n' + _FOOTER + ) + data = transactions[0].data + assert data['status'] == 'RC' + assert str(data['amount']) == '-10.50 EUR' + + +def test_statement_reversal_marks_parse(): + # RC/RD marks (2a subfield) parse and are preserved in `status`; the + # amount sign for RC is a pending decision, see the xfail above. + transactions = mt940.parse( + _HEADER + + ':61:2312290101RD10,50NTRFREF//BANK\n' + + ':61:2312290101RC10,50NTRFREF//BANK\n' + + _FOOTER + ) + assert transactions[0].data['status'] == 'RD' + assert str(transactions[0].data['amount']) == '10.50 EUR' + assert transactions[1].data['status'] == 'RC' + + +def test_statement_amount_without_decimals(): + # 15d allows a trailing decimal comma with no fraction digits. + transactions = mt940.parse( + _HEADER + ':61:2312290101C10,NTRFREF//BANK\n' + _FOOTER + ) + assert str(transactions[0].data['amount']) == '10 EUR' + + +def test_statement_lowercase_debit_mark_is_negative(): + # The tag patterns are compiled with re.IGNORECASE, so a lowercase 'd' + # debit mark is accepted. Amount must still treat it as a debit and + # negate the amount; otherwise a debit is silently stored as positive. + transactions = mt940.parse( + _HEADER + ':61:2312290101d10,50NTRFREF//BANK\n' + _FOOTER + ) + data = transactions[0].data + assert data['status'] == 'd' + assert str(data['amount']) == '-10.50 EUR' + + +def test_statement_second_double_slash_stays_in_bank_reference(): + # Only the first // separates the bank reference; a second one is kept + # as part of the reference content. + transactions = mt940.parse( + _HEADER + ':61:2312290101C10,50NTRFREF//BANK//EXTRA\n' + _FOOTER + ) + data = transactions[0].data + assert data['customer_reference'] == 'REF' + assert data['bank_reference'] == 'BANK//EXTRA' + + +def test_balance_on_leap_day(): + transactions = mt940.parse( + ':20:REF\n:25:ACC\n:28C:1\n:60F:C240229EUR0,00\n' + ':61:2402290229C10,50NTRFREF\n' + ':62F:C240229EUR10,50\n' + ) + balance = transactions.data['final_opening_balance'] + assert balance.date == models.Date(2024, 2, 29) + + +def test_date_time_indication_without_offset(): + # The offset is optional in the pattern; a bare 10-digit :13: must not + # crash and yields a naive datetime. + transactions = mt940.parse(':13:1701191815\n' + _HEADER + _FOOTER) + date = transactions.data['date'] + assert date == models.DateTime(2017, 1, 19, 18, 15) + assert date.tzinfo is None + @pytest.fixture def long_statement_number(): diff --git a/pyproject.toml b/pyproject.toml index 04a862c..58e8ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,9 @@ skip = '*/htmlcov,./docs/_build,*.asc,*.yml,*.sta,*.txt,*.html,./docs/html' [tool.pyright] include = ['mt940', 'mt940_tests'] exclude = ['dist/*', 'mt940_tests/*'] +# mt940/parser.py imports the package root (`import mt940`) by design, so the +# package has intentional import cycles; basedpyright reads this section too. +reportImportCycles = false strict = [ 'mt940/__about__.py', 'mt940/__init__.py',