diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5f68dce..c2515f5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.9", "3.14"] steps: - uses: actions/checkout@v4 with: @@ -27,13 +27,14 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Run Cargo Tests - run: cargo test --features python -- --nocapture --test-threads=1 + run: cargo test -- --nocapture --test-threads=1 - name: Run pytest run: | # just venv pytest rm -rf .venv python3 -m venv .venv . .venv/bin/activate + .venv/bin/pip install --upgrade pip .venv/bin/pip install wheel pytest maturin maturin develop .venv/bin/pytest python/tests @@ -46,11 +47,11 @@ jobs: matrix: platform: - target: x64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: armv7 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 steps: - uses: actions/checkout@v4 with: @@ -77,13 +78,13 @@ jobs: platform: - target: x86_64-unknown-linux-musl arch: x86_64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: i686-unknown-linux-musl arch: x86 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64-unknown-linux-musl arch: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 # all values: [x86_64, x86, aarch64, armhf, armv7, ppc64le, riscv64, s390x] # { target: "armv7-unknown-linux-musleabihf", image_tag: "armv7" }, # { target: "powerpc64le-unknown-linux-musl", image_tag: "ppc64le" }, @@ -113,7 +114,7 @@ jobs: strategy: matrix: target: [x64, x86] - interpreter: ["3.9", "3.10", "3.11", "3.12", "3.13"] + interpreter: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 with: @@ -151,9 +152,9 @@ jobs: matrix: platform: - target: x64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 426a62a..5678b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [1.1.0] - Unreleased + +### Added + +- Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. +- Support for Python 3.14. +- `__str__` and `__repr__` methods for `WeightedLevenshtein`. ### Changed -- Small runtime improvements in Rust backend. +- Cap `default_substitution_cost` at `default_insertion_cost + default_deletion_cost`, since a substitution can always be expressed as a deletion followed by an insertion. +- Treat empty-sided substitution costs as insertion/deletion aliases. +- Reject non-finite (NaN, infinite) costs during validation. + +### Fixed + +- Use the minimum cost for conflicting symmetric substitutions. ## [1.0.1] - 2025-09-21 diff --git a/Cargo.lock b/Cargo.lock index 65c789f..482bae7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,18 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +version = 4 [[package]] name = "crossbeam-deque" @@ -51,30 +39,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "ocr_stringdist" -version = "1.0.1" +version = "1.1.0" dependencies = [ "pyo3", "rayon", @@ -103,37 +76,32 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ - "cfg-if", - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -141,9 +109,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -153,9 +121,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -206,18 +174,12 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/Cargo.toml b/Cargo.toml index a5ff08d..34c8bbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ocr_stringdist" -version = "1.0.1" +version = "1.1.0" edition = "2021" description = "String distances considering OCR errors." authors = ["Niklas von Moers "] @@ -14,8 +14,5 @@ name = "ocr_stringdist" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.24.0", features = ["auto-initialize"] } +pyo3 = { version = "0.28.3", features = ["auto-initialize"] } rayon = "1.10.0" - -[features] -python = [] diff --git a/Justfile b/Justfile index ddc0da7..5d84390 100644 --- a/Justfile +++ b/Justfile @@ -8,15 +8,20 @@ pytest: uv run pytest --cov=python/ocr_stringdist python/tests test: - cargo llvm-cov --features python - #cargo test --features python + cargo llvm-cov + #cargo test mypy: uv run mypy . lint: + cargo clippy uv run ruff check . --fix +format: + cargo fmt + uv run ruff format + doc: uv run make -C docs html diff --git a/docs/source/index.rst b/docs/source/index.rst index 9948b55..06f90b0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -56,6 +56,7 @@ Contents getting-started examples end_to_end_example + transitive_closure cost_learning_model api/index changelog diff --git a/docs/source/transitive_closure.rst b/docs/source/transitive_closure.rst new file mode 100644 index 0000000..fb6f71b --- /dev/null +++ b/docs/source/transitive_closure.rst @@ -0,0 +1,37 @@ +=========================== + Transitive Cost Closure +=========================== + +By default, :class:`~ocr_stringdist.WeightedLevenshtein` only considers the +costs you explicitly provide. If `("6", "G")` costs `0.5` and deleting `"G"` +costs `0.01`, the engine does **not** automatically know that deleting `"6"` +in context costs `0.51`. + +:meth:`~ocr_stringdist.WeightedLevenshtein.transitive_closure` returns a new +instance whose cost dictionaries are filled with these effective (transitive) +costs. The closure also materializes mixed chains, e.g. a `del("y") + ins("x")` +sequence becoming an effective `("y", "x")` substitution. + +Example +======= + +.. code-block:: python + + from ocr_stringdist import WeightedLevenshtein + + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + ).transitive_closure() + + # The chain "6" -> "G" -> ε is now a single effective deletion at 0.51. + print(wl.distance("06", "0")) # 0.51 + +After closure, :meth:`~ocr_stringdist.WeightedLevenshtein.explain` returns a +single flat operation at the effective cost; the underlying chain is not +preserved. + +You may pass ``prune=True`` to the ``transitive_closure`` method to remove +generated substitutions whose costs are already represented by matches, +insertions, deletions, or shorter substitutions. This shrinks the resulting +cost map but is significantly more expensive to compute. diff --git a/pyproject.toml b/pyproject.toml index 0944694..bf6c044 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ documentation = "https://niklasvonm.github.io/ocr-stringdist/" [tool.maturin] -features = ["pyo3/extension-module", "python"] +features = ["pyo3/extension-module"] python-source = "python" module-name = "ocr_stringdist._rust_stringdist" diff --git a/python/ocr_stringdist/_observable_dict.py b/python/ocr_stringdist/_observable_dict.py new file mode 100644 index 0000000..ee69e3f --- /dev/null +++ b/python/ocr_stringdist/_observable_dict.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Callable, Optional, TypeVar + +K = TypeVar("K") +V = TypeVar("V") + + +class _ObservableDict(dict[K, V]): + """A dictionary that triggers a callback on mutation.""" + + _MISSING = object() + + def __init__( + self, + mapping: dict[K, V], + on_change: Callable[[], None], + validator: Optional[Callable[[K, V], None]] = None, + ) -> None: + self._on_change = on_change + self._validator = validator + if validator: + for k, v in mapping.items(): + validator(k, v) + super().__init__(mapping) + + def __setitem__(self, key: K, value: V) -> None: + if self._validator: + self._validator(key, value) + super().__setitem__(key, value) + self._on_change() + + def __delitem__(self, key: K) -> None: + super().__delitem__(key) + self._on_change() + + def clear(self) -> None: + super().clear() + self._on_change() + + def pop(self, key: K, default: Any = _MISSING) -> V: + res = super().pop(key) if default is self._MISSING else super().pop(key, default) + self._on_change() + return res + + def popitem(self) -> tuple[K, V]: + res = super().popitem() + self._on_change() + return res + + def update(self, other: Any = (), /, **kwargs: V) -> None: + items = self._items_from_update_args(other, kwargs) + if self._validator: + for key, value in items: + self._validator(key, value) + super().update(items) + self._on_change() + + def setdefault(self, key: K, default: Any = None) -> V: + if key in self: + return self[key] + if self._validator: + self._validator(key, default) + super().__setitem__(key, default) + self._on_change() + return default + + def __or__(self, other: object, /) -> Any: + if not isinstance(other, dict): + return NotImplemented + return dict(self) | other + + def __ior__(self, other: object, /) -> Any: + self.update(other) + return self + + def _items_from_update_args(self, other: Any, kwargs: dict[str, V]) -> list[tuple[Any, Any]]: + items = list(other.items()) if isinstance(other, Mapping) else list(other) + items.extend(kwargs.items()) + return items diff --git a/python/ocr_stringdist/default_ocr_distances.py b/python/ocr_stringdist/default_ocr_distances.py index 1db03b2..b3ed772 100644 --- a/python/ocr_stringdist/default_ocr_distances.py +++ b/python/ocr_stringdist/default_ocr_distances.py @@ -38,5 +38,5 @@ """ Pre-defined distance map between characters, considering common OCR errors. The distances are between 0 and 1. -This map is intended to be used with `symmetric=True`. +This map is intended to be used with `symmetric_substitution=True`. """ diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index b85c38a..bb92bf6 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -1,13 +1,11 @@ from __future__ import annotations +import math from collections.abc import Iterable from typing import Any, Optional -from ._rust_stringdist import ( - _batch_weighted_levenshtein_distance, - _explain_weighted_levenshtein_distance, - _weighted_levenshtein_distance, -) +from ._observable_dict import _ObservableDict +from ._rust_stringdist import RustLevenshteinCalculator from .default_ocr_distances import ocr_distance_map from .edit_operation import EditOperation @@ -33,14 +31,6 @@ class WeightedLevenshtein: :raises TypeError, ValueError: If the provided arguments are invalid. """ - substitution_costs: dict[tuple[str, str], float] - insertion_costs: dict[str, float] - deletion_costs: dict[str, float] - symmetric_substitution: bool - default_substitution_cost: float - default_insertion_cost: float - default_deletion_cost: float - def __init__( self, substitution_costs: Optional[dict[tuple[str, str], float]] = None, @@ -52,52 +42,236 @@ def __init__( default_insertion_cost: float = 1.0, default_deletion_cost: float = 1.0, ) -> None: - # Validate default costs - for cost_name, cost_val in [ - ("default_substitution_cost", default_substitution_cost), - ("default_insertion_cost", default_insertion_cost), - ("default_deletion_cost", default_deletion_cost), - ]: - if not isinstance(cost_val, (int, float)): - raise TypeError(f"{cost_name} must be a number, but got: {type(cost_val).__name__}") - if cost_val < 0: - raise ValueError(f"{cost_name} must be non-negative, got value: {cost_val}") - - # Validate substitution_costs dictionary - sub_costs = ocr_distance_map if substitution_costs is None else substitution_costs - for key, cost in sub_costs.items(): - if not ( - isinstance(key, tuple) - and len(key) == 2 - and isinstance(key[0], str) - and isinstance(key[1], str) - ): - raise TypeError( - f"substitution_costs keys must be tuples of two strings, but found: {key}" - ) - if not isinstance(cost, (int, float)): - raise TypeError( - f"Cost for substitution {key} must be a number, but got: {type(cost).__name__}" - ) - if cost < 0: - raise ValueError(f"Cost for substitution {key} cannot be negative, but got: {cost}") - - self.substitution_costs = sub_costs - self.insertion_costs = {} if insertion_costs is None else insertion_costs - self.deletion_costs = {} if deletion_costs is None else deletion_costs - self.symmetric_substitution = symmetric_substitution - self.default_substitution_cost = default_substitution_cost - self.default_insertion_cost = default_insertion_cost - self.default_deletion_cost = default_deletion_cost + self._symmetric_substitution = symmetric_substitution + self._default_insertion_cost = self._validate_cost( + "default_insertion_cost", default_insertion_cost + ) + self._default_deletion_cost = self._validate_cost( + "default_deletion_cost", default_deletion_cost + ) + # A substitution can always be expressed as a deletion + insertion, so + # capping here keeps the substitution default from being effectively + # ignored when the user supplies a value above the del/ins ceiling. + self._default_substitution_cost = min( + self._validate_cost("default_substitution_cost", default_substitution_cost), + self._default_insertion_cost + self._default_deletion_cost, + ) + + self._calculator = None + + # Initialize Observable Dicts + sub_init = ocr_distance_map if substitution_costs is None else substitution_costs + self._substitution_costs = _ObservableDict( + sub_init, self._invalidate_calculator, self._validate_sub_entry + ) + self._insertion_costs = _ObservableDict( + insertion_costs or {}, self._invalidate_calculator, self._validate_unary_entry + ) + self._deletion_costs = _ObservableDict( + deletion_costs or {}, self._invalidate_calculator, self._validate_unary_entry + ) + + def _invalidate_calculator(self) -> None: + """Mark the Rust backend as out of sync; it will be rebuilt on next use.""" + self._calculator = None + + def _get_calculator(self) -> RustLevenshteinCalculator: + """Return a Rust backend in sync with the current Python-side state.""" + if self._calculator is None: + substitution_costs, insertion_costs, deletion_costs = ( + self._effective_cost_maps_for_calculator() + ) + self._calculator = RustLevenshteinCalculator( + substitution_costs=substitution_costs, + insertion_costs=insertion_costs, + deletion_costs=deletion_costs, + symmetric_substitution=self._symmetric_substitution, + default_substitution_cost=self._default_substitution_cost, + default_insertion_cost=self._default_insertion_cost, + default_deletion_cost=self._default_deletion_cost, + ) + return self._calculator + + def _effective_cost_maps_for_calculator( + self, + ) -> tuple[dict[tuple[str, str], float], dict[str, float], dict[str, float]]: + """ + Split substitution entries with empty source/target into the + insertion/deletion maps, taking the minimum where they overlap. + """ + substitution_costs: dict[tuple[str, str], float] = {} + insertion_costs = dict(self._insertion_costs) + deletion_costs = dict(self._deletion_costs) + + for (source, target), cost in self._substitution_costs.items(): + if source == "": + self._set_min_cost(insertion_costs, target, cost) + elif target == "": + self._set_min_cost(deletion_costs, source, cost) + else: + substitution_costs[(source, target)] = cost + + return substitution_costs, insertion_costs, deletion_costs + + @staticmethod + def _set_min_cost(costs: dict[str, float], key: str, cost: float) -> None: + costs[key] = min(costs.get(key, cost), cost) + + # --- Properties --- + + @property + def substitution_costs(self) -> dict[tuple[str, str], float]: + return self._substitution_costs + + @substitution_costs.setter + def substitution_costs(self, value: dict[tuple[str, str], float]) -> None: + self._substitution_costs = _ObservableDict( + value, self._invalidate_calculator, self._validate_sub_entry + ) + self._invalidate_calculator() + + @property + def insertion_costs(self) -> dict[str, float]: + return self._insertion_costs + + @insertion_costs.setter + def insertion_costs(self, value: dict[str, float]) -> None: + self._insertion_costs = _ObservableDict( + value, self._invalidate_calculator, self._validate_unary_entry + ) + self._invalidate_calculator() + + @property + def deletion_costs(self) -> dict[str, float]: + return self._deletion_costs + + @deletion_costs.setter + def deletion_costs(self, value: dict[str, float]) -> None: + self._deletion_costs = _ObservableDict( + value, self._invalidate_calculator, self._validate_unary_entry + ) + self._invalidate_calculator() + + @property + def symmetric_substitution(self) -> bool: + return self._symmetric_substitution + + @symmetric_substitution.setter + def symmetric_substitution(self, value: bool) -> None: + self._symmetric_substitution = value + self._invalidate_calculator() + + @property + def default_substitution_cost(self) -> float: + return self._default_substitution_cost + + @default_substitution_cost.setter + def default_substitution_cost(self, value: float) -> None: + self._default_substitution_cost = self._validate_cost("default_substitution_cost", value) + self._invalidate_calculator() + + @property + def default_insertion_cost(self) -> float: + return self._default_insertion_cost + + @default_insertion_cost.setter + def default_insertion_cost(self, value: float) -> None: + self._default_insertion_cost = self._validate_cost("default_insertion_cost", value) + self._invalidate_calculator() + + @property + def default_deletion_cost(self) -> float: + return self._default_deletion_cost + + @default_deletion_cost.setter + def default_deletion_cost(self, value: float) -> None: + self._default_deletion_cost = self._validate_cost("default_deletion_cost", value) + self._invalidate_calculator() + + # --- Validation Helpers --- + + def _validate_cost(self, name: str, val: float) -> float: + if not isinstance(val, (int, float)): + raise TypeError(f"{name} must be a number, but got: {type(val).__name__}") + if not math.isfinite(val): + raise ValueError(f"{name} must be finite, got value: {val}") + if val < 0: + raise ValueError(f"{name} must be non-negative, got value: {val}") + return float(val) + + def _validate_sub_entry(self, key: Any, cost: Any) -> None: + if not (isinstance(key, tuple) and len(key) == 2 and all(isinstance(k, str) for k in key)): + raise TypeError(f"substitution_costs keys must be tuples of two strings, found: {key}") + if key == ("", ""): + raise ValueError('substitution_costs key ("", "") is not a meaningful edit operation') + self._validate_cost(f"Cost for {key}", cost) + + def _validate_unary_entry(self, key: Any, cost: Any) -> None: + if not isinstance(key, str): + raise TypeError(f"Cost keys must be strings, found: {key}") + self._validate_cost(f"Cost for {key}", cost) @classmethod def unweighted(cls) -> WeightedLevenshtein: """Creates an instance with all operations having equal cost of 1.0.""" return cls(substitution_costs={}, insertion_costs={}, deletion_costs={}) + def transitive_closure( + self, + *, + prune: bool = False, + max_node_length: Optional[int] = None, + ) -> WeightedLevenshtein: + """ + Returns a new instance whose cost dictionaries are filled with effective + (transitive) edit costs. + + If, for example, `substitution_costs[("a", "b")] = 0.1` and + `substitution_costs[("b", "c")] = 0.1`, the closed instance's + `substitution_costs[("a", "c")]` is `0.2` rather than the default. + Insertion and deletion chains, and chains that cross `ε` (e.g. + `del("y") + ins("x")` becoming an effective `("y", "x")` substitution), + are likewise materialized. + + :param prune: If True, remove generated substitutions whose costs are + already represented by matches, insertions, deletions, and + shorter substitutions. This can make the returned cost map + easier to inspect, but it is much more expensive for large + closures. + :param max_node_length: Maximum length (in characters) of intermediate + graph nodes the closure may construct. `None` + derives a sensible default from the input + (twice the longest raw token, with a small + floor); pass an `int` to override. The cap is + what guarantees termination - without it, + the graph could grow without bound. Floyd-Warshall is + :math:`O(N^3)` in the resulting node count, so a + higher cap can be substantially slower. + :raises ValueError: If the generated closure graph is too large to + process safely. + + ``explain()`` on the closed instance returns flat single-step ops; the + original chain that produced an effective cost is not preserved. + + For repeated use, save via :meth:`to_dict` and reload via + :meth:`from_dict` so the closure is computed once. + """ + sub_dict, ins_dict, del_dict = self._get_calculator().closed_cost_maps( + prune, max_node_length + ) + return WeightedLevenshtein( + substitution_costs=dict(sub_dict), + insertion_costs=dict(ins_dict), + deletion_costs=dict(del_dict), + symmetric_substitution=False, + default_substitution_cost=self.default_substitution_cost, + default_insertion_cost=self.default_insertion_cost, + default_deletion_cost=self.default_deletion_cost, + ) + def distance(self, s1: str, s2: str) -> float: """Calculates the weighted Levenshtein distance between two strings.""" - return _weighted_levenshtein_distance(s1, s2, **self.__dict__) # type: ignore[no-any-return] + return self._get_calculator().distance(s1, s2) # type: ignore[no-any-return] def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOperation]: """ @@ -108,7 +282,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe :param filter_matches: If True, 'match' operations are excluded from the result. :return: List of :class:`EditOperation` instances. """ - raw_path = _explain_weighted_levenshtein_distance(s1, s2, **self.__dict__) + raw_path = self._get_calculator().explain(s1, s2) parsed_path = [EditOperation(*op) for op in raw_path] if filter_matches: return list(filter(lambda op: op.op_type != "match", parsed_path)) @@ -116,7 +290,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe def batch_distance(self, s: str, candidates: list[str]) -> list[float]: """Calculates distances between a string and a list of candidates.""" - return _batch_weighted_levenshtein_distance(s, candidates, **self.__dict__) # type: ignore[no-any-return] + return self._get_calculator().batch_distance(s, candidates) # type: ignore[no-any-return] @classmethod def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein: @@ -147,6 +321,30 @@ def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein: return CostLearner().fit(pairs) + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(" + f"substitution_costs={dict(self._substitution_costs)!r}, " + f"insertion_costs={dict(self._insertion_costs)!r}, " + f"deletion_costs={dict(self._deletion_costs)!r}, " + f"symmetric_substitution={self._symmetric_substitution!r}, " + f"default_substitution_cost={self._default_substitution_cost!r}, " + f"default_insertion_cost={self._default_insertion_cost!r}, " + f"default_deletion_cost={self._default_deletion_cost!r})" + ) + + def __str__(self) -> str: + return ( + f"{type(self).__name__}(" + f"substitution_costs=<{len(self._substitution_costs)} entries>, " + f"insertion_costs=<{len(self._insertion_costs)} entries>, " + f"deletion_costs=<{len(self._deletion_costs)} entries>, " + f"symmetric_substitution={self._symmetric_substitution}, " + f"default_substitution_cost={self._default_substitution_cost}, " + f"default_insertion_cost={self._default_insertion_cost}, " + f"default_deletion_cost={self._default_deletion_cost})" + ) + def __eq__(self, other: object) -> bool: if not isinstance(other, WeightedLevenshtein): return NotImplemented diff --git a/python/tests/test_batch_weighted_levenshtein.py b/python/tests/test_batch_weighted_levenshtein.py index a0e03a7..4b97669 100644 --- a/python/tests/test_batch_weighted_levenshtein.py +++ b/python/tests/test_batch_weighted_levenshtein.py @@ -78,7 +78,6 @@ def test_batch_finds_best_match( distances = WeightedLevenshtein( substitution_costs=OCR_COST_MAP, ).batch_distance(source, candidates) - print(f"------------------------------------distances: {distances}") # Find the index with minimum distance min_index = distances.index(min(distances)) diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index b7470ac..eb83941 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -75,3 +75,129 @@ def test_explain_weighted_levenshtein( manually_filtered_operations = [op for op in full_operations if op.op_type != "match"] assert filtered_operations == manually_filtered_operations assert full_operations == expected_operations + assert sum(op.cost for op in full_operations) == wl.distance(s1, s2) + + +# Closure-flat explain tests +# +# After ``transitive_closure()``, the underlying chain that produced an +# effective cost is no longer preserved. ``explain()`` returns a single +# substitution / insertion / deletion at the effective cost. These tests +# verify the flat output and that the total cost equals ``distance()``. + + +def _flat_explain_assertions( + wl_closed: WeightedLevenshtein, s1: str, s2: str, expected_distance: float +) -> list[EditOperation]: + ops = wl_closed.explain(s1, s2) + assert sum(op.cost for op in ops) == pytest.approx(expected_distance) + assert wl_closed.distance(s1, s2) == pytest.approx(expected_distance) + return ops + + +def _assert_ops_equal(actual: list[EditOperation], expected: list[EditOperation]) -> None: + """Compare op sequences with float-tolerant cost equality.""" + assert len(actual) == len(expected), f"length mismatch: {actual} vs {expected}" + for a, e in zip(actual, expected): + assert a.op_type == e.op_type + assert a.source_token == e.source_token + assert a.target_token == e.target_token + assert a.cost == pytest.approx(e.cost) + + +def test_explain_transitive_deletion_chain_after_closure() -> None: + """After closure, '06' -> '0' is one effective deletion of '6' at 0.51.""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + symmetric_substitution=False, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "06", "0", 0.51) + assert ops == [EditOperation("delete", "6", None, 0.51)] + + +def test_explain_transitive_substitution_chain_after_closure() -> None: + """After closure, 'a' -> 'c' is one effective substitution at 0.2.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "a", "c", 0.2) + assert ops == [EditOperation("substitute", "a", "c", 0.2)] + + +def test_explain_transitive_insertion_chain_after_closure() -> None: + """After closure, inserting 'y' is one effective insertion at 0.3.""" + wl = WeightedLevenshtein( + substitution_costs={("x", "y"): 0.2}, + insertion_costs={"x": 0.1}, + symmetric_substitution=False, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "a", "ay", 0.3) + _assert_ops_equal(ops, [EditOperation("insert", None, "y", 0.3)]) + + +def test_explain_chain_with_expensive_direct_substitution_after_closure() -> None: + """A->B with cheaper A->AAA->B chain becomes a single sub at 0.5.""" + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "A", "B", 0.5) + assert ops == [EditOperation("substitute", "A", "B", 0.5)] + + +def test_explain_mixed_substitution_path_with_deletion_after_closure() -> None: + """AB->C: closure prefers sub(A,C) + del(B) = 0.3 over the direct AB->C = 0.5.""" + wl = WeightedLevenshtein( + substitution_costs={("A", "C"): 0.1, ("AB", "C"): 0.5}, + deletion_costs={"B": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "AB", "C", 0.3) + assert ops == [ + EditOperation("substitute", "A", "C", 0.1), + EditOperation("delete", "B", None, 0.2), + ] + + +def test_explain_direct_substitution_wins_over_chain_after_closure() -> None: + """A direct A->B at 0.4 beats A->AAA->B at 0.5; effective cost stays 0.4.""" + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.4}, + insertion_costs={"A": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "A", "B", 0.4) + assert ops == [EditOperation("substitute", "A", "B", 0.4)] + + +def test_explain_effective_deletion_with_insertion_then_deletion_after_closure() -> None: + """AC -> C via insert(B)+del(AB)=0.1 becomes a single effective del('A') at 0.1.""" + wl = WeightedLevenshtein( + insertion_costs={"B": 0.1}, + deletion_costs={"AB": 0.0}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "AC", "C", 0.1) + # The single effective op may be a del('A'), or another route at the same cost. + # Assert the explicit identity to lock down the canonical form: + assert ops == [EditOperation("delete", "A", None, 0.1)] + + +def test_explain_insert_delete_substitute_chain_after_closure() -> None: + """ADC -> Z via del(D)+ins(B)+sub(ABC,Z) becomes a single sub at 0.3.""" + wl = WeightedLevenshtein( + substitution_costs={("ABC", "Z"): 0.1}, + insertion_costs={"B": 0.1}, + deletion_costs={"D": 0.1}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "ADC", "Z", 0.3) + _assert_ops_equal(ops, [EditOperation("substitute", "ADC", "Z", 0.3)]) + + +def test_explain_single_char_composed_substitution_chain_after_closure() -> None: + """X -> Z via ins(AB)+sub(XAB,Z) becomes a single sub at 0.3.""" + wl = WeightedLevenshtein( + substitution_costs={("XAB", "Z"): 0.1}, + insertion_costs={"AB": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "X", "Z", 0.3) + _assert_ops_equal(ops, [EditOperation("substitute", "X", "Z", 0.3)]) diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index daf6005..adabbbd 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -457,6 +457,58 @@ def test_weighted_levenshtein_distance_with_insertion( ).distance(s1, s2) == pytest.approx(expected) +@pytest.mark.parametrize( + ["substitution_costs", "source", "target", "expected"], + [ + ({("", "ab"): 0.3}, "x", "xab", 0.3), + ({("cd", ""): 0.4}, "xcd", "x", 0.4), + ], +) +def test_empty_sided_substitution_costs_act_as_insertions_and_deletions( + substitution_costs: dict[tuple[str, str], float], source: str, target: str, expected: float +) -> None: + assert WeightedLevenshtein(substitution_costs=substitution_costs).distance( + source, target + ) == pytest.approx(expected) + + +@pytest.mark.parametrize( + ["substitution_costs", "insertion_costs", "deletion_costs", "source", "target", "expected"], + [ + ({("", "a"): 0.2}, {"a": 0.5}, {}, "", "a", 0.2), + ({("a", ""): 0.2}, {}, {"a": 0.5}, "a", "", 0.2), + ], +) +def test_empty_sided_substitution_costs_use_minimum_with_explicit_unary_costs( + substitution_costs: dict[tuple[str, str], float], + insertion_costs: dict[str, float], + deletion_costs: dict[str, float], + source: str, + target: str, + expected: float, +) -> None: + assert WeightedLevenshtein( + substitution_costs=substitution_costs, + insertion_costs=insertion_costs, + deletion_costs=deletion_costs, + ).distance(source, target) == pytest.approx(expected) + + +def test_empty_sided_substitution_costs_are_not_mirrored_by_symmetric_substitution() -> None: + wl = WeightedLevenshtein( + substitution_costs={("", "a"): 0.2}, + symmetric_substitution=True, + ) + + assert wl.distance("", "a") == pytest.approx(0.2) + assert wl.distance("a", "") == pytest.approx(1.0) + + +def test_empty_to_empty_substitution_cost_is_rejected() -> None: + with pytest.raises(ValueError, match="not a meaningful edit operation"): + WeightedLevenshtein(substitution_costs={("", ""): 0.0}) + + @pytest.mark.parametrize( ["s1", "s2", "ins_costs", "del_costs", "expected"], [ @@ -551,6 +603,172 @@ def test_costs_above_default_cost() -> None: assert actual_cost == configured_cost +def test_no_implicit_closure_in_constructor() -> None: + """Transitive paths are opt-in: the constructor does not run closure.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + # Without transitive_closure(), a->c falls back to the default 1.0. + assert wl.distance("a", "c") == pytest.approx(1.0) + + +def test_transitive_deletion_chain_distance() -> None: + """Issue #12: sub('6'->'G', 0.5) + del('G', 0.01) = 0.51 < direct del('6', 1.0).""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + symmetric_substitution=False, + ).transitive_closure() + assert wl.distance("06", "0") == pytest.approx(0.51) + + +def test_transitive_insertion_subtitution() -> None: + """ + A->AA->AAA->B + """ + wl = WeightedLevenshtein( + insertion_costs={"A": 0.2}, + substitution_costs={("AAA", "B"): 0.1}, + ).transitive_closure() + assert wl.distance("A", "B") == pytest.approx(0.5) + + +def test_transitive_closure_prunes_redundant_substitutions() -> None: + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure(prune=True) + + assert wl.distance("A", "B") == pytest.approx(0.5) + assert wl.distance("AA", "AAA") == pytest.approx(0.2) + assert ("A", "B") in wl.substitution_costs + assert ("AA", "AAA") not in wl.substitution_costs + assert ("B", "AA") not in wl.substitution_costs + + +def test_transitive_closure_does_not_prune_by_default() -> None: + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure() + + assert wl.distance("A", "B") == pytest.approx(0.5) + assert wl.distance("AA", "AAA") == pytest.approx(0.2) + assert ("AA", "AAA") in wl.substitution_costs + + +def test_transitive_insertion_subtitution2() -> None: + """ + A->AA->AAB->C + """ + wl = WeightedLevenshtein( + insertion_costs={"A": 0.2, "B": 0.3}, + substitution_costs={("AAB", "C"): 0.1}, + ).transitive_closure() + assert wl.distance("A", "C") == pytest.approx(0.6) + + +def test_transitive_insertion_deletion() -> None: + """ + AC->ABC->C + """ + wl = WeightedLevenshtein( + insertion_costs={"B": 0.1}, + deletion_costs={"AB": 0.0}, + ).transitive_closure() + assert wl.distance("AC", "C") == pytest.approx(0.1) + + +def test_transitive_insertion_chain_distance() -> None: + """Insertion analogue: ins('x', 0.1) + sub('x'->'y', 0.2) = 0.3 < direct ins('y', 1.0).""" + wl = WeightedLevenshtein( + substitution_costs={("x", "y"): 0.2}, + insertion_costs={"x": 0.1}, + symmetric_substitution=False, + ).transitive_closure() + assert wl.distance("a", "ay") == pytest.approx(0.3) + + +def test_direct_op_wins_when_chain_more_expensive() -> None: + """When the direct cost is already lower than any chain, the result is unchanged.""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"6": 0.2, "G": 0.01}, + symmetric_substitution=False, + ).transitive_closure() + assert wl.distance("06", "0") == pytest.approx(0.2) + + +def test_transitive_substitution_chain_distance() -> None: + """Triangle inequality: sub(a→b, 0.1) + sub(b→c, 0.1) = 0.2 < direct default 1.0.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ).transitive_closure() + assert wl.distance("a", "c") == pytest.approx(0.2) + + +def test_transitive_closure_returns_asymmetric_instance() -> None: + """Closure may break symmetry (e.g. via del+ins paths through ε), so the + returned instance is always asymmetric. Both directions of a symmetric + input are still preserved as explicit entries.""" + wl_closed = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1}, + symmetric_substitution=True, + ).transitive_closure() + assert wl_closed.symmetric_substitution is False + assert wl_closed.substitution_costs[("a", "b")] == pytest.approx(0.1) + assert wl_closed.substitution_costs[("b", "a")] == pytest.approx(0.1) + + +def test_transitive_closure_idempotent_on_distance() -> None: + """Applying closure twice yields the same distances as applying it once.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + wl1 = wl.transitive_closure() + wl2 = wl1.transitive_closure() + for s, t in [("a", "c"), ("a", "b"), ("b", "c"), ("c", "a"), ("xy", "yx")]: + assert wl1.distance(s, t) == pytest.approx(wl2.distance(s, t)) + + +def test_transitive_closure_round_trip_via_dict() -> None: + """Closed costs survive serialization, so users can compute closure once.""" + wl_orig = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + wl_closed = wl_orig.transitive_closure() + wl_reloaded = WeightedLevenshtein.from_dict(wl_closed.to_dict()) + assert wl_reloaded.distance("a", "c") == pytest.approx(0.2) + assert wl_reloaded == wl_closed + + +def test_transitive_closure_preserves_default_costs() -> None: + """The closed instance keeps the original default costs.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1}, + default_substitution_cost=2.0, + default_insertion_cost=3.0, + default_deletion_cost=4.0, + ).transitive_closure() + assert wl.default_substitution_cost == 2.0 + assert wl.default_insertion_cost == 3.0 + assert wl.default_deletion_cost == 4.0 + + +def test_insert_delete_substitute_chain_distance() -> None: + """ADC -> AC -> ABC -> Z via del(D)=0.1 + ins(B)=0.1 + sub(ABC,Z)=0.1.""" + wl = WeightedLevenshtein( + substitution_costs={("ABC", "Z"): 0.1}, + insertion_costs={"B": 0.1}, + deletion_costs={"D": 0.1}, + ).transitive_closure() + assert wl.distance("ADC", "Z") == pytest.approx(0.3) + + def test_serialization() -> None: wl_orig = WeightedLevenshtein( substitution_costs={("a", "b"): 0.5}, diff --git a/python/tests/test_weighted_levenshtein_mutation.py b/python/tests/test_weighted_levenshtein_mutation.py new file mode 100644 index 0000000..16b86e9 --- /dev/null +++ b/python/tests/test_weighted_levenshtein_mutation.py @@ -0,0 +1,283 @@ +import math +from collections.abc import Callable +from typing import Any + +import pytest +from ocr_stringdist import WeightedLevenshtein + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_cost_property_reassignment_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + assert wl.distance(source, target) == pytest.approx(1.0) + + setattr(wl, costs_attribute, costs) + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost", "source", "target", "expected"], + [ + ("substitution_costs", ("a", "b"), 0.2, "a", "b", 0.2), + ("substitution_costs", ("", "b"), 0.3, "", "b", 0.3), + ("substitution_costs", ("a", ""), 0.4, "a", "", 0.4), + ("insertion_costs", "b", 0.3, "", "b", 0.3), + ("deletion_costs", "a", 0.4, "a", "", 0.4), + ], +) +def test_in_place_cost_assignment_updates_calculator( + costs_attribute: str, + key: Any, + cost: float, + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + costs = getattr(wl, costs_attribute) + assert wl.distance(source, target) == pytest.approx(1.0) + + costs[key] = cost + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_in_place_cost_update_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + + getattr(wl, costs_attribute).update(costs) + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost", "source", "target", "expected"], + [ + ("substitution_costs", ("a", "b"), 0.2, "a", "b", 0.2), + ("insertion_costs", "b", 0.3, "", "b", 0.3), + ("deletion_costs", "a", 0.4, "a", "", 0.4), + ], +) +def test_in_place_cost_setdefault_updates_calculator( + costs_attribute: str, + key: Any, + cost: float, + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + + inserted = getattr(wl, costs_attribute).setdefault(key, cost) + + assert inserted == cost + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_in_place_cost_union_update_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + observable_costs = getattr(wl, costs_attribute) + + observable_costs |= costs + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "remove_cost", "source", "target"], + [ + ("substitution_costs", {("a", "b"): 0.2}, lambda costs: costs.pop(("a", "b")), "a", "b"), + ("insertion_costs", {"b": 0.3}, lambda costs: costs.pop("b"), "", "b"), + ("deletion_costs", {"a": 0.4}, lambda costs: costs.pop("a"), "a", ""), + ("substitution_costs", {("a", "b"): 0.2}, lambda costs: costs.clear(), "a", "b"), + ("insertion_costs", {"b": 0.3}, lambda costs: costs.clear(), "", "b"), + ("deletion_costs", {"a": 0.4}, lambda costs: costs.clear(), "a", ""), + ], +) +def test_in_place_cost_removal_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + remove_cost: Callable[[dict[Any, float]], Any], + source: str, + target: str, +) -> None: + wl = WeightedLevenshtein.unweighted() + setattr(wl, costs_attribute, costs) + assert wl.distance(source, target) < 1.0 + + remove_cost(getattr(wl, costs_attribute)) + + assert wl.distance(source, target) == pytest.approx(1.0) + assert wl.batch_distance(source, [target]) == pytest.approx([1.0]) + + +@pytest.mark.parametrize( + ["default_cost_attribute", "source", "target"], + [ + ("default_substitution_cost", "a", "b"), + ("default_insertion_cost", "", "b"), + ("default_deletion_cost", "a", ""), + ], +) +def test_default_cost_setters_update_calculator( + default_cost_attribute: str, source: str, target: str +) -> None: + wl = WeightedLevenshtein.unweighted() + assert wl.distance(source, target) == pytest.approx(1.0) + + setattr(wl, default_cost_attribute, 0.25) + + assert wl.distance(source, target) == pytest.approx(0.25) + assert wl.batch_distance(source, [target]) == pytest.approx([0.25]) + + +def test_symmetric_substitution_setter_updates_calculator() -> None: + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.2}, + symmetric_substitution=False, + ) + assert wl.distance("b", "a") == pytest.approx(1.0) + + wl.symmetric_substitution = True + + assert wl.distance("b", "a") == pytest.approx(0.2) + assert wl.batch_distance("b", ["a"]) == pytest.approx([0.2]) + + +def test_symmetric_substitution_conflicts_use_minimum_cost() -> None: + wl = WeightedLevenshtein( + substitution_costs={("Z", "2"): 0.3, ("2", "Z"): 0.5}, + symmetric_substitution=True, + ) + + assert wl.distance("Z", "2") == pytest.approx(0.3) + assert wl.distance("2", "Z") == pytest.approx(0.3) + + +@pytest.mark.parametrize( + ["default_cost_attribute", "invalid_cost"], + [ + ("default_substitution_cost", math.nan), + ("default_substitution_cost", math.inf), + ("default_insertion_cost", math.nan), + ("default_insertion_cost", math.inf), + ("default_deletion_cost", math.nan), + ("default_deletion_cost", math.inf), + ], +) +def test_invalid_default_cost_setter_rejects_non_finite_values( + default_cost_attribute: str, invalid_cost: float +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises(ValueError, match="must be finite"): + setattr(wl, default_cost_attribute, invalid_cost) + + assert wl.distance("a", "b") == pytest.approx(1.0) + + +@pytest.mark.parametrize( + ["costs_attribute", "invalid_costs"], + [ + ("substitution_costs", {("a", "b"): -0.1}), + ("substitution_costs", {("a", "b"): math.nan}), + ("substitution_costs", {("a", "b"): math.inf}), + ("substitution_costs", {("a",): 0.1}), + ("substitution_costs", {("", ""): 0.0}), + ("insertion_costs", {"b": -0.1}), + ("insertion_costs", {"b": math.nan}), + ("insertion_costs", {"b": math.inf}), + ("insertion_costs", {("b",): 0.1}), + ("deletion_costs", {"a": -0.1}), + ("deletion_costs", {"a": math.nan}), + ("deletion_costs", {"a": math.inf}), + ("deletion_costs", {("a",): 0.1}), + ], +) +def test_invalid_cost_property_reassignment_keeps_existing_calculator( + costs_attribute: str, invalid_costs: dict[Any, float] +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises((TypeError, ValueError)): + setattr(wl, costs_attribute, invalid_costs) + + assert getattr(wl, costs_attribute) == {} + assert wl.distance("a", "b") == pytest.approx(1.0) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost"], + [ + ("substitution_costs", ("a", "b"), -0.1), + ("substitution_costs", ("a", "b"), math.nan), + ("substitution_costs", ("a", "b"), math.inf), + ("substitution_costs", ("a",), 0.1), + ("substitution_costs", ("", ""), 0.0), + ("insertion_costs", "b", -0.1), + ("insertion_costs", "b", math.nan), + ("insertion_costs", "b", math.inf), + ("insertion_costs", ("b",), 0.1), + ("deletion_costs", "a", -0.1), + ("deletion_costs", "a", math.nan), + ("deletion_costs", "a", math.inf), + ("deletion_costs", ("a",), 0.1), + ], +) +def test_invalid_in_place_cost_assignment_keeps_existing_calculator( + costs_attribute: str, key: Any, cost: float +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises((TypeError, ValueError)): + getattr(wl, costs_attribute)[key] = cost + + assert getattr(wl, costs_attribute) == {} + assert wl.distance("a", "b") == pytest.approx(1.0) diff --git a/src/cost_map.rs b/src/cost_map.rs index 085cb91..8c1921b 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::hash::Hash; -#[cfg(feature = "python")] +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; /// A trait for cost map keys, allowing us to constrain the generic parameter @@ -16,19 +16,15 @@ impl CostKey for SubstitutionKey {} /// Generic cost map structure that works with different key types #[derive(Clone, Debug)] pub struct CostMap { - /// The costs map pub costs: HashMap, - /// Default cost for operations not found in the map default_cost: f64, - /// Maximum token length in the map - pub max_token_length: usize, + max_token_length: usize, } impl Default for CostMap where K: Default, { - /// Creates a new CostMap with default values fn default() -> Self { Self { costs: HashMap::new(), @@ -48,149 +44,135 @@ impl CostMap { symmetric: bool, ) -> Self { let mut costs = HashMap::with_capacity(custom_costs_input.len() * 2); - let mut max_length = 1; for ((s1, s2), cost) in custom_costs_input { - costs.entry((s1.clone(), s2.clone())).or_insert(cost); + insert_min_cost(&mut costs, (s1.clone(), s2.clone()), cost); if symmetric { - costs.entry((s2.clone(), s1.clone())).or_insert(cost); + insert_min_cost(&mut costs, (s2.clone(), s1.clone()), cost); } - - // Update max token length - max_length = max_length.max(s1.chars().count()).max(s2.chars().count()); } + let max_token_length = costs + .keys() + .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) + .fold(1, std::cmp::max); + CostMap { costs, default_cost, - max_token_length: max_length, + max_token_length, } } - /// Creates a new substitution CostMap with the specified custom costs. - /// Uses default values for other parameters. - pub fn with_costs(custom_costs: SubstitutionCostMap) -> Self { - Self::new(custom_costs, 1.0, true) - } - - #[cfg(feature = "python")] - /// Creates a substitution CostMap from a Python dictionary. - /// This method is only available when the "python" feature is enabled. - pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64, symmetric: bool) -> Self + pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64, symmetric: bool) -> PyResult where D: PyDictMethods<'a>, { let mut substitution_costs = SubstitutionCostMap::new(); - let mut max_length = 1; - // Convert Python dictionary to Rust HashMap for (key, value) in py_dict.iter() { - if let Ok(key_tuple) = key.extract::<(String, String)>() { - if let Ok(cost) = value.extract::() { - substitution_costs.insert((key_tuple.0.clone(), key_tuple.1.clone()), cost); - - // Update max token length - max_length = max_length - .max(key_tuple.0.chars().count()) - .max(key_tuple.1.chars().count()); - } - } + let key_tuple = key.extract::<(String, String)>()?; + let cost = value.extract::()?; + validate_cost( + cost, + &format!( + "Substitution cost for key ({}, {})", + key_tuple.0, key_tuple.1 + ), + )?; + substitution_costs.insert((key_tuple.0, key_tuple.1), cost); } - // Create the CostMap - Self::new(substitution_costs, default_cost, symmetric) + Ok(Self::new(substitution_costs, default_cost, symmetric)) } - /// Gets the substitution cost between two strings. - pub fn get_cost(&self, s1: &str, s2: &str) -> f64 { - if s1 == s2 { - 0.0 // No cost if strings are identical - } else { - let key_pair = (s1.to_string(), s2.to_string()); - - // Lookup the pair (symmetry is handled by storage in `new`) - // Use the map's configured default_cost as the fallback. - self.costs - .get(&key_pair) - .copied() - .unwrap_or(self.default_cost) - } + #[inline] + pub fn get_cost(&self, source: &str, target: &str) -> f64 { + self.costs + .get(&(source.to_string(), target.to_string())) + .copied() + .unwrap_or(self.default_cost) } - /// Checks if the cost map contains a specific substitution - pub fn has_key(&self, s1: &str, s2: &str) -> bool { - let key_pair = (s1.to_string(), s2.to_string()); - self.costs.contains_key(&key_pair) + #[inline] + pub fn has_key(&self, source: &str, target: &str) -> bool { + self.costs + .contains_key(&(source.to_string(), target.to_string())) } } // Implementation for SingleTokenKey (single string) impl CostMap { - /// Creates a new single token CostMap for insertion or deletion operations pub fn new(custom_costs_input: SingleTokenCostMap, default_cost: f64) -> Self { - let mut max_length = 1; - - // Calculate max token length - for key in custom_costs_input.keys() { - max_length = max_length.max(key.chars().count()); - } - + let max_token_length = custom_costs_input + .keys() + .map(|token| token.chars().count()) + .fold(1, std::cmp::max); CostMap { costs: custom_costs_input, default_cost, - max_token_length: max_length, + max_token_length, } } - /// Creates a new single token CostMap with the specified custom costs. - /// Uses default value for default cost. - pub fn with_costs(custom_costs: SingleTokenCostMap) -> Self { - Self::new(custom_costs, 1.0) - } - - #[cfg(feature = "python")] - /// Creates a single token CostMap from a Python dictionary. - /// This method is only available when the "python" feature is enabled. - pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64) -> Self + pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64) -> PyResult where D: PyDictMethods<'a>, { let mut single_token_costs = SingleTokenCostMap::new(); - let mut max_length = 1; - // Convert Python dictionary to Rust HashMap for (key, value) in py_dict.iter() { - if let Ok(token) = key.extract::() { - if let Ok(cost) = value.extract::() { - single_token_costs.insert(token.clone(), cost); - - // Update max token length - max_length = max_length.max(token.chars().count()); - } - } + let token = key.extract::()?; + let cost = value.extract::()?; + validate_cost(cost, "Cost")?; + single_token_costs.insert(token, cost); } - // Create the CostMap - Self::new(single_token_costs, default_cost) + Ok(Self::new(single_token_costs, default_cost)) } - /// Gets the cost for a single token (insertion or deletion). + #[inline] pub fn get_cost(&self, token: &str) -> f64 { self.costs.get(token).copied().unwrap_or(self.default_cost) } - /// Checks if the cost map contains a specific single token + #[inline] pub fn has_key(&self, token: &str) -> bool { self.costs.contains_key(token) } } +fn insert_min_cost(costs: &mut HashMap, key: K, cost: f64) { + costs + .entry(key) + .and_modify(|existing| *existing = (*existing).min(cost)) + .or_insert(cost); +} + +pub(crate) fn validate_cost(cost: f64, label: &str) -> PyResult<()> { + if !cost.is_finite() { + return Err(PyValueError::new_err(format!( + "{label} must be finite, got value: {cost}" + ))); + } + if cost < 0.0 { + return Err(PyValueError::new_err(format!( + "{label} must be non-negative, got value: {cost}" + ))); + } + Ok(()) +} + // Common methods for any type of CostMap impl CostMap { - /// Returns the default cost for this cost map pub fn default_cost(&self) -> f64 { self.default_cost } + + #[inline] + pub fn max_token_length(&self) -> usize { + self.max_token_length + } } #[cfg(test)] @@ -199,31 +181,9 @@ mod tests { #[test] fn test_single_token_map_default() { - // Test with default initialization let cost_map: CostMap = CostMap::default(); assert_eq!(cost_map.default_cost(), 1.0); - assert_eq!(cost_map.get_cost("any_token"), 1.0); - assert!(!cost_map.has_key("any_token")); - } - - #[test] - fn test_single_token_map_with_costs() { - let mut custom_costs = SingleTokenCostMap::new(); - custom_costs.insert("a".to_string(), 0.5); - custom_costs.insert("b".to_string(), 0.8); - - // Test with_costs constructor (default cost 1.0) - let cost_map = CostMap::::with_costs(custom_costs); - - // Test getting costs for tokens - assert_eq!(cost_map.get_cost("a"), 0.5); - assert_eq!(cost_map.get_cost("b"), 0.8); - assert_eq!(cost_map.get_cost("c"), 1.0); // Default cost - - // Test has_key - assert!(cost_map.has_key("a")); - assert!(cost_map.has_key("b")); - assert!(!cost_map.has_key("c")); + assert!(cost_map.costs.is_empty()); } #[test] @@ -231,75 +191,52 @@ mod tests { let mut custom_costs = SingleTokenCostMap::new(); custom_costs.insert("test".to_string(), 0.3); - // Test new constructor with custom default cost let cost_map = CostMap::::new(custom_costs, 2.0); assert_eq!(cost_map.default_cost(), 2.0); - assert_eq!(cost_map.get_cost("test"), 0.3); - assert_eq!(cost_map.get_cost("unknown"), 2.0); + assert_eq!(cost_map.costs["test"], 0.3); + assert!(!cost_map.costs.contains_key("unknown")); } #[test] fn test_substitution_map_default() { - let cost_map: CostMap = CostMap { - costs: HashMap::new(), - default_cost: 1.0, - max_token_length: 1, - }; + let cost_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); assert_eq!(cost_map.default_cost(), 1.0); - assert_eq!(cost_map.get_cost("a", "b"), 1.0); - assert!(!cost_map.has_key("a", "b")); + assert!(cost_map.costs.is_empty()); } #[test] - fn test_substitution_map_with_costs() { + fn test_substitution_map_asymmetric() { let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("0".to_string(), "o".to_string()), 0.2); - custom_costs.insert(("l".to_string(), "1".to_string()), 0.3); - - // Test with_costs constructor (symmetric by default) - let cost_map = CostMap::::with_costs(custom_costs); - - // Test getting costs - assert_eq!(cost_map.get_cost("0", "o"), 0.2); - assert_eq!(cost_map.get_cost("o", "0"), 0.2); // Symmetry check - assert_eq!(cost_map.get_cost("l", "1"), 0.3); - assert_eq!(cost_map.get_cost("1", "l"), 0.3); // Symmetry check - assert_eq!(cost_map.get_cost("a", "b"), 1.0); // Default - - // Test same character - assert_eq!(cost_map.get_cost("a", "a"), 0.0); // Same char = 0 cost - - // Test has_key - assert!(cost_map.has_key("0", "o")); - assert!(cost_map.has_key("o", "0")); // Symmetry check - assert!(!cost_map.has_key("a", "b")); + custom_costs.insert(("a".to_string(), "b".to_string()), 0.4); + + let cost_map = CostMap::::new(custom_costs, 1.5, false); + + assert_eq!(cost_map.costs[&("a".to_string(), "b".to_string())], 0.4); + assert!(!cost_map + .costs + .contains_key(&("b".to_string(), "a".to_string()))); + assert_eq!(cost_map.default_cost(), 1.5); } #[test] - fn test_substitution_map_asymmetric() { + fn test_symmetric_substitution_map_conflicts_use_minimum_cost() { let mut custom_costs = SubstitutionCostMap::new(); custom_costs.insert(("a".to_string(), "b".to_string()), 0.4); + custom_costs.insert(("b".to_string(), "a".to_string()), 0.2); - // Create with symmetric=false - let cost_map = CostMap::::new(custom_costs, 1.5, false); - - // Test asymmetry - assert_eq!(cost_map.get_cost("a", "b"), 0.4); - assert_eq!(cost_map.get_cost("b", "a"), 1.5); // Should be default cost + let cost_map = CostMap::::new(custom_costs, 1.0, true); - assert!(cost_map.has_key("a", "b")); - assert!(!cost_map.has_key("b", "a")); // Should not exist + assert_eq!(cost_map.costs[&("a".to_string(), "b".to_string())], 0.2); + assert_eq!(cost_map.costs[&("b".to_string(), "a".to_string())], 0.2); } #[test] fn test_default_cost_accessor() { - // Test for SubstitutionKey let sub_map = CostMap::::new(HashMap::new(), 2.5, true); assert_eq!(sub_map.default_cost(), 2.5); - // Test for SingleTokenKey let single_map = CostMap::::new(HashMap::new(), 3.0); assert_eq!(single_map.default_cost(), 3.0); } diff --git a/src/explanation.rs b/src/explanation.rs index 8688c2d..6c45e60 100644 --- a/src/explanation.rs +++ b/src/explanation.rs @@ -1,6 +1,6 @@ /// Represents a single operation in the edit path. /// This is the data structure that will be returned to Python. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum EditOperation { Substitute { source: String, diff --git a/src/lib.rs b/src/lib.rs index b19214b..cf011de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,8 @@ mod cost_map; mod explanation; +mod transitive_costs; mod types; mod weighted_levenshtein; -pub use cost_map::CostMap; -pub use types::*; -pub use weighted_levenshtein::{ - custom_levenshtein_distance_with_cost_maps, explain_custom_levenshtein_distance, -}; - -#[cfg(feature = "python")] mod rust_stringdist; -#[cfg(feature = "python")] pub use rust_stringdist::_rust_stringdist; diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index a2e82db..a1764bb 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,8 +1,8 @@ -use crate::cost_map::CostMap; +use crate::cost_map::{validate_cost, CostMap}; use crate::explanation::EditOperation; +use crate::transitive_costs::compute_closed_cost_maps; use crate::types::{SingleTokenKey, SubstitutionKey}; -use crate::weighted_levenshtein::custom_levenshtein_distance_with_cost_maps as calculate_core; -use crate::weighted_levenshtein::explain_custom_levenshtein_distance as explain_core; +use crate::weighted_levenshtein::{custom_levenshtein_distance, explain_custom_levenshtein}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; @@ -13,7 +13,6 @@ impl<'py> IntoPyObject<'py> for EditOperation { type Output = Bound<'py, Self::Target>; type Error = pyo3::PyErr; - /// Converts the `EditOperation` into a Python tuple fn into_pyobject(self, py: Python<'py>) -> Result { match self { EditOperation::Substitute { @@ -29,14 +28,28 @@ impl<'py> IntoPyObject<'py> for EditOperation { } } -/// Facade between the Python interface and the core algorithm implementation. -struct LevenshteinCalculator { - substitution_cost_map: CostMap, - insertion_cost_map: CostMap, - deletion_cost_map: CostMap, +/// Holds raw cost maps and runs the DP against them. Transitive closure is +/// opt-in via [`closed_cost_maps`]; the constructor never runs Floyd-Warshall. +#[pyclass] +#[derive(Debug)] +struct RustLevenshteinCalculator { + sub: CostMap, + ins: CostMap, + del: CostMap, } -impl LevenshteinCalculator { +#[pymethods] +impl RustLevenshteinCalculator { + #[new] + #[pyo3(signature = ( + substitution_costs, + insertion_costs, + deletion_costs, + symmetric_substitution = true, + default_substitution_cost = 1.0, + default_insertion_cost = 1.0, + default_deletion_cost = 1.0, + ))] fn new( substitution_costs: &Bound<'_, PyDict>, insertion_costs: &Bound<'_, PyDict>, @@ -46,493 +59,212 @@ impl LevenshteinCalculator { default_insertion_cost: f64, default_deletion_cost: f64, ) -> PyResult { - validate_default_cost(default_substitution_cost)?; - validate_default_cost(default_insertion_cost)?; - validate_default_cost(default_deletion_cost)?; + validate_cost(default_substitution_cost, "Default substitution cost")?; + validate_cost(default_insertion_cost, "Default insertion cost")?; + validate_cost(default_deletion_cost, "Default deletion cost")?; - let substitution_cost_map = CostMap::::from_py_dict( + let sub = CostMap::::from_py_dict( substitution_costs, default_substitution_cost, symmetric_substitution, - ); + )?; + let ins = CostMap::::from_py_dict(insertion_costs, default_insertion_cost)?; + let del = CostMap::::from_py_dict(deletion_costs, default_deletion_cost)?; - let insertion_cost_map = - CostMap::::from_py_dict(insertion_costs, default_insertion_cost); - - let deletion_cost_map = - CostMap::::from_py_dict(deletion_costs, default_deletion_cost); - - Ok(Self { - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, - }) + Ok(Self { sub, ins, del }) } fn distance(&self, a: &str, b: &str) -> f64 { - calculate_core( - a, - b, - &self.substitution_cost_map, - &self.insertion_cost_map, - &self.deletion_cost_map, - ) + custom_levenshtein_distance(a, b, &self.sub, &self.ins, &self.del) } - fn explain(&self, a: &str, b: &str) -> Vec { - explain_core( - a, - b, - &self.substitution_cost_map, - &self.insertion_cost_map, - &self.deletion_cost_map, - ) + fn batch_distance(&self, py: Python<'_>, s: String, candidates: Vec) -> Vec { + if candidates.is_empty() { + return Vec::new(); + } + py.detach(|| { + candidates + .par_iter() + .map(|c| custom_levenshtein_distance(&s, c, &self.sub, &self.ins, &self.del)) + .collect() + }) } -} -/// Validates that the default cost is non-negative -fn validate_default_cost(default_cost: f64) -> PyResult<()> { - if default_cost < 0.0 { - return Err(PyValueError::new_err(format!( - "Default cost must be non-negative, got value: {default_cost}" - ))); + fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult>> { + explain_custom_levenshtein(a, b, &self.sub, &self.ins, &self.del) + .into_iter() + .map(|op| op.into_pyobject(py).map(|bound| bound.into())) + .collect::>>>() } - Ok(()) -} -// Calculates the weighted Levenshtein distance with a custom cost map from Python. -#[pyfunction] -#[pyo3(signature = ( - a, - b, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _weighted_levenshtein_distance( - a: &str, - b: &str, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; - - Ok(calculator.distance(a, b)) -} - -#[pyfunction] -#[pyo3(signature = ( - a, - b, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _explain_weighted_levenshtein_distance( - py: Python, // For conversion - a: &str, - b: &str, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult> { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; - - let path = calculator.explain(a, b); + /// Computes effective edit costs via transitive closure and returns three + /// Python dicts: `(substitution_costs, insertion_costs, deletion_costs)`. + /// Generated substitutions are pruned only when `prune` is true. + /// `max_node_length` caps the length of intermediate graph nodes; pass + /// `None` to derive it from the input. + /// + /// The Python wrapper assembles these into a new `WeightedLevenshtein` + /// whose `.distance()` and `.explain()` use the closed costs directly. + #[pyo3(signature = (prune = false, max_node_length = None))] + fn closed_cost_maps<'py>( + &self, + py: Python<'py>, + prune: bool, + max_node_length: Option, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let (closed_sub, closed_ins, closed_del) = + compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune, max_node_length) + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let sub_dict = PyDict::new(py); + for ((source, target), cost) in closed_sub { + sub_dict.set_item((source, target), cost)?; + } - path.into_iter() - .map(|op| op.into_pyobject(py).map(|bound| bound.into())) - .collect::>>() -} + let ins_dict = PyDict::new(py); + for (token, cost) in closed_ins { + ins_dict.set_item(token, cost)?; + } -// Calculates the weighted Levenshtein distance between a string and a list of candidates. -#[pyfunction] -#[pyo3(signature = ( - s, - candidates, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _batch_weighted_levenshtein_distance( - s: &str, - candidates: Vec, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult> { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; + let del_dict = PyDict::new(py); + for (token, cost) in closed_del { + del_dict.set_item(token, cost)?; + } - if candidates.is_empty() { - return Ok(Vec::new()); + Ok((sub_dict, ins_dict, del_dict)) } - - // Calculate distances for each candidate in parallel - let distances: Vec = candidates - .par_iter() - .map(|candidate| calculator.distance(s, candidate)) - .collect(); - - Ok(distances) } -/// A Python module implemented in Rust. #[pymodule] pub fn _rust_stringdist(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(_weighted_levenshtein_distance, m)?)?; - m.add_function(wrap_pyfunction!(_batch_weighted_levenshtein_distance, m)?)?; - m.add_function(wrap_pyfunction!(_explain_weighted_levenshtein_distance, m)?)?; + m.add_class::()?; Ok(()) } #[cfg(test)] mod tests { use super::*; + use pyo3::exceptions::PyValueError; use pyo3::types::{PyDict, PyList, PyTuple}; - #[test] - fn test_levenshtein_distance_with_empty_costs() { - Python::with_gil(|py| { - let a = "hello"; - let b = "hxllo"; - - let substitution_costs = PyDict::new(py); - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); + fn make_calculator( + py: Python<'_>, + sub_costs: &[((&str, &str), f64)], + ins_costs: &[(&str, f64)], + del_costs: &[(&str, f64)], + symmetric: bool, + ) -> RustLevenshteinCalculator { + let sub = PyDict::new(py); + for ((a, b), c) in sub_costs { + sub.set_item((*a, *b), c).unwrap(); + } + let ins = PyDict::new(py); + for (k, v) in ins_costs { + ins.set_item(k, v).unwrap(); + } + let del = PyDict::new(py); + for (k, v) in del_costs { + del.set_item(k, v).unwrap(); + } + RustLevenshteinCalculator::new(&sub, &ins, &del, symmetric, 1.0, 1.0, 1.0).unwrap() + } - assert_eq!(distance, 1.0); + #[test] + fn test_distance_with_empty_costs() { + Python::attach(|py| { + let calc = make_calculator(py, &[], &[], &[], true); + assert_eq!(calc.distance("hello", "hxllo"), 1.0); }); } #[test] - fn test_levenshtein_with_custom_substitution_cost() { - Python::with_gil(|py| { - let a = "hello"; - let b = "hxllo"; - - let substitution_costs = PyDict::new(py); - substitution_costs.set_item(("e", "x"), 0.2).unwrap(); - - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert!((distance - 0.2).abs() < f64::EPSILON); + fn test_distance_with_custom_substitution_cost() { + Python::attach(|py| { + let calc = make_calculator(py, &[(("e", "x"), 0.2)], &[], &[], true); + assert!((calc.distance("hello", "hxllo") - 0.2).abs() < f64::EPSILON); }); } #[test] - fn test_levenshtein_asymmetric_substitution() { - Python::with_gil(|py| { - let a = "ab"; - let b = "ba"; - - let substitution_costs = PyDict::new(py); - substitution_costs.set_item(("a", "b"), 0.1).unwrap(); - - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - false, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - // Cost should be 0.1 (a->b) + 1.0 (b->a, default) - assert!((distance - 1.1).abs() < f64::EPSILON); + fn test_asymmetric_substitution() { + Python::attach(|py| { + let calc = make_calculator(py, &[(("a", "b"), 0.1)], &[], &[], false); + assert!((calc.distance("ab", "ba") - 1.1).abs() < f64::EPSILON); }); } #[test] fn test_negative_default_cost_errors() { - Python::with_gil(|py| { - let a = "test"; - let b = "toast"; - let empty_costs = PyDict::new(py); - - // Test negative substitution cost - let sub_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - -1.0, - 1.0, - 1.0, - ); + Python::attach(|py| { + let empty = PyDict::new(py); + let sub_err = + RustLevenshteinCalculator::new(&empty, &empty, &empty, true, -1.0, 1.0, 1.0); assert!(sub_err.is_err()); assert!(sub_err.unwrap_err().is_instance_of::(py)); - - // Test negative insertion cost - let ins_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - -1.0, - 1.0, - ); - assert!(ins_err.is_err()); - assert!(ins_err.unwrap_err().is_instance_of::(py)); - - // Test negative deletion cost - let del_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - -1.0, - ); - assert!(del_err.is_err()); - assert!(del_err.unwrap_err().is_instance_of::(py)); - }); - } - - #[test] - fn test_edit_op_substitute_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Substitute { - source: "a".to_string(), - target: "b".to_string(), - cost: 0.75, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('substitute', 'a', 'b', 0.75)"); }); } #[test] - fn test_edit_op_insert_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Insert { - target: "c".to_string(), - cost: 1.0, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('insert', None, 'c', 1.0)"); + fn test_constructor_does_not_apply_closure() { + // Without calling closed_cost_maps, transitive paths are not auto-applied. + // sub(a->b)=0.1, sub(b->c)=0.1: direct a->c lookup falls back to default 1.0. + Python::attach(|py| { + let calc = + make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); + assert!((calc.distance("a", "c") - 1.0).abs() < f64::EPSILON); }); } #[test] - fn test_edit_op_delete_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Delete { - source: "d".to_string(), - cost: 1.2, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('delete', 'd', None, 1.2)"); + fn test_closed_cost_maps_finds_chain() { + Python::attach(|py| { + let calc = + make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); + let (sub, _ins, _del) = calc.closed_cost_maps(py, false, None).unwrap(); + let cost: f64 = sub + .get_item(("a".to_string(), "c".to_string())) + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert!((cost - 0.2).abs() < 1e-9); }); } #[test] - fn test_edit_op_match_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Match { - token: "e".to_string(), + fn test_explain() { + Python::attach(|py| { + let calc = make_calculator(py, &[], &[], &[], true); + let result = calc.explain(py, "cat", "car").unwrap(); + let py_list = PyList::new(py, result).unwrap(); + assert_eq!(py_list.len(), 3); + let op = |i: usize| -> String { + py_list + .get_item(i) + .unwrap() + .cast_into::() + .unwrap() + .get_item(0) + .unwrap() + .extract::() + .unwrap() }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('match', 'e', 'e', 0.0)"); + assert_eq!(op(0), "match"); + assert_eq!(op(1), "match"); + assert_eq!(op(2), "substitute"); }); } #[test] - fn test_explain_weighted_levenshtein_distance() { - Python::with_gil(|py| { - let a = "cat"; - let b = "car"; - let empty_costs = PyDict::new(py); - - let result = _explain_weighted_levenshtein_distance( + fn test_batch_distance() { + Python::attach(|py| { + let calc = make_calculator(py, &[], &[], &[], true); + let distances = calc.batch_distance( py, - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - let py_list = PyList::new(py, result).unwrap(); - assert_eq!(py_list.clone().len(), 3); - - let first_op = py_list - .clone() - .get_item(0) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - first_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "match" - ); - - let second_op = py_list - .clone() - .get_item(1) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - second_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "match" + "book".to_string(), + vec!["back".to_string(), "books".to_string(), "look".to_string()], ); - - let third_op = py_list - .clone() - .get_item(2) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - third_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "substitute" - ); - assert_eq!(third_op.get_item(3).unwrap().extract::().unwrap(), 1.0); - }); - } - - #[test] - fn test_batch_weighted_levenshtein_distance() { - Python::with_gil(|py| { - let s = "book"; - let candidates = vec!["back".to_string(), "books".to_string(), "look".to_string()]; - let empty_costs = PyDict::new(py); - - let distances = _batch_weighted_levenshtein_distance( - s, - candidates, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert_eq!(distances.len(), 3); assert_eq!(distances, vec![2.0, 1.0, 1.0]); }); } - - #[test] - fn test_batch_with_empty_candidate_list() { - Python::with_gil(|py| { - let s = "test"; - let candidates: Vec = vec![]; - let empty_costs = PyDict::new(py); - - let distances = _batch_weighted_levenshtein_distance( - s, - candidates, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert!(distances.is_empty()); - }); - } } diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs new file mode 100644 index 0000000..5a35a6d --- /dev/null +++ b/src/transitive_costs.rs @@ -0,0 +1,1141 @@ +//! Closed (transitive) edit-cost maps from a unified token graph. +//! +//! [`compute_closed_cost_maps`] seeds one graph from the raw sub/ins/del maps, +//! grows the node set by applying configured ins/del transformations until +//! fixpoint, closes the graph with Floyd-Warshall, and projects distances back +//! into three plain cost maps: +//! - `dist[a][b]` for substitutions, +//! - `dist[a][ε]` for deletions, +//! - `dist[ε][b]` for insertions. +//! +//! Optional pruning removes generated substitutions that are already represented +//! by matches, insertions, deletions, and shorter substitutions. + +use crate::cost_map::CostMap; +use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; +use crate::weighted_levenshtein::custom_levenshtein_distance; +use std::collections::{HashMap, HashSet}; +use std::fmt; + +// When the caller passes `None` for `max_node_length`, the cap is derived as +// `max(longest raw token across all maps) * MAX_NODE_LENGTH_MULTIPLIER`, with a +// floor of `MIN_DERIVED_NODE_LENGTH` so trivial single-char maps still leave +// headroom for chained compositions. Length is what makes the graph finite — +// without any cap, configurations like `ins("A")=0.1` would grow the node set +// without bound (A → AA → AAA → …). +const MAX_NODE_LENGTH_MULTIPLIER: usize = 2; +const MIN_DERIVED_NODE_LENGTH: usize = 4; + +const REDUNDANT_SUBSTITUTION_EPSILON: f64 = 1e-9; + +/// `NodeId` interns indices as `u32`, so the number of graph nodes is bounded +/// by `u32::MAX`. Reaching this limit at human-sized inputs is implausible — +/// it requires the closure to construct ~4 billion distinct strings — but the +/// check exists so `NodeId::new` never panics from inside `compute_closed_cost_maps`. +const MAX_NODE_COUNT: usize = u32::MAX as usize; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TransitiveCostError { + NodeIdOverflow { node_count: usize }, + MatrixSizeOverflow { node_count: usize }, + MatrixAllocationFailed { node_count: usize, bytes: usize }, +} + +impl fmt::Display for TransitiveCostError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NodeIdOverflow { node_count } => write!( + f, + "transitive closure generated {node_count} graph nodes, exceeding the {MAX_NODE_COUNT}-node addressing limit; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), + Self::MatrixSizeOverflow { node_count } => write!( + f, + "transitive closure generated {node_count} graph nodes, too many to address in a dense matrix; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), + Self::MatrixAllocationFailed { node_count, bytes } => write!( + f, + "transitive closure generated {node_count} graph nodes and could not allocate a {bytes}-byte dense matrix; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), + } + } +} + +/// Interned identifier for a token graph node. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +struct NodeId(u32); + +impl NodeId { + fn new(index: usize) -> Self { + assert!(index < u32::MAX as usize, "too many token graph nodes"); + Self(index as u32) + } + + #[inline] + fn index(self) -> usize { + self.0 as usize + } +} + +/// Dense square matrix stored in row-major order. +#[derive(Debug)] +struct Matrix { + width: usize, + cells: Vec, +} + +impl Matrix { + fn try_filled(width: usize, value: T) -> Result { + let cell_count = width + .checked_mul(width) + .ok_or(TransitiveCostError::MatrixSizeOverflow { node_count: width })?; + let bytes = cell_count.saturating_mul(std::mem::size_of::()); + let mut cells = Vec::new(); + cells.try_reserve_exact(cell_count).map_err(|_| { + TransitiveCostError::MatrixAllocationFailed { + node_count: width, + bytes, + } + })?; + cells.resize(cell_count, value); + Ok(Self { width, cells }) + } +} + +impl Matrix { + #[inline] + fn idx(&self, row: NodeId, col: NodeId) -> usize { + row.index() * self.width + col.index() + } + + #[inline] + fn get(&self, row: NodeId, col: NodeId) -> &T { + &self.cells[self.idx(row, col)] + } + + #[inline] + fn set(&mut self, row: NodeId, col: NodeId, value: T) { + let idx = self.idx(row, col); + self.cells[idx] = value; + } +} + +/// Computes closed sub/ins/del cost maps via Floyd-Warshall on a unified graph. +/// +/// `max_node_length` caps the length (in characters) of intermediate nodes the +/// growth phase may construct and of substrings expanded from raw tokens. Pass +/// `None` to derive a sensible default from the input +/// (`max raw-token length × 2`, floored at `MIN_DERIVED_NODE_LENGTH`); pass +/// `Some(n)` to override. The cap is what guarantees termination - without it, +/// the graph could grow without bound. +/// +/// If `prune` is true, generated substitutions that the returned edit maps can +/// already express are removed from the substitution map. +pub fn compute_closed_cost_maps( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, + prune: bool, + max_node_length: Option, +) -> Result<(SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap), TransitiveCostError> { + let max_node_length = max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); + let tokens = collect_nodes(sub, ins, del, max_node_length); + let (token_to_id, epsilon_id) = build_token_index(&tokens)?; + let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del)?; + + let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); + let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); + let mut closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); + if prune { + closed_sub = + prune_redundant_substitutions(closed_sub, &closed_ins, &closed_del, sub, ins, del); + } + + Ok((closed_sub, closed_ins, closed_del)) +} + +/// Interns tokens as `NodeId`s and locates the ε node. Returns +/// `NodeIdOverflow` if the token set exceeds the addressable range. +fn build_token_index( + tokens: &[String], +) -> Result<(HashMap<&str, NodeId>, NodeId), TransitiveCostError> { + if tokens.len() > MAX_NODE_COUNT { + return Err(TransitiveCostError::NodeIdOverflow { + node_count: tokens.len(), + }); + } + let token_to_id: HashMap<&str, NodeId> = tokens + .iter() + .enumerate() + .map(|(i, token)| (token.as_str(), NodeId::new(i))) + .collect(); + let epsilon_id = *token_to_id.get("").expect("epsilon node must exist"); + Ok((token_to_id, epsilon_id)) +} + +/// Default `max_node_length` derivation: twice the longest raw token across all +/// three maps, floored at `MIN_DERIVED_NODE_LENGTH`. The doubling leaves room +/// for compositions like `sub("AB","C") + sub("CC","D")` that need an +/// intermediate (`"ABAB" → "CC" → "D"`) longer than any single raw token. +fn derive_max_node_length( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> usize { + let max_raw = sub + .costs + .keys() + .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) + .chain(ins.costs.keys().map(|k| k.chars().count())) + .chain(del.costs.keys().map(|k| k.chars().count())) + .max() + .unwrap_or(0); + (max_raw * MAX_NODE_LENGTH_MULTIPLIER).max(MIN_DERIVED_NODE_LENGTH) +} + +/// Collects graph nodes. Includes raw tokens, all substrings of raw tokens +/// (capped at `max_node_length`), and strings reachable by iteratively applying +/// configured ins/del transformations until fixpoint. The length cap is the +/// only termination guarantee — without it, `ins("A")` alone would grow the +/// node set without bound. +fn collect_nodes( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, + max_node_length: usize, +) -> Vec { + let mut tokens = gather_seed_tokens(sub, ins, del); + expand_with_substrings(&mut tokens, max_node_length); + grow_via_single_ops(&mut tokens, &single_op_tokens(ins, del), max_node_length); + tokens.into_iter().collect() +} + +/// Initial node set: ε plus every token that appears in any cost map. +fn gather_seed_tokens( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> HashSet { + let mut tokens: HashSet = HashSet::new(); + tokens.insert(String::new()); // ε + tokens.extend(ins.costs.keys().cloned()); + tokens.extend(del.costs.keys().cloned()); + for (s, t) in sub.costs.keys() { + tokens.insert(s.clone()); + tokens.insert(t.clone()); + } + tokens +} + +/// Add every contiguous substring (under the cap) of every current token. +fn expand_with_substrings(tokens: &mut HashSet, max_node_length: usize) { + let seeds: Vec = tokens.iter().cloned().collect(); + for token in &seeds { + for substring in substrings(token, max_node_length) { + tokens.insert(substring); + } + } +} + +/// Configured ins/del tokens, deduplicated, paired with their char count. +/// The char count is cached so the cap check in [`grow_via_single_ops`] +/// doesn't recompute it for every source token. +fn single_op_tokens( + ins: &CostMap, + del: &CostMap, +) -> Vec<(String, usize)> { + ins.costs + .keys() + .cloned() + .chain(del.costs.keys().cloned()) + .collect::>() + .into_iter() + .map(|token| { + let len = token.chars().count(); + (token, len) + }) + .collect() +} + +/// Grow the node set by applying each configured ins/del token in both +/// directions: inserting it produces forward-direction successors, removing it +/// produces predecessors that lie one configured ins/del edge away. This is +/// what lets the closure bridge a user-provided source like "ADC" through +/// intermediate nodes "AC" and "ABC" to a configured target "Z". +/// +/// Worklist-style: each round only processes tokens discovered in the previous +/// round, since older tokens already produced everything they can. Both gates +/// check the produced length against the cap — insertion can overrun by +/// lengthening, and deletion can overrun when an oversize raw seed (longer +/// than `max_node_length`) is shortened to something still above the cap. The +/// length cap bounds the total set, so the worklist drains in finitely many +/// rounds. +fn grow_via_single_ops( + tokens: &mut HashSet, + single_ops: &[(String, usize)], + max_node_length: usize, +) { + let mut frontier: Vec = tokens.iter().cloned().collect(); + while !frontier.is_empty() { + let mut next_frontier: Vec = Vec::new(); + for source in &frontier { + let source_len = source.chars().count(); + for (op_token, op_len) in single_ops { + if source_len + *op_len <= max_node_length { + add_new_variants( + tokens, + &mut next_frontier, + insert_token_variants(source, op_token), + ); + } + if source_len.saturating_sub(*op_len) <= max_node_length { + add_new_variants( + tokens, + &mut next_frontier, + delete_token_variants(source, op_token), + ); + } + } + } + frontier = next_frontier; + } +} + +fn add_new_variants( + tokens: &mut HashSet, + frontier: &mut Vec, + variants: Vec, +) { + for variant in variants { + if tokens.insert(variant.clone()) { + frontier.push(variant); + } + } +} + +fn substrings(token: &str, max_node_length: usize) -> Vec { + let char_count = token.chars().count(); + if char_count > max_node_length { + return Vec::new(); + } + let mut boundaries: Vec = token.char_indices().map(|(idx, _)| idx).collect(); + boundaries.push(token.len()); + + let mut out = Vec::with_capacity(char_count * (char_count + 1) / 2); + for start in 0..char_count { + for end in (start + 1)..=char_count { + out.push(token[boundaries[start]..boundaries[end]].to_owned()); + } + } + out +} + +/// All strings obtainable by inserting `inserted` at a char boundary in `source`. +fn insert_token_variants(source: &str, inserted: &str) -> Vec { + source + .char_indices() + .map(|(idx, _)| idx) + .chain(std::iter::once(source.len())) + .map(|idx| { + let mut target = String::with_capacity(source.len() + inserted.len()); + target.push_str(&source[..idx]); + target.push_str(inserted); + target.push_str(&source[idx..]); + target + }) + .collect() +} + +/// All strings obtainable by deleting one exact `deleted` occurrence from `source`. +/// +/// Enumerates every char-aligned offset where `deleted` matches, including +/// overlapping ones: `delete_token_variants("ABABA", "ABA")` yields both `"BA"` +/// (delete at 0) and `"AB"` (delete at 2). `str::match_indices` would only +/// return the non-overlapping leftmost match, so we walk char boundaries +/// manually instead. +fn delete_token_variants(source: &str, deleted: &str) -> Vec { + if deleted.is_empty() || deleted.len() > source.len() { + return Vec::new(); + } + let mut out = Vec::new(); + for (idx, _) in source.char_indices() { + let end = idx + deleted.len(); + if end > source.len() { + break; + } + if !source.is_char_boundary(end) { + continue; + } + if &source[idx..end] != deleted { + continue; + } + let mut target = String::with_capacity(source.len() - deleted.len()); + target.push_str(&source[..idx]); + target.push_str(&source[end..]); + out.push(target); + } + out +} + +fn run_closure( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + epsilon_id: NodeId, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> Result, TransitiveCostError> { + let mut dist = Matrix::try_filled(tokens.len(), f64::INFINITY)?; + set_zero_diagonal(&mut dist); + seed_substitution_edges(&mut dist, sub, token_to_id); + seed_insertion_edges(&mut dist, ins, token_to_id, epsilon_id); + seed_deletion_edges(&mut dist, del, token_to_id, epsilon_id); + seed_embedded_edges(tokens, token_to_id, ins, del, &mut dist); + floyd_warshall(&mut dist); + Ok(dist) +} + +fn set_zero_diagonal(dist: &mut Matrix) { + for index in 0..dist.width { + let node = NodeId::new(index); + dist.set(node, node, 0.0); + } +} + +fn seed_substitution_edges( + dist: &mut Matrix, + sub: &CostMap, + token_to_id: &HashMap<&str, NodeId>, +) { + for ((source, target), &cost) in &sub.costs { + if let (Some(&s), Some(&t)) = ( + token_to_id.get(source.as_str()), + token_to_id.get(target.as_str()), + ) { + relax(dist, s, t, cost); + } + } +} + +fn seed_insertion_edges( + dist: &mut Matrix, + ins: &CostMap, + token_to_id: &HashMap<&str, NodeId>, + epsilon_id: NodeId, +) { + for (token, &cost) in &ins.costs { + if let Some(&id) = token_to_id.get(token.as_str()) { + relax(dist, epsilon_id, id, cost); + } + } +} + +fn seed_deletion_edges( + dist: &mut Matrix, + del: &CostMap, + token_to_id: &HashMap<&str, NodeId>, + epsilon_id: NodeId, +) { + for (token, &cost) in &del.costs { + if let Some(&id) = token_to_id.get(token.as_str()) { + relax(dist, id, epsilon_id, cost); + } + } +} + +#[inline] +fn relax(dist: &mut Matrix, source: NodeId, target: NodeId, cost: f64) { + if cost < *dist.get(source, target) { + dist.set(source, target, cost); + } +} + +/// Adds direct edges between two existing nodes that differ by exactly one +/// configured insertion or deletion. +fn seed_embedded_edges( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + ins: &CostMap, + del: &CostMap, + dist: &mut Matrix, +) { + let insertions: Vec<(&str, f64)> = ins + .costs + .iter() + .map(|(token, &cost)| (token.as_str(), cost)) + .collect(); + let deletions: Vec<(&str, f64)> = del + .costs + .iter() + .map(|(token, &cost)| (token.as_str(), cost)) + .collect(); + + for source in tokens { + let source_id = token_to_id[source.as_str()]; + for &(inserted, cost) in &insertions { + for variant in insert_token_variants(source, inserted) { + if let Some(&target_id) = token_to_id.get(variant.as_str()) { + relax(dist, source_id, target_id, cost); + } + } + } + for &(deleted, cost) in &deletions { + for variant in delete_token_variants(source, deleted) { + if let Some(&target_id) = token_to_id.get(variant.as_str()) { + relax(dist, source_id, target_id, cost); + } + } + } + } +} + +fn floyd_warshall(dist: &mut Matrix) { + let n = dist.width; + for via in 0..n { + let via = NodeId::new(via); + for source in 0..n { + let source = NodeId::new(source); + let source_to_via = *dist.get(source, via); + if !source_to_via.is_finite() { + continue; + } + for target in 0..n { + let target = NodeId::new(target); + let candidate = source_to_via + *dist.get(via, target); + if candidate < *dist.get(source, target) { + dist.set(source, target, candidate); + } + } + } + } +} + +fn project_substitutions( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + dist: &Matrix, + raw: &CostMap, +) -> SubstitutionCostMap { + let default_cost = raw.default_cost(); + let mut closed = SubstitutionCostMap::new(); + + for source in tokens { + if source.is_empty() { + continue; + } + let source_id = token_to_id[source.as_str()]; + for target in tokens { + if target.is_empty() || source == target { + continue; + } + let target_id = token_to_id[target.as_str()]; + let key = (source.clone(), target.clone()); + + let raw_cost = raw.costs.get(&key).copied(); + let closure_cost = *dist.get(source_id, target_id); + let direct = raw_cost.unwrap_or(default_cost); + let effective = direct.min(closure_cost); + + if raw_cost.is_some() || effective < default_cost { + closed.insert(key, effective); + } + } + } + + closed +} + +fn prune_redundant_substitutions( + closed_sub: SubstitutionCostMap, + closed_ins: &SingleTokenCostMap, + closed_del: &SingleTokenCostMap, + raw_sub: &CostMap, + raw_ins: &CostMap, + raw_del: &CostMap, +) -> SubstitutionCostMap { + // Shortest-first iteration is safe: redundancy is monotone under removal of + // redundant edges. If a short substitution is dropped because an ins/del/sub + // alternative matches its cost, any longer substitution that relied on it in + // the closure can fall back to the same alternative at the same total cost, + // so its redundancy verdict is unchanged. + let mut keys: Vec = closed_sub.keys().cloned().collect(); + keys.sort_by(|(source_a, target_a), (source_b, target_b)| { + ( + source_a.chars().count() + target_a.chars().count(), + source_a, + target_a, + ) + .cmp(&( + source_b.chars().count() + target_b.chars().count(), + source_b, + target_b, + )) + }); + + // `max_token_length` is computed once at construction and is not updated as + // keys are removed below. The DP will scan longer windows than strictly + // necessary; correctness is preserved because overestimating the window only + // costs CPU. + let mut sub_map = CostMap::::new(closed_sub, raw_sub.default_cost(), false); + let ins_map = CostMap::::new(closed_ins.clone(), raw_ins.default_cost()); + let del_map = CostMap::::new(closed_del.clone(), raw_del.default_cost()); + + for key in keys { + if raw_sub.costs.contains_key(&key) { + continue; + } + let Some(cost) = sub_map.costs.remove(&key) else { + continue; + }; + let alternative = custom_levenshtein_distance(&key.0, &key.1, &sub_map, &ins_map, &del_map); + if alternative > cost + REDUNDANT_SUBSTITUTION_EPSILON { + sub_map.costs.insert(key, cost); + } + } + + sub_map.costs +} + +fn project_single_token( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + dist: &Matrix, + raw: &CostMap, + epsilon_id: NodeId, + is_insertion: bool, +) -> SingleTokenCostMap { + let default_cost = raw.default_cost(); + let mut closed = SingleTokenCostMap::new(); + + for token in tokens { + if token.is_empty() { + continue; + } + let id = token_to_id[token.as_str()]; + let raw_cost = raw.costs.get(token).copied(); + let closure_cost = if is_insertion { + *dist.get(epsilon_id, id) + } else { + *dist.get(id, epsilon_id) + }; + let direct = raw_cost.unwrap_or(default_cost); + let effective = direct.min(closure_cost); + + if raw_cost.is_some() || effective < default_cost { + closed.insert(token.clone(), effective); + } + } + + closed +} + +#[cfg(test)] +mod tests { + use super::*; + + fn approx(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-9 + } + + fn make_sub( + pairs: &[((&str, &str), f64)], + default: f64, + symmetric: bool, + ) -> CostMap { + let mut map = SubstitutionCostMap::new(); + for ((s, t), c) in pairs { + map.insert(((*s).to_string(), (*t).to_string()), *c); + } + CostMap::::new(map, default, symmetric) + } + + fn make_single(pairs: &[(&str, f64)], default: f64) -> CostMap { + let mut map = SingleTokenCostMap::new(); + for (k, v) in pairs { + map.insert((*k).to_string(), *v); + } + CostMap::::new(map, default) + } + + fn compute_closed_cost_maps( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, + prune: bool, + max_node_length: Option, + ) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { + super::compute_closed_cost_maps(sub, ins, del, prune, max_node_length).unwrap() + } + + #[test] + fn closure_finds_substitution_chain() { + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); + } + + #[test] + fn closure_finds_deletion_chain() { + let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("G", 0.01)], 1.0); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_del["6"], 0.51)); + } + + #[test] + fn closure_finds_insertion_chain() { + let sub = make_sub(&[(("x", "y"), 0.2)], 1.0, false); + let ins = make_single(&[("x", 0.1)], 1.0); + let del = make_single(&[], 1.0); + let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_ins["y"], 0.3)); + } + + #[test] + fn closure_finds_repeated_insertion_substitution() { + // A -> AA -> AAA -> B at cost 0.2 + 0.2 + 0.1 = 0.5 + let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); + } + + #[test] + fn closure_prunes_substitutions_represented_by_insertions() { + let sub = make_sub(&[(("AAA", "B"), 0.1), (("A", "B"), 0.6)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + + let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + + assert!(approx(closed_ins["A"], 0.2)); + assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); + assert!(!closed_sub.contains_key(&("AA".to_string(), "AAA".to_string()))); + assert!(!closed_sub.contains_key(&("B".to_string(), "AA".to_string()))); + } + + #[test] + fn closure_preserves_raw_substitutions_even_when_redundant() { + let sub = make_sub(&[(("AA", "AAA"), 0.2)], 1.0, false); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + + assert!(approx( + closed_sub[&("AA".to_string(), "AAA".to_string())], + 0.2 + )); + } + + #[test] + fn closure_composes_del_ins_sub() { + // ADC -> AC -> ABC -> Z at cost 0.1 + 0.1 + 0.1 = 0.3 + let sub = make_sub(&[(("ABC", "Z"), 0.1)], 1.0, true); + let ins = make_single(&[("B", 0.1)], 1.0); + let del = make_single(&[("D", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx( + closed_sub[&("ADC".to_string(), "Z".to_string())], + 0.3 + )); + } + + #[test] + fn closure_preserves_direct_when_chain_more_expensive() { + let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("6", 0.2), ("G", 0.01)], 1.0); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_del["6"], 0.2)); + } + + // --- helper-function tests -------------------------------------------------- + + fn sorted(mut v: Vec) -> Vec { + v.sort(); + v + } + + #[test] + fn substrings_includes_all_contiguous_slices() { + let mut got = sorted(substrings("abc", 8)); + got.dedup(); + assert_eq!(got, vec!["a", "ab", "abc", "b", "bc", "c"]); + } + + #[test] + fn substrings_empty_for_empty_input() { + assert!(substrings("", 8).is_empty()); + } + + #[test] + fn substrings_handles_unicode() { + let got = sorted(substrings("café", 8)); + // Chars are 'c', 'a', 'f', 'é'; substrings are all contiguous slices on + // char boundaries. + assert!(got.contains(&"é".to_string())); + assert!(got.contains(&"fé".to_string())); + assert!(got.contains(&"café".to_string())); + // No partial-byte slice should appear. + for s in &got { + assert!(s.is_char_boundary(0) && s.is_char_boundary(s.len())); + } + } + + #[test] + fn substrings_skipped_when_token_exceeds_cap() { + // A token longer than the cap returns no substrings. + let long_token = "A".repeat(9); + assert!(substrings(&long_token, 8).is_empty()); + } + + #[test] + fn insert_token_variants_inserts_at_every_boundary() { + let mut got = sorted(insert_token_variants("ab", "X")); + assert_eq!(got, vec!["Xab", "aXb", "abX"]); + got.dedup(); + assert_eq!(got.len(), 3); + } + + #[test] + fn insert_token_variants_into_empty() { + assert_eq!(insert_token_variants("", "AB"), vec!["AB".to_string()]); + } + + #[test] + fn insert_token_variants_handles_unicode() { + let got = sorted(insert_token_variants("é", "X")); + assert_eq!(got, vec!["Xé".to_string(), "éX".to_string()]); + } + + #[test] + fn delete_token_variants_enumerates_overlapping_matches() { + let mut variants = delete_token_variants("ABABA", "ABA"); + variants.sort(); + assert_eq!(variants, vec!["AB".to_string(), "BA".to_string()]); + } + + #[test] + fn delete_token_variants_no_match() { + assert!(delete_token_variants("xyz", "ABA").is_empty()); + } + + #[test] + fn delete_token_variants_full_match_yields_empty_string() { + assert_eq!(delete_token_variants("ABA", "ABA"), vec![String::new()]); + } + + #[test] + fn delete_token_variants_deleted_longer_than_source() { + assert!(delete_token_variants("AB", "ABCDE").is_empty()); + } + + #[test] + fn delete_token_variants_empty_deleted_string() { + // Deleting nothing produces no graph edges (we'd otherwise loop on ε). + assert!(delete_token_variants("ABC", "").is_empty()); + } + + #[test] + fn delete_token_variants_single_char_repeated() { + // For single chars, every position matches; we get one variant per + // occurrence (deduplicated downstream by the HashSet). + let mut got = delete_token_variants("AAAA", "A"); + got.sort(); + // All four deletions produce "AAA". + assert_eq!(got, vec!["AAA"; 4]); + } + + #[test] + fn delete_token_variants_handles_unicode() { + // "café" minus "fé" yields "ca". A non-overlapping single-byte search + // at byte index 2 ('f') would be wrong if it weren't gated by + // is_char_boundary — confirm the unicode path works end-to-end. + let got = delete_token_variants("café", "fé"); + assert_eq!(got, vec!["ca".to_string()]); + } + + #[test] + fn delete_token_variants_unicode_overlap() { + // "🙂🙃🙂🙃🙂" minus "🙂🙃🙂": matches at char positions 0 and 2 (both + // overlap). Expect both variants. + let mut got = delete_token_variants("🙂🙃🙂🙃🙂", "🙂🙃🙂"); + got.sort(); + assert_eq!(got, vec!["🙂🙃".to_string(), "🙃🙂".to_string()]); + } + + // --- closure correctness ---------------------------------------------------- + + #[test] + fn closure_finds_chain_through_overlapping_deletion() { + // ABABA can become AB by deleting the "ABA" starting at index 2. + // sub("AB","X")=0.1 then closes to sub("ABABA","X")=0.2. + let sub = make_sub(&[(("AB", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("ABA", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx( + closed_sub[&("ABABA".to_string(), "X".to_string())], + 0.2 + )); + } + + #[test] + fn closure_emits_overlapping_deletion_intermediate_edge() { + // Pin the projected substitution that the previous match_indices-based + // implementation silently dropped. Without overlap support, this entry + // would be missing from closed_sub. + let sub = make_sub(&[(("AB", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("ABA", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + let expected_key = ("ABABA".to_string(), "AB".to_string()); + assert!( + closed_sub.contains_key(&expected_key), + "expected overlapping-delete edge to be projected as a substitution" + ); + assert!(approx(closed_sub[&expected_key], 0.1)); + } + + #[test] + fn closure_handles_empty_cost_maps() { + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, closed_ins, closed_del) = + compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(closed_sub.is_empty()); + assert!(closed_ins.is_empty()); + assert!(closed_del.is_empty()); + } + + #[test] + fn closure_does_not_emit_pairs_above_default() { + // No raw entry, no chain — projection should not insert anything that + // would just equal the default cost. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + // ("a","z") has no path, falls back to default — must not be inserted. + assert!(!closed_sub.contains_key(&("a".to_string(), "z".to_string()))); + } + + #[test] + fn closure_keeps_raw_substitution_even_when_above_default() { + // User-set 2.0 > default 1.0. The raw entry must survive so that the + // round-trip preserves the user's explicit intent. + let sub = make_sub(&[(("a", "b"), 2.0)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + let key = ("a".to_string(), "b".to_string()); + assert!(closed_sub.contains_key(&key)); + // No cheaper chain exists, so the raw 2.0 stays. + assert!(approx(closed_sub[&key], 2.0)); + } + + #[test] + fn closure_skips_self_substitution() { + // (a,a) is never a useful substitution; projection must skip it. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + for ((s, t), _) in &closed_sub { + assert_ne!(s, t, "self-substitution leaked into closed map"); + } + } + + #[test] + fn closure_symmetric_propagates_both_directions() { + // Symmetric chain: (a,b)+ (b,c) at 0.1 each gives (a,c) and (c,a) both at 0.2. + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, true); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); + assert!(approx(closed_sub[&("c".to_string(), "a".to_string())], 0.2)); + } + + #[test] + fn closure_asymmetric_does_not_invent_reverse() { + // Asymmetric: only (a,b) is given. (b,a) must not appear because no + // path exists in that direction. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(closed_sub.contains_key(&("a".to_string(), "b".to_string()))); + assert!(!closed_sub.contains_key(&("b".to_string(), "a".to_string()))); + } + + #[test] + fn closure_materializes_del_then_ins_as_substitution() { + // del("y")=0.1 + ins("x")=0.2 should fold into an effective sub("y","x")=0.3. + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[("x", 0.2)], 1.0); + let del = make_single(&[("y", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("y".to_string(), "x".to_string())], 0.3)); + } + + #[test] + fn closure_finds_long_substitution_chain() { + // a -> b -> c -> d at 0.1 + 0.1 + 0.1 = 0.3. + let sub = make_sub( + &[(("a", "b"), 0.1), (("b", "c"), 0.1), (("c", "d"), 0.1)], + 1.0, + false, + ); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("a".to_string(), "d".to_string())], 0.3)); + } + + #[test] + fn closure_handles_zero_cost_edges() { + // Zero-cost ins should still appear in closed map. + let sub = make_sub(&[(("x", "y"), 0.0)], 1.0, false); + let ins = make_single(&[("x", 0.0)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_ins["y"], 0.0)); + assert!(approx(closed_sub[&("x".to_string(), "y".to_string())], 0.0)); + } + + #[test] + fn closure_handles_unicode_tokens() { + // Same chain shape as the basic ASCII test, but with non-ASCII chars. + let sub = make_sub(&[(("é", "ê"), 0.1), (("ê", "è"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("é".to_string(), "è".to_string())], 0.2)); + } + + #[test] + fn matrix_allocation_checks_size_overflow() { + let err = Matrix::::try_filled(usize::MAX, 0.0).unwrap_err(); + assert!(matches!( + err, + TransitiveCostError::MatrixSizeOverflow { + node_count: usize::MAX + } + )); + } + + #[test] + fn growth_does_not_overrun_cap_via_deletion_from_oversize_seed() { + // "LONGSTRING" (10 chars) is a raw substitution endpoint and is + // always seeded into the node set. With a configured `del("LON")` + // the previous, ungated deletion would have spawned "GSTRING" + // (7 chars) from it — still over the 4-char cap. The symmetric + // deletion gate prevents that. + let sub = make_sub(&[(("LONGSTRING", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("LON", 0.1)], 1.0); + + let nodes = super::collect_nodes(&sub, &ins, &del, 4); + let raw_seeds: HashSet<&str> = ["LONGSTRING"].into_iter().collect(); + for token in &nodes { + let len = token.chars().count(); + assert!( + len <= 4 || raw_seeds.contains(token.as_str()), + "node {token:?} of length {len} exceeds cap and is not a raw seed" + ); + } + } + + // --- pruning ---------------------------------------------------------------- + + #[test] + fn pruning_on_empty_input_is_noop() { + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.is_empty()); + } + + #[test] + fn pruning_keeps_strictly_cheaper_substitution() { + // sub(a,b)=0.1 with ins(a)=0.5, del(b)=0.5: alternative path through ε + // costs 1.0, so the substitution is strictly cheaper and must be kept. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[("a", 0.5)], 1.0); + let del = make_single(&[("b", 0.5)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.contains_key(&("a".to_string(), "b".to_string()))); + } + + #[test] + fn pruning_preserves_distances_for_chains() { + // Closure-then-pruning must not change distances on the strings the + // chains describe. We verify by running the DP with the closed maps + // before and after pruning and comparing. + let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (sub_unpruned, ins_unpruned, del_unpruned) = + compute_closed_cost_maps(&sub, &ins, &del, false, None); + let (sub_pruned, ins_pruned, del_pruned) = + compute_closed_cost_maps(&sub, &ins, &del, true, None); + + let unpruned_subs = CostMap::::new(sub_unpruned, 1.0, false); + let unpruned_ins = CostMap::::new(ins_unpruned, 1.0); + let unpruned_del = CostMap::::new(del_unpruned, 1.0); + let pruned_subs = CostMap::::new(sub_pruned, 1.0, false); + let pruned_ins = CostMap::::new(ins_pruned, 1.0); + let pruned_del = CostMap::::new(del_pruned, 1.0); + + for (s, t) in [ + ("A", "B"), + ("AA", "B"), + ("AAA", "B"), + ("B", "A"), + ("AA", "AAA"), + ] { + let unpruned = + custom_levenshtein_distance(s, t, &unpruned_subs, &unpruned_ins, &unpruned_del); + let pruned = custom_levenshtein_distance(s, t, &pruned_subs, &pruned_ins, &pruned_del); + assert!( + approx(unpruned, pruned), + "distance({s:?},{t:?}) drifted after pruning: {unpruned} vs {pruned}" + ); + } + } + + #[test] + fn pruning_raw_entries_are_never_dropped() { + // ("AA","AAA") at 0.2 is exactly representable as ins("A"), so it would + // be redundant if generated. But it's user-provided, so prune must keep it. + let sub = make_sub(&[(("AA", "AAA"), 0.2)], 1.0, false); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.contains_key(&("AA".to_string(), "AAA".to_string()))); + } + + #[test] + fn closure_idempotent_pure_substitution() { + // Pure substitution chains converge in one closure round: round 1 adds + // (a,c) at 0.2; round 2 sees it directly, finds the same path through b. + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + let sub2 = CostMap::::new(s1.clone(), 1.0, false); + let ins2 = CostMap::::new(i1.clone(), 1.0); + let del2 = CostMap::::new(d1.clone(), 1.0); + let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2, false, None); + assert_eq!(s1, s2); + assert_eq!(i1, i2); + assert_eq!(d1, d2); + } +} diff --git a/src/weighted_levenshtein.rs b/src/weighted_levenshtein.rs index 7c536e2..0b9b51e 100644 --- a/src/weighted_levenshtein.rs +++ b/src/weighted_levenshtein.rs @@ -2,36 +2,27 @@ use crate::cost_map::CostMap; use crate::explanation::{EditOperation, Predecessor}; use crate::types::{SingleTokenKey, SubstitutionKey}; -// --- Public Functions --- - -pub fn custom_levenshtein_distance_with_cost_maps( +pub(crate) fn custom_levenshtein_distance( source: &str, target: &str, - substitution_cost_map: &CostMap, - insertion_cost_map: &CostMap, - deletion_cost_map: &CostMap, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, ) -> f64 { if source == target { return 0.0; } - let mut processor = LevenshteinProcessor::new( - source, - target, - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, - false, - ); + let mut processor = LevenshteinProcessor::new(source, target, sub, ins, del, false); processor.run(); processor.distance() } -pub fn explain_custom_levenshtein_distance( +pub(crate) fn explain_custom_levenshtein( source: &str, target: &str, - substitution_cost_map: &CostMap, - insertion_cost_map: &CostMap, - deletion_cost_map: &CostMap, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, ) -> Vec { if source == target { return source @@ -41,26 +32,17 @@ pub fn explain_custom_levenshtein_distance( }) .collect(); } - let mut processor = LevenshteinProcessor::new( - source, - target, - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, - true, - ); + let mut processor = LevenshteinProcessor::new(source, target, sub, ins, del, true); processor.run(); processor.into_result() } -// --- Algorithm Implementation --- - struct LevenshteinProcessor<'a> { source_chars: Vec, target_chars: Vec, - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, + sub: &'a CostMap, + ins: &'a CostMap, + del: &'a CostMap, dp: Vec>, predecessors: Option>>, multi_char_ops: bool, @@ -70,9 +52,9 @@ impl<'a> LevenshteinProcessor<'a> { fn new( source: &str, target: &str, - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, + sub: &'a CostMap, + ins: &'a CostMap, + del: &'a CostMap, explain: bool, ) -> Self { let source_chars: Vec = source.chars().collect(); @@ -83,9 +65,12 @@ impl<'a> LevenshteinProcessor<'a> { let mut processor = Self { source_chars, target_chars, - sub_map, - ins_map, - del_map, + multi_char_ops: sub.max_token_length() > 1 + || ins.max_token_length() > 1 + || del.max_token_length() > 1, + sub, + ins, + del, dp: vec![vec![0.0; len_target + 1]; len_source + 1], predecessors: if explain { Some(vec![ @@ -95,15 +80,11 @@ impl<'a> LevenshteinProcessor<'a> { } else { None }, - multi_char_ops: sub_map.max_token_length > 1 - || ins_map.max_token_length > 1 - || del_map.max_token_length > 1, }; processor.initialize(); processor } - /// Fill the DP table. fn run(&mut self) { for i in 1..=self.source_chars.len() { for j in 1..=self.target_chars.len() { @@ -112,13 +93,11 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Get the final computed distance. #[inline] fn distance(&self) -> f64 { self.dp[self.source_chars.len()][self.target_chars.len()] } - /// Convert the computed predecessors into a sequence of edit operations. fn into_result(self) -> Vec { match self.predecessors.as_ref() { Some(preds) => self.backtrack(preds), @@ -133,21 +112,19 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Compute the cost for cell (i, j) in the DP table. #[inline] fn compute_cell(&mut self, i: usize, j: usize) { let source_char = self.source_chars[i - 1]; let target_char = self.target_chars[j - 1]; - let source_char_str = self.source_chars[i - 1].to_string(); - let target_char_str = self.target_chars[j - 1].to_string(); + let source_char_str = source_char.to_string(); + let target_char_str = target_char.to_string(); - let deletion_cost = self.dp[i - 1][j] + self.del_map.get_cost(&source_char_str); - let insertion_cost = self.dp[i][j - 1] + self.ins_map.get_cost(&target_char_str); - let sub_cost = self.sub_map.get_cost(&source_char_str, &target_char_str); + let deletion_cost = self.dp[i - 1][j] + self.del.get_cost(&source_char_str); + let insertion_cost = self.dp[i][j - 1] + self.ins.get_cost(&target_char_str); + let sub_cost = self.sub.get_cost(&source_char_str, &target_char_str); let substitution_cost = self.dp[i - 1][j - 1] + sub_cost; - // Check for exact match let match_cost = self.dp[i - 1][j - 1]; let (mut min_cost, mut best_op) = if source_char == target_char { (match_cost, Predecessor::Match(1)) @@ -172,24 +149,22 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Initialize the first row and column of the DP table. fn initialize(&mut self) { let len_source = self.source_chars.len(); let len_target = self.target_chars.len(); self.dp[0][0] = 0.0; - // First row (insertions) for j in 1..=len_target { let char_str = self.target_chars[j - 1].to_string(); - self.dp[0][j] = self.dp[0][j - 1] + self.ins_map.get_cost(&char_str); + self.dp[0][j] = self.dp[0][j - 1] + self.ins.get_cost(&char_str); self.record(0, j, Predecessor::Insert(1)); - let max_len = self.ins_map.max_token_length.min(j); + let max_len = self.ins.max_token_length().min(j); for token_len in 2..=max_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.ins_map.has_key(&token) { - let new_cost = self.dp[0][token_start] + self.ins_map.get_cost(&token); + if self.ins.has_key(&token) { + let new_cost = self.dp[0][token_start] + self.ins.get_cost(&token); if new_cost < self.dp[0][j] { self.dp[0][j] = new_cost; self.record(0, j, Predecessor::Insert(token_len)); @@ -197,18 +172,17 @@ impl<'a> LevenshteinProcessor<'a> { } } } - // First column (deletions) for i in 1..=len_source { let char_str = self.source_chars[i - 1].to_string(); - self.dp[i][0] = self.dp[i - 1][0] + self.del_map.get_cost(&char_str); + self.dp[i][0] = self.dp[i - 1][0] + self.del.get_cost(&char_str); self.record(i, 0, Predecessor::Delete(1)); - let max_len = self.del_map.max_token_length.min(i); + let max_len = self.del.max_token_length().min(i); for token_len in 2..=max_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.del_map.has_key(&token) { - let new_cost = self.dp[token_start][0] + self.del_map.get_cost(&token); + if self.del.has_key(&token) { + let new_cost = self.dp[token_start][0] + self.del.get_cost(&token); if new_cost < self.dp[i][0] { self.dp[i][0] = new_cost; self.record(i, 0, Predecessor::Delete(token_len)); @@ -219,8 +193,8 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_substitutions(&mut self, i: usize, j: usize) { - let max_source_len = self.sub_map.max_token_length.min(i); - let max_target_len = self.sub_map.max_token_length.min(j); + let max_source_len = self.sub.max_token_length().min(i); + let max_target_len = self.sub.max_token_length().min(j); for source_len in 1..=max_source_len { for target_len in 1..=max_target_len { if source_len == 1 && target_len == 1 { @@ -230,9 +204,9 @@ impl<'a> LevenshteinProcessor<'a> { let target_start = j - target_len; let source_substr: String = self.source_chars[source_start..i].iter().collect(); let target_substr: String = self.target_chars[target_start..j].iter().collect(); - if self.sub_map.has_key(&source_substr, &target_substr) { + if self.sub.has_key(&source_substr, &target_substr) { let new_cost = self.dp[source_start][target_start] - + self.sub_map.get_cost(&source_substr, &target_substr); + + self.sub.get_cost(&source_substr, &target_substr); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Substitute(source_len, target_len)); @@ -243,12 +217,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_insertions(&mut self, i: usize, j: usize) { - let max_ins_len = self.ins_map.max_token_length.min(j); + let max_ins_len = self.ins.max_token_length().min(j); for token_len in 2..=max_ins_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.ins_map.has_key(&token) { - let new_cost = self.dp[i][token_start] + self.ins_map.get_cost(&token); + if self.ins.has_key(&token) { + let new_cost = self.dp[i][token_start] + self.ins.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Insert(token_len)); @@ -258,12 +232,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_deletions(&mut self, i: usize, j: usize) { - let max_del_len = self.del_map.max_token_length.min(i); + let max_del_len = self.del.max_token_length().min(i); for token_len in 2..=max_del_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.del_map.has_key(&token) { - let new_cost = self.dp[token_start][j] + self.del_map.get_cost(&token); + if self.del.has_key(&token) { + let new_cost = self.dp[token_start][j] + self.del.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Delete(token_len)); @@ -272,14 +246,12 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Check for multi-character operations (substitutions, insertions, deletions). fn check_multi_char_ops(&mut self, i: usize, j: usize) { self.check_multi_char_substitutions(i, j); self.check_multi_char_insertions(i, j); self.check_multi_char_deletions(i, j); } - /// Backtrack through the predecessors to construct the edit path. fn backtrack(&self, preds: &[Vec]) -> Vec { let mut path = Vec::new(); let mut i = self.source_chars.len(); @@ -291,7 +263,7 @@ impl<'a> LevenshteinProcessor<'a> { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); let target_token: String = self.target_chars[j - t_len..j].iter().collect(); if source_token != target_token { - let cost = self.sub_map.get_cost(&source_token, &target_token); + let cost = self.sub.get_cost(&source_token, &target_token); path.push(EditOperation::Substitute { source: source_token, target: target_token, @@ -303,7 +275,7 @@ impl<'a> LevenshteinProcessor<'a> { } Predecessor::Insert(t_len) => { let target_token: String = self.target_chars[j - t_len..j].iter().collect(); - let cost = self.ins_map.get_cost(&target_token); + let cost = self.ins.get_cost(&target_token); path.push(EditOperation::Insert { target: target_token, cost, @@ -312,7 +284,7 @@ impl<'a> LevenshteinProcessor<'a> { } Predecessor::Delete(s_len) => { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); - let cost = self.del_map.get_cost(&source_token); + let cost = self.del.get_cost(&source_token); path.push(EditOperation::Delete { source: source_token, cost, @@ -350,7 +322,6 @@ mod test { ); } - // Helper function to create default cost maps for testing fn create_default_cost_maps() -> ( CostMap, CostMap, @@ -362,19 +333,26 @@ mod test { (sub_map, ins_map, del_map) } + fn calc_distance( + source: &str, + target: &str, + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, + ) -> f64 { + custom_levenshtein_distance(source, target, sub_map, ins_map, del_map) + } + #[test] fn test_custom_levenshtein_with_custom_sub_map() { let (_, ins_map, del_map) = create_default_cost_maps(); - - // Create a custom substitution map with specific a->b cost let sub_map = CostMap::::new( SubstitutionCostMap::from([(("a".to_string(), "b".to_string()), 0.1)]), 1.0, true, ); - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "bbc", &sub_map, &ins_map, &del_map), + calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); @@ -382,29 +360,23 @@ mod test { #[test] fn test_mixed_custom_costs() { - // Create cost maps let sub_map = CostMap::::new( SubstitutionCostMap::from([(("a".to_string(), "b".to_string()), 0.1)]), 1.0, true, ); - let ins_map = CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.3)]), 1.0); - let del_map = CostMap::::new(SingleTokenCostMap::from([("y".to_string(), 0.4)]), 1.0); - // Test with all three maps: delete 'y' (0.4) + insert 'x' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("aby", "abx", &sub_map, &ins_map, &del_map), + calc_distance("aby", "abx", &sub_map, &ins_map, &del_map), 0.7, 1e-9, ); - - // Test substitution: substitute 'a' with 'b' (0.1) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "bbc", &sub_map, &ins_map, &del_map), + calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); @@ -413,600 +385,88 @@ mod test { #[test] fn test_multi_character_substitutions() { let (_, ins_map, del_map) = create_default_cost_maps(); - let sub_map = CostMap::::new( SubstitutionCostMap::from([(("h".to_string(), "In".to_string()), 0.2)]), 1.0, true, ); - - // Test that "hi" with "Ini" has a low cost due to the special substitution - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("hi", "Ini", &sub_map, &ins_map, &del_map), - 0.2, // Only the h->In substitution cost - 1e-9, - ); - - // Test another example - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hello", "Inello", &sub_map, &ins_map, &del_map, - ), - 0.2, // Only the h->In substitution cost - 1e-9, - ); - } - - #[test] - fn test_multiple_substitutions_in_same_string() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("h".to_string(), "In".to_string()), 0.2); - custom_costs.insert(("l".to_string(), "1".to_string()), 0.3); - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test multiple substitutions in the same string - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hello", "Ine11o", &sub_map, &ins_map, &del_map, - ), - 0.8, // 0.2 for h->In and 0.3+0.3 for l->1 twice - 1e-9, - ); - } - - #[test] - fn test_overlapping_substitution_patterns() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("rn".to_string(), "m".to_string()), 0.1); // common OCR confusion - custom_costs.insert(("cl".to_string(), "d".to_string()), 0.2); // another common confusion - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test the rn->m substitution - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "corner", "comer", &sub_map, &ins_map, &del_map, - ), - 0.1, - 1e-9, - ); - - // Test the cl->d substitution assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "class", "dass", &sub_map, &ins_map, &del_map, - ), + calc_distance("hi", "Ini", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - } - - #[test] - fn test_asymmetric_costs() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - // Sometimes OCR errors aren't symmetric - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("0".to_string(), "O".to_string()), 0.1); // 0->O is common - custom_costs.insert(("O".to_string(), "0".to_string()), 0.5); // O->0 is less common - let sub_map = CostMap::::new(custom_costs, 1.0, false); // asymmetric costs - - // Test 0->O substitution (lower cost) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "R0AD", "ROAD", &sub_map, &ins_map, &del_map, - ), - 0.1, - 1e-9, - ); - - // Test O->0 substitution (higher cost) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "rOad", "r0ad", &sub_map, &ins_map, &del_map, - ), - 0.5, - 1e-9, - ); - } - - #[test] - fn test_substitution_at_word_boundaries() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("rn".to_string(), "m".to_string()), 0.1); - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test substitution at start of word - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("rnat", "mat", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test substitution at end of word - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("burn", "bum", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - } - - #[test] - fn test_specific_custom_ins_del_costs() { - let sub_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); - - // Test with custom insertion cost - let ins_map_custom = CostMap::::new( - SingleTokenCostMap::from([("a".to_string(), 0.2), ("b".to_string(), 0.3)]), - 1.0, - ); - let del_map_default = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - // Test insertion with custom cost: Insert 'a' with cost 0.2 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "bc", - "abc", - &sub_map, - &ins_map_custom, - &del_map_default, - ), + calc_distance("hello", "Inello", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - - // Test with custom deletion cost - let ins_map_default = CostMap::::new(SingleTokenCostMap::new(), 1.0); - let del_map_custom = CostMap::::new( - SingleTokenCostMap::from([("a".to_string(), 0.4), ("c".to_string(), 0.5)]), - 1.0, - ); - - // Test deletion with custom cost: Delete 'a' with cost 0.4 - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abc", - "bc", - &sub_map, - &ins_map_default, - &del_map_custom, - ), - 0.4, - 1e-9, - ); - - // Test with both custom insertion and deletion costs, forcing ins/del - let ins_map_force = - CostMap::::new(SingleTokenCostMap::from([("b".to_string(), 0.3)]), 1.0); - let del_map_force = - CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.5)]), 1.0); - - // Create a substitution map with very high cost to force deletion+insertion - let high_cost_sub_map = CostMap::::new( - SubstitutionCostMap::new(), // Empty map uses default cost - 2.0, // High default cost to ensure deletion+insertion is preferred - true, - ); - - // Test combined operations: Delete 'x' (0.5) + insert 'b' (0.3) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "axc", - "abc", - &high_cost_sub_map, - &ins_map_force, - &del_map_force, - ), - 0.8, - 1e-9, - ); } #[test] fn test_edge_cases() { let (sub_map, ins_map, del_map) = create_default_cost_maps(); - - // Test empty strings: Empty strings have zero distance assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("", "", &sub_map, &ins_map, &del_map), + calc_distance("", "", &sub_map, &ins_map, &del_map), 0.0, 1e-9, ); - - // Test source empty, target not empty: Insert 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("", "abc", &sub_map, &ins_map, &del_map), + calc_distance("", "abc", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); - - // Test source not empty, target empty: Delete 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "", &sub_map, &ins_map, &del_map), + calc_distance("abc", "", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); - - // Test with custom insertion costs for empty source - let custom_ins_map = CostMap::::new( - SingleTokenCostMap::from([ - ("a".to_string(), 0.2), - ("b".to_string(), 0.3), - ("c".to_string(), 0.4), - ]), - 1.0, - ); - - // Test with custom insertion costs: Insert 'a' (0.2) + 'b' (0.3) + 'c' (0.4) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "", - "abc", - &sub_map, - &custom_ins_map, - &del_map, - ), - 0.9, - 1e-9, - ); - - // Test with custom deletion costs for empty target - let custom_del_map = CostMap::::new( - SingleTokenCostMap::from([ - ("a".to_string(), 0.5), - ("b".to_string(), 0.6), - ("c".to_string(), 0.7), - ]), - 1.0, - ); - - // Test with custom deletion costs: Delete 'a' (0.5) + 'b' (0.6) + 'c' (0.7) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abc", - "", - &sub_map, - &ins_map, - &custom_del_map, - ), - 1.8, - 1e-9, - ); - } - - #[test] - fn test_overall_mixed_operations() { - // Create maps with various custom costs - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("a".to_string(), "A".to_string()), 0.1), - (("b".to_string(), "B".to_string()), 0.2), - ]), - 1.0, - true, - ); - - let ins_map = CostMap::::new( - SingleTokenCostMap::from([("x".to_string(), 0.3), ("y".to_string(), 0.4)]), - 1.0, - ); - - let del_map = CostMap::::new( - SingleTokenCostMap::from([("m".to_string(), 0.5), ("n".to_string(), 0.6)]), - 1.0, - ); - - // Test with a mix of operations: Sub 'a'→'A' (0.1) + Sub 'b'→'B' (0.2) + delete 'm' (0.5) + delete 'n' (0.6) + insert 'x' (0.3) + insert 'y' (0.4) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abmn", "ABxy", &sub_map, &ins_map, &del_map, - ), - 2.1, - 1e-9, - ); } #[test] fn test_unicode_handling() { let (sub_map, ins_map, del_map) = create_default_cost_maps(); - - // Test with Unicode characters: Substitute 'é' with 'e' with default cost 1.0 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "café", "cafe", &sub_map, &ins_map, &del_map, - ), + calc_distance("café", "cafe", &sub_map, &ins_map, &del_map), 1.0, 1e-9, ); - - // Test with emoji: Delete ' ' and '😊' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("hi 😊", "hi", &sub_map, &ins_map, &del_map), + calc_distance("hi 😊", "hi", &sub_map, &ins_map, &del_map), 2.0, 1e-9, ); - - // Test with custom costs for Unicode - let sub_map_unicode = CostMap::::new( - SubstitutionCostMap::from([(("e".to_string(), "é".to_string()), 0.1)]), // Custom e->é cost - 1.0, - true, - ); - let ins_map_unicode = CostMap::::new( - SingleTokenCostMap::from([("é".to_string(), 0.2), ("😊".to_string(), 0.3)]), - 1.0, - ); - let del_map_unicode = CostMap::::new( - SingleTokenCostMap::from([("é".to_string(), 0.4), ("😊".to_string(), 0.5)]), - 1.0, - ); - - // Test substitution of Unicode with custom cost - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "cafe", - "café", - &sub_map_unicode, - &ins_map_unicode, - &del_map_unicode, - ), - 0.1, // Custom substitution cost for 'e'->'é' - 1e-9, - ); - - // Test deletion of Unicode with custom cost - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hi 😊", - "hi", - &sub_map, - &ins_map_unicode, - &del_map_unicode, - ), - 1.5, // Delete ' ' (default 1.0) and '😊' (custom 0.5) - 1e-9, - ); - } - - #[test] - fn test_various_multi_char_substitutions() { - // Test multi-character substitutions with different lengths - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("th".to_string(), "T".to_string()), 0.2), // 2 -> 1 - (("ing".to_string(), "in'".to_string()), 0.3), // 3 -> 3 - (("o".to_string(), "ou".to_string()), 0.1), // 1 -> 2 - ]), - 1.0, - true, - ); - let (_, ins_map, del_map) = create_default_cost_maps(); - - // Test 2-to-1 character substitution: Substitute "th" with "T" with cost 0.2 - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("this", "Tis", &sub_map, &ins_map, &del_map), - 0.2, - 1e-9, - ); - - // Test 3-to-3 character substitution: Substitute "ing" with "in'" with cost 0.3 - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "singing", "singin'", &sub_map, &ins_map, &del_map, - ), - 0.3, - 1e-9, - ); - - // Test 1-to-2 character substitution: Substitute "o" with "ou" with cost 0.1 - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("go", "gou", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test multiple multi-character substitutions: Sub "th"->"T" (0.2) + Sub "ing"->"in'" (0.3) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "thinking", "Tinkin'", &sub_map, &ins_map, &del_map, - ), - 0.5, - 1e-9, - ); } #[test] fn test_multi_character_insertions_and_deletions() { let (sub_map, _, _) = create_default_cost_maps(); - let ins_map = CostMap::::new( - SingleTokenCostMap::from([ - ("ab".to_string(), 0.3), - ("xyz".to_string(), 0.2), - ("123".to_string(), 0.1), - ("bc".to_string(), 0.25), - ]), + SingleTokenCostMap::from([("ab".to_string(), 0.3), ("xyz".to_string(), 0.2)]), 1.0, ); - let del_map = CostMap::::new( - SingleTokenCostMap::from([ - ("cd".to_string(), 0.4), - ("ef".to_string(), 0.5), - ("789".to_string(), 0.6), - ("bc".to_string(), 0.35), - ]), + SingleTokenCostMap::from([("cd".to_string(), 0.4), ("ef".to_string(), 0.5)]), 1.0, ); - - // Test multi-character insertion: insert 'ab' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("x", "xab", &sub_map, &ins_map, &del_map), + calc_distance("x", "xab", &sub_map, &ins_map, &del_map), 0.3, 1e-9, ); - - // Test multi-character deletion: delete 'cd' (0.4) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("ycd", "y", &sub_map, &ins_map, &del_map), + calc_distance("ycd", "y", &sub_map, &ins_map, &del_map), 0.4, 1e-9, ); - - // Test both insertion and deletion: delete 'ef' (0.5) + insert 'ab' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("aef", "aab", &sub_map, &ins_map, &del_map), + calc_distance("aef", "aab", &sub_map, &ins_map, &del_map), 0.8, 1e-9, ); - - // Test with longer token insertion: insert 'xyz' (0.2) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "test", "testxyz", &sub_map, &ins_map, &del_map, - ), + calc_distance("test", "testxyz", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - - // Test with mixed operations: delete '789' (0.6) + insert 'xyz' (0.2) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "a789b", "axyzb", &sub_map, &ins_map, &del_map, - ), - 0.8, - 1e-9, - ); - - // Test multi-character deletion "bc" at the beginning: delete 'bc' (cost 0.35) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("bcd", "d", &sub_map, &ins_map, &del_map), - 0.35, - 1e-9, - ); - - // Test multi-character insertion "bc" at the beginning: insert 'bc' (cost 0.25) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("c", "bcc", &sub_map, &ins_map, &del_map), - 0.25, - 1e-9, - ); - } - - #[test] - fn test_fallback_to_default_costs_when_multi_char_sub_missing() { - // Create cost maps with multi-character substitutions - let sub_map_full = CostMap::::new( - SubstitutionCostMap::from([ - (("abc".to_string(), "xyz".to_string()), 0.1), - (("de".to_string(), "uv".to_string()), 0.2), - ]), - 1.0, - true, - ); - // Create map with only the 2-char substitution - let sub_map_partial = CostMap::::new( - SubstitutionCostMap::from([(("de".to_string(), "uv".to_string()), 0.2)]), - 1.0, - true, - ); - let (sub_map_empty, ins_map, del_map) = create_default_cost_maps(); - - // Test with full map (allows abc->xyz and de->uv): Sub "abc"->"xyz" (0.1) + Sub "de"->"uv" (0.2) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_full, - &ins_map, - &del_map, - ), - 0.3, - 1e-9, - ); - - // Test with partial map (does not allow abc->xyz, forces default): Sub a->x(1.0) + b->y(1.0) + c->z(1.0) + Sub "de"->"uv"(0.2) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_partial, - &ins_map, - &del_map, - ), - 3.2, - 1e-9, - ); - - // Test with empty map (only single character default operations): 5 * default sub cost (1.0) - assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_empty, - &ins_map, - &del_map, - ), - 5.0, - 1e-9, - ); - } - - #[test] - fn test_check_multi_char_ops_with_empty_maps() { - let (sub_map, ins_map, del_map) = create_default_cost_maps(); - - let mut processor = - LevenshteinProcessor::new("abcd", "xyz", &sub_map, &ins_map, &del_map, true); - - // Simulate the DP state before the operation - let original_dp_3_2 = processor.dp[3][2]; - processor.check_multi_char_ops(3, 2); - - // Verify that the DP value remains unchanged - assert_approx_eq(processor.dp[3][2], original_dp_3_2, 1e-9); - } - - #[test] - fn test_main_function_with_multi_char_ins_del() { - // Define source and target strings - let source = "hello"; - let target = "helloxyz"; - - // Create a custom insertion cost map - let ins_map = CostMap::::new( - SingleTokenCostMap::from([("xyz".to_string(), 0.2)]), - 1.0, - ); - let sub_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); - let del_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - // Test multi-char insertion via main function - let dist = custom_levenshtein_distance_with_cost_maps( - source, target, &sub_map, &ins_map, &del_map, - ); - assert_approx_eq(dist, 0.2, 1e-9); // Should be 0.2 (insert "xyz") - - // Now test a multi-character deletion via main function - let source2 = "helloxyz"; - let target2 = "hello"; - - // Create a custom deletion cost map - let del_map2 = CostMap::::new( - SingleTokenCostMap::from([("xyz".to_string(), 0.3)]), - 1.0, - ); - // Use default insertion map for this test - let ins_map2 = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - let dist2 = custom_levenshtein_distance_with_cost_maps( - source2, target2, &sub_map, &ins_map2, &del_map2, - ); - assert_approx_eq(dist2, 0.3, 1e-9); // Should be 0.3 (delete "xyz") } }