From a4b8c207a0d108bbc62e93f153f41e90a9c2aa0d Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Wed, 29 Apr 2026 21:45:13 +0200 Subject: [PATCH 01/21] consider transitive costs --- .github/workflows/CI.yml | 22 +- CHANGELOG.md | 8 +- Cargo.toml | 3 - Justfile | 9 +- benchmarks/benchmark.py | 143 ++++ pyproject.toml | 2 +- python/ocr_stringdist/levenshtein.py | 21 +- .../test_explain_weighted_levenshtein.py | 43 + python/tests/test_weighted_levenshtein.py | 50 ++ src/cost_map.rs | 152 +--- src/lib.rs | 9 +- src/rust_stringdist.rs | 495 +++-------- src/transitive_costs.rs | 782 ++++++++++++++++++ src/weighted_levenshtein.rs | 542 +++++++----- 14 files changed, 1545 insertions(+), 736 deletions(-) create mode 100644 benchmarks/benchmark.py create mode 100644 src/transitive_costs.rs diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5f68dce..9e49ce9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.9", "3.14"] steps: - uses: actions/checkout@v4 with: @@ -27,7 +27,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Run Cargo Tests - run: cargo test --features python -- --nocapture --test-threads=1 + run: cargo test -- --nocapture --test-threads=1 - name: Run pytest run: | # just venv pytest @@ -46,11 +46,11 @@ jobs: matrix: platform: - target: x64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: armv7 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 steps: - uses: actions/checkout@v4 with: @@ -77,13 +77,13 @@ jobs: platform: - target: x86_64-unknown-linux-musl arch: x86_64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: i686-unknown-linux-musl arch: x86 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64-unknown-linux-musl arch: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 # all values: [x86_64, x86, aarch64, armhf, armv7, ppc64le, riscv64, s390x] # { target: "armv7-unknown-linux-musleabihf", image_tag: "armv7" }, # { target: "powerpc64le-unknown-linux-musl", image_tag: "ppc64le" }, @@ -113,7 +113,7 @@ jobs: strategy: matrix: target: [x64, x86] - interpreter: ["3.9", "3.10", "3.11", "3.12", "3.13"] + interpreter: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 with: @@ -151,9 +151,9 @@ jobs: matrix: platform: - target: x64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 - target: aarch64 - interpreter: 3.9 3.10 3.11 3.12 3.13 + interpreter: 3.9 3.10 3.11 3.12 3.13 3.14 steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 426a62a..1c13791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [1.1.0] - Unreleased ### Changed -- Small runtime improvements in Rust backend. +- Consider transitive costs, making the weighted Levenshtein distance satisfy the triangle inequality. + +### Added + +- Support for Python 3.14. ## [1.0.1] - 2025-09-21 diff --git a/Cargo.toml b/Cargo.toml index a5ff08d..5f4f3ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,3 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.24.0", features = ["auto-initialize"] } rayon = "1.10.0" - -[features] -python = [] diff --git a/Justfile b/Justfile index ddc0da7..5d84390 100644 --- a/Justfile +++ b/Justfile @@ -8,15 +8,20 @@ pytest: uv run pytest --cov=python/ocr_stringdist python/tests test: - cargo llvm-cov --features python - #cargo test --features python + cargo llvm-cov + #cargo test mypy: uv run mypy . lint: + cargo clippy uv run ruff check . --fix +format: + cargo fmt + uv run ruff format + doc: uv run make -C docs html diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py new file mode 100644 index 0000000..20d3b3b --- /dev/null +++ b/benchmarks/benchmark.py @@ -0,0 +1,143 @@ +""" +Benchmark for WeightedLevenshtein distance computation. + +Run with: + uv run python benchmarks/benchmark.py + +The results are printed to stdout. Save them before and after a code change to +compare throughput. +""" + +from __future__ import annotations + +import sys +import timeit +from dataclasses import dataclass + +sys.path.insert(0, "python") + +from ocr_stringdist import WeightedLevenshtein + +REPEAT = 5 +NUMBER = 200 + + +@dataclass +class Case: + label: str + wl: WeightedLevenshtein + s1: str + s2: str + + +# ── Cost maps used across cases ──────────────────────────────────────────────── + +_WL_DEFAULT = WeightedLevenshtein.unweighted() + +_WL_OCR = WeightedLevenshtein( + substitution_costs={ + ("6", "G"): 0.5, + ("0", "O"): 0.1, + ("rn", "m"): 0.15, + ("cl", "d"): 0.2, + ("l", "1"): 0.2, + ("h", "In"): 0.25, + ("vv", "w"): 0.15, + }, + deletion_costs={"G": 0.01, "O": 0.05}, + default_substitution_cost=1.0, + default_deletion_cost=1.0, + default_insertion_cost=1.0, +) + +# ── Benchmark cases ──────────────────────────────────────────────────────────── + +CASES: list[Case] = [ + # Issue #12 — transitive chain: sub("6"→"G", 0.5) + del("G", 0.01) = 0.51 + Case( + "issue-12: transitive chain '06'→'0'", + WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + ), + "06", + "0", + ), + # Short strings, no custom costs + Case("short identical (no-op)", _WL_DEFAULT, "hello", "hello"), + Case("short similar (1 sub)", _WL_DEFAULT, "kitten", "sitten"), + Case("short dissimilar", _WL_DEFAULT, "abc", "xyz"), + # Medium strings + Case( + "medium OCR-like", + _WL_OCR, + "The man ran down the hill at 10 km/h.", + "Tine rnan ram dovvn tine Ini11 at 1O krn/In.", + ), + Case( + "medium no-match", + _WL_DEFAULT, + "abcdefghij", + "zyxwvutsrq", + ), + # Long strings + Case( + "long similar", + _WL_DEFAULT, + "a" * 200 + "b" * 50, + "a" * 198 + "c" * 52, + ), + Case( + "long OCR-like", + _WL_OCR, + "The man ran down the hill at 10 km/h. " * 5, + "Tine rnan ram dovvn tine Ini11 at 1O krn/In. " * 5, + ), + # Batch distance (1 source vs. 100 candidates) +] + +BATCH_CANDIDATES = [f"word{i}" for i in range(100)] +_WL_BATCH = WeightedLevenshtein.unweighted() + + +def run_batch() -> None: + _WL_BATCH.batch_distance("word50", BATCH_CANDIDATES) + + +# Runner + + +def bench_case(case: Case) -> tuple[float, float]: + """Returns (best_ms_per_call, calls_per_second).""" + stmt = lambda: case.wl.distance(case.s1, case.s2) # noqa: E731 + times = timeit.repeat(stmt, repeat=REPEAT, number=NUMBER) + best_total_s = min(times) + best_ms = best_total_s / NUMBER * 1000 + cps = NUMBER / best_total_s + return best_ms, cps + + +def main() -> None: + col_w = max(len(c.label) for c in CASES) + 2 + header = f"{'Case':<{col_w}} {'Best ms/call':>14} {'calls/sec':>12}" + print(header) + print("-" * len(header)) + + for case in CASES: + ms, cps = bench_case(case) + print(f"{case.label:<{col_w}} {ms:>14.4f} {cps:>12,.0f}") + + # Batch benchmark + batch_times = timeit.repeat(run_batch, repeat=REPEAT, number=NUMBER) + best_batch_s = min(batch_times) + batch_ms = best_batch_s / NUMBER * 1000 + batch_cps = NUMBER / best_batch_s + label = "batch_distance (100 candidates)" + print(f"{label:<{col_w}} {batch_ms:>14.4f} {batch_cps:>12,.0f}") + + print() + print(f"Settings: repeat={REPEAT}, number={NUMBER} calls per timing") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 0944694..bf6c044 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ documentation = "https://niklasvonm.github.io/ocr-stringdist/" [tool.maturin] -features = ["pyo3/extension-module", "python"] +features = ["pyo3/extension-module"] python-source = "python" module-name = "ocr_stringdist._rust_stringdist" diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index b85c38a..54867ff 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -3,11 +3,7 @@ from collections.abc import Iterable from typing import Any, Optional -from ._rust_stringdist import ( - _batch_weighted_levenshtein_distance, - _explain_weighted_levenshtein_distance, - _weighted_levenshtein_distance, -) +from ._rust_stringdist import RustLevenshteinCalculator from .default_ocr_distances import ocr_distance_map from .edit_operation import EditOperation @@ -89,6 +85,15 @@ def __init__( self.default_substitution_cost = default_substitution_cost self.default_insertion_cost = default_insertion_cost self.default_deletion_cost = default_deletion_cost + self._calculator = RustLevenshteinCalculator( + substitution_costs=self.substitution_costs, + insertion_costs=self.insertion_costs, + deletion_costs=self.deletion_costs, + symmetric_substitution=symmetric_substitution, + default_substitution_cost=default_substitution_cost, + default_insertion_cost=default_insertion_cost, + default_deletion_cost=default_deletion_cost, + ) @classmethod def unweighted(cls) -> WeightedLevenshtein: @@ -97,7 +102,7 @@ def unweighted(cls) -> WeightedLevenshtein: def distance(self, s1: str, s2: str) -> float: """Calculates the weighted Levenshtein distance between two strings.""" - return _weighted_levenshtein_distance(s1, s2, **self.__dict__) # type: ignore[no-any-return] + return self._calculator.distance(s1, s2) # type: ignore[no-any-return] def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOperation]: """ @@ -108,7 +113,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe :param filter_matches: If True, 'match' operations are excluded from the result. :return: List of :class:`EditOperation` instances. """ - raw_path = _explain_weighted_levenshtein_distance(s1, s2, **self.__dict__) + raw_path = self._calculator.explain(s1, s2) parsed_path = [EditOperation(*op) for op in raw_path] if filter_matches: return list(filter(lambda op: op.op_type != "match", parsed_path)) @@ -116,7 +121,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe def batch_distance(self, s: str, candidates: list[str]) -> list[float]: """Calculates distances between a string and a list of candidates.""" - return _batch_weighted_levenshtein_distance(s, candidates, **self.__dict__) # type: ignore[no-any-return] + return self._calculator.batch_distance(s, candidates) # type: ignore[no-any-return] @classmethod def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein: diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index b7470ac..267b5c1 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -75,3 +75,46 @@ def test_explain_weighted_levenshtein( manually_filtered_operations = [op for op in full_operations if op.op_type != "match"] assert filtered_operations == manually_filtered_operations assert full_operations == expected_operations + + +def test_explain_transitive_deletion_chain() -> None: + """Issue #12: the explain path for '06'->'0' should expose the sub+del chain.""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + symmetric_substitution=False, + ) + ops = wl.explain("06", "0", filter_matches=False) + assert ops == [ + EditOperation("match", "0", "0", 0.0), + EditOperation("substitute", "6", "G", 0.5), + EditOperation("delete", "G", None, 0.01), + ] + + +def test_explain_transitive_substitution_chain() -> None: + """Triangle inequality: sub(a->b, 0.1) + sub(b->c, 0.1) should expand to two ops.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + ops = wl.explain("a", "c", filter_matches=False) + assert ops == [ + EditOperation("substitute", "a", "b", 0.1), + EditOperation("substitute", "b", "c", 0.1), + ] + + +def test_explain_transitive_insertion_chain() -> None: + """Insertion analogue: ins('x') + sub('x'->'y') chain should appear in the path.""" + wl = WeightedLevenshtein( + substitution_costs={("x", "y"): 0.2}, + insertion_costs={"x": 0.1}, + symmetric_substitution=False, + ) + ops = wl.explain("a", "ay", filter_matches=False) + assert ops == [ + EditOperation("match", "a", "a", 0.0), + EditOperation("insert", None, "x", 0.1), + EditOperation("substitute", "x", "y", 0.2), + ] diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index daf6005..127eb4f 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -551,6 +551,56 @@ def test_costs_above_default_cost() -> None: assert actual_cost == configured_cost +def test_transitive_deletion_chain_distance() -> None: + """Issue #12: sub('6'->'G', 0.5) + del('G', 0.01) = 0.51 < direct del('6', 1.0).""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + symmetric_substitution=False, + ) + assert wl.distance("06", "0") == pytest.approx(0.51) + + +def test_transitive_insertion_subtitution() -> None: + """ + A->AA->AAA->B + """ + wl = WeightedLevenshtein( + insertion_costs={"A": 0.2}, + substitution_costs={("AAA", "B"): 0.1}, + ) + assert wl.distance("A", "B") == pytest.approx(0.5) + + +def test_transitive_insertion_chain_distance() -> None: + """Insertion analogue: ins('x', 0.1) + sub('x'->'y', 0.2) = 0.3 < direct ins('y', 1.0).""" + wl = WeightedLevenshtein( + substitution_costs={("x", "y"): 0.2}, + insertion_costs={"x": 0.1}, + symmetric_substitution=False, + ) + assert wl.distance("a", "ay") == pytest.approx(0.3) + + +def test_direct_op_wins_when_chain_more_expensive() -> None: + """When the direct cost is already lower than any chain, the result is unchanged.""" + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"6": 0.2, "G": 0.01}, + symmetric_substitution=False, + ) + assert wl.distance("06", "0") == pytest.approx(0.2) + + +def test_transitive_substitution_chain_distance() -> None: + """Triangle inequality: sub(a→b, 0.1) + sub(b→c, 0.1) = 0.2 < direct default 1.0.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + assert wl.distance("a", "c") == pytest.approx(0.2) + + def test_serialization() -> None: wl_orig = WeightedLevenshtein( substitution_costs={("a", "b"): 0.5}, diff --git a/src/cost_map.rs b/src/cost_map.rs index 085cb91..bb874af 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use std::fmt::Debug; use std::hash::Hash; -#[cfg(feature = "python")] use pyo3::prelude::*; /// A trait for cost map keys, allowing us to constrain the generic parameter @@ -16,24 +15,18 @@ impl CostKey for SubstitutionKey {} /// Generic cost map structure that works with different key types #[derive(Clone, Debug)] pub struct CostMap { - /// The costs map pub costs: HashMap, - /// Default cost for operations not found in the map default_cost: f64, - /// Maximum token length in the map - pub max_token_length: usize, } impl Default for CostMap where K: Default, { - /// Creates a new CostMap with default values fn default() -> Self { Self { costs: HashMap::new(), default_cost: 1.0, - max_token_length: 1, } } } @@ -48,138 +41,68 @@ impl CostMap { symmetric: bool, ) -> Self { let mut costs = HashMap::with_capacity(custom_costs_input.len() * 2); - let mut max_length = 1; for ((s1, s2), cost) in custom_costs_input { costs.entry((s1.clone(), s2.clone())).or_insert(cost); if symmetric { costs.entry((s2.clone(), s1.clone())).or_insert(cost); } - - // Update max token length - max_length = max_length.max(s1.chars().count()).max(s2.chars().count()); } CostMap { costs, default_cost, - max_token_length: max_length, } } - /// Creates a new substitution CostMap with the specified custom costs. - /// Uses default values for other parameters. - pub fn with_costs(custom_costs: SubstitutionCostMap) -> Self { - Self::new(custom_costs, 1.0, true) - } - - #[cfg(feature = "python")] - /// Creates a substitution CostMap from a Python dictionary. - /// This method is only available when the "python" feature is enabled. pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64, symmetric: bool) -> Self where D: PyDictMethods<'a>, { let mut substitution_costs = SubstitutionCostMap::new(); - let mut max_length = 1; - // Convert Python dictionary to Rust HashMap for (key, value) in py_dict.iter() { if let Ok(key_tuple) = key.extract::<(String, String)>() { if let Ok(cost) = value.extract::() { - substitution_costs.insert((key_tuple.0.clone(), key_tuple.1.clone()), cost); - - // Update max token length - max_length = max_length - .max(key_tuple.0.chars().count()) - .max(key_tuple.1.chars().count()); + substitution_costs.insert((key_tuple.0, key_tuple.1), cost); } } } - // Create the CostMap Self::new(substitution_costs, default_cost, symmetric) } - - /// Gets the substitution cost between two strings. - pub fn get_cost(&self, s1: &str, s2: &str) -> f64 { - if s1 == s2 { - 0.0 // No cost if strings are identical - } else { - let key_pair = (s1.to_string(), s2.to_string()); - - // Lookup the pair (symmetry is handled by storage in `new`) - // Use the map's configured default_cost as the fallback. - self.costs - .get(&key_pair) - .copied() - .unwrap_or(self.default_cost) - } - } - - /// Checks if the cost map contains a specific substitution - pub fn has_key(&self, s1: &str, s2: &str) -> bool { - let key_pair = (s1.to_string(), s2.to_string()); - self.costs.contains_key(&key_pair) - } } // Implementation for SingleTokenKey (single string) impl CostMap { - /// Creates a new single token CostMap for insertion or deletion operations pub fn new(custom_costs_input: SingleTokenCostMap, default_cost: f64) -> Self { - let mut max_length = 1; - - // Calculate max token length - for key in custom_costs_input.keys() { - max_length = max_length.max(key.chars().count()); - } - CostMap { costs: custom_costs_input, default_cost, - max_token_length: max_length, } } - /// Creates a new single token CostMap with the specified custom costs. - /// Uses default value for default cost. - pub fn with_costs(custom_costs: SingleTokenCostMap) -> Self { - Self::new(custom_costs, 1.0) - } - - #[cfg(feature = "python")] - /// Creates a single token CostMap from a Python dictionary. - /// This method is only available when the "python" feature is enabled. pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64) -> Self where D: PyDictMethods<'a>, { let mut single_token_costs = SingleTokenCostMap::new(); - let mut max_length = 1; - // Convert Python dictionary to Rust HashMap for (key, value) in py_dict.iter() { if let Ok(token) = key.extract::() { if let Ok(cost) = value.extract::() { - single_token_costs.insert(token.clone(), cost); - - // Update max token length - max_length = max_length.max(token.chars().count()); + single_token_costs.insert(token, cost); } } } - // Create the CostMap Self::new(single_token_costs, default_cost) } - /// Gets the cost for a single token (insertion or deletion). pub fn get_cost(&self, token: &str) -> f64 { self.costs.get(token).copied().unwrap_or(self.default_cost) } - /// Checks if the cost map contains a specific single token pub fn has_key(&self, token: &str) -> bool { self.costs.contains_key(token) } @@ -187,7 +110,6 @@ impl CostMap { // Common methods for any type of CostMap impl CostMap { - /// Returns the default cost for this cost map pub fn default_cost(&self) -> f64 { self.default_cost } @@ -199,39 +121,17 @@ mod tests { #[test] fn test_single_token_map_default() { - // Test with default initialization let cost_map: CostMap = CostMap::default(); assert_eq!(cost_map.default_cost(), 1.0); assert_eq!(cost_map.get_cost("any_token"), 1.0); assert!(!cost_map.has_key("any_token")); } - #[test] - fn test_single_token_map_with_costs() { - let mut custom_costs = SingleTokenCostMap::new(); - custom_costs.insert("a".to_string(), 0.5); - custom_costs.insert("b".to_string(), 0.8); - - // Test with_costs constructor (default cost 1.0) - let cost_map = CostMap::::with_costs(custom_costs); - - // Test getting costs for tokens - assert_eq!(cost_map.get_cost("a"), 0.5); - assert_eq!(cost_map.get_cost("b"), 0.8); - assert_eq!(cost_map.get_cost("c"), 1.0); // Default cost - - // Test has_key - assert!(cost_map.has_key("a")); - assert!(cost_map.has_key("b")); - assert!(!cost_map.has_key("c")); - } - #[test] fn test_single_token_map_with_custom_default() { let mut custom_costs = SingleTokenCostMap::new(); custom_costs.insert("test".to_string(), 0.3); - // Test new constructor with custom default cost let cost_map = CostMap::::new(custom_costs, 2.0); assert_eq!(cost_map.default_cost(), 2.0); @@ -241,40 +141,10 @@ mod tests { #[test] fn test_substitution_map_default() { - let cost_map: CostMap = CostMap { - costs: HashMap::new(), - default_cost: 1.0, - max_token_length: 1, - }; + let cost_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); assert_eq!(cost_map.default_cost(), 1.0); - assert_eq!(cost_map.get_cost("a", "b"), 1.0); - assert!(!cost_map.has_key("a", "b")); - } - - #[test] - fn test_substitution_map_with_costs() { - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("0".to_string(), "o".to_string()), 0.2); - custom_costs.insert(("l".to_string(), "1".to_string()), 0.3); - - // Test with_costs constructor (symmetric by default) - let cost_map = CostMap::::with_costs(custom_costs); - - // Test getting costs - assert_eq!(cost_map.get_cost("0", "o"), 0.2); - assert_eq!(cost_map.get_cost("o", "0"), 0.2); // Symmetry check - assert_eq!(cost_map.get_cost("l", "1"), 0.3); - assert_eq!(cost_map.get_cost("1", "l"), 0.3); // Symmetry check - assert_eq!(cost_map.get_cost("a", "b"), 1.0); // Default - - // Test same character - assert_eq!(cost_map.get_cost("a", "a"), 0.0); // Same char = 0 cost - - // Test has_key - assert!(cost_map.has_key("0", "o")); - assert!(cost_map.has_key("o", "0")); // Symmetry check - assert!(!cost_map.has_key("a", "b")); + assert!(cost_map.costs.is_empty()); } #[test] @@ -282,24 +152,20 @@ mod tests { let mut custom_costs = SubstitutionCostMap::new(); custom_costs.insert(("a".to_string(), "b".to_string()), 0.4); - // Create with symmetric=false let cost_map = CostMap::::new(custom_costs, 1.5, false); - // Test asymmetry - assert_eq!(cost_map.get_cost("a", "b"), 0.4); - assert_eq!(cost_map.get_cost("b", "a"), 1.5); // Should be default cost - - assert!(cost_map.has_key("a", "b")); - assert!(!cost_map.has_key("b", "a")); // Should not exist + assert_eq!(cost_map.costs[&("a".to_string(), "b".to_string())], 0.4); + assert!(!cost_map + .costs + .contains_key(&("b".to_string(), "a".to_string()))); + assert_eq!(cost_map.default_cost(), 1.5); } #[test] fn test_default_cost_accessor() { - // Test for SubstitutionKey let sub_map = CostMap::::new(HashMap::new(), 2.5, true); assert_eq!(sub_map.default_cost(), 2.5); - // Test for SingleTokenKey let single_map = CostMap::::new(HashMap::new(), 3.0); assert_eq!(single_map.default_cost(), 3.0); } diff --git a/src/lib.rs b/src/lib.rs index b19214b..cf011de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,8 @@ mod cost_map; mod explanation; +mod transitive_costs; mod types; mod weighted_levenshtein; -pub use cost_map::CostMap; -pub use types::*; -pub use weighted_levenshtein::{ - custom_levenshtein_distance_with_cost_maps, explain_custom_levenshtein_distance, -}; - -#[cfg(feature = "python")] mod rust_stringdist; -#[cfg(feature = "python")] pub use rust_stringdist::_rust_stringdist; diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index a2e82db..d51409f 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,8 +1,12 @@ use crate::cost_map::CostMap; use crate::explanation::EditOperation; +use crate::transitive_costs::{ + compute_effective_deletion_costs, compute_effective_insertion_costs, + compute_effective_substitution_costs, EffectiveSingleTokenCosts, EffectiveSubstitutionCosts, +}; use crate::types::{SingleTokenKey, SubstitutionKey}; -use crate::weighted_levenshtein::custom_levenshtein_distance_with_cost_maps as calculate_core; -use crate::weighted_levenshtein::explain_custom_levenshtein_distance as explain_core; +use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; +use crate::weighted_levenshtein::explain_custom_levenshtein_precomputed; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; @@ -29,14 +33,30 @@ impl<'py> IntoPyObject<'py> for EditOperation { } } -/// Facade between the Python interface and the core algorithm implementation. -struct LevenshteinCalculator { - substitution_cost_map: CostMap, - insertion_cost_map: CostMap, - deletion_cost_map: CostMap, +/// Precomputes the transitive closure once and reuses it across all distance calls. +/// +/// Exposed to Python so that `WeightedLevenshtein.__init__` can pay the Dijkstra +/// cost once and avoid recomputing it on every `.distance()` / `.batch_distance()` call. +#[pyclass] +#[derive(Debug)] +struct RustLevenshteinCalculator { + eff_sub: EffectiveSubstitutionCosts, + eff_del: EffectiveSingleTokenCosts, + eff_ins: EffectiveSingleTokenCosts, } -impl LevenshteinCalculator { +#[pymethods] +impl RustLevenshteinCalculator { + #[new] + #[pyo3(signature = ( + substitution_costs, + insertion_costs, + deletion_costs, + symmetric_substitution = true, + default_substitution_cost = 1.0, + default_insertion_cost = 1.0, + default_deletion_cost = 1.0, + ))] fn new( substitution_costs: &Bound<'_, PyDict>, insertion_costs: &Bound<'_, PyDict>, @@ -50,43 +70,56 @@ impl LevenshteinCalculator { validate_default_cost(default_insertion_cost)?; validate_default_cost(default_deletion_cost)?; - let substitution_cost_map = CostMap::::from_py_dict( + let sub_map = CostMap::::from_py_dict( substitution_costs, default_substitution_cost, symmetric_substitution, ); - - let insertion_cost_map = + let ins_map = CostMap::::from_py_dict(insertion_costs, default_insertion_cost); - - let deletion_cost_map = + let del_map = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); + let eff_sub = compute_effective_substitution_costs(&sub_map); + let eff_del = compute_effective_deletion_costs(&del_map, &sub_map); + let eff_ins = compute_effective_insertion_costs(&ins_map, &sub_map); + Ok(Self { - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, + eff_sub, + eff_del, + eff_ins, }) } fn distance(&self, a: &str, b: &str) -> f64 { - calculate_core( - a, - b, - &self.substitution_cost_map, - &self.insertion_cost_map, - &self.deletion_cost_map, - ) + custom_levenshtein_distance_precomputed(a, b, &self.eff_sub, &self.eff_ins, &self.eff_del) } - fn explain(&self, a: &str, b: &str) -> Vec { - explain_core( - a, - b, - &self.substitution_cost_map, - &self.insertion_cost_map, - &self.deletion_cost_map, - ) + fn batch_distance(&self, py: Python<'_>, s: String, candidates: Vec) -> Vec { + if candidates.is_empty() { + return Vec::new(); + } + py.allow_threads(|| { + candidates + .par_iter() + .map(|c| { + custom_levenshtein_distance_precomputed( + &s, + c, + &self.eff_sub, + &self.eff_ins, + &self.eff_del, + ) + }) + .collect() + }) + } + + fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult> { + explain_custom_levenshtein_precomputed(a, b, &self.eff_sub, &self.eff_ins, &self.eff_del) + .into_iter() + .map(|op| op.into_pyobject(py).map(|bound| bound.into())) + .collect::>>() } } @@ -100,137 +133,10 @@ fn validate_default_cost(default_cost: f64) -> PyResult<()> { Ok(()) } -// Calculates the weighted Levenshtein distance with a custom cost map from Python. -#[pyfunction] -#[pyo3(signature = ( - a, - b, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _weighted_levenshtein_distance( - a: &str, - b: &str, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; - - Ok(calculator.distance(a, b)) -} - -#[pyfunction] -#[pyo3(signature = ( - a, - b, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _explain_weighted_levenshtein_distance( - py: Python, // For conversion - a: &str, - b: &str, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult> { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; - - let path = calculator.explain(a, b); - - path.into_iter() - .map(|op| op.into_pyobject(py).map(|bound| bound.into())) - .collect::>>() -} - -// Calculates the weighted Levenshtein distance between a string and a list of candidates. -#[pyfunction] -#[pyo3(signature = ( - s, - candidates, - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution = true, - default_substitution_cost = 1.0, - default_insertion_cost = 1.0, - default_deletion_cost = 1.0, -))] -fn _batch_weighted_levenshtein_distance( - s: &str, - candidates: Vec, - substitution_costs: &Bound<'_, PyDict>, - insertion_costs: &Bound<'_, PyDict>, - deletion_costs: &Bound<'_, PyDict>, - symmetric_substitution: bool, - default_substitution_cost: f64, - default_insertion_cost: f64, - default_deletion_cost: f64, -) -> PyResult> { - let calculator = LevenshteinCalculator::new( - substitution_costs, - insertion_costs, - deletion_costs, - symmetric_substitution, - default_substitution_cost, - default_insertion_cost, - default_deletion_cost, - )?; - - if candidates.is_empty() { - return Ok(Vec::new()); - } - - // Calculate distances for each candidate in parallel - let distances: Vec = candidates - .par_iter() - .map(|candidate| calculator.distance(s, candidate)) - .collect(); - - Ok(distances) -} - /// A Python module implemented in Rust. #[pymodule] pub fn _rust_stringdist(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(_weighted_levenshtein_distance, m)?)?; - m.add_function(wrap_pyfunction!(_batch_weighted_levenshtein_distance, m)?)?; - m.add_function(wrap_pyfunction!(_explain_weighted_levenshtein_distance, m)?)?; + m.add_class::()?; Ok(()) } @@ -239,141 +145,70 @@ mod tests { use super::*; use pyo3::types::{PyDict, PyList, PyTuple}; + fn make_calculator<'py>( + py: Python<'py>, + sub_costs: &[((&str, &str), f64)], + ins_costs: &[(&str, f64)], + del_costs: &[(&str, f64)], + symmetric: bool, + ) -> RustLevenshteinCalculator { + let sub = PyDict::new(py); + for ((a, b), c) in sub_costs { + sub.set_item((*a, *b), c).unwrap(); + } + let ins = PyDict::new(py); + for (k, v) in ins_costs { + ins.set_item(k, v).unwrap(); + } + let del = PyDict::new(py); + for (k, v) in del_costs { + del.set_item(k, v).unwrap(); + } + RustLevenshteinCalculator::new(&sub, &ins, &del, symmetric, 1.0, 1.0, 1.0).unwrap() + } + #[test] - fn test_levenshtein_distance_with_empty_costs() { + fn test_distance_with_empty_costs() { Python::with_gil(|py| { - let a = "hello"; - let b = "hxllo"; - - let substitution_costs = PyDict::new(py); - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert_eq!(distance, 1.0); + let calc = make_calculator(py, &[], &[], &[], true); + assert_eq!(calc.distance("hello", "hxllo"), 1.0); }); } #[test] - fn test_levenshtein_with_custom_substitution_cost() { + fn test_distance_with_custom_substitution_cost() { Python::with_gil(|py| { - let a = "hello"; - let b = "hxllo"; - - let substitution_costs = PyDict::new(py); - substitution_costs.set_item(("e", "x"), 0.2).unwrap(); - - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert!((distance - 0.2).abs() < f64::EPSILON); + let calc = make_calculator(py, &[(("e", "x"), 0.2)], &[], &[], true); + assert!((calc.distance("hello", "hxllo") - 0.2).abs() < f64::EPSILON); }); } #[test] - fn test_levenshtein_asymmetric_substitution() { + fn test_asymmetric_substitution() { Python::with_gil(|py| { - let a = "ab"; - let b = "ba"; - - let substitution_costs = PyDict::new(py); - substitution_costs.set_item(("a", "b"), 0.1).unwrap(); - - let insertion_costs = PyDict::new(py); - let deletion_costs = PyDict::new(py); - - let distance = _weighted_levenshtein_distance( - a, - b, - &substitution_costs, - &insertion_costs, - &deletion_costs, - false, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - // Cost should be 0.1 (a->b) + 1.0 (b->a, default) - assert!((distance - 1.1).abs() < f64::EPSILON); + let calc = make_calculator(py, &[(("a", "b"), 0.1)], &[], &[], false); + // a->b costs 0.1; b->a uses default 1.0 -> total 1.1 + assert!((calc.distance("ab", "ba") - 1.1).abs() < f64::EPSILON); }); } #[test] fn test_negative_default_cost_errors() { Python::with_gil(|py| { - let a = "test"; - let b = "toast"; - let empty_costs = PyDict::new(py); + let empty = PyDict::new(py); - // Test negative substitution cost - let sub_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - -1.0, - 1.0, - 1.0, - ); + let sub_err = + RustLevenshteinCalculator::new(&empty, &empty, &empty, true, -1.0, 1.0, 1.0); assert!(sub_err.is_err()); assert!(sub_err.unwrap_err().is_instance_of::(py)); - // Test negative insertion cost - let ins_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - -1.0, - 1.0, - ); + let ins_err = + RustLevenshteinCalculator::new(&empty, &empty, &empty, true, 1.0, -1.0, 1.0); assert!(ins_err.is_err()); assert!(ins_err.unwrap_err().is_instance_of::(py)); - // Test negative deletion cost - let del_err = _weighted_levenshtein_distance( - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - -1.0, - ); + let del_err = + RustLevenshteinCalculator::new(&empty, &empty, &empty, true, 1.0, 1.0, -1.0); assert!(del_err.is_err()); assert!(del_err.unwrap_err().is_instance_of::(py)); }); @@ -428,111 +263,51 @@ mod tests { } #[test] - fn test_explain_weighted_levenshtein_distance() { + fn test_explain() { Python::with_gil(|py| { - let a = "cat"; - let b = "car"; - let empty_costs = PyDict::new(py); - - let result = _explain_weighted_levenshtein_distance( - py, - a, - b, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); + let calc = make_calculator(py, &[], &[], &[], true); + let result = calc.explain(py, "cat", "car").unwrap(); let py_list = PyList::new(py, result).unwrap(); - assert_eq!(py_list.clone().len(), 3); - - let first_op = py_list - .clone() - .get_item(0) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - first_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "match" - ); - - let second_op = py_list - .clone() - .get_item(1) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - second_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "match" - ); - - let third_op = py_list - .clone() - .get_item(2) - .unwrap() - .downcast_into::() - .unwrap(); - assert_eq!( - third_op.get_item(0).unwrap().extract::<&str>().unwrap(), - "substitute" - ); - assert_eq!(third_op.get_item(3).unwrap().extract::().unwrap(), 1.0); + assert_eq!(py_list.len(), 3); + + let op = |i: usize| -> String { + py_list + .get_item(i) + .unwrap() + .downcast_into::() + .unwrap() + .get_item(0) + .unwrap() + .extract::() + .unwrap() + }; + assert_eq!(op(0), "match"); + assert_eq!(op(1), "match"); + assert_eq!(op(2), "substitute"); }); } #[test] - fn test_batch_weighted_levenshtein_distance() { + fn test_batch_distance() { Python::with_gil(|py| { - let s = "book"; - let candidates = vec!["back".to_string(), "books".to_string(), "look".to_string()]; - let empty_costs = PyDict::new(py); - - let distances = _batch_weighted_levenshtein_distance( - s, - candidates, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert_eq!(distances.len(), 3); + let calc = make_calculator(py, &[], &[], &[], true); + let distances = calc.batch_distance( + py, + "book".to_string(), + vec!["back".to_string(), "books".to_string(), "look".to_string()], + ); assert_eq!(distances, vec![2.0, 1.0, 1.0]); }); } #[test] - fn test_batch_with_empty_candidate_list() { + fn test_batch_distance_empty() { Python::with_gil(|py| { - let s = "test"; - let candidates: Vec = vec![]; - let empty_costs = PyDict::new(py); - - let distances = _batch_weighted_levenshtein_distance( - s, - candidates, - &empty_costs, - &empty_costs, - &empty_costs, - true, - 1.0, - 1.0, - 1.0, - ) - .unwrap(); - - assert!(distances.is_empty()); + let calc = make_calculator(py, &[], &[], &[], true); + assert!(calc + .batch_distance(py, "test".to_string(), vec![]) + .is_empty()); }); } } diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs new file mode 100644 index 0000000..728778d --- /dev/null +++ b/src/transitive_costs.rs @@ -0,0 +1,782 @@ +//! Transitive closure of edit costs via Dijkstra. +//! +//! This module precomputes the globally cheapest cost for every operation via +//! Dijkstra, so the DP uses optimal paths without per-cell search. +//! +//! Results are cached in [`EffectiveSingleTokenCosts`] (del/ins) and +//! [`EffectiveSubstitutionCosts`] (sub), computed once at construction. + +use crate::cost_map::CostMap; +use crate::types::{SingleTokenKey, SubstitutionKey}; +use std::collections::{BinaryHeap, HashMap, HashSet}; + +type DistMap = HashMap; +type PrevMap = HashMap>; + +// Public types + +/// How the minimum effective cost for a single-token operation was achieved. +#[derive(Clone, Debug, PartialEq)] +pub enum EffectiveOpChain { + /// The token is deleted / inserted directly at its mapped or default cost. + Direct, + + /// A cheaper path exists through a chain of substitutions. + /// + /// `steps` holds the substitution edges in forward order: + /// - **Deletion**: `(source -> x1, c1), (x1 -> x2, c2), …, (xn-1 -> terminal, cn)` then + /// `delete(terminal)` at `terminal_cost`. + /// - **Insertion**: `insert(initial)` at `terminal_cost`, then + /// `(initial -> x1, c1), …, (xn -> target, cn)`. + Via { + /// Substitution edges `(from, to, cost)` in forward order. + steps: Vec<(String, String, f64)>, + /// Cost of the direct `del` / `ins` at the terminal (deletion) or initial + /// (insertion) node of the chain. + terminal_cost: f64, + }, +} + +/// Precomputed effective single-token operation costs (deletion or insertion). +/// +/// Replaces the raw [`CostMap`] inside the DP so the algorithm +/// automatically uses the globally cheapest edit path. +#[derive(Debug)] +pub struct EffectiveSingleTokenCosts { + entries: HashMap, + default_cost: f64, + pub max_token_length: usize, +} + +impl EffectiveSingleTokenCosts { + #[inline] + pub fn get_cost(&self, token: &str) -> f64 { + self.entries + .get(token) + .map(|(c, _)| *c) + .unwrap_or(self.default_cost) + } + + #[inline] + pub fn get_chain(&self, token: &str) -> EffectiveOpChain { + self.entries + .get(token) + .map(|(_, ch)| ch.clone()) + .unwrap_or(EffectiveOpChain::Direct) + } + + #[inline] + pub fn has_key(&self, token: &str) -> bool { + self.entries.contains_key(token) + } +} + +/// How the minimum effective substitution cost was achieved. +#[derive(Clone, Debug, PartialEq)] +pub enum EffectiveSubChain { + /// Direct substitution at the mapped cost. + Direct, + + /// A cheaper path was found through a chain of substitutions. + /// + /// `steps` holds edges `(from, to, cost)` in forward order, covering the + /// full path from the source token to the target token. + Via { + /// Substitution edges `(from, to, cost)` in forward order. + steps: Vec<(String, String, f64)>, + }, +} + +/// Precomputed effective substitution costs (all-pairs shortest paths). +/// +/// Replaces the raw [`CostMap`] inside the DP so the algorithm +/// automatically uses the globally cheapest substitution path. +#[derive(Debug)] +pub struct EffectiveSubstitutionCosts { + entries: HashMap<(String, String), (f64, EffectiveSubChain)>, + default_cost: f64, + pub max_token_length: usize, +} + +impl EffectiveSubstitutionCosts { + #[inline] + pub fn get_cost(&self, source: &str, target: &str) -> f64 { + self.entries + .get(&(source.to_owned(), target.to_owned())) + .map(|(c, _)| *c) + .unwrap_or(self.default_cost) + } + + #[inline] + pub fn get_chain(&self, source: &str, target: &str) -> EffectiveSubChain { + self.entries + .get(&(source.to_owned(), target.to_owned())) + .map(|(_, ch)| ch.clone()) + .unwrap_or(EffectiveSubChain::Direct) + } + + #[inline] + pub fn has_key(&self, source: &str, target: &str) -> bool { + self.entries + .contains_key(&(source.to_owned(), target.to_owned())) + } +} + +// Public constructors + +/// Precomputes effective deletion costs. +/// +/// Runs multi-source Dijkstra on the **reversed** substitution graph, seeded +/// with `del(x)` per node: +/// +/// ```text +/// eff_del(s) = min_x { shortest_sub_path(s -> x) + del(x) } +/// ``` +pub fn compute_effective_deletion_costs( + del_map: &CostMap, + sub_map: &CostMap, +) -> EffectiveSingleTokenCosts { + let default_cost = del_map.default_cost(); + + let all_tokens = collect_tokens(del_map, sub_map); + + // Initial distance for every token = its direct deletion cost. + let initial: HashMap = all_tokens + .iter() + .map(|t| (t.clone(), del_map.get_cost(t))) + .collect(); + + let rev_graph = build_reversed_sub_graph(sub_map); + let (dist, prev) = dijkstra(&rev_graph, &initial); + + build_entries( + &all_tokens, + &dist, + &initial, + &prev, + |token| del_map.has_key(token), + |terminal| del_map.get_cost(terminal), + default_cost, + ChainDirection::Deletion, + ) +} + +/// Precomputes effective insertion costs. +/// +/// Runs multi-source Dijkstra on the **forward** substitution graph, seeded +/// with `ins(x)` per node: +/// +/// ```text +/// eff_ins(t) = min_x { ins(x) + shortest_sub_path(x -> t) } +/// ``` +pub fn compute_effective_insertion_costs( + ins_map: &CostMap, + sub_map: &CostMap, +) -> EffectiveSingleTokenCosts { + let default_cost = ins_map.default_cost(); + + let all_tokens = collect_tokens(ins_map, sub_map); + + let initial: HashMap = all_tokens + .iter() + .map(|t| (t.clone(), ins_map.get_cost(t))) + .collect(); + + let fwd_graph = build_forward_sub_graph(sub_map); + let (dist, prev) = dijkstra(&fwd_graph, &initial); + + build_entries( + &all_tokens, + &dist, + &initial, + &prev, + |token| ins_map.has_key(token), + |initial_node| ins_map.get_cost(initial_node), + default_cost, + ChainDirection::Insertion, + ) +} + +/// Precomputes effective substitution costs via all-pairs shortest paths. +/// +/// For every source token in the substitution graph, runs Dijkstra on the +/// forward graph. When a chain `sub(a->b) + sub(b->c)` is cheaper than the +/// default cost for `sub(a->c)`, the result is stored so the DP automatically +/// uses the cheaper path. +/// +/// ```text +/// eff_sub(s, t) = min_path { sum of edge costs along s -> t } +/// ``` +pub fn compute_effective_substitution_costs( + sub_map: &CostMap, +) -> EffectiveSubstitutionCosts { + let default_cost = sub_map.default_cost(); + + let all_tokens: HashSet = sub_map + .costs + .keys() + .flat_map(|(s, t)| [s.clone(), t.clone()]) + .collect(); + + let fwd_graph = build_forward_sub_graph(sub_map); + let mut entries: HashMap<(String, String), (f64, EffectiveSubChain)> = HashMap::new(); + + for source in &all_tokens { + let initial: DistMap = [(source.clone(), 0.0)].into_iter().collect(); + let (dist, prev) = dijkstra(&fwd_graph, &initial); + + for target in &all_tokens { + if target == source { + continue; + } + let chain_cost = dist.get(target).copied().unwrap_or(f64::INFINITY); + let direct_cost_opt = sub_map + .costs + .get(&(source.clone(), target.clone())) + .copied(); + if let Some(entry) = + classify_sub_pair(chain_cost, direct_cost_opt, default_cost, target, &prev) + { + entries.insert((source.clone(), target.clone()), entry); + } + } + } + + let max_len = entries + .keys() + .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) + .max() + .unwrap_or(1) + .max(1); + + EffectiveSubstitutionCosts { + entries, + default_cost, + max_token_length: max_len, + } +} + +// Dijkstra + +/// Min-heap entry (BinaryHeap is a max-heap; reversed comparison gives min behaviour). +#[derive(Debug, Clone, PartialEq)] +struct HeapEntry { + cost: f64, + token: String, +} + +impl Eq for HeapEntry {} + +impl PartialOrd for HeapEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other + .cost + .partial_cmp(&self.cost) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| self.token.cmp(&other.token)) + } +} + +/// Generic Dijkstra. +/// +/// Returns `(dist, prev)` where `prev[v] = Some((u, edge_cost))` means node +/// `v` was relaxed via the edge `u -> v`. +fn dijkstra(graph: &HashMap>, initial: &DistMap) -> (DistMap, PrevMap) { + let mut dist = initial.clone(); + let mut prev: PrevMap = initial.keys().map(|k| (k.clone(), None)).collect(); + + let mut heap: BinaryHeap = dist + .iter() + .map(|(t, &c)| HeapEntry { + cost: c, + token: t.clone(), + }) + .collect(); + + while let Some(HeapEntry { cost, token }) = heap.pop() { + if cost > dist[&token] { + continue; // stale entry + } + for (nbr, edge_cost) in graph.get(&token).into_iter().flatten() { + let new_cost = cost + edge_cost; + let cur = dist.get(nbr).copied().unwrap_or(f64::INFINITY); + if new_cost < cur { + dist.insert(nbr.clone(), new_cost); + prev.insert(nbr.clone(), Some((token.clone(), *edge_cost))); + heap.push(HeapEntry { + cost: new_cost, + token: nbr.clone(), + }); + } + } + } + + (dist, prev) +} + +// Chain reconstruction & entry building + +enum ChainDirection { + Deletion, + Insertion, +} + +/// Builds the `EffectiveSingleTokenCosts` entries after Dijkstra. +/// +/// A token is added iff it was explicitly configured in the raw map OR the +/// Dijkstra found a chain that is strictly cheaper than the direct cost. +/// This prevents tokens that only appear in the substitution graph (but whose +/// chain is NOT cheaper) from being registered as "explicitly deletable/ +/// insertable", which would otherwise wrongly enable multi-char DP operations. +#[allow(clippy::too_many_arguments)] +fn build_entries( + all_tokens: &HashSet, + dist: &DistMap, + initial: &DistMap, + prev: &PrevMap, + in_raw_map: impl Fn(&str) -> bool, + terminal_cost_fn: impl Fn(&str) -> f64, + default_cost: f64, + direction: ChainDirection, +) -> EffectiveSingleTokenCosts { + let mut entries: HashMap = HashMap::new(); + + for token in all_tokens { + let final_dist = dist[token]; + let initial_cost = initial[token]; + let improved = final_dist < initial_cost; + + if !improved && !in_raw_map(token) { + continue; + } + + let chain = if prev[token].is_none() { + EffectiveOpChain::Direct + } else { + match direction { + ChainDirection::Deletion => { + let steps = reconstruct_deletion_steps(token, prev); + let terminal = steps.last().map(|(_, t, _)| t.as_str()).unwrap_or(token); + EffectiveOpChain::Via { + terminal_cost: terminal_cost_fn(terminal), + steps, + } + } + ChainDirection::Insertion => { + let (steps, initial_token) = reconstruct_insertion_steps(token, prev); + EffectiveOpChain::Via { + terminal_cost: terminal_cost_fn(&initial_token), + steps, + } + } + } + }; + + entries.insert(token.clone(), (final_dist, chain)); + } + + let max_len = entries + .keys() + .map(|k| k.chars().count()) + .max() + .unwrap_or(1) + .max(1); + + EffectiveSingleTokenCosts { + entries, + default_cost, + max_token_length: max_len, + } +} + +/// Follows `prev` forward from `source` (deletion direction). +/// Returns steps `[(s, x1, c1), (x1, x2, c2), …]` in forward order. +fn reconstruct_deletion_steps(source: &str, prev: &PrevMap) -> Vec<(String, String, f64)> { + let mut steps = Vec::new(); + let mut cur = source.to_string(); + while let Some((next, c)) = prev.get(&cur).and_then(|o| o.as_ref()) { + steps.push((cur.clone(), next.clone(), *c)); + cur = next.clone(); + } + steps +} + +/// Follows `prev` backward from `target`, returns `(steps, seed)` where +/// `steps` are in forward order and `seed` is the path origin (the node +/// with `prev[seed] = None`). +/// +/// Used for both insertion chains and substitution chains. +fn reconstruct_steps_to(target: &str, prev: &PrevMap) -> Vec<(String, String, f64)> { + let mut steps_rev = Vec::new(); + let mut cur = target.to_string(); + while let Some((from, c)) = prev.get(&cur).and_then(|o| o.as_ref()) { + steps_rev.push((from.clone(), cur.clone(), *c)); + cur = from.clone(); + } + steps_rev.reverse(); + steps_rev +} + +/// Wrapper that also returns the seed (initial) token — used for insertion chains. +fn reconstruct_insertion_steps( + target: &str, + prev: &PrevMap, +) -> (Vec<(String, String, f64)>, String) { + let steps = reconstruct_steps_to(target, prev); + let initial = steps + .first() + .map(|(f, _, _)| f.clone()) + .unwrap_or_else(|| target.to_string()); + (steps, initial) +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +fn collect_tokens( + single_map: &CostMap, + sub_map: &CostMap, +) -> HashSet { + let mut tokens: HashSet = single_map.costs.keys().cloned().collect(); + for (src, tgt) in sub_map.costs.keys() { + tokens.insert(src.clone()); + tokens.insert(tgt.clone()); + } + tokens +} + +/// Builds the forward substitution graph: an edge `src -> tgt` has the sub cost. +fn build_forward_sub_graph( + sub_map: &CostMap, +) -> HashMap> { + let mut graph: HashMap> = HashMap::new(); + for ((src, tgt), &c) in &sub_map.costs { + graph.entry(src.clone()).or_default().push((tgt.clone(), c)); + } + graph +} + +/// Builds the reversed substitution graph: an edge `tgt -> src` has the sub cost. +fn build_reversed_sub_graph( + sub_map: &CostMap, +) -> HashMap> { + let mut graph: HashMap> = HashMap::new(); + for ((src, tgt), &c) in &sub_map.costs { + graph.entry(tgt.clone()).or_default().push((src.clone(), c)); + } + graph +} + +/// Decides how to record a substitution pair `(source, target)` given the +/// Dijkstra result for `source`. +/// +/// Returns `None` if the pair should be omitted (not in the raw map and no +/// improvement over the default). Otherwise returns the effective cost and chain. +fn classify_sub_pair( + chain_cost: f64, + direct_cost_opt: Option, + default_cost: f64, + target: &str, + prev: &PrevMap, +) -> Option<(f64, EffectiveSubChain)> { + let in_raw_map = direct_cost_opt.is_some(); + if !in_raw_map && chain_cost >= default_cost { + return None; + } + let direct = direct_cost_opt.unwrap_or(f64::INFINITY); + if chain_cost < direct { + let steps = reconstruct_steps_to(target, prev); + Some((chain_cost, EffectiveSubChain::Via { steps })) + } else { + Some((direct, EffectiveSubChain::Direct)) + } +} + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{SingleTokenCostMap, SubstitutionCostMap}; + + fn make_del_map(entries: &[(&str, f64)]) -> CostMap { + let costs: SingleTokenCostMap = entries.iter().map(|(k, v)| (k.to_string(), *v)).collect(); + CostMap::::new(costs, 1.0) + } + + fn make_ins_map(entries: &[(&str, f64)]) -> CostMap { + make_del_map(entries) + } + + fn make_sub_map(entries: &[((&str, &str), f64)]) -> CostMap { + let costs: SubstitutionCostMap = entries + .iter() + .map(|((a, b), v)| ((a.to_string(), b.to_string()), *v)) + .collect(); + CostMap::::new(costs, 1.0, false) + } + + fn approx_eq(a: f64, b: f64) -> bool { + (a - b).abs() < f64::EPSILON * 100.0 + } + + // ── Deletion ────────────────────────────────────────────────────────────── + + #[test] + fn direct_deletion_without_substitution_map() { + let eff = + compute_effective_deletion_costs(&make_del_map(&[("a", 0.3)]), &make_sub_map(&[])); + assert!(approx_eq(eff.get_cost("a"), 0.3)); + assert_eq!(eff.get_chain("a"), EffectiveOpChain::Direct); + assert!(approx_eq(eff.get_cost("z"), 1.0)); + assert_eq!(eff.get_chain("z"), EffectiveOpChain::Direct); + } + + #[test] + fn one_hop_deletion_chain() { + // Issue #12: sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 < direct del("6", 1.0) + let eff = compute_effective_deletion_costs( + &make_del_map(&[("G", 0.01)]), + &make_sub_map(&[(("6", "G"), 0.5)]), + ); + + // "6": chain is strictly cheaper — must be present with Via chain + assert!(approx_eq(eff.get_cost("6"), 0.51)); + assert_eq!( + eff.get_chain("6"), + EffectiveOpChain::Via { + steps: vec![("6".to_string(), "G".to_string(), 0.5)], + terminal_cost: 0.01, + } + ); + assert!(eff.has_key("6")); + + // "G": explicit in del_map, direct — must not be wrapped in a chain + assert!(approx_eq(eff.get_cost("G"), 0.01)); + assert_eq!(eff.get_chain("G"), EffectiveOpChain::Direct); + assert!(eff.has_key("G")); + + // unknown token falls back to default cost, Direct chain, not registered + assert!(approx_eq(eff.get_cost("X"), 1.0)); + assert_eq!(eff.get_chain("X"), EffectiveOpChain::Direct); + assert!(!eff.has_key("X")); + + // all tokens are single-character + assert_eq!(eff.max_token_length, 1); + } + + #[test] + fn three_hop_deletion_chain() { + // A->B (0.3), B->C (0.2), del(C)=0.01 -> chain(A) = 0.51 < 1.0 + let eff = compute_effective_deletion_costs( + &make_del_map(&[("C", 0.01)]), + &make_sub_map(&[(("A", "B"), 0.3), (("B", "C"), 0.2)]), + ); + assert!(approx_eq(eff.get_cost("A"), 0.51)); + assert_eq!( + eff.get_chain("A"), + EffectiveOpChain::Via { + steps: vec![ + ("A".to_string(), "B".to_string(), 0.3), + ("B".to_string(), "C".to_string(), 0.2), + ], + terminal_cost: 0.01, + } + ); + // B also improves: 0.2 + 0.01 = 0.21 < 1.0 + assert!(approx_eq(eff.get_cost("B"), 0.21)); + } + + #[test] + fn direct_deletion_preserved_when_chain_is_more_expensive() { + // sub("a"->"b") = 0.5, del("b") = 0.8 -> chain = 1.3 > del("a") = 0.2 + let eff = compute_effective_deletion_costs( + &make_del_map(&[("a", 0.2), ("b", 0.8)]), + &make_sub_map(&[(("a", "b"), 0.5)]), + ); + assert!(approx_eq(eff.get_cost("a"), 0.2)); + assert_eq!(eff.get_chain("a"), EffectiveOpChain::Direct); + } + + #[test] + fn multiple_substitution_targets_best_chain_wins() { + // sub("X"->"A") = 0.4, del("A") = 0.3 -> 0.7 + // sub("X"->"B") = 0.1, del("B") = 0.5 -> 0.6 ← winner + let eff = compute_effective_deletion_costs( + &make_del_map(&[("A", 0.3), ("B", 0.5)]), + &make_sub_map(&[(("X", "A"), 0.4), (("X", "B"), 0.1)]), + ); + assert!(approx_eq(eff.get_cost("X"), 0.6)); + assert_eq!( + eff.get_chain("X"), + EffectiveOpChain::Via { + steps: vec![("X".to_string(), "B".to_string(), 0.1)], + terminal_cost: 0.5, + } + ); + } + + // ── Insertion ───────────────────────────────────────────────────────────── + + #[test] + fn one_hop_insertion_chain() { + // ins("x") = 0.1, sub("x"->"y") = 0.2 -> chain ins("y") = 0.3 < 1.0 + let eff = compute_effective_insertion_costs( + &make_ins_map(&[("x", 0.1)]), + &make_sub_map(&[(("x", "y"), 0.2)]), + ); + assert!(approx_eq(eff.get_cost("y"), 0.3)); + assert_eq!( + eff.get_chain("y"), + EffectiveOpChain::Via { + steps: vec![("x".to_string(), "y".to_string(), 0.2)], + terminal_cost: 0.1, + } + ); + } + + #[test] + fn three_hop_insertion_chain() { + // ins(A)=0.05, sub(A->B)=0.3, sub(B->C)=0.2 -> chain(C)=0.55 < 1.0 + let eff = compute_effective_insertion_costs( + &make_ins_map(&[("A", 0.05)]), + &make_sub_map(&[(("A", "B"), 0.3), (("B", "C"), 0.2)]), + ); + assert!(approx_eq(eff.get_cost("C"), 0.55)); + assert_eq!( + eff.get_chain("C"), + EffectiveOpChain::Via { + steps: vec![ + ("A".to_string(), "B".to_string(), 0.3), + ("B".to_string(), "C".to_string(), 0.2), + ], + terminal_cost: 0.05, + } + ); + } + + #[test] + fn direct_insertion_preserved_when_chain_is_more_expensive() { + // ins("y") = 0.1, ins("x") = 0.9, sub("x"->"y") = 0.5 -> chain = 1.4 > 0.1 + let eff = compute_effective_insertion_costs( + &make_ins_map(&[("y", 0.1), ("x", 0.9)]), + &make_sub_map(&[(("x", "y"), 0.5)]), + ); + assert!(approx_eq(eff.get_cost("y"), 0.1)); + assert_eq!(eff.get_chain("y"), EffectiveOpChain::Direct); + } + + // ── has_key semantics ───────────────────────────────────────────────────── + + #[test] + fn has_key_only_when_explicit_or_chain_improves() { + // sub("b"->"c", 0.2) + del("c", 1.0) = 1.2 > default 1.0 -> "b" NOT added + let eff = compute_effective_deletion_costs( + &make_del_map(&[("a", 0.5)]), + &make_sub_map(&[(("b", "c"), 0.2)]), + ); + assert!(eff.has_key("a")); + assert!(!eff.has_key("b")); // chain not cheaper + assert!(!eff.has_key("z")); + + // sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 < 1.0 -> "6" IS added + let eff2 = compute_effective_deletion_costs( + &make_del_map(&[("G", 0.01)]), + &make_sub_map(&[(("6", "G"), 0.5)]), + ); + assert!(eff2.has_key("6")); + } + + #[test] + fn max_token_length_reflects_longest_key() { + let eff = + compute_effective_deletion_costs(&make_del_map(&[("ab", 0.5)]), &make_sub_map(&[])); + assert_eq!(eff.max_token_length, 2); + } + + // ── Substitution ────────────────────────────────────────────────────────── + + #[test] + fn direct_substitution() { + let eff = compute_effective_substitution_costs(&make_sub_map(&[(("a", "b"), 0.3)])); + assert!(approx_eq(eff.get_cost("a", "b"), 0.3)); + assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); + assert!(eff.has_key("a", "b")); + // Unknown pair falls back to default. + assert!(approx_eq(eff.get_cost("a", "c"), 1.0)); + assert!(!eff.has_key("a", "c")); + } + + #[test] + fn two_hop_substitution_chain() { + // sub(a->b)=0.1 + sub(b->c)=0.1 -> eff_sub(a->c)=0.2 < default 1.0 + let eff = compute_effective_substitution_costs(&make_sub_map(&[ + (("a", "b"), 0.1), + (("b", "c"), 0.1), + ])); + assert!(approx_eq(eff.get_cost("a", "c"), 0.2)); + assert_eq!( + eff.get_chain("a", "c"), + EffectiveSubChain::Via { + steps: vec![ + ("a".to_string(), "b".to_string(), 0.1), + ("b".to_string(), "c".to_string(), 0.1), + ], + } + ); + assert!(eff.has_key("a", "c")); + // Direct pair is still Direct. + assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); + assert!(approx_eq(eff.get_cost("a", "b"), 0.1)); + } + + #[test] + fn direct_substitution_beats_chain() { + // sub(a->b)=0.1 (direct), sub(a->c)=0.3, sub(c->b)=0.1 + // chain(a->b) via c = 0.4 > direct 0.1 -> Direct wins + let eff = compute_effective_substitution_costs(&make_sub_map(&[ + (("a", "b"), 0.1), + (("a", "c"), 0.3), + (("c", "b"), 0.1), + ])); + assert!(approx_eq(eff.get_cost("a", "b"), 0.1)); + assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); + } + + #[test] + fn chain_substitution_beats_direct() { + // sub(a->b)=0.5 (direct), sub(a->c)=0.1, sub(c->b)=0.1 + // chain(a->b) via c = 0.2 < direct 0.5 -> Via wins + let eff = compute_effective_substitution_costs(&make_sub_map(&[ + (("a", "b"), 0.5), + (("a", "c"), 0.1), + (("c", "b"), 0.1), + ])); + assert!(approx_eq(eff.get_cost("a", "b"), 0.2)); + assert_eq!( + eff.get_chain("a", "b"), + EffectiveSubChain::Via { + steps: vec![ + ("a".to_string(), "c".to_string(), 0.1), + ("c".to_string(), "b".to_string(), 0.1), + ], + } + ); + } + + #[test] + fn chain_not_added_when_no_improvement_over_default() { + // sub(a->b)=0.6, sub(b->c)=0.6 -> chain(a->c)=1.2 >= default 1.0 -> NOT added + let eff = compute_effective_substitution_costs(&make_sub_map(&[ + (("a", "b"), 0.6), + (("b", "c"), 0.6), + ])); + assert!(!eff.has_key("a", "c")); + assert!(approx_eq(eff.get_cost("a", "c"), 1.0)); // default + } +} diff --git a/src/weighted_levenshtein.rs b/src/weighted_levenshtein.rs index 7c536e2..9806a7c 100644 --- a/src/weighted_levenshtein.rs +++ b/src/weighted_levenshtein.rs @@ -1,37 +1,29 @@ -use crate::cost_map::CostMap; use crate::explanation::{EditOperation, Predecessor}; -use crate::types::{SingleTokenKey, SubstitutionKey}; +use crate::transitive_costs::{ + EffectiveOpChain, EffectiveSingleTokenCosts, EffectiveSubChain, EffectiveSubstitutionCosts, +}; -// --- Public Functions --- - -pub fn custom_levenshtein_distance_with_cost_maps( +pub(crate) fn custom_levenshtein_distance_precomputed( source: &str, target: &str, - substitution_cost_map: &CostMap, - insertion_cost_map: &CostMap, - deletion_cost_map: &CostMap, + eff_sub: &EffectiveSubstitutionCosts, + eff_ins: &EffectiveSingleTokenCosts, + eff_del: &EffectiveSingleTokenCosts, ) -> f64 { if source == target { return 0.0; } - let mut processor = LevenshteinProcessor::new( - source, - target, - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, - false, - ); + let mut processor = LevenshteinProcessor::new(source, target, eff_sub, eff_ins, eff_del, false); processor.run(); processor.distance() } -pub fn explain_custom_levenshtein_distance( +pub(crate) fn explain_custom_levenshtein_precomputed( source: &str, target: &str, - substitution_cost_map: &CostMap, - insertion_cost_map: &CostMap, - deletion_cost_map: &CostMap, + eff_sub: &EffectiveSubstitutionCosts, + eff_ins: &EffectiveSingleTokenCosts, + eff_del: &EffectiveSingleTokenCosts, ) -> Vec { if source == target { return source @@ -41,14 +33,7 @@ pub fn explain_custom_levenshtein_distance( }) .collect(); } - let mut processor = LevenshteinProcessor::new( - source, - target, - substitution_cost_map, - insertion_cost_map, - deletion_cost_map, - true, - ); + let mut processor = LevenshteinProcessor::new(source, target, eff_sub, eff_ins, eff_del, true); processor.run(); processor.into_result() } @@ -58,9 +43,9 @@ pub fn explain_custom_levenshtein_distance( struct LevenshteinProcessor<'a> { source_chars: Vec, target_chars: Vec, - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, + eff_sub: &'a EffectiveSubstitutionCosts, + eff_del: &'a EffectiveSingleTokenCosts, + eff_ins: &'a EffectiveSingleTokenCosts, dp: Vec>, predecessors: Option>>, multi_char_ops: bool, @@ -70,9 +55,9 @@ impl<'a> LevenshteinProcessor<'a> { fn new( source: &str, target: &str, - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, + eff_sub: &'a EffectiveSubstitutionCosts, + eff_ins: &'a EffectiveSingleTokenCosts, + eff_del: &'a EffectiveSingleTokenCosts, explain: bool, ) -> Self { let source_chars: Vec = source.chars().collect(); @@ -83,9 +68,12 @@ impl<'a> LevenshteinProcessor<'a> { let mut processor = Self { source_chars, target_chars, - sub_map, - ins_map, - del_map, + eff_sub, + multi_char_ops: eff_sub.max_token_length > 1 + || eff_ins.max_token_length > 1 + || eff_del.max_token_length > 1, + eff_del, + eff_ins, dp: vec![vec![0.0; len_target + 1]; len_source + 1], predecessors: if explain { Some(vec![ @@ -95,9 +83,6 @@ impl<'a> LevenshteinProcessor<'a> { } else { None }, - multi_char_ops: sub_map.max_token_length > 1 - || ins_map.max_token_length > 1 - || del_map.max_token_length > 1, }; processor.initialize(); processor @@ -142,9 +127,9 @@ impl<'a> LevenshteinProcessor<'a> { let source_char_str = self.source_chars[i - 1].to_string(); let target_char_str = self.target_chars[j - 1].to_string(); - let deletion_cost = self.dp[i - 1][j] + self.del_map.get_cost(&source_char_str); - let insertion_cost = self.dp[i][j - 1] + self.ins_map.get_cost(&target_char_str); - let sub_cost = self.sub_map.get_cost(&source_char_str, &target_char_str); + let deletion_cost = self.dp[i - 1][j] + self.eff_del.get_cost(&source_char_str); + let insertion_cost = self.dp[i][j - 1] + self.eff_ins.get_cost(&target_char_str); + let sub_cost = self.eff_sub.get_cost(&source_char_str, &target_char_str); let substitution_cost = self.dp[i - 1][j - 1] + sub_cost; // Check for exact match @@ -181,15 +166,15 @@ impl<'a> LevenshteinProcessor<'a> { // First row (insertions) for j in 1..=len_target { let char_str = self.target_chars[j - 1].to_string(); - self.dp[0][j] = self.dp[0][j - 1] + self.ins_map.get_cost(&char_str); + self.dp[0][j] = self.dp[0][j - 1] + self.eff_ins.get_cost(&char_str); self.record(0, j, Predecessor::Insert(1)); - let max_len = self.ins_map.max_token_length.min(j); + let max_len = self.eff_ins.max_token_length.min(j); for token_len in 2..=max_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.ins_map.has_key(&token) { - let new_cost = self.dp[0][token_start] + self.ins_map.get_cost(&token); + if self.eff_ins.has_key(&token) { + let new_cost = self.dp[0][token_start] + self.eff_ins.get_cost(&token); if new_cost < self.dp[0][j] { self.dp[0][j] = new_cost; self.record(0, j, Predecessor::Insert(token_len)); @@ -200,15 +185,15 @@ impl<'a> LevenshteinProcessor<'a> { // First column (deletions) for i in 1..=len_source { let char_str = self.source_chars[i - 1].to_string(); - self.dp[i][0] = self.dp[i - 1][0] + self.del_map.get_cost(&char_str); + self.dp[i][0] = self.dp[i - 1][0] + self.eff_del.get_cost(&char_str); self.record(i, 0, Predecessor::Delete(1)); - let max_len = self.del_map.max_token_length.min(i); + let max_len = self.eff_del.max_token_length.min(i); for token_len in 2..=max_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.del_map.has_key(&token) { - let new_cost = self.dp[token_start][0] + self.del_map.get_cost(&token); + if self.eff_del.has_key(&token) { + let new_cost = self.dp[token_start][0] + self.eff_del.get_cost(&token); if new_cost < self.dp[i][0] { self.dp[i][0] = new_cost; self.record(i, 0, Predecessor::Delete(token_len)); @@ -219,8 +204,8 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_substitutions(&mut self, i: usize, j: usize) { - let max_source_len = self.sub_map.max_token_length.min(i); - let max_target_len = self.sub_map.max_token_length.min(j); + let max_source_len = self.eff_sub.max_token_length.min(i); + let max_target_len = self.eff_sub.max_token_length.min(j); for source_len in 1..=max_source_len { for target_len in 1..=max_target_len { if source_len == 1 && target_len == 1 { @@ -230,9 +215,9 @@ impl<'a> LevenshteinProcessor<'a> { let target_start = j - target_len; let source_substr: String = self.source_chars[source_start..i].iter().collect(); let target_substr: String = self.target_chars[target_start..j].iter().collect(); - if self.sub_map.has_key(&source_substr, &target_substr) { + if self.eff_sub.has_key(&source_substr, &target_substr) { let new_cost = self.dp[source_start][target_start] - + self.sub_map.get_cost(&source_substr, &target_substr); + + self.eff_sub.get_cost(&source_substr, &target_substr); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Substitute(source_len, target_len)); @@ -243,12 +228,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_insertions(&mut self, i: usize, j: usize) { - let max_ins_len = self.ins_map.max_token_length.min(j); + let max_ins_len = self.eff_ins.max_token_length.min(j); for token_len in 2..=max_ins_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.ins_map.has_key(&token) { - let new_cost = self.dp[i][token_start] + self.ins_map.get_cost(&token); + if self.eff_ins.has_key(&token) { + let new_cost = self.dp[i][token_start] + self.eff_ins.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Insert(token_len)); @@ -258,12 +243,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_deletions(&mut self, i: usize, j: usize) { - let max_del_len = self.del_map.max_token_length.min(i); + let max_del_len = self.eff_del.max_token_length.min(i); for token_len in 2..=max_del_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.del_map.has_key(&token) { - let new_cost = self.dp[token_start][j] + self.del_map.get_cost(&token); + if self.eff_del.has_key(&token) { + let new_cost = self.dp[token_start][j] + self.eff_del.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Delete(token_len)); @@ -291,32 +276,99 @@ impl<'a> LevenshteinProcessor<'a> { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); let target_token: String = self.target_chars[j - t_len..j].iter().collect(); if source_token != target_token { - let cost = self.sub_map.get_cost(&source_token, &target_token); - path.push(EditOperation::Substitute { - source: source_token, - target: target_token, - cost, - }); + // path is reversed at end; push in reverse so after reversal + // the chain appears in forward order. + match self.eff_sub.get_chain(&source_token, &target_token) { + EffectiveSubChain::Direct => { + let cost = self.eff_sub.get_cost(&source_token, &target_token); + path.push(EditOperation::Substitute { + source: source_token, + target: target_token, + cost, + }); + } + EffectiveSubChain::Via { steps } => { + for (from, to, cost) in steps.iter().rev() { + path.push(EditOperation::Substitute { + source: from.clone(), + target: to.clone(), + cost: *cost, + }); + } + } + } } i -= s_len; j -= t_len; } Predecessor::Insert(t_len) => { let target_token: String = self.target_chars[j - t_len..j].iter().collect(); - let cost = self.ins_map.get_cost(&target_token); - path.push(EditOperation::Insert { - target: target_token, - cost, - }); + match self.eff_ins.get_chain(&target_token) { + EffectiveOpChain::Direct => { + let cost = self.eff_ins.get_cost(&target_token); + path.push(EditOperation::Insert { + target: target_token, + cost, + }); + } + EffectiveOpChain::Via { + steps, + terminal_cost, + } => { + // path is reversed at end; push in reverse of forward order so + // after reversal: Insert(initial) -> Sub(…) -> … -> Sub(…->target) + for (from, to, cost) in steps.iter().rev() { + path.push(EditOperation::Substitute { + source: from.clone(), + target: to.clone(), + cost: *cost, + }); + } + let initial = steps + .first() + .map(|(f, _, _)| f.as_str()) + .unwrap_or(&target_token); + path.push(EditOperation::Insert { + target: initial.to_string(), + cost: terminal_cost, + }); + } + } j -= t_len; } Predecessor::Delete(s_len) => { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); - let cost = self.del_map.get_cost(&source_token); - path.push(EditOperation::Delete { - source: source_token, - cost, - }); + match self.eff_del.get_chain(&source_token) { + EffectiveOpChain::Direct => { + let cost = self.eff_del.get_cost(&source_token); + path.push(EditOperation::Delete { + source: source_token, + cost, + }); + } + EffectiveOpChain::Via { + steps, + terminal_cost, + } => { + // path is reversed at end; push in reverse of forward order so + // after reversal: Sub(source->…) -> … -> Sub(…->terminal) -> Del(terminal) + let terminal = steps + .last() + .map(|(_, t, _)| t.as_str()) + .unwrap_or(&source_token); + path.push(EditOperation::Delete { + source: terminal.to_string(), + cost: terminal_cost, + }); + for (from, to, cost) in steps.iter().rev() { + path.push(EditOperation::Substitute { + source: from.clone(), + target: to.clone(), + cost: *cost, + }); + } + } + } i -= s_len; } Predecessor::Match(t_len) => { @@ -338,7 +390,12 @@ impl<'a> LevenshteinProcessor<'a> { #[cfg(test)] mod test { use super::*; - use crate::types::{SingleTokenCostMap, SubstitutionCostMap}; + use crate::cost_map::CostMap; + use crate::transitive_costs::{ + compute_effective_deletion_costs, compute_effective_insertion_costs, + compute_effective_substitution_costs, + }; + use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; fn assert_approx_eq(a: f64, b: f64, epsilon: f64) { assert!( @@ -350,7 +407,6 @@ mod test { ); } - // Helper function to create default cost maps for testing fn create_default_cost_maps() -> ( CostMap, CostMap, @@ -362,6 +418,32 @@ mod test { (sub_map, ins_map, del_map) } + fn calc_distance( + source: &str, + target: &str, + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, + ) -> f64 { + let eff_sub = compute_effective_substitution_costs(sub_map); + let eff_del = compute_effective_deletion_costs(del_map, sub_map); + let eff_ins = compute_effective_insertion_costs(ins_map, sub_map); + custom_levenshtein_distance_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) + } + + fn calc_explain( + source: &str, + target: &str, + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, + ) -> Vec { + let eff_sub = compute_effective_substitution_costs(sub_map); + let eff_del = compute_effective_deletion_costs(del_map, sub_map); + let eff_ins = compute_effective_insertion_costs(ins_map, sub_map); + explain_custom_levenshtein_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) + } + #[test] fn test_custom_levenshtein_with_custom_sub_map() { let (_, ins_map, del_map) = create_default_cost_maps(); @@ -374,7 +456,7 @@ mod test { ); assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "bbc", &sub_map, &ins_map, &del_map), + calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); @@ -397,14 +479,14 @@ mod test { // Test with all three maps: delete 'y' (0.4) + insert 'x' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("aby", "abx", &sub_map, &ins_map, &del_map), + calc_distance("aby", "abx", &sub_map, &ins_map, &del_map), 0.7, 1e-9, ); // Test substitution: substitute 'a' with 'b' (0.1) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "bbc", &sub_map, &ins_map, &del_map), + calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); @@ -422,16 +504,14 @@ mod test { // Test that "hi" with "Ini" has a low cost due to the special substitution assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("hi", "Ini", &sub_map, &ins_map, &del_map), + calc_distance("hi", "Ini", &sub_map, &ins_map, &del_map), 0.2, // Only the h->In substitution cost 1e-9, ); // Test another example assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hello", "Inello", &sub_map, &ins_map, &del_map, - ), + calc_distance("hello", "Inello", &sub_map, &ins_map, &del_map), 0.2, // Only the h->In substitution cost 1e-9, ); @@ -448,9 +528,7 @@ mod test { // Test multiple substitutions in the same string assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hello", "Ine11o", &sub_map, &ins_map, &del_map, - ), + calc_distance("hello", "Ine11o", &sub_map, &ins_map, &del_map), 0.8, // 0.2 for h->In and 0.3+0.3 for l->1 twice 1e-9, ); @@ -467,18 +545,14 @@ mod test { // Test the rn->m substitution assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "corner", "comer", &sub_map, &ins_map, &del_map, - ), + calc_distance("corner", "comer", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); // Test the cl->d substitution assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "class", "dass", &sub_map, &ins_map, &del_map, - ), + calc_distance("class", "dass", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); @@ -496,18 +570,14 @@ mod test { // Test 0->O substitution (lower cost) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "R0AD", "ROAD", &sub_map, &ins_map, &del_map, - ), + calc_distance("R0AD", "ROAD", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); // Test O->0 substitution (higher cost) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "rOad", "r0ad", &sub_map, &ins_map, &del_map, - ), + calc_distance("rOad", "r0ad", &sub_map, &ins_map, &del_map), 0.5, 1e-9, ); @@ -523,14 +593,14 @@ mod test { // Test substitution at start of word assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("rnat", "mat", &sub_map, &ins_map, &del_map), + calc_distance("rnat", "mat", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); // Test substitution at end of word assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("burn", "bum", &sub_map, &ins_map, &del_map), + calc_distance("burn", "bum", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); @@ -549,13 +619,7 @@ mod test { // Test insertion with custom cost: Insert 'a' with cost 0.2 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "bc", - "abc", - &sub_map, - &ins_map_custom, - &del_map_default, - ), + calc_distance("bc", "abc", &sub_map, &ins_map_custom, &del_map_default), 0.2, 1e-9, ); @@ -569,13 +633,7 @@ mod test { // Test deletion with custom cost: Delete 'a' with cost 0.4 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abc", - "bc", - &sub_map, - &ins_map_default, - &del_map_custom, - ), + calc_distance("abc", "bc", &sub_map, &ins_map_default, &del_map_custom), 0.4, 1e-9, ); @@ -595,7 +653,7 @@ mod test { // Test combined operations: Delete 'x' (0.5) + insert 'b' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( + calc_distance( "axc", "abc", &high_cost_sub_map, @@ -613,21 +671,21 @@ mod test { // Test empty strings: Empty strings have zero distance assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("", "", &sub_map, &ins_map, &del_map), + calc_distance("", "", &sub_map, &ins_map, &del_map), 0.0, 1e-9, ); // Test source empty, target not empty: Insert 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("", "abc", &sub_map, &ins_map, &del_map), + calc_distance("", "abc", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); // Test source not empty, target empty: Delete 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("abc", "", &sub_map, &ins_map, &del_map), + calc_distance("abc", "", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); @@ -644,13 +702,7 @@ mod test { // Test with custom insertion costs: Insert 'a' (0.2) + 'b' (0.3) + 'c' (0.4) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "", - "abc", - &sub_map, - &custom_ins_map, - &del_map, - ), + calc_distance("", "abc", &sub_map, &custom_ins_map, &del_map), 0.9, 1e-9, ); @@ -667,13 +719,7 @@ mod test { // Test with custom deletion costs: Delete 'a' (0.5) + 'b' (0.6) + 'c' (0.7) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abc", - "", - &sub_map, - &ins_map, - &custom_del_map, - ), + calc_distance("abc", "", &sub_map, &ins_map, &custom_del_map), 1.8, 1e-9, ); @@ -701,11 +747,9 @@ mod test { 1.0, ); - // Test with a mix of operations: Sub 'a'→'A' (0.1) + Sub 'b'→'B' (0.2) + delete 'm' (0.5) + delete 'n' (0.6) + insert 'x' (0.3) + insert 'y' (0.4) + // Test with a mix of operations: Sub 'a'->'A' (0.1) + Sub 'b'->'B' (0.2) + delete 'm' (0.5) + delete 'n' (0.6) + insert 'x' (0.3) + insert 'y' (0.4) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abmn", "ABxy", &sub_map, &ins_map, &del_map, - ), + calc_distance("abmn", "ABxy", &sub_map, &ins_map, &del_map), 2.1, 1e-9, ); @@ -717,16 +761,14 @@ mod test { // Test with Unicode characters: Substitute 'é' with 'e' with default cost 1.0 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "café", "cafe", &sub_map, &ins_map, &del_map, - ), + calc_distance("café", "cafe", &sub_map, &ins_map, &del_map), 1.0, 1e-9, ); // Test with emoji: Delete ' ' and '😊' with default cost 1.0 each assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("hi 😊", "hi", &sub_map, &ins_map, &del_map), + calc_distance("hi 😊", "hi", &sub_map, &ins_map, &del_map), 2.0, 1e-9, ); @@ -748,7 +790,7 @@ mod test { // Test substitution of Unicode with custom cost assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( + calc_distance( "cafe", "café", &sub_map_unicode, @@ -761,13 +803,7 @@ mod test { // Test deletion of Unicode with custom cost assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "hi 😊", - "hi", - &sub_map, - &ins_map_unicode, - &del_map_unicode, - ), + calc_distance("hi 😊", "hi", &sub_map, &ins_map_unicode, &del_map_unicode), 1.5, // Delete ' ' (default 1.0) and '😊' (custom 0.5) 1e-9, ); @@ -789,32 +825,28 @@ mod test { // Test 2-to-1 character substitution: Substitute "th" with "T" with cost 0.2 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("this", "Tis", &sub_map, &ins_map, &del_map), + calc_distance("this", "Tis", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); // Test 3-to-3 character substitution: Substitute "ing" with "in'" with cost 0.3 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "singing", "singin'", &sub_map, &ins_map, &del_map, - ), + calc_distance("singing", "singin'", &sub_map, &ins_map, &del_map), 0.3, 1e-9, ); // Test 1-to-2 character substitution: Substitute "o" with "ou" with cost 0.1 assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("go", "gou", &sub_map, &ins_map, &del_map), + calc_distance("go", "gou", &sub_map, &ins_map, &del_map), 0.1, 1e-9, ); // Test multiple multi-character substitutions: Sub "th"->"T" (0.2) + Sub "ing"->"in'" (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "thinking", "Tinkin'", &sub_map, &ins_map, &del_map, - ), + calc_distance("thinking", "Tinkin'", &sub_map, &ins_map, &del_map), 0.5, 1e-9, ); @@ -846,53 +878,49 @@ mod test { // Test multi-character insertion: insert 'ab' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("x", "xab", &sub_map, &ins_map, &del_map), + calc_distance("x", "xab", &sub_map, &ins_map, &del_map), 0.3, 1e-9, ); // Test multi-character deletion: delete 'cd' (0.4) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("ycd", "y", &sub_map, &ins_map, &del_map), + calc_distance("ycd", "y", &sub_map, &ins_map, &del_map), 0.4, 1e-9, ); // Test both insertion and deletion: delete 'ef' (0.5) + insert 'ab' (0.3) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("aef", "aab", &sub_map, &ins_map, &del_map), + calc_distance("aef", "aab", &sub_map, &ins_map, &del_map), 0.8, 1e-9, ); // Test with longer token insertion: insert 'xyz' (0.2) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "test", "testxyz", &sub_map, &ins_map, &del_map, - ), + calc_distance("test", "testxyz", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); // Test with mixed operations: delete '789' (0.6) + insert 'xyz' (0.2) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "a789b", "axyzb", &sub_map, &ins_map, &del_map, - ), + calc_distance("a789b", "axyzb", &sub_map, &ins_map, &del_map), 0.8, 1e-9, ); // Test multi-character deletion "bc" at the beginning: delete 'bc' (cost 0.35) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("bcd", "d", &sub_map, &ins_map, &del_map), + calc_distance("bcd", "d", &sub_map, &ins_map, &del_map), 0.35, 1e-9, ); // Test multi-character insertion "bc" at the beginning: insert 'bc' (cost 0.25) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps("c", "bcc", &sub_map, &ins_map, &del_map), + calc_distance("c", "bcc", &sub_map, &ins_map, &del_map), 0.25, 1e-9, ); @@ -919,39 +947,21 @@ mod test { // Test with full map (allows abc->xyz and de->uv): Sub "abc"->"xyz" (0.1) + Sub "de"->"uv" (0.2) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_full, - &ins_map, - &del_map, - ), + calc_distance("abcde", "xyzuv", &sub_map_full, &ins_map, &del_map), 0.3, 1e-9, ); // Test with partial map (does not allow abc->xyz, forces default): Sub a->x(1.0) + b->y(1.0) + c->z(1.0) + Sub "de"->"uv"(0.2) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_partial, - &ins_map, - &del_map, - ), + calc_distance("abcde", "xyzuv", &sub_map_partial, &ins_map, &del_map), 3.2, 1e-9, ); // Test with empty map (only single character default operations): 5 * default sub cost (1.0) assert_approx_eq( - custom_levenshtein_distance_with_cost_maps( - "abcde", - "xyzuv", - &sub_map_empty, - &ins_map, - &del_map, - ), + calc_distance("abcde", "xyzuv", &sub_map_empty, &ins_map, &del_map), 5.0, 1e-9, ); @@ -959,10 +969,17 @@ mod test { #[test] fn test_check_multi_char_ops_with_empty_maps() { + use crate::transitive_costs::{ + compute_effective_deletion_costs, compute_effective_insertion_costs, + compute_effective_substitution_costs, + }; let (sub_map, ins_map, del_map) = create_default_cost_maps(); + let eff_sub = compute_effective_substitution_costs(&sub_map); + let eff_del = compute_effective_deletion_costs(&del_map, &sub_map); + let eff_ins = compute_effective_insertion_costs(&ins_map, &sub_map); let mut processor = - LevenshteinProcessor::new("abcd", "xyz", &sub_map, &ins_map, &del_map, true); + LevenshteinProcessor::new("abcd", "xyz", &eff_sub, &eff_ins, &eff_del, true); // Simulate the DP state before the operation let original_dp_3_2 = processor.dp[3][2]; @@ -987,9 +1004,7 @@ mod test { let del_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); // Test multi-char insertion via main function - let dist = custom_levenshtein_distance_with_cost_maps( - source, target, &sub_map, &ins_map, &del_map, - ); + let dist = calc_distance(source, target, &sub_map, &ins_map, &del_map); assert_approx_eq(dist, 0.2, 1e-9); // Should be 0.2 (insert "xyz") // Now test a multi-character deletion via main function @@ -1004,9 +1019,140 @@ mod test { // Use default insertion map for this test let ins_map2 = CostMap::::new(SingleTokenCostMap::new(), 1.0); - let dist2 = custom_levenshtein_distance_with_cost_maps( - source2, target2, &sub_map, &ins_map2, &del_map2, - ); + let dist2 = calc_distance(source2, target2, &sub_map, &ins_map2, &del_map2); assert_approx_eq(dist2, 0.3, 1e-9); // Should be 0.3 (delete "xyz") } + + // Transitive substitution + + #[test] + fn test_transitive_substitution_chain() { + // sub(a->b)=0.1, sub(b->c)=0.1, default=1.0 -> eff_sub(a->c)=0.2 + let sub_map = CostMap::::new( + SubstitutionCostMap::from([ + (("a".to_string(), "b".to_string()), 0.1), + (("b".to_string(), "c".to_string()), 0.1), + ]), + 1.0, + false, + ); + let (_, ins_map, del_map) = create_default_cost_maps(); + assert_approx_eq( + calc_distance("a", "c", &sub_map, &ins_map, &del_map), + 0.2, + 1e-9, + ); + } + + #[test] + fn test_transitive_substitution_explain() { + let sub_map = CostMap::::new( + SubstitutionCostMap::from([ + (("a".to_string(), "b".to_string()), 0.1), + (("b".to_string(), "c".to_string()), 0.1), + ]), + 1.0, + false, + ); + let (_, ins_map, del_map) = create_default_cost_maps(); + let ops = calc_explain("a", "c", &sub_map, &ins_map, &del_map); + assert_eq!(ops.len(), 2); + assert!( + matches!(&ops[0], EditOperation::Substitute { source, target, cost } + if source == "a" && target == "b" && (*cost - 0.1).abs() < 1e-9) + ); + assert!( + matches!(&ops[1], EditOperation::Substitute { source, target, cost } + if source == "b" && target == "c" && (*cost - 0.1).abs() < 1e-9) + ); + } + + // ── Issue #12: transitive chain tests ───────────────────────────────────── + + #[test] + fn test_transitive_deletion_chain() { + // sub("6"->"G") = 0.5, del("G") = 0.01 -> chain = 0.51 < direct del("6") = 1.0 + let sub_map = CostMap::::new( + SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), + 1.0, + false, + ); + let del_map = CostMap::::new( + SingleTokenCostMap::from([("G".to_string(), 0.01)]), + 1.0, + ); + let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); + + assert_approx_eq( + calc_distance("06", "0", &sub_map, &ins_map, &del_map), + 0.51, + 1e-9, + ); + } + + #[test] + fn test_transitive_deletion_chain_explain() { + let sub_map = CostMap::::new( + SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), + 1.0, + false, + ); + let del_map = CostMap::::new( + SingleTokenCostMap::from([("G".to_string(), 0.01)]), + 1.0, + ); + let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); + + let ops = calc_explain("06", "0", &sub_map, &ins_map, &del_map); + + // Match("0"), Substitute("6"->"G", 0.5), Delete("G", 0.01) + assert_eq!(ops.len(), 3); + assert!(matches!(&ops[0], EditOperation::Match { token } if token == "0")); + assert!( + matches!(&ops[1], EditOperation::Substitute { source, target, cost } + if source == "6" && target == "G" && (*cost - 0.5).abs() < 1e-9) + ); + assert!(matches!(&ops[2], EditOperation::Delete { source, cost } + if source == "G" && (*cost - 0.01).abs() < 1e-9)); + } + + #[test] + fn test_transitive_insertion_chain() { + // ins("x") = 0.1, sub("x"->"y") = 0.2 -> chain ins("y") = 0.3 < direct ins("y") = 1.0 + let sub_map = CostMap::::new( + SubstitutionCostMap::from([(("x".to_string(), "y".to_string()), 0.2)]), + 1.0, + false, + ); + let ins_map = + CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.1)]), 1.0); + let del_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); + + assert_approx_eq( + calc_distance("a", "ay", &sub_map, &ins_map, &del_map), + 0.3, + 1e-9, + ); + } + + #[test] + fn test_direct_op_wins_when_chain_is_more_expensive() { + // del("6") = 0.2 < sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 -> direct wins + let sub_map = CostMap::::new( + SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), + 1.0, + false, + ); + let del_map = CostMap::::new( + SingleTokenCostMap::from([("6".to_string(), 0.2), ("G".to_string(), 0.01)]), + 1.0, + ); + let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); + + assert_approx_eq( + calc_distance("06", "0", &sub_map, &ins_map, &del_map), + 0.2, + 1e-9, + ); + } } From 9158c04dda2733291614bb4e328d07c77f9a0f62 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Wed, 29 Apr 2026 22:59:00 +0200 Subject: [PATCH 02/21] chore --- python/tests/test_weighted_levenshtein.py | 22 + src/rust_stringdist.rs | 15 +- src/transitive_costs.rs | 990 +++++++++------------- src/weighted_levenshtein.rs | 24 +- 4 files changed, 431 insertions(+), 620 deletions(-) diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index 127eb4f..a2357f3 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -572,6 +572,28 @@ def test_transitive_insertion_subtitution() -> None: assert wl.distance("A", "B") == pytest.approx(0.5) +def test_transitive_insertion_subtitution2() -> None: + """ + A->AA->AAB->C + """ + wl = WeightedLevenshtein( + insertion_costs={"A": 0.2, "B": 0.3}, + substitution_costs={("AAB", "C"): 0.1}, + ) + assert wl.distance("A", "C") == pytest.approx(0.6) + + +def test_transitive_insertion_deletion() -> None: + """ + AC->ABC->C + """ + wl = WeightedLevenshtein( + insertion_costs={"B": 0.1}, + deletion_costs={"AB": 0.0}, + ) + assert wl.distance("AC", "C") == pytest.approx(0.1) + + def test_transitive_insertion_chain_distance() -> None: """Insertion analogue: ins('x', 0.1) + sub('x'->'y', 0.2) = 0.3 < direct ins('y', 1.0).""" wl = WeightedLevenshtein( diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index d51409f..41dec0b 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,8 +1,7 @@ use crate::cost_map::CostMap; use crate::explanation::EditOperation; use crate::transitive_costs::{ - compute_effective_deletion_costs, compute_effective_insertion_costs, - compute_effective_substitution_costs, EffectiveSingleTokenCosts, EffectiveSubstitutionCosts, + compute_effective_costs_unified, EffectiveSingleTokenCosts, EffectiveSubstitutionCosts, }; use crate::types::{SingleTokenKey, SubstitutionKey}; use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; @@ -33,10 +32,11 @@ impl<'py> IntoPyObject<'py> for EditOperation { } } -/// Precomputes the transitive closure once and reuses it across all distance calls. +/// Precomputes effective substitution, insertion, and deletion costs once and reuses them +/// for every `.distance()` / `.batch_distance()` call. /// -/// Exposed to Python so that `WeightedLevenshtein.__init__` can pay the Dijkstra -/// cost once and avoid recomputing it on every `.distance()` / `.batch_distance()` call. +/// Building the calculator runs Dijkstra on the substitution graph and then token-graph +/// closure passes (see `transitive_costs`), so per-call distance stays linear in string length. #[pyclass] #[derive(Debug)] struct RustLevenshteinCalculator { @@ -80,9 +80,8 @@ impl RustLevenshteinCalculator { let del_map = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); - let eff_sub = compute_effective_substitution_costs(&sub_map); - let eff_del = compute_effective_deletion_costs(&del_map, &sub_map); - let eff_ins = compute_effective_insertion_costs(&ins_map, &sub_map); + let (eff_sub, eff_del, eff_ins) = + compute_effective_costs_unified(&sub_map, &ins_map, &del_map); Ok(Self { eff_sub, diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 728778d..2287376 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -1,17 +1,19 @@ -//! Transitive closure of edit costs via Dijkstra. +//! Effective edit costs under substitution-only transitivity and token-graph closure. //! -//! This module precomputes the globally cheapest cost for every operation via -//! Dijkstra, so the DP uses optimal paths without per-cell search. +//! The Python-facing calculator uses [`compute_effective_costs_unified`], which +//! seeds one token graph from raw insertion/deletion/substitution maps and then +//! runs a single Floyd-Warshall closure. The older Dijkstra-based constructors +//! remain as small building blocks for tests and for substitution-chain +//! provenance in Rust-only callers. //! -//! Results are cached in [`EffectiveSingleTokenCosts`] (del/ins) and -//! [`EffectiveSubstitutionCosts`] (sub), computed once at construction. +//! Effective costs are stored in [`EffectiveSingleTokenCosts`] (del/ins) and +//! [`EffectiveSubstitutionCosts`] (sub), computed once at construction (for example when +//! building the Rust/Python calculator). use crate::cost_map::CostMap; use crate::types::{SingleTokenKey, SubstitutionKey}; -use std::collections::{BinaryHeap, HashMap, HashSet}; - -type DistMap = HashMap; -type PrevMap = HashMap>; +use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; +use std::collections::{HashMap, HashSet}; // Public types @@ -122,661 +124,459 @@ impl EffectiveSubstitutionCosts { } } -// Public constructors +// Unified token graph solver -/// Precomputes effective deletion costs. -/// -/// Runs multi-source Dijkstra on the **reversed** substitution graph, seeded -/// with `del(x)` per node: +/// Interned identifier for a token graph node. /// -/// ```text -/// eff_del(s) = min_x { shortest_sub_path(s -> x) + del(x) } -/// ``` -pub fn compute_effective_deletion_costs( - del_map: &CostMap, - sub_map: &CostMap, -) -> EffectiveSingleTokenCosts { - let default_cost = del_map.default_cost(); - - let all_tokens = collect_tokens(del_map, sub_map); +/// A node represents either a configured/derived token or the distinguished +/// epsilon node (`""`). Keeping this as a newtype instead of a bare `usize` +/// makes graph indexing sites explicit. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +struct NodeId(usize); - // Initial distance for every token = its direct deletion cost. - let initial: HashMap = all_tokens - .iter() - .map(|t| (t.clone(), del_map.get_cost(t))) - .collect(); - - let rev_graph = build_reversed_sub_graph(sub_map); - let (dist, prev) = dijkstra(&rev_graph, &initial); - - build_entries( - &all_tokens, - &dist, - &initial, - &prev, - |token| del_map.has_key(token), - |terminal| del_map.get_cost(terminal), - default_cost, - ChainDirection::Deletion, - ) +impl NodeId { + #[inline] + fn index(self) -> usize { + self.0 + } } -/// Precomputes effective insertion costs. +/// All base costs used to seed the unified token graph. /// -/// Runs multi-source Dijkstra on the **forward** substitution graph, seeded -/// with `ins(x)` per node: -/// -/// ```text -/// eff_ins(t) = min_x { ins(x) + shortest_sub_path(x -> t) } -/// ``` -pub fn compute_effective_insertion_costs( - ins_map: &CostMap, - sub_map: &CostMap, -) -> EffectiveSingleTokenCosts { - let default_cost = ins_map.default_cost(); - - let all_tokens = collect_tokens(ins_map, sub_map); - - let initial: HashMap = all_tokens - .iter() - .map(|t| (t.clone(), ins_map.get_cost(t))) - .collect(); - - let fwd_graph = build_forward_sub_graph(sub_map); - let (dist, prev) = dijkstra(&fwd_graph, &initial); - - build_entries( - &all_tokens, - &dist, - &initial, - &prev, - |token| ins_map.has_key(token), - |initial_node| ins_map.get_cost(initial_node), - default_cost, - ChainDirection::Insertion, - ) +/// These are direct wrappers around the raw cost maps. They intentionally do +/// not apply transitive closure; [`CostSolver`] owns the single closure pass. +struct BaseEffectiveCosts { + sub: EffectiveSubstitutionCosts, + ins: EffectiveSingleTokenCosts, + del: EffectiveSingleTokenCosts, } -/// Precomputes effective substitution costs via all-pairs shortest paths. -/// -/// For every source token in the substitution graph, runs Dijkstra on the -/// forward graph. When a chain `sub(a->b) + sub(b->c)` is cheaper than the -/// default cost for `sub(a->c)`, the result is stored so the DP automatically -/// uses the cheaper path. -/// -/// ```text -/// eff_sub(s, t) = min_path { sum of edge costs along s -> t } -/// ``` -pub fn compute_effective_substitution_costs( - sub_map: &CostMap, -) -> EffectiveSubstitutionCosts { - let default_cost = sub_map.default_cost(); - - let all_tokens: HashSet = sub_map +fn raw_effective_substitution_costs(sub_map: &CostMap) -> EffectiveSubstitutionCosts { + let entries = sub_map .costs - .keys() - .flat_map(|(s, t)| [s.clone(), t.clone()]) + .iter() + .map(|((source, target), &cost)| { + ( + (source.clone(), target.clone()), + (cost, EffectiveSubChain::Direct), + ) + }) .collect(); - let fwd_graph = build_forward_sub_graph(sub_map); - let mut entries: HashMap<(String, String), (f64, EffectiveSubChain)> = HashMap::new(); - - for source in &all_tokens { - let initial: DistMap = [(source.clone(), 0.0)].into_iter().collect(); - let (dist, prev) = dijkstra(&fwd_graph, &initial); - - for target in &all_tokens { - if target == source { - continue; - } - let chain_cost = dist.get(target).copied().unwrap_or(f64::INFINITY); - let direct_cost_opt = sub_map - .costs - .get(&(source.clone(), target.clone())) - .copied(); - if let Some(entry) = - classify_sub_pair(chain_cost, direct_cost_opt, default_cost, target, &prev) - { - entries.insert((source.clone(), target.clone()), entry); - } - } - } - - let max_len = entries - .keys() - .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) - .max() - .unwrap_or(1) - .max(1); - EffectiveSubstitutionCosts { + max_token_length: sub_map + .costs + .keys() + .flat_map(|(source, target)| [source.chars().count(), target.chars().count()]) + .max() + .unwrap_or(1) + .max(1), entries, - default_cost, - max_token_length: max_len, + default_cost: sub_map.default_cost(), } } -// Dijkstra - -/// Min-heap entry (BinaryHeap is a max-heap; reversed comparison gives min behaviour). -#[derive(Debug, Clone, PartialEq)] -struct HeapEntry { - cost: f64, - token: String, -} - -impl Eq for HeapEntry {} - -impl PartialOrd for HeapEntry { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for HeapEntry { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - other - .cost - .partial_cmp(&self.cost) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| self.token.cmp(&other.token)) - } -} - -/// Generic Dijkstra. -/// -/// Returns `(dist, prev)` where `prev[v] = Some((u, edge_cost))` means node -/// `v` was relaxed via the edge `u -> v`. -fn dijkstra(graph: &HashMap>, initial: &DistMap) -> (DistMap, PrevMap) { - let mut dist = initial.clone(); - let mut prev: PrevMap = initial.keys().map(|k| (k.clone(), None)).collect(); - - let mut heap: BinaryHeap = dist +fn raw_effective_single_token_costs( + map: &CostMap, +) -> EffectiveSingleTokenCosts { + let entries = map + .costs .iter() - .map(|(t, &c)| HeapEntry { - cost: c, - token: t.clone(), - }) + .map(|(token, &cost)| (token.clone(), (cost, EffectiveOpChain::Direct))) .collect(); - while let Some(HeapEntry { cost, token }) = heap.pop() { - if cost > dist[&token] { - continue; // stale entry - } - for (nbr, edge_cost) in graph.get(&token).into_iter().flatten() { - let new_cost = cost + edge_cost; - let cur = dist.get(nbr).copied().unwrap_or(f64::INFINITY); - if new_cost < cur { - dist.insert(nbr.clone(), new_cost); - prev.insert(nbr.clone(), Some((token.clone(), *edge_cost))); - heap.push(HeapEntry { - cost: new_cost, - token: nbr.clone(), - }); - } - } + EffectiveSingleTokenCosts { + max_token_length: map + .costs + .keys() + .map(|token| token.chars().count()) + .max() + .unwrap_or(1) + .max(1), + entries, + default_cost: map.default_cost(), } - - (dist, prev) } -// Chain reconstruction & entry building - -enum ChainDirection { - Deletion, - Insertion, -} - -/// Builds the `EffectiveSingleTokenCosts` entries after Dijkstra. +/// Unified state-transition solver for effective edit costs. /// -/// A token is added iff it was explicitly configured in the raw map OR the -/// Dijkstra found a chain that is strictly cheaper than the direct cost. -/// This prevents tokens that only appear in the substitution graph (but whose -/// chain is NOT cheaper) from being registered as "explicitly deletable/ -/// insertable", which would otherwise wrongly enable multi-char DP operations. -#[allow(clippy::too_many_arguments)] -fn build_entries( - all_tokens: &HashSet, - dist: &DistMap, - initial: &DistMap, - prev: &PrevMap, - in_raw_map: impl Fn(&str) -> bool, - terminal_cost_fn: impl Fn(&str) -> f64, - default_cost: f64, - direction: ChainDirection, -) -> EffectiveSingleTokenCosts { - let mut entries: HashMap = HashMap::new(); +/// The graph has one node per relevant token plus a distinguished epsilon node. +/// Its dense adjacency matrix is initialized with direct weighted token-to-token +/// distances under the base effective costs, then closed with Floyd-Warshall. +/// +/// After closure, every operation is a projection from the same matrix: +/// - substitution `a -> b`: `dist[a][b]` +/// - deletion `a`: `dist[a][epsilon]` +/// - insertion `b`: `dist[epsilon][b]` +struct CostSolver<'a> { + sub_map: &'a CostMap, + ins_map: &'a CostMap, + del_map: &'a CostMap, + base: BaseEffectiveCosts, + token_to_id: HashMap, + tokens: Vec, + epsilon_id: NodeId, + dist: Vec>, + next: Vec>>, +} - for token in all_tokens { - let final_dist = dist[token]; - let initial_cost = initial[token]; - let improved = final_dist < initial_cost; +impl<'a> CostSolver<'a> { + fn new( + sub_map: &'a CostMap, + ins_map: &'a CostMap, + del_map: &'a CostMap, + ) -> Self { + let base = BaseEffectiveCosts { + sub: raw_effective_substitution_costs(sub_map), + ins: raw_effective_single_token_costs(ins_map), + del: raw_effective_single_token_costs(del_map), + }; - if !improved && !in_raw_map(token) { - continue; + let tokens = collect_solver_tokens(sub_map, ins_map, del_map, &base); + let token_to_id: HashMap = tokens + .iter() + .enumerate() + .map(|(i, token)| (token.clone(), NodeId(i))) + .collect(); + let epsilon_id = token_to_id[""]; + let (dist, next) = Self::seed_distances(&tokens, &base); + + Self { + sub_map, + ins_map, + del_map, + base, + token_to_id, + tokens, + epsilon_id, + dist, + next, } + } - let chain = if prev[token].is_none() { - EffectiveOpChain::Direct - } else { - match direction { - ChainDirection::Deletion => { - let steps = reconstruct_deletion_steps(token, prev); - let terminal = steps.last().map(|(_, t, _)| t.as_str()).unwrap_or(token); - EffectiveOpChain::Via { - terminal_cost: terminal_cost_fn(terminal), - steps, - } + fn compute_effective_costs(mut self) -> ( + EffectiveSubstitutionCosts, + EffectiveSingleTokenCosts, + EffectiveSingleTokenCosts, + ) { + self.close_all_pairs(); + ( + self.effective_substitutions(), + self.effective_deletions(), + self.effective_insertions(), + ) + } + + /// Initialize graph edges from direct weighted token-to-token distances. + fn seed_distances( + tokens: &[String], + base: &BaseEffectiveCosts, + ) -> (Vec>, Vec>>) { + let n = tokens.len(); + let mut dist = vec![vec![f64::INFINITY; n]; n]; + let mut next = vec![vec![None; n]; n]; + for source in 0..n { + dist[source][source] = 0.0; + next[source][source] = Some(NodeId(source)); + for target in 0..n { + if source == target { + continue; + } + dist[source][target] = custom_levenshtein_distance_precomputed( + &tokens[source], + &tokens[target], + &base.sub, + &base.ins, + &base.del, + ); + if dist[source][target].is_finite() { + next[source][target] = Some(NodeId(target)); + } + } + } + (dist, next) + } + + /// Floyd-Warshall all-pairs shortest paths over the unified graph. + #[allow(clippy::needless_range_loop)] // Indexed `target` avoids simultaneous borrows of `dist`. + fn close_all_pairs(&mut self) { + let n = self.dist.len(); + for via in 0..n { + for source in 0..n { + let source_to_via = self.dist[source][via]; + if !source_to_via.is_finite() { + continue; } - ChainDirection::Insertion => { - let (steps, initial_token) = reconstruct_insertion_steps(token, prev); - EffectiveOpChain::Via { - terminal_cost: terminal_cost_fn(&initial_token), - steps, + for target in 0..n { + let candidate = source_to_via + self.dist[via][target]; + if candidate < self.dist[source][target] { + self.dist[source][target] = candidate; + self.next[source][target] = self.next[source][via]; } } } - }; - - entries.insert(token.clone(), (final_dist, chain)); - } - - let max_len = entries - .keys() - .map(|k| k.chars().count()) - .max() - .unwrap_or(1) - .max(1); - - EffectiveSingleTokenCosts { - entries, - default_cost, - max_token_length: max_len, + } } -} -/// Follows `prev` forward from `source` (deletion direction). -/// Returns steps `[(s, x1, c1), (x1, x2, c2), …]` in forward order. -fn reconstruct_deletion_steps(source: &str, prev: &PrevMap) -> Vec<(String, String, f64)> { - let mut steps = Vec::new(); - let mut cur = source.to_string(); - while let Some((next, c)) = prev.get(&cur).and_then(|o| o.as_ref()) { - steps.push((cur.clone(), next.clone(), *c)); - cur = next.clone(); + fn id(&self, token: &str) -> NodeId { + self.token_to_id[token] } - steps -} -/// Follows `prev` backward from `target`, returns `(steps, seed)` where -/// `steps` are in forward order and `seed` is the path origin (the node -/// with `prev[seed] = None`). -/// -/// Used for both insertion chains and substitution chains. -fn reconstruct_steps_to(target: &str, prev: &PrevMap) -> Vec<(String, String, f64)> { - let mut steps_rev = Vec::new(); - let mut cur = target.to_string(); - while let Some((from, c)) = prev.get(&cur).and_then(|o| o.as_ref()) { - steps_rev.push((from.clone(), cur.clone(), *c)); - cur = from.clone(); - } - steps_rev.reverse(); - steps_rev -} - -/// Wrapper that also returns the seed (initial) token — used for insertion chains. -fn reconstruct_insertion_steps( - target: &str, - prev: &PrevMap, -) -> (Vec<(String, String, f64)>, String) { - let steps = reconstruct_steps_to(target, prev); - let initial = steps - .first() - .map(|(f, _, _)| f.clone()) - .unwrap_or_else(|| target.to_string()); - (steps, initial) -} - -// ─── Helpers ────────────────────────────────────────────────────────────────── - -fn collect_tokens( - single_map: &CostMap, - sub_map: &CostMap, -) -> HashSet { - let mut tokens: HashSet = single_map.costs.keys().cloned().collect(); - for (src, tgt) in sub_map.costs.keys() { - tokens.insert(src.clone()); - tokens.insert(tgt.clone()); + fn cost(&self, source: NodeId, target: NodeId) -> f64 { + self.dist[source.index()][target.index()] } - tokens -} -/// Builds the forward substitution graph: an edge `src -> tgt` has the sub cost. -fn build_forward_sub_graph( - sub_map: &CostMap, -) -> HashMap> { - let mut graph: HashMap> = HashMap::new(); - for ((src, tgt), &c) in &sub_map.costs { - graph.entry(src.clone()).or_default().push((tgt.clone(), c)); + fn path(&self, source: NodeId, target: NodeId) -> Option> { + self.next[source.index()][target.index()]?; + let mut path = vec![source]; + let mut current = source; + while current != target { + current = self.next[current.index()][target.index()]?; + path.push(current); + } + Some(path) } - graph -} -/// Builds the reversed substitution graph: an edge `tgt -> src` has the sub cost. -fn build_reversed_sub_graph( - sub_map: &CostMap, -) -> HashMap> { - let mut graph: HashMap> = HashMap::new(); - for ((src, tgt), &c) in &sub_map.costs { - graph.entry(tgt.clone()).or_default().push((src.clone(), c)); + fn token(&self, node: NodeId) -> &str { + &self.tokens[node.index()] } - graph -} -/// Decides how to record a substitution pair `(source, target)` given the -/// Dijkstra result for `source`. -/// -/// Returns `None` if the pair should be omitted (not in the raw map and no -/// improvement over the default). Otherwise returns the effective cost and chain. -fn classify_sub_pair( - chain_cost: f64, - direct_cost_opt: Option, - default_cost: f64, - target: &str, - prev: &PrevMap, -) -> Option<(f64, EffectiveSubChain)> { - let in_raw_map = direct_cost_opt.is_some(); - if !in_raw_map && chain_cost >= default_cost { - return None; - } - let direct = direct_cost_opt.unwrap_or(f64::INFINITY); - if chain_cost < direct { - let steps = reconstruct_steps_to(target, prev); - Some((chain_cost, EffectiveSubChain::Via { steps })) - } else { - Some((direct, EffectiveSubChain::Direct)) - } -} - -// ─── Tests ──────────────────────────────────────────────────────────────────── + fn effective_substitutions(&self) -> EffectiveSubstitutionCosts { + let mut entries: HashMap<(String, String), (f64, EffectiveSubChain)> = HashMap::new(); + for source in self.non_epsilon_tokens() { + for target in self.non_epsilon_tokens() { + if source == target { + continue; + } + let best = self.cost(self.id(source), self.id(target)); + let raw_direct = self + .sub_map + .costs + .get(&(source.clone(), target.clone())) + .copied() + .unwrap_or(self.base.sub.default_cost); + let in_raw_map = self + .sub_map + .costs + .contains_key(&(source.clone(), target.clone())); + + if !in_raw_map && best >= self.base.sub.default_cost { + continue; + } -#[cfg(test)] -mod tests { - use super::*; - use crate::types::{SingleTokenCostMap, SubstitutionCostMap}; + if best < raw_direct { + let chain = self + .substitution_chain(self.id(source), self.id(target)) + .unwrap_or(EffectiveSubChain::Direct); + entries.insert((source.clone(), target.clone()), (best, chain)); + } else { + entries.insert( + (source.clone(), target.clone()), + (raw_direct, EffectiveSubChain::Direct), + ); + } + } + } - fn make_del_map(entries: &[(&str, f64)]) -> CostMap { - let costs: SingleTokenCostMap = entries.iter().map(|(k, v)| (k.to_string(), *v)).collect(); - CostMap::::new(costs, 1.0) + EffectiveSubstitutionCosts { + max_token_length: max_pair_token_len(&entries), + entries, + default_cost: self.base.sub.default_cost, + } } - fn make_ins_map(entries: &[(&str, f64)]) -> CostMap { - make_del_map(entries) - } + fn effective_deletions(&self) -> EffectiveSingleTokenCosts { + let mut entries: HashMap = HashMap::new(); + for token in self.non_epsilon_tokens() { + let node = self.id(token); + let best = self.cost(node, self.epsilon_id); + let direct = self.base.del.get_cost(token); + if self.del_map.has_key(token) || self.base.del.has_key(token) || best < direct { + let chain = if best < direct { + self.deletion_chain(node).unwrap_or(EffectiveOpChain::Direct) + } else { + self.base.del.get_chain(token) + }; + entries.insert(token.clone(), (best, chain)); + } + } - fn make_sub_map(entries: &[((&str, &str), f64)]) -> CostMap { - let costs: SubstitutionCostMap = entries - .iter() - .map(|((a, b), v)| ((a.to_string(), b.to_string()), *v)) - .collect(); - CostMap::::new(costs, 1.0, false) + EffectiveSingleTokenCosts { + max_token_length: max_single_token_len(&entries), + entries, + default_cost: self.base.del.default_cost, + } } - fn approx_eq(a: f64, b: f64) -> bool { - (a - b).abs() < f64::EPSILON * 100.0 - } + fn substitution_chain(&self, source: NodeId, target: NodeId) -> Option { + let path = self.path(source, target)?; + if path.len() <= 2 { + return Some(EffectiveSubChain::Direct); + } - // ── Deletion ────────────────────────────────────────────────────────────── + let mut steps = Vec::new(); + for edge in path.windows(2) { + let from = self.token(edge[0]); + let to = self.token(edge[1]); + let cost = self.raw_substitution_cost(from, to)?; + steps.push((from.to_string(), to.to_string(), cost)); + } - #[test] - fn direct_deletion_without_substitution_map() { - let eff = - compute_effective_deletion_costs(&make_del_map(&[("a", 0.3)]), &make_sub_map(&[])); - assert!(approx_eq(eff.get_cost("a"), 0.3)); - assert_eq!(eff.get_chain("a"), EffectiveOpChain::Direct); - assert!(approx_eq(eff.get_cost("z"), 1.0)); - assert_eq!(eff.get_chain("z"), EffectiveOpChain::Direct); + Some(EffectiveSubChain::Via { steps }) } - #[test] - fn one_hop_deletion_chain() { - // Issue #12: sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 < direct del("6", 1.0) - let eff = compute_effective_deletion_costs( - &make_del_map(&[("G", 0.01)]), - &make_sub_map(&[(("6", "G"), 0.5)]), - ); - - // "6": chain is strictly cheaper — must be present with Via chain - assert!(approx_eq(eff.get_cost("6"), 0.51)); - assert_eq!( - eff.get_chain("6"), - EffectiveOpChain::Via { - steps: vec![("6".to_string(), "G".to_string(), 0.5)], - terminal_cost: 0.01, - } - ); - assert!(eff.has_key("6")); - - // "G": explicit in del_map, direct — must not be wrapped in a chain - assert!(approx_eq(eff.get_cost("G"), 0.01)); - assert_eq!(eff.get_chain("G"), EffectiveOpChain::Direct); - assert!(eff.has_key("G")); - - // unknown token falls back to default cost, Direct chain, not registered - assert!(approx_eq(eff.get_cost("X"), 1.0)); - assert_eq!(eff.get_chain("X"), EffectiveOpChain::Direct); - assert!(!eff.has_key("X")); + fn deletion_chain(&self, source: NodeId) -> Option { + let path = self.path(source, self.epsilon_id)?; + if path.len() <= 2 { + return Some(EffectiveOpChain::Direct); + } - // all tokens are single-character - assert_eq!(eff.max_token_length, 1); - } + let terminal = *path.get(path.len() - 2)?; + let terminal_token = self.token(terminal); + let terminal_cost = self.del_map.get_cost(terminal_token); + if !self.del_map.has_key(terminal_token) { + return None; + } - #[test] - fn three_hop_deletion_chain() { - // A->B (0.3), B->C (0.2), del(C)=0.01 -> chain(A) = 0.51 < 1.0 - let eff = compute_effective_deletion_costs( - &make_del_map(&[("C", 0.01)]), - &make_sub_map(&[(("A", "B"), 0.3), (("B", "C"), 0.2)]), - ); - assert!(approx_eq(eff.get_cost("A"), 0.51)); - assert_eq!( - eff.get_chain("A"), - EffectiveOpChain::Via { - steps: vec![ - ("A".to_string(), "B".to_string(), 0.3), - ("B".to_string(), "C".to_string(), 0.2), - ], - terminal_cost: 0.01, - } - ); - // B also improves: 0.2 + 0.01 = 0.21 < 1.0 - assert!(approx_eq(eff.get_cost("B"), 0.21)); + let mut steps = Vec::new(); + for edge in path[..path.len() - 1].windows(2) { + let from = self.token(edge[0]); + let to = self.token(edge[1]); + let cost = self.raw_substitution_cost(from, to)?; + steps.push((from.to_string(), to.to_string(), cost)); + } + Some(EffectiveOpChain::Via { + steps, + terminal_cost, + }) } - #[test] - fn direct_deletion_preserved_when_chain_is_more_expensive() { - // sub("a"->"b") = 0.5, del("b") = 0.8 -> chain = 1.3 > del("a") = 0.2 - let eff = compute_effective_deletion_costs( - &make_del_map(&[("a", 0.2), ("b", 0.8)]), - &make_sub_map(&[(("a", "b"), 0.5)]), - ); - assert!(approx_eq(eff.get_cost("a"), 0.2)); - assert_eq!(eff.get_chain("a"), EffectiveOpChain::Direct); - } + fn insertion_chain(&self, target: NodeId) -> Option { + let path = self.path(self.epsilon_id, target)?; + if path.len() <= 2 { + return Some(EffectiveOpChain::Direct); + } - #[test] - fn multiple_substitution_targets_best_chain_wins() { - // sub("X"->"A") = 0.4, del("A") = 0.3 -> 0.7 - // sub("X"->"B") = 0.1, del("B") = 0.5 -> 0.6 ← winner - let eff = compute_effective_deletion_costs( - &make_del_map(&[("A", 0.3), ("B", 0.5)]), - &make_sub_map(&[(("X", "A"), 0.4), (("X", "B"), 0.1)]), - ); - assert!(approx_eq(eff.get_cost("X"), 0.6)); - assert_eq!( - eff.get_chain("X"), - EffectiveOpChain::Via { - steps: vec![("X".to_string(), "B".to_string(), 0.1)], - terminal_cost: 0.5, - } - ); - } + let initial = *path.get(1)?; + let initial_token = self.token(initial); + let terminal_cost = self.ins_map.get_cost(initial_token); + if !self.ins_map.has_key(initial_token) { + return None; + } - // ── Insertion ───────────────────────────────────────────────────────────── - - #[test] - fn one_hop_insertion_chain() { - // ins("x") = 0.1, sub("x"->"y") = 0.2 -> chain ins("y") = 0.3 < 1.0 - let eff = compute_effective_insertion_costs( - &make_ins_map(&[("x", 0.1)]), - &make_sub_map(&[(("x", "y"), 0.2)]), - ); - assert!(approx_eq(eff.get_cost("y"), 0.3)); - assert_eq!( - eff.get_chain("y"), - EffectiveOpChain::Via { - steps: vec![("x".to_string(), "y".to_string(), 0.2)], - terminal_cost: 0.1, - } - ); + let mut steps = Vec::new(); + for edge in path[1..].windows(2) { + let from = self.token(edge[0]); + let to = self.token(edge[1]); + let cost = self.raw_substitution_cost(from, to)?; + steps.push((from.to_string(), to.to_string(), cost)); + } + Some(EffectiveOpChain::Via { + steps, + terminal_cost, + }) } - #[test] - fn three_hop_insertion_chain() { - // ins(A)=0.05, sub(A->B)=0.3, sub(B->C)=0.2 -> chain(C)=0.55 < 1.0 - let eff = compute_effective_insertion_costs( - &make_ins_map(&[("A", 0.05)]), - &make_sub_map(&[(("A", "B"), 0.3), (("B", "C"), 0.2)]), - ); - assert!(approx_eq(eff.get_cost("C"), 0.55)); - assert_eq!( - eff.get_chain("C"), - EffectiveOpChain::Via { - steps: vec![ - ("A".to_string(), "B".to_string(), 0.3), - ("B".to_string(), "C".to_string(), 0.2), - ], - terminal_cost: 0.05, + fn raw_substitution_cost(&self, source: &str, target: &str) -> Option { + self.sub_map + .costs + .get(&(source.to_owned(), target.to_owned())) + .copied() + } + + fn effective_insertions(&self) -> EffectiveSingleTokenCosts { + let mut entries: HashMap = HashMap::new(); + for token in self.non_epsilon_tokens() { + let node = self.id(token); + let best = self.cost(self.epsilon_id, node); + let direct = self.base.ins.get_cost(token); + if self.ins_map.has_key(token) || self.base.ins.has_key(token) || best < direct { + let chain = if best < direct { + self.insertion_chain(node).unwrap_or(EffectiveOpChain::Direct) + } else { + self.base.ins.get_chain(token) + }; + entries.insert(token.clone(), (best, chain)); } - ); - } - - #[test] - fn direct_insertion_preserved_when_chain_is_more_expensive() { - // ins("y") = 0.1, ins("x") = 0.9, sub("x"->"y") = 0.5 -> chain = 1.4 > 0.1 - let eff = compute_effective_insertion_costs( - &make_ins_map(&[("y", 0.1), ("x", 0.9)]), - &make_sub_map(&[(("x", "y"), 0.5)]), - ); - assert!(approx_eq(eff.get_cost("y"), 0.1)); - assert_eq!(eff.get_chain("y"), EffectiveOpChain::Direct); - } - - // ── has_key semantics ───────────────────────────────────────────────────── - - #[test] - fn has_key_only_when_explicit_or_chain_improves() { - // sub("b"->"c", 0.2) + del("c", 1.0) = 1.2 > default 1.0 -> "b" NOT added - let eff = compute_effective_deletion_costs( - &make_del_map(&[("a", 0.5)]), - &make_sub_map(&[(("b", "c"), 0.2)]), - ); - assert!(eff.has_key("a")); - assert!(!eff.has_key("b")); // chain not cheaper - assert!(!eff.has_key("z")); - - // sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 < 1.0 -> "6" IS added - let eff2 = compute_effective_deletion_costs( - &make_del_map(&[("G", 0.01)]), - &make_sub_map(&[(("6", "G"), 0.5)]), - ); - assert!(eff2.has_key("6")); - } + } - #[test] - fn max_token_length_reflects_longest_key() { - let eff = - compute_effective_deletion_costs(&make_del_map(&[("ab", 0.5)]), &make_sub_map(&[])); - assert_eq!(eff.max_token_length, 2); + EffectiveSingleTokenCosts { + max_token_length: max_single_token_len(&entries), + entries, + default_cost: self.base.ins.default_cost, + } } - // ── Substitution ────────────────────────────────────────────────────────── - - #[test] - fn direct_substitution() { - let eff = compute_effective_substitution_costs(&make_sub_map(&[(("a", "b"), 0.3)])); - assert!(approx_eq(eff.get_cost("a", "b"), 0.3)); - assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); - assert!(eff.has_key("a", "b")); - // Unknown pair falls back to default. - assert!(approx_eq(eff.get_cost("a", "c"), 1.0)); - assert!(!eff.has_key("a", "c")); + fn non_epsilon_tokens(&self) -> impl Iterator { + self.tokens.iter().filter(|token| !token.is_empty()) } +} - #[test] - fn two_hop_substitution_chain() { - // sub(a->b)=0.1 + sub(b->c)=0.1 -> eff_sub(a->c)=0.2 < default 1.0 - let eff = compute_effective_substitution_costs(&make_sub_map(&[ - (("a", "b"), 0.1), - (("b", "c"), 0.1), - ])); - assert!(approx_eq(eff.get_cost("a", "c"), 0.2)); - assert_eq!( - eff.get_chain("a", "c"), - EffectiveSubChain::Via { - steps: vec![ - ("a".to_string(), "b".to_string(), 0.1), - ("b".to_string(), "c".to_string(), 0.1), - ], +fn collect_solver_tokens( + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, + base: &BaseEffectiveCosts, +) -> Vec { + let mut tokens: HashSet = HashSet::new(); + tokens.insert(String::new()); // epsilon + tokens.extend(ins_map.costs.keys().cloned()); + tokens.extend(del_map.costs.keys().cloned()); + tokens.extend(base.ins.entries.keys().cloned()); + tokens.extend(base.del.entries.keys().cloned()); + tokens.extend( + sub_map + .costs + .keys() + .flat_map(|(source, target)| [source.clone(), target.clone()]), + ); + tokens.extend( + base.sub + .entries + .keys() + .flat_map(|(source, target)| [source.clone(), target.clone()]), + ); + + let originals: Vec = tokens.iter().cloned().collect(); + for token in originals { + let chars: Vec = token.chars().collect(); + for start in 0..chars.len() { + for end in (start + 1)..=chars.len() { + tokens.insert(chars[start..end].iter().collect()); } - ); - assert!(eff.has_key("a", "c")); - // Direct pair is still Direct. - assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); - assert!(approx_eq(eff.get_cost("a", "b"), 0.1)); + } } - #[test] - fn direct_substitution_beats_chain() { - // sub(a->b)=0.1 (direct), sub(a->c)=0.3, sub(c->b)=0.1 - // chain(a->b) via c = 0.4 > direct 0.1 -> Direct wins - let eff = compute_effective_substitution_costs(&make_sub_map(&[ - (("a", "b"), 0.1), - (("a", "c"), 0.3), - (("c", "b"), 0.1), - ])); - assert!(approx_eq(eff.get_cost("a", "b"), 0.1)); - assert_eq!(eff.get_chain("a", "b"), EffectiveSubChain::Direct); - } + tokens.into_iter().collect() +} - #[test] - fn chain_substitution_beats_direct() { - // sub(a->b)=0.5 (direct), sub(a->c)=0.1, sub(c->b)=0.1 - // chain(a->b) via c = 0.2 < direct 0.5 -> Via wins - let eff = compute_effective_substitution_costs(&make_sub_map(&[ - (("a", "b"), 0.5), - (("a", "c"), 0.1), - (("c", "b"), 0.1), - ])); - assert!(approx_eq(eff.get_cost("a", "b"), 0.2)); - assert_eq!( - eff.get_chain("a", "b"), - EffectiveSubChain::Via { - steps: vec![ - ("a".to_string(), "c".to_string(), 0.1), - ("c".to_string(), "b".to_string(), 0.1), - ], - } - ); - } +fn max_pair_token_len(entries: &HashMap<(String, String), (f64, EffectiveSubChain)>) -> usize { + entries + .keys() + .flat_map(|(source, target)| [source.chars().count(), target.chars().count()]) + .max() + .unwrap_or(1) + .max(1) +} - #[test] - fn chain_not_added_when_no_improvement_over_default() { - // sub(a->b)=0.6, sub(b->c)=0.6 -> chain(a->c)=1.2 >= default 1.0 -> NOT added - let eff = compute_effective_substitution_costs(&make_sub_map(&[ - (("a", "b"), 0.6), - (("b", "c"), 0.6), - ])); - assert!(!eff.has_key("a", "c")); - assert!(approx_eq(eff.get_cost("a", "c"), 1.0)); // default - } +fn max_single_token_len(entries: &HashMap) -> usize { + entries + .keys() + .map(|token| token.chars().count()) + .max() + .unwrap_or(1) + .max(1) +} + +/// Computes all effective costs with one unified state-transition graph. +pub(crate) fn compute_effective_costs_unified( + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, +) -> ( + EffectiveSubstitutionCosts, + EffectiveSingleTokenCosts, + EffectiveSingleTokenCosts, +) { + CostSolver::new(sub_map, ins_map, del_map).compute_effective_costs() } diff --git a/src/weighted_levenshtein.rs b/src/weighted_levenshtein.rs index 9806a7c..9ecfd97 100644 --- a/src/weighted_levenshtein.rs +++ b/src/weighted_levenshtein.rs @@ -391,10 +391,7 @@ impl<'a> LevenshteinProcessor<'a> { mod test { use super::*; use crate::cost_map::CostMap; - use crate::transitive_costs::{ - compute_effective_deletion_costs, compute_effective_insertion_costs, - compute_effective_substitution_costs, - }; + use crate::transitive_costs::compute_effective_costs_unified; use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; fn assert_approx_eq(a: f64, b: f64, epsilon: f64) { @@ -425,9 +422,8 @@ mod test { ins_map: &CostMap, del_map: &CostMap, ) -> f64 { - let eff_sub = compute_effective_substitution_costs(sub_map); - let eff_del = compute_effective_deletion_costs(del_map, sub_map); - let eff_ins = compute_effective_insertion_costs(ins_map, sub_map); + let (eff_sub, eff_del, eff_ins) = + compute_effective_costs_unified(sub_map, ins_map, del_map); custom_levenshtein_distance_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) } @@ -438,9 +434,8 @@ mod test { ins_map: &CostMap, del_map: &CostMap, ) -> Vec { - let eff_sub = compute_effective_substitution_costs(sub_map); - let eff_del = compute_effective_deletion_costs(del_map, sub_map); - let eff_ins = compute_effective_insertion_costs(ins_map, sub_map); + let (eff_sub, eff_del, eff_ins) = + compute_effective_costs_unified(sub_map, ins_map, del_map); explain_custom_levenshtein_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) } @@ -969,14 +964,9 @@ mod test { #[test] fn test_check_multi_char_ops_with_empty_maps() { - use crate::transitive_costs::{ - compute_effective_deletion_costs, compute_effective_insertion_costs, - compute_effective_substitution_costs, - }; let (sub_map, ins_map, del_map) = create_default_cost_maps(); - let eff_sub = compute_effective_substitution_costs(&sub_map); - let eff_del = compute_effective_deletion_costs(&del_map, &sub_map); - let eff_ins = compute_effective_insertion_costs(&ins_map, &sub_map); + let (eff_sub, eff_del, eff_ins) = + compute_effective_costs_unified(&sub_map, &ins_map, &del_map); let mut processor = LevenshteinProcessor::new("abcd", "xyz", &eff_sub, &eff_ins, &eff_del, true); From db0346578ae17452d2a6d457a2ce7e6b13ae2fa9 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Thu, 30 Apr 2026 21:45:40 +0200 Subject: [PATCH 03/21] updates --- Cargo.lock | 4 +- Cargo.toml | 2 +- .../test_explain_weighted_levenshtein.py | 61 ++ python/tests/test_weighted_levenshtein.py | 24 + src/cost_map.rs | 15 +- src/explanation.rs | 2 +- src/rust_stringdist.rs | 38 +- src/transitive_costs.rs | 637 ++++++++++++------ src/weighted_levenshtein.rs | 126 ++-- 9 files changed, 597 insertions(+), 312 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65c789f..0b00281 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" @@ -74,7 +74,7 @@ dependencies = [ [[package]] name = "ocr_stringdist" -version = "1.0.1" +version = "1.1.0" dependencies = [ "pyo3", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 5f4f3ca..d21cfb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ocr_stringdist" -version = "1.0.1" +version = "1.1.0" edition = "2021" description = "String distances considering OCR errors." authors = ["Niklas von Moers "] diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index 267b5c1..7001be3 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -75,6 +75,7 @@ def test_explain_weighted_levenshtein( manually_filtered_operations = [op for op in full_operations if op.op_type != "match"] assert filtered_operations == manually_filtered_operations assert full_operations == expected_operations + assert sum(op.cost for op in full_operations) == wl.distance(s1, s2) def test_explain_transitive_deletion_chain() -> None: @@ -118,3 +119,63 @@ def test_explain_transitive_insertion_chain() -> None: EditOperation("insert", None, "x", 0.1), EditOperation("substitute", "x", "y", 0.2), ] + + +def test_explain_chain_with_expensive_direct_substitution() -> None: + """ + Test that A->AA->AAA->B is explained instead of the more expensive A->B. + """ + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, insertion_costs={"A": 0.2} + ) + ops = wl.explain("A", "B", filter_matches=True) + assert ops == [ + EditOperation("insert", None, "A", 0.2), + EditOperation("insert", None, "A", 0.2), + EditOperation("substitute", "AAA", "B", 0.1), + ] + + +def test_explain_mixed_substitution_path_with_deletion() -> None: + """ + Test that AB->A->C is expanded when it beats the direct AB->C substitution. + """ + wl = WeightedLevenshtein( + substitution_costs={("A", "C"): 0.1, ("AB", "C"): 0.5}, + deletion_costs={"B": 0.2}, + ) + ops = wl.explain("AB", "C", filter_matches=True) + assert ops == [ + EditOperation("substitute", "A", "C", 0.1), + EditOperation("delete", "B", None, 0.2), + ] + + +def test_explain_direct_substitution_wins_over_mixed_chain() -> None: + """ + Test that a cheaper direct A->B substitution is not expanded into A->AAA->B. + """ + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.4}, + insertion_costs={"A": 0.2}, + ) + ops = wl.explain("A", "B", filter_matches=True) + assert ops == [ + EditOperation("substitute", "A", "B", 0.4), + ] + + +def test_explain_effective_deletion_with_insertion_then_deletion() -> None: + """ + Test that AC->ABC->C is expanded as insert(B), delete(AB), match(C). + """ + wl = WeightedLevenshtein( + insertion_costs={"B": 0.1}, + deletion_costs={"AB": 0.0}, + ) + ops = wl.explain("AC", "C", filter_matches=False) + assert ops == [ + EditOperation("insert", None, "B", 0.1), + EditOperation("delete", "AB", None, 0.0), + EditOperation("match", "C", "C", 0.0), + ] diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index a2357f3..3ee7a4a 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -623,6 +623,30 @@ def test_transitive_substitution_chain_distance() -> None: assert wl.distance("a", "c") == pytest.approx(0.2) +@pytest.mark.xfail( + reason=( + "Optimized transitive seeding does not create arbitrary token-to-token " + "shortcuts from full weighted DP alignments." + ), + strict=True, +) +def test_full_dp_seed_would_create_multi_edit_token_shortcut() -> None: + """ + We only seed raw operations and targeted one-edit embedded + edges. Because these tokens are longer than the subtoken expansion cap, the + intermediate `source + "A"` node is absent, so the shortcut is not present. + """ + source = "abcdefghijklmnopq" # 17 chars: above MAX_SUBTOKEN_EXPANSION_CHARS + bridge = f"{source}AB" + target = "Z" + wl = WeightedLevenshtein( + insertion_costs={"A": 0.2, "B": 0.3}, + deletion_costs={source: 10.0}, # make `source` a graph token + substitution_costs={(bridge, target): 0.1}, + ) + assert wl.distance(source, target) == pytest.approx(0.6) + + def test_serialization() -> None: wl_orig = WeightedLevenshtein( substitution_costs={("a", "b"): 0.5}, diff --git a/src/cost_map.rs b/src/cost_map.rs index bb874af..0609f45 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -98,14 +98,6 @@ impl CostMap { Self::new(single_token_costs, default_cost) } - - pub fn get_cost(&self, token: &str) -> f64 { - self.costs.get(token).copied().unwrap_or(self.default_cost) - } - - pub fn has_key(&self, token: &str) -> bool { - self.costs.contains_key(token) - } } // Common methods for any type of CostMap @@ -123,8 +115,7 @@ mod tests { fn test_single_token_map_default() { let cost_map: CostMap = CostMap::default(); assert_eq!(cost_map.default_cost(), 1.0); - assert_eq!(cost_map.get_cost("any_token"), 1.0); - assert!(!cost_map.has_key("any_token")); + assert!(cost_map.costs.is_empty()); } #[test] @@ -135,8 +126,8 @@ mod tests { let cost_map = CostMap::::new(custom_costs, 2.0); assert_eq!(cost_map.default_cost(), 2.0); - assert_eq!(cost_map.get_cost("test"), 0.3); - assert_eq!(cost_map.get_cost("unknown"), 2.0); + assert_eq!(cost_map.costs["test"], 0.3); + assert!(!cost_map.costs.contains_key("unknown")); } #[test] diff --git a/src/explanation.rs b/src/explanation.rs index 8688c2d..6c45e60 100644 --- a/src/explanation.rs +++ b/src/explanation.rs @@ -1,6 +1,6 @@ /// Represents a single operation in the edit path. /// This is the data structure that will be returned to Python. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum EditOperation { Substitute { source: String, diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index 41dec0b..b26acfc 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,8 +1,6 @@ use crate::cost_map::CostMap; use crate::explanation::EditOperation; -use crate::transitive_costs::{ - compute_effective_costs_unified, EffectiveSingleTokenCosts, EffectiveSubstitutionCosts, -}; +use crate::transitive_costs::{compute_effective_costs, EffectiveCosts}; use crate::types::{SingleTokenKey, SubstitutionKey}; use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; use crate::weighted_levenshtein::explain_custom_levenshtein_precomputed; @@ -32,17 +30,12 @@ impl<'py> IntoPyObject<'py> for EditOperation { } } -/// Precomputes effective substitution, insertion, and deletion costs once and reuses them -/// for every `.distance()` / `.batch_distance()` call. -/// -/// Building the calculator runs Dijkstra on the substitution graph and then token-graph -/// closure passes (see `transitive_costs`), so per-call distance stays linear in string length. +/// Precomputes effective substitution, insertion, and deletion costs once and +/// reuses them for every `.distance()` / `.batch_distance()` call. #[pyclass] #[derive(Debug)] struct RustLevenshteinCalculator { - eff_sub: EffectiveSubstitutionCosts, - eff_del: EffectiveSingleTokenCosts, - eff_ins: EffectiveSingleTokenCosts, + costs: EffectiveCosts, } #[pymethods] @@ -80,18 +73,13 @@ impl RustLevenshteinCalculator { let del_map = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); - let (eff_sub, eff_del, eff_ins) = - compute_effective_costs_unified(&sub_map, &ins_map, &del_map); + let costs = compute_effective_costs(&sub_map, &ins_map, &del_map); - Ok(Self { - eff_sub, - eff_del, - eff_ins, - }) + Ok(Self { costs }) } fn distance(&self, a: &str, b: &str) -> f64 { - custom_levenshtein_distance_precomputed(a, b, &self.eff_sub, &self.eff_ins, &self.eff_del) + custom_levenshtein_distance_precomputed(a, b, &self.costs) } fn batch_distance(&self, py: Python<'_>, s: String, candidates: Vec) -> Vec { @@ -101,21 +89,13 @@ impl RustLevenshteinCalculator { py.allow_threads(|| { candidates .par_iter() - .map(|c| { - custom_levenshtein_distance_precomputed( - &s, - c, - &self.eff_sub, - &self.eff_ins, - &self.eff_del, - ) - }) + .map(|c| custom_levenshtein_distance_precomputed(&s, c, &self.costs)) .collect() }) } fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult> { - explain_custom_levenshtein_precomputed(a, b, &self.eff_sub, &self.eff_ins, &self.eff_del) + explain_custom_levenshtein_precomputed(a, b, &self.costs) .into_iter() .map(|op| op.into_pyobject(py).map(|bound| bound.into())) .collect::>>() diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 2287376..83e02bf 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -1,20 +1,22 @@ -//! Effective edit costs under substitution-only transitivity and token-graph closure. +//! Effective edit costs from a unified token graph. //! -//! The Python-facing calculator uses [`compute_effective_costs_unified`], which -//! seeds one token graph from raw insertion/deletion/substitution maps and then -//! runs a single Floyd-Warshall closure. The older Dijkstra-based constructors -//! remain as small building blocks for tests and for substitution-chain -//! provenance in Rust-only callers. -//! -//! Effective costs are stored in [`EffectiveSingleTokenCosts`] (del/ins) and -//! [`EffectiveSubstitutionCosts`] (sub), computed once at construction (for example when -//! building the Rust/Python calculator). +//! [`compute_effective_costs`] seeds one graph from the raw sub/ins/del maps, +//! closes it with Floyd-Warshall, and projects distances back: `dist[a][b]` for +//! substitutions, `dist[a][ε]` for deletions, `dist[ε][b]` for insertions. use crate::cost_map::CostMap; +use crate::explanation::EditOperation; use crate::types::{SingleTokenKey, SubstitutionKey}; -use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; +use crate::weighted_levenshtein::explain_custom_levenshtein_precomputed; use std::collections::{HashMap, HashSet}; +const NO_NEXT_NODE: u32 = u32::MAX; + +// Configured tokens up to this length are expanded into all of their substrings +// so closure can route through intermediate strings (e.g. `A -> AA -> AAA`). +// Capped because Floyd-Warshall is O(N³) over the resulting node set. +const MAX_SUBTOKEN_EXPANSION_CHARS: usize = 16; + // Public types /// How the minimum effective cost for a single-token operation was achieved. @@ -37,6 +39,9 @@ pub enum EffectiveOpChain { /// (insertion) node of the chain. terminal_cost: f64, }, + + /// A cheaper path exists through mixed edit operations. + EditPath { operations: Vec }, } /// Precomputed effective single-token operation costs (deletion or insertion). @@ -53,10 +58,13 @@ pub struct EffectiveSingleTokenCosts { impl EffectiveSingleTokenCosts { #[inline] pub fn get_cost(&self, token: &str) -> f64 { - self.entries - .get(token) - .map(|(c, _)| *c) - .unwrap_or(self.default_cost) + self.get_explicit_cost(token).unwrap_or(self.default_cost) + } + + /// Cost of an explicit entry, or `None` if `token` is not in the map. + #[inline] + pub fn get_explicit_cost(&self, token: &str) -> Option { + self.entries.get(token).map(|(c, _)| *c) } #[inline] @@ -87,6 +95,12 @@ pub enum EffectiveSubChain { /// Substitution edges `(from, to, cost)` in forward order. steps: Vec<(String, String, f64)>, }, + + /// A cheaper path was found through mixed edit operations. + /// + /// This is needed when an effective substitution path contains insertions + /// or deletions between graph nodes, for example `A -> AAA -> B`. + EditPath { operations: Vec }, } /// Precomputed effective substitution costs (all-pairs shortest paths). @@ -95,7 +109,8 @@ pub enum EffectiveSubChain { /// automatically uses the globally cheapest substitution path. #[derive(Debug)] pub struct EffectiveSubstitutionCosts { - entries: HashMap<(String, String), (f64, EffectiveSubChain)>, + /// Entries are indexed by source token, then by target token. + entries: HashMap>, default_cost: f64, pub max_token_length: usize, } @@ -103,16 +118,24 @@ pub struct EffectiveSubstitutionCosts { impl EffectiveSubstitutionCosts { #[inline] pub fn get_cost(&self, source: &str, target: &str) -> f64 { + self.get_explicit_cost(source, target) + .unwrap_or(self.default_cost) + } + + /// Cost of an explicit entry, or `None` if `(source, target)` is not in the map. + #[inline] + pub fn get_explicit_cost(&self, source: &str, target: &str) -> Option { self.entries - .get(&(source.to_owned(), target.to_owned())) + .get(source) + .and_then(|targets| targets.get(target)) .map(|(c, _)| *c) - .unwrap_or(self.default_cost) } #[inline] pub fn get_chain(&self, source: &str, target: &str) -> EffectiveSubChain { self.entries - .get(&(source.to_owned(), target.to_owned())) + .get(source) + .and_then(|targets| targets.get(target)) .map(|(_, ch)| ch.clone()) .unwrap_or(EffectiveSubChain::Direct) } @@ -120,7 +143,8 @@ impl EffectiveSubstitutionCosts { #[inline] pub fn has_key(&self, source: &str, target: &str) -> bool { self.entries - .contains_key(&(source.to_owned(), target.to_owned())) + .get(source) + .is_some_and(|targets| targets.contains_key(target)) } } @@ -132,36 +156,105 @@ impl EffectiveSubstitutionCosts { /// epsilon node (`""`). Keeping this as a newtype instead of a bare `usize` /// makes graph indexing sites explicit. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -struct NodeId(usize); +struct NodeId(u32); impl NodeId { + fn new(index: usize) -> Self { + assert!(index < NO_NEXT_NODE as usize, "too many token graph nodes"); + Self(index as u32) + } + #[inline] fn index(self) -> usize { + self.0 as usize + } + + #[inline] + fn raw(self) -> u32 { self.0 } } -/// All base costs used to seed the unified token graph. +#[derive(Debug)] +struct EdgeResolution { + operations: Vec, + substitution_steps: Vec<(String, String, f64)>, + all_edges_are_raw_substitutions: bool, +} + +/// Dense square matrix stored in row-major order. /// -/// These are direct wrappers around the raw cost maps. They intentionally do -/// not apply transitive closure; [`CostSolver`] owns the single closure pass. -struct BaseEffectiveCosts { - sub: EffectiveSubstitutionCosts, - ins: EffectiveSingleTokenCosts, - del: EffectiveSingleTokenCosts, +/// Floyd-Warshall touches the matrix in tight nested loops; a flat vector avoids +/// the pointer chasing and per-row allocations of `Vec>`. +#[derive(Debug)] +struct Matrix { + width: usize, + cells: Vec, } -fn raw_effective_substitution_costs(sub_map: &CostMap) -> EffectiveSubstitutionCosts { - let entries = sub_map - .costs - .iter() - .map(|((source, target), &cost)| { - ( - (source.clone(), target.clone()), - (cost, EffectiveSubChain::Direct), - ) - }) - .collect(); +impl Matrix { + fn filled(width: usize, value: T) -> Self { + Self { + width, + cells: vec![value; width * width], + } + } +} + +impl Matrix { + #[inline] + fn idx(&self, row: NodeId, col: NodeId) -> usize { + row.index() * self.width + col.index() + } + + #[inline] + fn get(&self, row: NodeId, col: NodeId) -> &T { + &self.cells[self.idx(row, col)] + } + + #[inline] + fn set(&mut self, row: NodeId, col: NodeId, value: T) { + let idx = self.idx(row, col); + self.cells[idx] = value; + } +} + +/// Bundle of effective sub/ins/del costs. +/// +/// Used both as direct wrappers around the raw cost maps (input to closure) and +/// as the closed-graph projection returned by [`compute_effective_costs`]. +#[derive(Debug)] +pub struct EffectiveCosts { + pub sub: EffectiveSubstitutionCosts, + pub ins: EffectiveSingleTokenCosts, + pub del: EffectiveSingleTokenCosts, +} + +impl EffectiveCosts { + /// Direct wrappers around the raw cost maps, no closure applied. + fn raw( + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, + ) -> Self { + Self { + sub: raw_effective_substitution_costs(sub_map), + ins: raw_effective_single_token_costs(ins_map), + del: raw_effective_single_token_costs(del_map), + } + } +} + +fn raw_effective_substitution_costs( + sub_map: &CostMap, +) -> EffectiveSubstitutionCosts { + let mut entries: HashMap> = HashMap::new(); + for ((source, target), &cost) in &sub_map.costs { + entries + .entry(source.clone()) + .or_default() + .insert(target.clone(), (cost, EffectiveSubChain::Direct)); + } EffectiveSubstitutionCosts { max_token_length: sub_map @@ -169,16 +262,14 @@ fn raw_effective_substitution_costs(sub_map: &CostMap) -> Effec .keys() .flat_map(|(source, target)| [source.chars().count(), target.chars().count()]) .max() - .unwrap_or(1) + .unwrap_or(0) .max(1), entries, default_cost: sub_map.default_cost(), } } -fn raw_effective_single_token_costs( - map: &CostMap, -) -> EffectiveSingleTokenCosts { +fn raw_effective_single_token_costs(map: &CostMap) -> EffectiveSingleTokenCosts { let entries = map .costs .iter() @@ -191,60 +282,133 @@ fn raw_effective_single_token_costs( .keys() .map(|token| token.chars().count()) .max() - .unwrap_or(1) + .unwrap_or(0) .max(1), entries, default_cost: map.default_cost(), } } +/// Adds or improves one directed graph edge and records its first hop. +fn set_seed_edge( + dist: &mut Matrix, + next: &mut Matrix, + source: NodeId, + target: NodeId, + cost: f64, +) { + if cost < *dist.get(source, target) { + dist.set(source, target, cost); + next.set(source, target, target.raw()); + } +} + +/// Adds token-to-token edges that can be made by one explicit insertion/deletion. +/// +/// This captures paths like `A -> AA` or `AB -> A` without running full +/// Levenshtein DP for every token pair during graph construction. +fn seed_embedded_single_token_edges( + tokens: &[String], + token_to_id: &HashMap, + base: &EffectiveCosts, + dist: &mut Matrix, + next: &mut Matrix, +) { + let insertions: Vec<(&str, f64)> = base + .ins + .entries + .iter() + .map(|(token, (cost, _))| (token.as_str(), *cost)) + .collect(); + let deletions: Vec<(&str, f64)> = base + .del + .entries + .iter() + .map(|(token, (cost, _))| (token.as_str(), *cost)) + .collect(); + + for source in tokens { + let source_id = token_to_id[source.as_str()]; + for (inserted, cost) in &insertions { + for target in insert_token_variants(source, inserted) { + if let Some(&target_id) = token_to_id.get(target.as_str()) { + set_seed_edge(dist, next, source_id, target_id, *cost); + } + } + } + + for (deleted, cost) in &deletions { + for target in delete_token_variants(source, deleted) { + if let Some(&target_id) = token_to_id.get(target.as_str()) { + set_seed_edge(dist, next, source_id, target_id, *cost); + } + } + } + } +} + +/// All strings obtainable by inserting `inserted` at a char boundary in `source`. +fn insert_token_variants(source: &str, inserted: &str) -> Vec { + source + .char_indices() + .map(|(idx, _)| idx) + .chain(std::iter::once(source.len())) + .map(|idx| { + let mut target = String::with_capacity(source.len() + inserted.len()); + target.push_str(&source[..idx]); + target.push_str(inserted); + target.push_str(&source[idx..]); + target + }) + .collect() +} + +/// All strings obtainable by deleting one exact `deleted` occurrence from `source`. +fn delete_token_variants(source: &str, deleted: &str) -> Vec { + source + .match_indices(deleted) + .map(|(idx, _)| { + let end = idx + deleted.len(); + let mut target = String::with_capacity(source.len() - deleted.len()); + target.push_str(&source[..idx]); + target.push_str(&source[end..]); + target + }) + .collect() +} + /// Unified state-transition solver for effective edit costs. /// /// The graph has one node per relevant token plus a distinguished epsilon node. /// Its dense adjacency matrix is initialized with direct weighted token-to-token /// distances under the base effective costs, then closed with Floyd-Warshall. -/// -/// After closure, every operation is a projection from the same matrix: -/// - substitution `a -> b`: `dist[a][b]` -/// - deletion `a`: `dist[a][epsilon]` -/// - insertion `b`: `dist[epsilon][b]` -struct CostSolver<'a> { - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, - base: BaseEffectiveCosts, +struct CostSolver { + base: EffectiveCosts, token_to_id: HashMap, tokens: Vec, epsilon_id: NodeId, - dist: Vec>, - next: Vec>>, + dist: Matrix, + next: Matrix, } -impl<'a> CostSolver<'a> { +impl CostSolver { fn new( - sub_map: &'a CostMap, - ins_map: &'a CostMap, - del_map: &'a CostMap, + sub_map: &CostMap, + ins_map: &CostMap, + del_map: &CostMap, ) -> Self { - let base = BaseEffectiveCosts { - sub: raw_effective_substitution_costs(sub_map), - ins: raw_effective_single_token_costs(ins_map), - del: raw_effective_single_token_costs(del_map), - }; + let base = EffectiveCosts::raw(sub_map, ins_map, del_map); - let tokens = collect_solver_tokens(sub_map, ins_map, del_map, &base); + let tokens = collect_solver_tokens(sub_map, ins_map, del_map); let token_to_id: HashMap = tokens .iter() .enumerate() - .map(|(i, token)| (token.clone(), NodeId(i))) + .map(|(i, token)| (token.clone(), NodeId::new(i))) .collect(); let epsilon_id = token_to_id[""]; - let (dist, next) = Self::seed_distances(&tokens, &base); + let (dist, next) = Self::seed_distances(&tokens, &token_to_id, &base); Self { - sub_map, - ins_map, - del_map, base, token_to_id, tokens, @@ -254,64 +418,84 @@ impl<'a> CostSolver<'a> { } } - fn compute_effective_costs(mut self) -> ( - EffectiveSubstitutionCosts, - EffectiveSingleTokenCosts, - EffectiveSingleTokenCosts, - ) { + fn compute_effective_costs(mut self) -> EffectiveCosts { self.close_all_pairs(); - ( - self.effective_substitutions(), - self.effective_deletions(), - self.effective_insertions(), - ) + EffectiveCosts { + sub: self.effective_substitutions(), + ins: self.effective_insertions(), + del: self.effective_deletions(), + } } - /// Initialize graph edges from direct weighted token-to-token distances. + /// Initializes direct graph edges from raw edit operations. + /// + /// This deliberately avoids all-pairs weighted DP. The closure pass can + /// discover multi-step paths from raw substitutions, epsilon insertions/ + /// deletions, and the targeted embedded single-token edges. fn seed_distances( tokens: &[String], - base: &BaseEffectiveCosts, - ) -> (Vec>, Vec>>) { + token_to_id: &HashMap, + base: &EffectiveCosts, + ) -> (Matrix, Matrix) { let n = tokens.len(); - let mut dist = vec![vec![f64::INFINITY; n]; n]; - let mut next = vec![vec![None; n]; n]; - for source in 0..n { - dist[source][source] = 0.0; - next[source][source] = Some(NodeId(source)); - for target in 0..n { - if source == target { - continue; - } - dist[source][target] = custom_levenshtein_distance_precomputed( - &tokens[source], - &tokens[target], - &base.sub, - &base.ins, - &base.del, - ); - if dist[source][target].is_finite() { - next[source][target] = Some(NodeId(target)); - } + let mut dist = Matrix::filled(n, f64::INFINITY); + let mut next = Matrix::filled(n, NO_NEXT_NODE); + let epsilon = token_to_id[""]; + + for node in 0..n { + let node = NodeId::new(node); + set_seed_edge(&mut dist, &mut next, node, node, 0.0); + } + + for (source, targets) in &base.sub.entries { + let source_id = token_to_id[source.as_str()]; + for (target, (cost, _)) in targets { + let target_id = token_to_id[target.as_str()]; + set_seed_edge(&mut dist, &mut next, source_id, target_id, *cost); } } + + for (token, (cost, _)) in &base.ins.entries { + set_seed_edge( + &mut dist, + &mut next, + epsilon, + token_to_id[token.as_str()], + *cost, + ); + } + + for (token, (cost, _)) in &base.del.entries { + set_seed_edge( + &mut dist, + &mut next, + token_to_id[token.as_str()], + epsilon, + *cost, + ); + } + + seed_embedded_single_token_edges(tokens, token_to_id, base, &mut dist, &mut next); (dist, next) } /// Floyd-Warshall all-pairs shortest paths over the unified graph. - #[allow(clippy::needless_range_loop)] // Indexed `target` avoids simultaneous borrows of `dist`. fn close_all_pairs(&mut self) { - let n = self.dist.len(); + let n = self.dist.width; for via in 0..n { + let via = NodeId::new(via); for source in 0..n { - let source_to_via = self.dist[source][via]; + let source = NodeId::new(source); + let source_to_via = *self.dist.get(source, via); if !source_to_via.is_finite() { continue; } for target in 0..n { - let candidate = source_to_via + self.dist[via][target]; - if candidate < self.dist[source][target] { - self.dist[source][target] = candidate; - self.next[source][target] = self.next[source][via]; + let target = NodeId::new(target); + let candidate = source_to_via + *self.dist.get(via, target); + if candidate < *self.dist.get(source, target) { + self.dist.set(source, target, candidate); + self.next.set(source, target, *self.next.get(source, via)); } } } @@ -323,15 +507,21 @@ impl<'a> CostSolver<'a> { } fn cost(&self, source: NodeId, target: NodeId) -> f64 { - self.dist[source.index()][target.index()] + *self.dist.get(source, target) } fn path(&self, source: NodeId, target: NodeId) -> Option> { - self.next[source.index()][target.index()]?; + if *self.next.get(source, target) == NO_NEXT_NODE { + return None; + } let mut path = vec![source]; let mut current = source; while current != target { - current = self.next[current.index()][target.index()]?; + let next = *self.next.get(current, target); + if next == NO_NEXT_NODE { + return None; + } + current = NodeId(next); path.push(current); } Some(path) @@ -341,40 +531,37 @@ impl<'a> CostSolver<'a> { &self.tokens[node.index()] } + /// Projects closed token-to-token distances into effective substitutions. fn effective_substitutions(&self) -> EffectiveSubstitutionCosts { - let mut entries: HashMap<(String, String), (f64, EffectiveSubChain)> = HashMap::new(); + let mut entries: HashMap> = + HashMap::new(); for source in self.non_epsilon_tokens() { for target in self.non_epsilon_tokens() { if source == target { continue; } let best = self.cost(self.id(source), self.id(target)); - let raw_direct = self - .sub_map - .costs - .get(&(source.clone(), target.clone())) - .copied() - .unwrap_or(self.base.sub.default_cost); - let in_raw_map = self - .sub_map - .costs - .contains_key(&(source.clone(), target.clone())); + let raw_direct = self.raw_substitution_cost(source, target); + let in_raw_map = raw_direct.is_some(); + let direct_cost = raw_direct.unwrap_or(self.base.sub.default_cost); if !in_raw_map && best >= self.base.sub.default_cost { continue; } - if best < raw_direct { + let entry = if best < direct_cost { let chain = self .substitution_chain(self.id(source), self.id(target)) .unwrap_or(EffectiveSubChain::Direct); - entries.insert((source.clone(), target.clone()), (best, chain)); + (best, chain) } else { - entries.insert( - (source.clone(), target.clone()), - (raw_direct, EffectiveSubChain::Direct), - ); - } + (direct_cost, EffectiveSubChain::Direct) + }; + + entries + .entry(source.to_owned()) + .or_default() + .insert(target.to_owned(), entry); } } @@ -385,19 +572,21 @@ impl<'a> CostSolver<'a> { } } + /// Projects token-to-epsilon distances into effective deletions. fn effective_deletions(&self) -> EffectiveSingleTokenCosts { let mut entries: HashMap = HashMap::new(); for token in self.non_epsilon_tokens() { let node = self.id(token); let best = self.cost(node, self.epsilon_id); let direct = self.base.del.get_cost(token); - if self.del_map.has_key(token) || self.base.del.has_key(token) || best < direct { + if self.base.del.has_key(token) || best < direct { let chain = if best < direct { - self.deletion_chain(node).unwrap_or(EffectiveOpChain::Direct) + self.deletion_chain(node) + .unwrap_or(EffectiveOpChain::Direct) } else { self.base.del.get_chain(token) }; - entries.insert(token.clone(), (best, chain)); + entries.insert(token.to_owned(), (best, chain)); } } @@ -408,49 +597,56 @@ impl<'a> CostSolver<'a> { } } + /// Builds the explanation chain for an effective substitution. fn substitution_chain(&self, source: NodeId, target: NodeId) -> Option { let path = self.path(source, target)?; if path.len() <= 2 { return Some(EffectiveSubChain::Direct); } - let mut steps = Vec::new(); - for edge in path.windows(2) { - let from = self.token(edge[0]); - let to = self.token(edge[1]); - let cost = self.raw_substitution_cost(from, to)?; - steps.push((from.to_string(), to.to_string(), cost)); - } + let resolution = self.resolve_path_edges(&path); - Some(EffectiveSubChain::Via { steps }) + if resolution.all_edges_are_raw_substitutions { + Some(EffectiveSubChain::Via { + steps: resolution.substitution_steps, + }) + } else { + Some(EffectiveSubChain::EditPath { + operations: resolution.operations, + }) + } } + /// Builds the explanation chain for an effective deletion. fn deletion_chain(&self, source: NodeId) -> Option { let path = self.path(source, self.epsilon_id)?; if path.len() <= 2 { return Some(EffectiveOpChain::Direct); } + let mut resolution = self.resolve_path_edges(&path[..path.len() - 1]); + let terminal = *path.get(path.len() - 2)?; let terminal_token = self.token(terminal); - let terminal_cost = self.del_map.get_cost(terminal_token); - if !self.del_map.has_key(terminal_token) { - return None; - } - - let mut steps = Vec::new(); - for edge in path[..path.len() - 1].windows(2) { - let from = self.token(edge[0]); - let to = self.token(edge[1]); - let cost = self.raw_substitution_cost(from, to)?; - steps.push((from.to_string(), to.to_string(), cost)); + let terminal_cost = self.base.del.get_explicit_cost(terminal_token)?; + + if resolution.all_edges_are_raw_substitutions { + Some(EffectiveOpChain::Via { + steps: resolution.substitution_steps, + terminal_cost, + }) + } else { + resolution.operations.push(EditOperation::Delete { + source: terminal_token.to_string(), + cost: terminal_cost, + }); + Some(EffectiveOpChain::EditPath { + operations: resolution.operations, + }) } - Some(EffectiveOpChain::Via { - steps, - terminal_cost, - }) } + /// Builds the explanation chain for an effective insertion. fn insertion_chain(&self, target: NodeId) -> Option { let path = self.path(self.epsilon_id, target)?; if path.len() <= 2 { @@ -459,44 +655,78 @@ impl<'a> CostSolver<'a> { let initial = *path.get(1)?; let initial_token = self.token(initial); - let terminal_cost = self.ins_map.get_cost(initial_token); - if !self.ins_map.has_key(initial_token) { - return None; + let terminal_cost = self.base.ins.get_explicit_cost(initial_token)?; + + let mut operations = vec![EditOperation::Insert { + target: initial_token.to_string(), + cost: terminal_cost, + }]; + let mut resolution = self.resolve_path_edges(&path[1..]); + + if resolution.all_edges_are_raw_substitutions { + Some(EffectiveOpChain::Via { + steps: resolution.substitution_steps, + terminal_cost, + }) + } else { + operations.append(&mut resolution.operations); + Some(EffectiveOpChain::EditPath { operations }) } + } + + /// Converts graph edges back into user-visible edit operations. + fn resolve_path_edges(&self, path: &[NodeId]) -> EdgeResolution { + let mut operations = Vec::new(); + let mut substitution_steps = Vec::new(); + let mut all_edges_are_raw_substitutions = true; - let mut steps = Vec::new(); - for edge in path[1..].windows(2) { + for edge in path.windows(2) { let from = self.token(edge[0]); let to = self.token(edge[1]); - let cost = self.raw_substitution_cost(from, to)?; - steps.push((from.to_string(), to.to_string(), cost)); + if let Some(cost) = self.raw_substitution_cost(from, to) { + substitution_steps.push((from.to_owned(), to.to_owned(), cost)); + operations.push(EditOperation::Substitute { + source: from.to_owned(), + target: to.to_owned(), + cost, + }); + } else { + all_edges_are_raw_substitutions = false; + operations.extend( + explain_custom_levenshtein_precomputed(from, to, &self.base) + .into_iter() + .filter(|op| !matches!(op, EditOperation::Match { .. })), + ); + } + } + + EdgeResolution { + operations, + substitution_steps, + all_edges_are_raw_substitutions, } - Some(EffectiveOpChain::Via { - steps, - terminal_cost, - }) } + /// Raw substitution edge cost (pre-closure), or `None` if not configured. fn raw_substitution_cost(&self, source: &str, target: &str) -> Option { - self.sub_map - .costs - .get(&(source.to_owned(), target.to_owned())) - .copied() + self.base.sub.get_explicit_cost(source, target) } + /// Projects epsilon-to-token distances into effective insertions. fn effective_insertions(&self) -> EffectiveSingleTokenCosts { let mut entries: HashMap = HashMap::new(); for token in self.non_epsilon_tokens() { let node = self.id(token); let best = self.cost(self.epsilon_id, node); let direct = self.base.ins.get_cost(token); - if self.ins_map.has_key(token) || self.base.ins.has_key(token) || best < direct { + if self.base.ins.has_key(token) || best < direct { let chain = if best < direct { - self.insertion_chain(node).unwrap_or(EffectiveOpChain::Direct) + self.insertion_chain(node) + .unwrap_or(EffectiveOpChain::Direct) } else { self.base.ins.get_chain(token) }; - entries.insert(token.clone(), (best, chain)); + entries.insert(token.to_owned(), (best, chain)); } } @@ -507,42 +737,43 @@ impl<'a> CostSolver<'a> { } } - fn non_epsilon_tokens(&self) -> impl Iterator { - self.tokens.iter().filter(|token| !token.is_empty()) + fn non_epsilon_tokens(&self) -> impl Iterator { + self.tokens + .iter() + .filter(|token| !token.is_empty()) + .map(String::as_str) } } +/// Collects graph nodes and bounded substrings needed for transitive closure. fn collect_solver_tokens( sub_map: &CostMap, ins_map: &CostMap, del_map: &CostMap, - base: &BaseEffectiveCosts, ) -> Vec { let mut tokens: HashSet = HashSet::new(); tokens.insert(String::new()); // epsilon tokens.extend(ins_map.costs.keys().cloned()); tokens.extend(del_map.costs.keys().cloned()); - tokens.extend(base.ins.entries.keys().cloned()); - tokens.extend(base.del.entries.keys().cloned()); tokens.extend( sub_map .costs .keys() .flat_map(|(source, target)| [source.clone(), target.clone()]), ); - tokens.extend( - base.sub - .entries - .keys() - .flat_map(|(source, target)| [source.clone(), target.clone()]), - ); let originals: Vec = tokens.iter().cloned().collect(); for token in originals { - let chars: Vec = token.chars().collect(); - for start in 0..chars.len() { - for end in (start + 1)..=chars.len() { - tokens.insert(chars[start..end].iter().collect()); + let char_count = token.chars().count(); + if char_count > MAX_SUBTOKEN_EXPANSION_CHARS { + continue; + } + + let mut boundaries: Vec = token.char_indices().map(|(idx, _)| idx).collect(); + boundaries.push(token.len()); + for start in 0..char_count { + for end in (start + 1)..=char_count { + tokens.insert(token[boundaries[start]..boundaries[end]].to_owned()); } } } @@ -550,33 +781,37 @@ fn collect_solver_tokens( tokens.into_iter().collect() } -fn max_pair_token_len(entries: &HashMap<(String, String), (f64, EffectiveSubChain)>) -> usize { +/// Longest token length in effective substitution entries. +fn max_pair_token_len( + entries: &HashMap>, +) -> usize { entries - .keys() - .flat_map(|(source, target)| [source.chars().count(), target.chars().count()]) + .iter() + .flat_map(|(source, targets)| { + targets + .keys() + .flat_map(move |target| [source.chars().count(), target.chars().count()]) + }) .max() - .unwrap_or(1) + .unwrap_or(0) .max(1) } +/// Longest token length in effective insertion/deletion entries. fn max_single_token_len(entries: &HashMap) -> usize { entries .keys() .map(|token| token.chars().count()) .max() - .unwrap_or(1) + .unwrap_or(0) .max(1) } /// Computes all effective costs with one unified state-transition graph. -pub(crate) fn compute_effective_costs_unified( +pub(crate) fn compute_effective_costs( sub_map: &CostMap, ins_map: &CostMap, del_map: &CostMap, -) -> ( - EffectiveSubstitutionCosts, - EffectiveSingleTokenCosts, - EffectiveSingleTokenCosts, -) { +) -> EffectiveCosts { CostSolver::new(sub_map, ins_map, del_map).compute_effective_costs() } diff --git a/src/weighted_levenshtein.rs b/src/weighted_levenshtein.rs index 9ecfd97..c0a35e6 100644 --- a/src/weighted_levenshtein.rs +++ b/src/weighted_levenshtein.rs @@ -1,19 +1,15 @@ use crate::explanation::{EditOperation, Predecessor}; -use crate::transitive_costs::{ - EffectiveOpChain, EffectiveSingleTokenCosts, EffectiveSubChain, EffectiveSubstitutionCosts, -}; +use crate::transitive_costs::{EffectiveCosts, EffectiveOpChain, EffectiveSubChain}; pub(crate) fn custom_levenshtein_distance_precomputed( source: &str, target: &str, - eff_sub: &EffectiveSubstitutionCosts, - eff_ins: &EffectiveSingleTokenCosts, - eff_del: &EffectiveSingleTokenCosts, + costs: &EffectiveCosts, ) -> f64 { if source == target { return 0.0; } - let mut processor = LevenshteinProcessor::new(source, target, eff_sub, eff_ins, eff_del, false); + let mut processor = LevenshteinProcessor::new(source, target, costs, false); processor.run(); processor.distance() } @@ -21,9 +17,7 @@ pub(crate) fn custom_levenshtein_distance_precomputed( pub(crate) fn explain_custom_levenshtein_precomputed( source: &str, target: &str, - eff_sub: &EffectiveSubstitutionCosts, - eff_ins: &EffectiveSingleTokenCosts, - eff_del: &EffectiveSingleTokenCosts, + costs: &EffectiveCosts, ) -> Vec { if source == target { return source @@ -33,7 +27,7 @@ pub(crate) fn explain_custom_levenshtein_precomputed( }) .collect(); } - let mut processor = LevenshteinProcessor::new(source, target, eff_sub, eff_ins, eff_del, true); + let mut processor = LevenshteinProcessor::new(source, target, costs, true); processor.run(); processor.into_result() } @@ -43,23 +37,14 @@ pub(crate) fn explain_custom_levenshtein_precomputed( struct LevenshteinProcessor<'a> { source_chars: Vec, target_chars: Vec, - eff_sub: &'a EffectiveSubstitutionCosts, - eff_del: &'a EffectiveSingleTokenCosts, - eff_ins: &'a EffectiveSingleTokenCosts, + costs: &'a EffectiveCosts, dp: Vec>, predecessors: Option>>, multi_char_ops: bool, } impl<'a> LevenshteinProcessor<'a> { - fn new( - source: &str, - target: &str, - eff_sub: &'a EffectiveSubstitutionCosts, - eff_ins: &'a EffectiveSingleTokenCosts, - eff_del: &'a EffectiveSingleTokenCosts, - explain: bool, - ) -> Self { + fn new(source: &str, target: &str, costs: &'a EffectiveCosts, explain: bool) -> Self { let source_chars: Vec = source.chars().collect(); let target_chars: Vec = target.chars().collect(); let len_source = source_chars.len(); @@ -68,12 +53,10 @@ impl<'a> LevenshteinProcessor<'a> { let mut processor = Self { source_chars, target_chars, - eff_sub, - multi_char_ops: eff_sub.max_token_length > 1 - || eff_ins.max_token_length > 1 - || eff_del.max_token_length > 1, - eff_del, - eff_ins, + multi_char_ops: costs.sub.max_token_length > 1 + || costs.ins.max_token_length > 1 + || costs.del.max_token_length > 1, + costs, dp: vec![vec![0.0; len_target + 1]; len_source + 1], predecessors: if explain { Some(vec![ @@ -127,9 +110,9 @@ impl<'a> LevenshteinProcessor<'a> { let source_char_str = self.source_chars[i - 1].to_string(); let target_char_str = self.target_chars[j - 1].to_string(); - let deletion_cost = self.dp[i - 1][j] + self.eff_del.get_cost(&source_char_str); - let insertion_cost = self.dp[i][j - 1] + self.eff_ins.get_cost(&target_char_str); - let sub_cost = self.eff_sub.get_cost(&source_char_str, &target_char_str); + let deletion_cost = self.dp[i - 1][j] + self.costs.del.get_cost(&source_char_str); + let insertion_cost = self.dp[i][j - 1] + self.costs.ins.get_cost(&target_char_str); + let sub_cost = self.costs.sub.get_cost(&source_char_str, &target_char_str); let substitution_cost = self.dp[i - 1][j - 1] + sub_cost; // Check for exact match @@ -166,15 +149,15 @@ impl<'a> LevenshteinProcessor<'a> { // First row (insertions) for j in 1..=len_target { let char_str = self.target_chars[j - 1].to_string(); - self.dp[0][j] = self.dp[0][j - 1] + self.eff_ins.get_cost(&char_str); + self.dp[0][j] = self.dp[0][j - 1] + self.costs.ins.get_cost(&char_str); self.record(0, j, Predecessor::Insert(1)); - let max_len = self.eff_ins.max_token_length.min(j); + let max_len = self.costs.ins.max_token_length.min(j); for token_len in 2..=max_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.eff_ins.has_key(&token) { - let new_cost = self.dp[0][token_start] + self.eff_ins.get_cost(&token); + if self.costs.ins.has_key(&token) { + let new_cost = self.dp[0][token_start] + self.costs.ins.get_cost(&token); if new_cost < self.dp[0][j] { self.dp[0][j] = new_cost; self.record(0, j, Predecessor::Insert(token_len)); @@ -185,15 +168,15 @@ impl<'a> LevenshteinProcessor<'a> { // First column (deletions) for i in 1..=len_source { let char_str = self.source_chars[i - 1].to_string(); - self.dp[i][0] = self.dp[i - 1][0] + self.eff_del.get_cost(&char_str); + self.dp[i][0] = self.dp[i - 1][0] + self.costs.del.get_cost(&char_str); self.record(i, 0, Predecessor::Delete(1)); - let max_len = self.eff_del.max_token_length.min(i); + let max_len = self.costs.del.max_token_length.min(i); for token_len in 2..=max_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.eff_del.has_key(&token) { - let new_cost = self.dp[token_start][0] + self.eff_del.get_cost(&token); + if self.costs.del.has_key(&token) { + let new_cost = self.dp[token_start][0] + self.costs.del.get_cost(&token); if new_cost < self.dp[i][0] { self.dp[i][0] = new_cost; self.record(i, 0, Predecessor::Delete(token_len)); @@ -204,8 +187,8 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_substitutions(&mut self, i: usize, j: usize) { - let max_source_len = self.eff_sub.max_token_length.min(i); - let max_target_len = self.eff_sub.max_token_length.min(j); + let max_source_len = self.costs.sub.max_token_length.min(i); + let max_target_len = self.costs.sub.max_token_length.min(j); for source_len in 1..=max_source_len { for target_len in 1..=max_target_len { if source_len == 1 && target_len == 1 { @@ -215,9 +198,9 @@ impl<'a> LevenshteinProcessor<'a> { let target_start = j - target_len; let source_substr: String = self.source_chars[source_start..i].iter().collect(); let target_substr: String = self.target_chars[target_start..j].iter().collect(); - if self.eff_sub.has_key(&source_substr, &target_substr) { + if self.costs.sub.has_key(&source_substr, &target_substr) { let new_cost = self.dp[source_start][target_start] - + self.eff_sub.get_cost(&source_substr, &target_substr); + + self.costs.sub.get_cost(&source_substr, &target_substr); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Substitute(source_len, target_len)); @@ -228,12 +211,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_insertions(&mut self, i: usize, j: usize) { - let max_ins_len = self.eff_ins.max_token_length.min(j); + let max_ins_len = self.costs.ins.max_token_length.min(j); for token_len in 2..=max_ins_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.eff_ins.has_key(&token) { - let new_cost = self.dp[i][token_start] + self.eff_ins.get_cost(&token); + if self.costs.ins.has_key(&token) { + let new_cost = self.dp[i][token_start] + self.costs.ins.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Insert(token_len)); @@ -243,12 +226,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_deletions(&mut self, i: usize, j: usize) { - let max_del_len = self.eff_del.max_token_length.min(i); + let max_del_len = self.costs.del.max_token_length.min(i); for token_len in 2..=max_del_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.eff_del.has_key(&token) { - let new_cost = self.dp[token_start][j] + self.eff_del.get_cost(&token); + if self.costs.del.has_key(&token) { + let new_cost = self.dp[token_start][j] + self.costs.del.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Delete(token_len)); @@ -278,9 +261,9 @@ impl<'a> LevenshteinProcessor<'a> { if source_token != target_token { // path is reversed at end; push in reverse so after reversal // the chain appears in forward order. - match self.eff_sub.get_chain(&source_token, &target_token) { + match self.costs.sub.get_chain(&source_token, &target_token) { EffectiveSubChain::Direct => { - let cost = self.eff_sub.get_cost(&source_token, &target_token); + let cost = self.costs.sub.get_cost(&source_token, &target_token); path.push(EditOperation::Substitute { source: source_token, target: target_token, @@ -296,6 +279,11 @@ impl<'a> LevenshteinProcessor<'a> { }); } } + EffectiveSubChain::EditPath { operations } => { + for op in operations.iter().rev() { + path.push(op.clone()); + } + } } } i -= s_len; @@ -303,9 +291,9 @@ impl<'a> LevenshteinProcessor<'a> { } Predecessor::Insert(t_len) => { let target_token: String = self.target_chars[j - t_len..j].iter().collect(); - match self.eff_ins.get_chain(&target_token) { + match self.costs.ins.get_chain(&target_token) { EffectiveOpChain::Direct => { - let cost = self.eff_ins.get_cost(&target_token); + let cost = self.costs.ins.get_cost(&target_token); path.push(EditOperation::Insert { target: target_token, cost, @@ -333,14 +321,19 @@ impl<'a> LevenshteinProcessor<'a> { cost: terminal_cost, }); } + EffectiveOpChain::EditPath { operations } => { + for op in operations.iter().rev() { + path.push(op.clone()); + } + } } j -= t_len; } Predecessor::Delete(s_len) => { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); - match self.eff_del.get_chain(&source_token) { + match self.costs.del.get_chain(&source_token) { EffectiveOpChain::Direct => { - let cost = self.eff_del.get_cost(&source_token); + let cost = self.costs.del.get_cost(&source_token); path.push(EditOperation::Delete { source: source_token, cost, @@ -368,6 +361,11 @@ impl<'a> LevenshteinProcessor<'a> { }); } } + EffectiveOpChain::EditPath { operations } => { + for op in operations.iter().rev() { + path.push(op.clone()); + } + } } i -= s_len; } @@ -391,7 +389,7 @@ impl<'a> LevenshteinProcessor<'a> { mod test { use super::*; use crate::cost_map::CostMap; - use crate::transitive_costs::compute_effective_costs_unified; + use crate::transitive_costs::compute_effective_costs; use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; fn assert_approx_eq(a: f64, b: f64, epsilon: f64) { @@ -422,9 +420,8 @@ mod test { ins_map: &CostMap, del_map: &CostMap, ) -> f64 { - let (eff_sub, eff_del, eff_ins) = - compute_effective_costs_unified(sub_map, ins_map, del_map); - custom_levenshtein_distance_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) + let costs = compute_effective_costs(sub_map, ins_map, del_map); + custom_levenshtein_distance_precomputed(source, target, &costs) } fn calc_explain( @@ -434,9 +431,8 @@ mod test { ins_map: &CostMap, del_map: &CostMap, ) -> Vec { - let (eff_sub, eff_del, eff_ins) = - compute_effective_costs_unified(sub_map, ins_map, del_map); - explain_custom_levenshtein_precomputed(source, target, &eff_sub, &eff_ins, &eff_del) + let costs = compute_effective_costs(sub_map, ins_map, del_map); + explain_custom_levenshtein_precomputed(source, target, &costs) } #[test] @@ -965,11 +961,9 @@ mod test { #[test] fn test_check_multi_char_ops_with_empty_maps() { let (sub_map, ins_map, del_map) = create_default_cost_maps(); - let (eff_sub, eff_del, eff_ins) = - compute_effective_costs_unified(&sub_map, &ins_map, &del_map); + let costs = compute_effective_costs(&sub_map, &ins_map, &del_map); - let mut processor = - LevenshteinProcessor::new("abcd", "xyz", &eff_sub, &eff_ins, &eff_del, true); + let mut processor = LevenshteinProcessor::new("abcd", "xyz", &costs, true); // Simulate the DP state before the operation let original_dp_3_2 = processor.dp[3][2]; From d028e5e685ebacc91e25fe9ffe976517c5b00933 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Thu, 30 Apr 2026 22:07:23 +0200 Subject: [PATCH 04/21] add unittest --- python/tests/test_explain_weighted_levenshtein.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index 7001be3..f7ced1a 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -179,3 +179,17 @@ def test_explain_effective_deletion_with_insertion_then_deletion() -> None: EditOperation("delete", "AB", None, 0.0), EditOperation("match", "C", "C", 0.0), ] + + +def test_insert_delete_substitute_chain() -> None: + wl = WeightedLevenshtein( + substitution_costs={("ABC", "Z"): 0.1}, + insertion_costs={"B": 0.1}, + deletion_costs={"D": 0.1}, + ) + ops = wl.explain("ADC", "Z") + assert ops == [ + EditOperation("delete", "D", None, 0.1), + EditOperation("insert", None, "B", 0.1), + EditOperation("substitute", "ABC", "Z", 0.1), + ] From 3c2f28b9d0e72dbe87ff937852544ac2ab7cfcfa Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 11:12:01 +0200 Subject: [PATCH 05/21] initial transitive_costs impl --- python/ocr_stringdist/levenshtein.py | 38 + .../test_explain_weighted_levenshtein.py | 150 +-- python/tests/test_weighted_levenshtein.py | 100 +- src/cost_map.rs | 46 + src/rust_stringdist.rs | 155 ++- src/transitive_costs.rs | 1001 ++++++----------- src/weighted_levenshtein.rs | 816 ++------------ 7 files changed, 718 insertions(+), 1588 deletions(-) diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 54867ff..651a575 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -100,6 +100,44 @@ def unweighted(cls) -> WeightedLevenshtein: """Creates an instance with all operations having equal cost of 1.0.""" return cls(substitution_costs={}, insertion_costs={}, deletion_costs={}) + def transitive_closure(self) -> WeightedLevenshtein: + """ + Returns a new instance whose cost dictionaries are filled with effective + (transitive) edit costs. + + If, for example, ``substitution_costs[("a", "b")] = 0.1`` and + ``substitution_costs[("b", "c")] = 0.1``, the closed instance's + ``substitution_costs[("a", "c")]`` is ``0.2`` rather than the default. + Insertion and deletion chains, and chains that cross ``ε`` (e.g. + ``del("y") + ins("x")`` becoming an effective ``("y", "x")`` substitution), + are likewise materialized. + + The returned instance has ``symmetric_substitution=False`` because + closure may produce asymmetric pairs even when the input is symmetric. + Symmetric input is mirrored before closure, so both directions of every + original pair are still present in the result. + + Closure is bounded: very large or pathological cost maps may not be + fully closed. The DP falls back to the configured default costs for any + ``(s, t)`` not in the resulting map. + + ``explain()`` on the closed instance returns flat single-step ops; the + original chain that produced an effective cost is not preserved. + + For repeated use, save via :meth:`to_dict` and reload via + :meth:`from_dict` so the closure is computed once. + """ + sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps() + return WeightedLevenshtein( + substitution_costs=dict(sub_dict), + insertion_costs=dict(ins_dict), + deletion_costs=dict(del_dict), + symmetric_substitution=False, + default_substitution_cost=self.default_substitution_cost, + default_insertion_cost=self.default_insertion_cost, + default_deletion_cost=self.default_deletion_cost, + ) + def distance(self, s1: str, s2: str) -> float: """Calculates the weighted Levenshtein distance between two strings.""" return self._calculator.distance(s1, s2) # type: ignore[no-any-return] diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index f7ced1a..026be7a 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -78,118 +78,128 @@ def test_explain_weighted_levenshtein( assert sum(op.cost for op in full_operations) == wl.distance(s1, s2) -def test_explain_transitive_deletion_chain() -> None: - """Issue #12: the explain path for '06'->'0' should expose the sub+del chain.""" +# Closure-flat explain tests +# +# After ``transitive_closure()``, the underlying chain that produced an +# effective cost is no longer preserved. ``explain()`` returns a single +# substitution / insertion / deletion at the effective cost. These tests +# verify the flat output and that the total cost equals ``distance()``. + + +def _flat_explain_assertions( + wl_closed: WeightedLevenshtein, s1: str, s2: str, expected_distance: float +) -> list[EditOperation]: + ops = wl_closed.explain(s1, s2) + assert sum(op.cost for op in ops) == pytest.approx(expected_distance) + assert wl_closed.distance(s1, s2) == pytest.approx(expected_distance) + return ops + + +def _assert_ops_equal(actual: list[EditOperation], expected: list[EditOperation]) -> None: + """Compare op sequences with float-tolerant cost equality.""" + assert len(actual) == len(expected), f"length mismatch: {actual} vs {expected}" + for a, e in zip(actual, expected): + assert a.op_type == e.op_type + assert a.source_token == e.source_token + assert a.target_token == e.target_token + assert a.cost == pytest.approx(e.cost) + + +def test_explain_transitive_deletion_chain_after_closure() -> None: + """After closure, '06' -> '0' is one effective deletion of '6' at 0.51.""" wl = WeightedLevenshtein( substitution_costs={("6", "G"): 0.5}, deletion_costs={"G": 0.01}, symmetric_substitution=False, - ) - ops = wl.explain("06", "0", filter_matches=False) - assert ops == [ - EditOperation("match", "0", "0", 0.0), - EditOperation("substitute", "6", "G", 0.5), - EditOperation("delete", "G", None, 0.01), - ] + ).transitive_closure() + ops = _flat_explain_assertions(wl, "06", "0", 0.51) + assert ops == [EditOperation("delete", "6", None, 0.51)] -def test_explain_transitive_substitution_chain() -> None: - """Triangle inequality: sub(a->b, 0.1) + sub(b->c, 0.1) should expand to two ops.""" +def test_explain_transitive_substitution_chain_after_closure() -> None: + """After closure, 'a' -> 'c' is one effective substitution at 0.2.""" wl = WeightedLevenshtein( substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, symmetric_substitution=False, - ) - ops = wl.explain("a", "c", filter_matches=False) - assert ops == [ - EditOperation("substitute", "a", "b", 0.1), - EditOperation("substitute", "b", "c", 0.1), - ] + ).transitive_closure() + ops = _flat_explain_assertions(wl, "a", "c", 0.2) + assert ops == [EditOperation("substitute", "a", "c", 0.2)] -def test_explain_transitive_insertion_chain() -> None: - """Insertion analogue: ins('x') + sub('x'->'y') chain should appear in the path.""" +def test_explain_transitive_insertion_chain_after_closure() -> None: + """After closure, inserting 'y' is one effective insertion at 0.3.""" wl = WeightedLevenshtein( substitution_costs={("x", "y"): 0.2}, insertion_costs={"x": 0.1}, symmetric_substitution=False, - ) - ops = wl.explain("a", "ay", filter_matches=False) + ).transitive_closure() + ops = _flat_explain_assertions(wl, "a", "ay", 0.3) assert ops == [ - EditOperation("match", "a", "a", 0.0), - EditOperation("insert", None, "x", 0.1), - EditOperation("substitute", "x", "y", 0.2), + EditOperation("insert", None, "y", 0.3), ] -def test_explain_chain_with_expensive_direct_substitution() -> None: - """ - Test that A->AA->AAA->B is explained instead of the more expensive A->B. - """ +def test_explain_chain_with_expensive_direct_substitution_after_closure() -> None: + """A->B with cheaper A->AAA->B chain becomes a single sub at 0.5.""" wl = WeightedLevenshtein( - substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, insertion_costs={"A": 0.2} - ) - ops = wl.explain("A", "B", filter_matches=True) - assert ops == [ - EditOperation("insert", None, "A", 0.2), - EditOperation("insert", None, "A", 0.2), - EditOperation("substitute", "AAA", "B", 0.1), - ] + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "A", "B", 0.5) + assert ops == [EditOperation("substitute", "A", "B", 0.5)] -def test_explain_mixed_substitution_path_with_deletion() -> None: - """ - Test that AB->A->C is expanded when it beats the direct AB->C substitution. - """ +def test_explain_mixed_substitution_path_with_deletion_after_closure() -> None: + """AB->C: closure prefers sub(A,C) + del(B) = 0.3 over the direct AB->C = 0.5.""" wl = WeightedLevenshtein( substitution_costs={("A", "C"): 0.1, ("AB", "C"): 0.5}, deletion_costs={"B": 0.2}, - ) - ops = wl.explain("AB", "C", filter_matches=True) + ).transitive_closure() + ops = _flat_explain_assertions(wl, "AB", "C", 0.3) assert ops == [ EditOperation("substitute", "A", "C", 0.1), EditOperation("delete", "B", None, 0.2), ] -def test_explain_direct_substitution_wins_over_mixed_chain() -> None: - """ - Test that a cheaper direct A->B substitution is not expanded into A->AAA->B. - """ +def test_explain_direct_substitution_wins_over_chain_after_closure() -> None: + """A direct A->B at 0.4 beats A->AAA->B at 0.5; effective cost stays 0.4.""" wl = WeightedLevenshtein( substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.4}, insertion_costs={"A": 0.2}, - ) - ops = wl.explain("A", "B", filter_matches=True) - assert ops == [ - EditOperation("substitute", "A", "B", 0.4), - ] + ).transitive_closure() + ops = _flat_explain_assertions(wl, "A", "B", 0.4) + assert ops == [EditOperation("substitute", "A", "B", 0.4)] -def test_explain_effective_deletion_with_insertion_then_deletion() -> None: - """ - Test that AC->ABC->C is expanded as insert(B), delete(AB), match(C). - """ +def test_explain_effective_deletion_with_insertion_then_deletion_after_closure() -> None: + """AC -> C via insert(B)+del(AB)=0.1 becomes a single effective del('A') at 0.1.""" wl = WeightedLevenshtein( insertion_costs={"B": 0.1}, deletion_costs={"AB": 0.0}, - ) - ops = wl.explain("AC", "C", filter_matches=False) - assert ops == [ - EditOperation("insert", None, "B", 0.1), - EditOperation("delete", "AB", None, 0.0), - EditOperation("match", "C", "C", 0.0), - ] + ).transitive_closure() + ops = _flat_explain_assertions(wl, "AC", "C", 0.1) + # The single effective op may be a del('A'), or another route at the same cost. + # Assert the explicit identity to lock down the canonical form: + assert ops == [EditOperation("delete", "A", None, 0.1)] -def test_insert_delete_substitute_chain() -> None: +def test_explain_insert_delete_substitute_chain_after_closure() -> None: + """ADC -> Z via del(D)+ins(B)+sub(ABC,Z) becomes a single sub at 0.3.""" wl = WeightedLevenshtein( substitution_costs={("ABC", "Z"): 0.1}, insertion_costs={"B": 0.1}, deletion_costs={"D": 0.1}, - ) - ops = wl.explain("ADC", "Z") - assert ops == [ - EditOperation("delete", "D", None, 0.1), - EditOperation("insert", None, "B", 0.1), - EditOperation("substitute", "ABC", "Z", 0.1), - ] + ).transitive_closure() + ops = _flat_explain_assertions(wl, "ADC", "Z", 0.3) + assert ops == [EditOperation("substitute", "ADC", "Z", 0.3)] + + +def test_explain_single_char_composed_substitution_chain_after_closure() -> None: + """X -> Z via ins(AB)+sub(XAB,Z) becomes a single sub at 0.3.""" + wl = WeightedLevenshtein( + substitution_costs={("XAB", "Z"): 0.1}, + insertion_costs={"AB": 0.2}, + ).transitive_closure() + ops = _flat_explain_assertions(wl, "X", "Z", 0.3) + assert ops == [EditOperation("substitute", "X", "Z", 0.3)] diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index 3ee7a4a..1212890 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -551,13 +551,23 @@ def test_costs_above_default_cost() -> None: assert actual_cost == configured_cost +def test_no_implicit_closure_in_constructor() -> None: + """Transitive paths are opt-in: the constructor does not run closure.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + # Without transitive_closure(), a->c falls back to the default 1.0. + assert wl.distance("a", "c") == pytest.approx(1.0) + + def test_transitive_deletion_chain_distance() -> None: """Issue #12: sub('6'->'G', 0.5) + del('G', 0.01) = 0.51 < direct del('6', 1.0).""" wl = WeightedLevenshtein( substitution_costs={("6", "G"): 0.5}, deletion_costs={"G": 0.01}, symmetric_substitution=False, - ) + ).transitive_closure() assert wl.distance("06", "0") == pytest.approx(0.51) @@ -568,7 +578,7 @@ def test_transitive_insertion_subtitution() -> None: wl = WeightedLevenshtein( insertion_costs={"A": 0.2}, substitution_costs={("AAA", "B"): 0.1}, - ) + ).transitive_closure() assert wl.distance("A", "B") == pytest.approx(0.5) @@ -579,7 +589,7 @@ def test_transitive_insertion_subtitution2() -> None: wl = WeightedLevenshtein( insertion_costs={"A": 0.2, "B": 0.3}, substitution_costs={("AAB", "C"): 0.1}, - ) + ).transitive_closure() assert wl.distance("A", "C") == pytest.approx(0.6) @@ -590,7 +600,7 @@ def test_transitive_insertion_deletion() -> None: wl = WeightedLevenshtein( insertion_costs={"B": 0.1}, deletion_costs={"AB": 0.0}, - ) + ).transitive_closure() assert wl.distance("AC", "C") == pytest.approx(0.1) @@ -600,7 +610,7 @@ def test_transitive_insertion_chain_distance() -> None: substitution_costs={("x", "y"): 0.2}, insertion_costs={"x": 0.1}, symmetric_substitution=False, - ) + ).transitive_closure() assert wl.distance("a", "ay") == pytest.approx(0.3) @@ -610,7 +620,7 @@ def test_direct_op_wins_when_chain_more_expensive() -> None: substitution_costs={("6", "G"): 0.5}, deletion_costs={"6": 0.2, "G": 0.01}, symmetric_substitution=False, - ) + ).transitive_closure() assert wl.distance("06", "0") == pytest.approx(0.2) @@ -619,32 +629,68 @@ def test_transitive_substitution_chain_distance() -> None: wl = WeightedLevenshtein( substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, symmetric_substitution=False, - ) + ).transitive_closure() assert wl.distance("a", "c") == pytest.approx(0.2) -@pytest.mark.xfail( - reason=( - "Optimized transitive seeding does not create arbitrary token-to-token " - "shortcuts from full weighted DP alignments." - ), - strict=True, -) -def test_full_dp_seed_would_create_multi_edit_token_shortcut() -> None: - """ - We only seed raw operations and targeted one-edit embedded - edges. Because these tokens are longer than the subtoken expansion cap, the - intermediate `source + "A"` node is absent, so the shortcut is not present. - """ - source = "abcdefghijklmnopq" # 17 chars: above MAX_SUBTOKEN_EXPANSION_CHARS - bridge = f"{source}AB" - target = "Z" +def test_transitive_closure_returns_asymmetric_instance() -> None: + """Closure may break symmetry (e.g. via del+ins paths through ε), so the + returned instance is always asymmetric. Both directions of a symmetric + input are still preserved as explicit entries.""" + wl_closed = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1}, + symmetric_substitution=True, + ).transitive_closure() + assert wl_closed.symmetric_substitution is False + assert wl_closed.substitution_costs[("a", "b")] == pytest.approx(0.1) + assert wl_closed.substitution_costs[("b", "a")] == pytest.approx(0.1) + + +def test_transitive_closure_idempotent_on_distance() -> None: + """Applying closure twice yields the same distances as applying it once.""" wl = WeightedLevenshtein( - insertion_costs={"A": 0.2, "B": 0.3}, - deletion_costs={source: 10.0}, # make `source` a graph token - substitution_costs={(bridge, target): 0.1}, + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, + ) + wl1 = wl.transitive_closure() + wl2 = wl1.transitive_closure() + for s, t in [("a", "c"), ("a", "b"), ("b", "c"), ("c", "a"), ("xy", "yx")]: + assert wl1.distance(s, t) == pytest.approx(wl2.distance(s, t)) + + +def test_transitive_closure_round_trip_via_dict() -> None: + """Closed costs survive serialization, so users can compute closure once.""" + wl_orig = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1, ("b", "c"): 0.1}, + symmetric_substitution=False, ) - assert wl.distance(source, target) == pytest.approx(0.6) + wl_closed = wl_orig.transitive_closure() + wl_reloaded = WeightedLevenshtein.from_dict(wl_closed.to_dict()) + assert wl_reloaded.distance("a", "c") == pytest.approx(0.2) + assert wl_reloaded == wl_closed + + +def test_transitive_closure_preserves_default_costs() -> None: + """The closed instance keeps the original default costs.""" + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.1}, + default_substitution_cost=2.0, + default_insertion_cost=3.0, + default_deletion_cost=4.0, + ).transitive_closure() + assert wl.default_substitution_cost == 2.0 + assert wl.default_insertion_cost == 3.0 + assert wl.default_deletion_cost == 4.0 + + +def test_insert_delete_substitute_chain_distance() -> None: + """ADC -> AC -> ABC -> Z via del(D)=0.1 + ins(B)=0.1 + sub(ABC,Z)=0.1.""" + wl = WeightedLevenshtein( + substitution_costs={("ABC", "Z"): 0.1}, + insertion_costs={"B": 0.1}, + deletion_costs={"D": 0.1}, + ).transitive_closure() + assert wl.distance("ADC", "Z") == pytest.approx(0.3) def test_serialization() -> None: diff --git a/src/cost_map.rs b/src/cost_map.rs index 0609f45..1c8c7ea 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -17,6 +17,7 @@ impl CostKey for SubstitutionKey {} pub struct CostMap { pub costs: HashMap, default_cost: f64, + max_token_length: usize, } impl Default for CostMap @@ -27,6 +28,7 @@ where Self { costs: HashMap::new(), default_cost: 1.0, + max_token_length: 1, } } } @@ -49,9 +51,17 @@ impl CostMap { } } + let max_token_length = costs + .keys() + .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) + .max() + .unwrap_or(0) + .max(1); + CostMap { costs, default_cost, + max_token_length, } } @@ -71,14 +81,35 @@ impl CostMap { Self::new(substitution_costs, default_cost, symmetric) } + + #[inline] + pub fn get_cost(&self, source: &str, target: &str) -> f64 { + self.costs + .get(&(source.to_string(), target.to_string())) + .copied() + .unwrap_or(self.default_cost) + } + + #[inline] + pub fn has_key(&self, source: &str, target: &str) -> bool { + self.costs + .contains_key(&(source.to_string(), target.to_string())) + } } // Implementation for SingleTokenKey (single string) impl CostMap { pub fn new(custom_costs_input: SingleTokenCostMap, default_cost: f64) -> Self { + let max_token_length = custom_costs_input + .keys() + .map(|token| token.chars().count()) + .max() + .unwrap_or(0) + .max(1); CostMap { costs: custom_costs_input, default_cost, + max_token_length, } } @@ -98,6 +129,16 @@ impl CostMap { Self::new(single_token_costs, default_cost) } + + #[inline] + pub fn get_cost(&self, token: &str) -> f64 { + self.costs.get(token).copied().unwrap_or(self.default_cost) + } + + #[inline] + pub fn has_key(&self, token: &str) -> bool { + self.costs.contains_key(token) + } } // Common methods for any type of CostMap @@ -105,6 +146,11 @@ impl CostMap { pub fn default_cost(&self) -> f64 { self.default_cost } + + #[inline] + pub fn max_token_length(&self) -> usize { + self.max_token_length + } } #[cfg(test)] diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index b26acfc..47da8a2 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,9 +1,8 @@ use crate::cost_map::CostMap; use crate::explanation::EditOperation; -use crate::transitive_costs::{compute_effective_costs, EffectiveCosts}; +use crate::transitive_costs::compute_closed_cost_maps; use crate::types::{SingleTokenKey, SubstitutionKey}; -use crate::weighted_levenshtein::custom_levenshtein_distance_precomputed; -use crate::weighted_levenshtein::explain_custom_levenshtein_precomputed; +use crate::weighted_levenshtein::{custom_levenshtein_distance, explain_custom_levenshtein}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; @@ -14,7 +13,6 @@ impl<'py> IntoPyObject<'py> for EditOperation { type Output = Bound<'py, Self::Target>; type Error = pyo3::PyErr; - /// Converts the `EditOperation` into a Python tuple fn into_pyobject(self, py: Python<'py>) -> Result { match self { EditOperation::Substitute { @@ -30,12 +28,14 @@ impl<'py> IntoPyObject<'py> for EditOperation { } } -/// Precomputes effective substitution, insertion, and deletion costs once and -/// reuses them for every `.distance()` / `.batch_distance()` call. +/// Holds raw cost maps and runs the DP against them. Transitive closure is +/// opt-in via [`closed_cost_maps`]; the constructor never runs Floyd-Warshall. #[pyclass] #[derive(Debug)] struct RustLevenshteinCalculator { - costs: EffectiveCosts, + sub: CostMap, + ins: CostMap, + del: CostMap, } #[pymethods] @@ -63,23 +63,21 @@ impl RustLevenshteinCalculator { validate_default_cost(default_insertion_cost)?; validate_default_cost(default_deletion_cost)?; - let sub_map = CostMap::::from_py_dict( + let sub = CostMap::::from_py_dict( substitution_costs, default_substitution_cost, symmetric_substitution, ); - let ins_map = + let ins = CostMap::::from_py_dict(insertion_costs, default_insertion_cost); - let del_map = + let del = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); - let costs = compute_effective_costs(&sub_map, &ins_map, &del_map); - - Ok(Self { costs }) + Ok(Self { sub, ins, del }) } fn distance(&self, a: &str, b: &str) -> f64 { - custom_levenshtein_distance_precomputed(a, b, &self.costs) + custom_levenshtein_distance(a, b, &self.sub, &self.ins, &self.del) } fn batch_distance(&self, py: Python<'_>, s: String, candidates: Vec) -> Vec { @@ -89,20 +87,49 @@ impl RustLevenshteinCalculator { py.allow_threads(|| { candidates .par_iter() - .map(|c| custom_levenshtein_distance_precomputed(&s, c, &self.costs)) + .map(|c| custom_levenshtein_distance(&s, c, &self.sub, &self.ins, &self.del)) .collect() }) } fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult> { - explain_custom_levenshtein_precomputed(a, b, &self.costs) + explain_custom_levenshtein(a, b, &self.sub, &self.ins, &self.del) .into_iter() .map(|op| op.into_pyobject(py).map(|bound| bound.into())) .collect::>>() } + + /// Computes effective edit costs via transitive closure and returns three + /// Python dicts: `(substitution_costs, insertion_costs, deletion_costs)`. + /// + /// The Python wrapper assembles these into a new `WeightedLevenshtein` + /// whose `.distance()` and `.explain()` use the closed costs directly. + fn closed_cost_maps<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let (closed_sub, closed_ins, closed_del) = + compute_closed_cost_maps(&self.sub, &self.ins, &self.del); + + let sub_dict = PyDict::new(py); + for ((source, target), cost) in closed_sub { + sub_dict.set_item((source, target), cost)?; + } + + let ins_dict = PyDict::new(py); + for (token, cost) in closed_ins { + ins_dict.set_item(token, cost)?; + } + + let del_dict = PyDict::new(py); + for (token, cost) in closed_del { + del_dict.set_item(token, cost)?; + } + + Ok((sub_dict, ins_dict, del_dict)) + } } -/// Validates that the default cost is non-negative fn validate_default_cost(default_cost: f64) -> PyResult<()> { if default_cost < 0.0 { return Err(PyValueError::new_err(format!( @@ -112,7 +139,6 @@ fn validate_default_cost(default_cost: f64) -> PyResult<()> { Ok(()) } -/// A Python module implemented in Rust. #[pymodule] pub fn _rust_stringdist(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -124,8 +150,8 @@ mod tests { use super::*; use pyo3::types::{PyDict, PyList, PyTuple}; - fn make_calculator<'py>( - py: Python<'py>, + fn make_calculator( + py: Python<'_>, sub_costs: &[((&str, &str), f64)], ins_costs: &[(&str, f64)], del_costs: &[(&str, f64)], @@ -166,7 +192,6 @@ mod tests { fn test_asymmetric_substitution() { Python::with_gil(|py| { let calc = make_calculator(py, &[(("a", "b"), 0.1)], &[], &[], false); - // a->b costs 0.1; b->a uses default 1.0 -> total 1.1 assert!((calc.distance("ab", "ba") - 1.1).abs() < f64::EPSILON); }); } @@ -175,69 +200,47 @@ mod tests { fn test_negative_default_cost_errors() { Python::with_gil(|py| { let empty = PyDict::new(py); - let sub_err = RustLevenshteinCalculator::new(&empty, &empty, &empty, true, -1.0, 1.0, 1.0); assert!(sub_err.is_err()); assert!(sub_err.unwrap_err().is_instance_of::(py)); - - let ins_err = - RustLevenshteinCalculator::new(&empty, &empty, &empty, true, 1.0, -1.0, 1.0); - assert!(ins_err.is_err()); - assert!(ins_err.unwrap_err().is_instance_of::(py)); - - let del_err = - RustLevenshteinCalculator::new(&empty, &empty, &empty, true, 1.0, 1.0, -1.0); - assert!(del_err.is_err()); - assert!(del_err.unwrap_err().is_instance_of::(py)); - }); - } - - #[test] - fn test_edit_op_substitute_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Substitute { - source: "a".to_string(), - target: "b".to_string(), - cost: 0.75, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('substitute', 'a', 'b', 0.75)"); }); } #[test] - fn test_edit_op_insert_into_pyobject() { + fn test_constructor_does_not_apply_closure() { + // Without calling closed_cost_maps, transitive paths are not auto-applied. + // sub(a->b)=0.1, sub(b->c)=0.1: direct a->c lookup falls back to default 1.0. Python::with_gil(|py| { - let op = EditOperation::Insert { - target: "c".to_string(), - cost: 1.0, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('insert', None, 'c', 1.0)"); - }); - } - - #[test] - fn test_edit_op_delete_into_pyobject() { - Python::with_gil(|py| { - let op = EditOperation::Delete { - source: "d".to_string(), - cost: 1.2, - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('delete', 'd', None, 1.2)"); + let calc = make_calculator( + py, + &[(("a", "b"), 0.1), (("b", "c"), 0.1)], + &[], + &[], + false, + ); + assert!((calc.distance("a", "c") - 1.0).abs() < f64::EPSILON); }); } #[test] - fn test_edit_op_match_into_pyobject() { + fn test_closed_cost_maps_finds_chain() { Python::with_gil(|py| { - let op = EditOperation::Match { - token: "e".to_string(), - }; - let tuple = op.into_pyobject(py).unwrap(); - assert_eq!(tuple.to_string(), "('match', 'e', 'e', 0.0)"); + let calc = make_calculator( + py, + &[(("a", "b"), 0.1), (("b", "c"), 0.1)], + &[], + &[], + false, + ); + let (sub, _ins, _del) = calc.closed_cost_maps(py).unwrap(); + let cost: f64 = sub + .get_item(("a".to_string(), "c".to_string())) + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert!((cost - 0.2).abs() < 1e-9); }); } @@ -246,10 +249,8 @@ mod tests { Python::with_gil(|py| { let calc = make_calculator(py, &[], &[], &[], true); let result = calc.explain(py, "cat", "car").unwrap(); - let py_list = PyList::new(py, result).unwrap(); assert_eq!(py_list.len(), 3); - let op = |i: usize| -> String { py_list .get_item(i) @@ -279,14 +280,4 @@ mod tests { assert_eq!(distances, vec![2.0, 1.0, 1.0]); }); } - - #[test] - fn test_batch_distance_empty() { - Python::with_gil(|py| { - let calc = make_calculator(py, &[], &[], &[], true); - assert!(calc - .batch_distance(py, "test".to_string(), vec![]) - .is_empty()); - }); - } } diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 83e02bf..979a8fe 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -1,166 +1,34 @@ -//! Effective edit costs from a unified token graph. +//! Closed (transitive) edit-cost maps from a unified token graph. //! -//! [`compute_effective_costs`] seeds one graph from the raw sub/ins/del maps, -//! closes it with Floyd-Warshall, and projects distances back: `dist[a][b]` for -//! substitutions, `dist[a][ε]` for deletions, `dist[ε][b]` for insertions. +//! [`compute_closed_cost_maps`] seeds one graph from the raw sub/ins/del maps, +//! grows the node set by applying configured ins/del transformations until +//! fixpoint, closes the graph with Floyd-Warshall, and projects distances back +//! into three plain cost maps: +//! - `dist[a][b]` for substitutions, +//! - `dist[a][ε]` for deletions, +//! - `dist[ε][b]` for insertions. use crate::cost_map::CostMap; -use crate::explanation::EditOperation; -use crate::types::{SingleTokenKey, SubstitutionKey}; -use crate::weighted_levenshtein::explain_custom_levenshtein_precomputed; +use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; use std::collections::{HashMap, HashSet}; -const NO_NEXT_NODE: u32 = u32::MAX; - -// Configured tokens up to this length are expanded into all of their substrings -// so closure can route through intermediate strings (e.g. `A -> AA -> AAA`). -// Capped because Floyd-Warshall is O(N³) over the resulting node set. -const MAX_SUBTOKEN_EXPANSION_CHARS: usize = 16; - -// Public types - -/// How the minimum effective cost for a single-token operation was achieved. -#[derive(Clone, Debug, PartialEq)] -pub enum EffectiveOpChain { - /// The token is deleted / inserted directly at its mapped or default cost. - Direct, - - /// A cheaper path exists through a chain of substitutions. - /// - /// `steps` holds the substitution edges in forward order: - /// - **Deletion**: `(source -> x1, c1), (x1 -> x2, c2), …, (xn-1 -> terminal, cn)` then - /// `delete(terminal)` at `terminal_cost`. - /// - **Insertion**: `insert(initial)` at `terminal_cost`, then - /// `(initial -> x1, c1), …, (xn -> target, cn)`. - Via { - /// Substitution edges `(from, to, cost)` in forward order. - steps: Vec<(String, String, f64)>, - /// Cost of the direct `del` / `ins` at the terminal (deletion) or initial - /// (insertion) node of the chain. - terminal_cost: f64, - }, - - /// A cheaper path exists through mixed edit operations. - EditPath { operations: Vec }, -} - -/// Precomputed effective single-token operation costs (deletion or insertion). -/// -/// Replaces the raw [`CostMap`] inside the DP so the algorithm -/// automatically uses the globally cheapest edit path. -#[derive(Debug)] -pub struct EffectiveSingleTokenCosts { - entries: HashMap, - default_cost: f64, - pub max_token_length: usize, -} - -impl EffectiveSingleTokenCosts { - #[inline] - pub fn get_cost(&self, token: &str) -> f64 { - self.get_explicit_cost(token).unwrap_or(self.default_cost) - } - - /// Cost of an explicit entry, or `None` if `token` is not in the map. - #[inline] - pub fn get_explicit_cost(&self, token: &str) -> Option { - self.entries.get(token).map(|(c, _)| *c) - } - - #[inline] - pub fn get_chain(&self, token: &str) -> EffectiveOpChain { - self.entries - .get(token) - .map(|(_, ch)| ch.clone()) - .unwrap_or(EffectiveOpChain::Direct) - } - - #[inline] - pub fn has_key(&self, token: &str) -> bool { - self.entries.contains_key(token) - } -} - -/// How the minimum effective substitution cost was achieved. -#[derive(Clone, Debug, PartialEq)] -pub enum EffectiveSubChain { - /// Direct substitution at the mapped cost. - Direct, - - /// A cheaper path was found through a chain of substitutions. - /// - /// `steps` holds edges `(from, to, cost)` in forward order, covering the - /// full path from the source token to the target token. - Via { - /// Substitution edges `(from, to, cost)` in forward order. - steps: Vec<(String, String, f64)>, - }, - - /// A cheaper path was found through mixed edit operations. - /// - /// This is needed when an effective substitution path contains insertions - /// or deletions between graph nodes, for example `A -> AAA -> B`. - EditPath { operations: Vec }, -} - -/// Precomputed effective substitution costs (all-pairs shortest paths). -/// -/// Replaces the raw [`CostMap`] inside the DP so the algorithm -/// automatically uses the globally cheapest substitution path. -#[derive(Debug)] -pub struct EffectiveSubstitutionCosts { - /// Entries are indexed by source token, then by target token. - entries: HashMap>, - default_cost: f64, - pub max_token_length: usize, -} +// Configured tokens up to this length are expanded into all of their substrings, +// and intermediate nodes generated by ins/del growth are also capped at this +// length. Floyd-Warshall is O(N³) over the resulting node set. +const MAX_NODE_LENGTH_CHARS: usize = 8; -impl EffectiveSubstitutionCosts { - #[inline] - pub fn get_cost(&self, source: &str, target: &str) -> f64 { - self.get_explicit_cost(source, target) - .unwrap_or(self.default_cost) - } - - /// Cost of an explicit entry, or `None` if `(source, target)` is not in the map. - #[inline] - pub fn get_explicit_cost(&self, source: &str, target: &str) -> Option { - self.entries - .get(source) - .and_then(|targets| targets.get(target)) - .map(|(c, _)| *c) - } - - #[inline] - pub fn get_chain(&self, source: &str, target: &str) -> EffectiveSubChain { - self.entries - .get(source) - .and_then(|targets| targets.get(target)) - .map(|(_, ch)| ch.clone()) - .unwrap_or(EffectiveSubChain::Direct) - } - - #[inline] - pub fn has_key(&self, source: &str, target: &str) -> bool { - self.entries - .get(source) - .is_some_and(|targets| targets.contains_key(target)) - } -} - -// Unified token graph solver +// Hard caps on iterative node growth so pathological inputs cannot explode the +// graph. Once these are hit, growth stops; closure runs on whatever nodes exist. +const MAX_NODES: usize = 2048; +const MAX_GROWTH_ROUNDS: usize = 3; /// Interned identifier for a token graph node. -/// -/// A node represents either a configured/derived token or the distinguished -/// epsilon node (`""`). Keeping this as a newtype instead of a bare `usize` -/// makes graph indexing sites explicit. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] struct NodeId(u32); impl NodeId { fn new(index: usize) -> Self { - assert!(index < NO_NEXT_NODE as usize, "too many token graph nodes"); + assert!(index < u32::MAX as usize, "too many token graph nodes"); Self(index as u32) } @@ -168,24 +36,9 @@ impl NodeId { fn index(self) -> usize { self.0 as usize } - - #[inline] - fn raw(self) -> u32 { - self.0 - } -} - -#[derive(Debug)] -struct EdgeResolution { - operations: Vec, - substitution_steps: Vec<(String, String, f64)>, - all_edges_are_raw_substitutions: bool, } /// Dense square matrix stored in row-major order. -/// -/// Floyd-Warshall touches the matrix in tight nested loops; a flat vector avoids -/// the pointer chasing and per-row allocations of `Vec>`. #[derive(Debug)] struct Matrix { width: usize, @@ -219,132 +72,113 @@ impl Matrix { } } -/// Bundle of effective sub/ins/del costs. -/// -/// Used both as direct wrappers around the raw cost maps (input to closure) and -/// as the closed-graph projection returned by [`compute_effective_costs`]. -#[derive(Debug)] -pub struct EffectiveCosts { - pub sub: EffectiveSubstitutionCosts, - pub ins: EffectiveSingleTokenCosts, - pub del: EffectiveSingleTokenCosts, -} - -impl EffectiveCosts { - /// Direct wrappers around the raw cost maps, no closure applied. - fn raw( - sub_map: &CostMap, - ins_map: &CostMap, - del_map: &CostMap, - ) -> Self { - Self { - sub: raw_effective_substitution_costs(sub_map), - ins: raw_effective_single_token_costs(ins_map), - del: raw_effective_single_token_costs(del_map), - } - } -} - -fn raw_effective_substitution_costs( - sub_map: &CostMap, -) -> EffectiveSubstitutionCosts { - let mut entries: HashMap> = HashMap::new(); - for ((source, target), &cost) in &sub_map.costs { - entries - .entry(source.clone()) - .or_default() - .insert(target.clone(), (cost, EffectiveSubChain::Direct)); - } - - EffectiveSubstitutionCosts { - max_token_length: sub_map - .costs - .keys() - .flat_map(|(source, target)| [source.chars().count(), target.chars().count()]) - .max() - .unwrap_or(0) - .max(1), - entries, - default_cost: sub_map.default_cost(), - } -} - -fn raw_effective_single_token_costs(map: &CostMap) -> EffectiveSingleTokenCosts { - let entries = map - .costs +/// Computes closed sub/ins/del cost maps via Floyd-Warshall on a unified graph. +pub fn compute_closed_cost_maps( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { + let tokens = collect_nodes(sub, ins, del); + let token_to_id: HashMap<&str, NodeId> = tokens .iter() - .map(|(token, &cost)| (token.clone(), (cost, EffectiveOpChain::Direct))) + .enumerate() + .map(|(i, token)| (token.as_str(), NodeId::new(i))) .collect(); + let epsilon_id = token_to_id[""]; + let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del); - EffectiveSingleTokenCosts { - max_token_length: map - .costs - .keys() - .map(|token| token.chars().count()) - .max() - .unwrap_or(0) - .max(1), - entries, - default_cost: map.default_cost(), - } + let closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); + let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); + let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); + + (closed_sub, closed_ins, closed_del) } -/// Adds or improves one directed graph edge and records its first hop. -fn set_seed_edge( - dist: &mut Matrix, - next: &mut Matrix, - source: NodeId, - target: NodeId, - cost: f64, -) { - if cost < *dist.get(source, target) { - dist.set(source, target, cost); - next.set(source, target, target.raw()); +/// Collects graph nodes. Includes raw tokens, all substrings of raw tokens, and +/// strings reachable by iteratively applying configured ins/del transformations +/// (bounded by length, total count, and round cap). +fn collect_nodes( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> Vec { + let mut tokens: HashSet = HashSet::new(); + tokens.insert(String::new()); // ε + tokens.extend(ins.costs.keys().cloned()); + tokens.extend(del.costs.keys().cloned()); + for (s, t) in sub.costs.keys() { + tokens.insert(s.clone()); + tokens.insert(t.clone()); + } + + let seeds: Vec = tokens.iter().cloned().collect(); + for token in &seeds { + for substring in substrings(token) { + tokens.insert(substring); + } } -} -/// Adds token-to-token edges that can be made by one explicit insertion/deletion. -/// -/// This captures paths like `A -> AA` or `AB -> A` without running full -/// Levenshtein DP for every token pair during graph construction. -fn seed_embedded_single_token_edges( - tokens: &[String], - token_to_id: &HashMap, - base: &EffectiveCosts, - dist: &mut Matrix, - next: &mut Matrix, -) { - let insertions: Vec<(&str, f64)> = base - .ins - .entries - .iter() - .map(|(token, (cost, _))| (token.as_str(), *cost)) - .collect(); - let deletions: Vec<(&str, f64)> = base - .del - .entries - .iter() - .map(|(token, (cost, _))| (token.as_str(), *cost)) + // Configured ins/del tokens are applied in BOTH directions to grow the set: + // inserting them produces forward-direction successors, removing them + // produces predecessors that lie one configured ins/del edge away. This is + // what lets the closure bridge a user-provided source like "ADC" through + // intermediate nodes "AC" and "ABC" to a configured target "Z". + let single_op_tokens: Vec = ins + .costs + .keys() + .cloned() + .chain(del.costs.keys().cloned()) + .collect::>() + .into_iter() .collect(); - for source in tokens { - let source_id = token_to_id[source.as_str()]; - for (inserted, cost) in &insertions { - for target in insert_token_variants(source, inserted) { - if let Some(&target_id) = token_to_id.get(target.as_str()) { - set_seed_edge(dist, next, source_id, target_id, *cost); + 'rounds: for _ in 0..MAX_GROWTH_ROUNDS { + let snapshot: Vec = tokens.iter().cloned().collect(); + let prev_size = snapshot.len(); + + for source in &snapshot { + let source_len = source.chars().count(); + for op_token in &single_op_tokens { + if source_len + op_token.chars().count() <= MAX_NODE_LENGTH_CHARS { + for variant in insert_token_variants(source, op_token) { + tokens.insert(variant); + if tokens.len() >= MAX_NODES { + break 'rounds; + } + } + } + for variant in delete_token_variants(source, op_token) { + tokens.insert(variant); + if tokens.len() >= MAX_NODES { + break 'rounds; + } } } } - for (deleted, cost) in &deletions { - for target in delete_token_variants(source, deleted) { - if let Some(&target_id) = token_to_id.get(target.as_str()) { - set_seed_edge(dist, next, source_id, target_id, *cost); - } - } + if tokens.len() == prev_size { + break; } } + + tokens.into_iter().collect() +} + +fn substrings(token: &str) -> Vec { + let char_count = token.chars().count(); + if char_count > MAX_NODE_LENGTH_CHARS { + return Vec::new(); + } + let mut boundaries: Vec = token.char_indices().map(|(idx, _)| idx).collect(); + boundaries.push(token.len()); + + let mut out = Vec::with_capacity(char_count * (char_count + 1) / 2); + for start in 0..char_count { + for end in (start + 1)..=char_count { + out.push(token[boundaries[start]..boundaries[end]].to_owned()); + } + } + out } /// All strings obtainable by inserting `inserted` at a char boundary in `source`. @@ -377,441 +211,276 @@ fn delete_token_variants(source: &str, deleted: &str) -> Vec { .collect() } -/// Unified state-transition solver for effective edit costs. -/// -/// The graph has one node per relevant token plus a distinguished epsilon node. -/// Its dense adjacency matrix is initialized with direct weighted token-to-token -/// distances under the base effective costs, then closed with Floyd-Warshall. -struct CostSolver { - base: EffectiveCosts, - token_to_id: HashMap, - tokens: Vec, +fn run_closure( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, epsilon_id: NodeId, - dist: Matrix, - next: Matrix, -} - -impl CostSolver { - fn new( - sub_map: &CostMap, - ins_map: &CostMap, - del_map: &CostMap, - ) -> Self { - let base = EffectiveCosts::raw(sub_map, ins_map, del_map); - - let tokens = collect_solver_tokens(sub_map, ins_map, del_map); - let token_to_id: HashMap = tokens - .iter() - .enumerate() - .map(|(i, token)| (token.clone(), NodeId::new(i))) - .collect(); - let epsilon_id = token_to_id[""]; - let (dist, next) = Self::seed_distances(&tokens, &token_to_id, &base); - - Self { - base, - token_to_id, - tokens, - epsilon_id, - dist, - next, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> Matrix { + let n = tokens.len(); + let mut dist = Matrix::filled(n, f64::INFINITY); + + for index in 0..n { + let node = NodeId::new(index); + dist.set(node, node, 0.0); + } + + for ((source, target), &cost) in &sub.costs { + if let (Some(&s), Some(&t)) = + (token_to_id.get(source.as_str()), token_to_id.get(target.as_str())) + { + relax(&mut dist, s, t, cost); } } - fn compute_effective_costs(mut self) -> EffectiveCosts { - self.close_all_pairs(); - EffectiveCosts { - sub: self.effective_substitutions(), - ins: self.effective_insertions(), - del: self.effective_deletions(), + for (token, &cost) in &ins.costs { + if let Some(&id) = token_to_id.get(token.as_str()) { + relax(&mut dist, epsilon_id, id, cost); } } - /// Initializes direct graph edges from raw edit operations. - /// - /// This deliberately avoids all-pairs weighted DP. The closure pass can - /// discover multi-step paths from raw substitutions, epsilon insertions/ - /// deletions, and the targeted embedded single-token edges. - fn seed_distances( - tokens: &[String], - token_to_id: &HashMap, - base: &EffectiveCosts, - ) -> (Matrix, Matrix) { - let n = tokens.len(); - let mut dist = Matrix::filled(n, f64::INFINITY); - let mut next = Matrix::filled(n, NO_NEXT_NODE); - let epsilon = token_to_id[""]; - - for node in 0..n { - let node = NodeId::new(node); - set_seed_edge(&mut dist, &mut next, node, node, 0.0); - } - - for (source, targets) in &base.sub.entries { - let source_id = token_to_id[source.as_str()]; - for (target, (cost, _)) in targets { - let target_id = token_to_id[target.as_str()]; - set_seed_edge(&mut dist, &mut next, source_id, target_id, *cost); - } - } - - for (token, (cost, _)) in &base.ins.entries { - set_seed_edge( - &mut dist, - &mut next, - epsilon, - token_to_id[token.as_str()], - *cost, - ); - } - - for (token, (cost, _)) in &base.del.entries { - set_seed_edge( - &mut dist, - &mut next, - token_to_id[token.as_str()], - epsilon, - *cost, - ); + for (token, &cost) in &del.costs { + if let Some(&id) = token_to_id.get(token.as_str()) { + relax(&mut dist, id, epsilon_id, cost); } - - seed_embedded_single_token_edges(tokens, token_to_id, base, &mut dist, &mut next); - (dist, next) } - /// Floyd-Warshall all-pairs shortest paths over the unified graph. - fn close_all_pairs(&mut self) { - let n = self.dist.width; - for via in 0..n { - let via = NodeId::new(via); - for source in 0..n { - let source = NodeId::new(source); - let source_to_via = *self.dist.get(source, via); - if !source_to_via.is_finite() { - continue; - } - for target in 0..n { - let target = NodeId::new(target); - let candidate = source_to_via + *self.dist.get(via, target); - if candidate < *self.dist.get(source, target) { - self.dist.set(source, target, candidate); - self.next.set(source, target, *self.next.get(source, via)); - } - } - } - } - } + seed_embedded_edges(tokens, token_to_id, ins, del, &mut dist); + floyd_warshall(&mut dist); + dist +} - fn id(&self, token: &str) -> NodeId { - self.token_to_id[token] +#[inline] +fn relax(dist: &mut Matrix, source: NodeId, target: NodeId, cost: f64) { + if cost < *dist.get(source, target) { + dist.set(source, target, cost); } +} - fn cost(&self, source: NodeId, target: NodeId) -> f64 { - *self.dist.get(source, target) - } +/// Adds direct edges between two existing nodes that differ by exactly one +/// configured insertion or deletion. +fn seed_embedded_edges( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + ins: &CostMap, + del: &CostMap, + dist: &mut Matrix, +) { + let insertions: Vec<(&str, f64)> = ins + .costs + .iter() + .map(|(token, &cost)| (token.as_str(), cost)) + .collect(); + let deletions: Vec<(&str, f64)> = del + .costs + .iter() + .map(|(token, &cost)| (token.as_str(), cost)) + .collect(); - fn path(&self, source: NodeId, target: NodeId) -> Option> { - if *self.next.get(source, target) == NO_NEXT_NODE { - return None; - } - let mut path = vec![source]; - let mut current = source; - while current != target { - let next = *self.next.get(current, target); - if next == NO_NEXT_NODE { - return None; + for source in tokens { + let source_id = token_to_id[source.as_str()]; + for &(inserted, cost) in &insertions { + for variant in insert_token_variants(source, inserted) { + if let Some(&target_id) = token_to_id.get(variant.as_str()) { + relax(dist, source_id, target_id, cost); + } } - current = NodeId(next); - path.push(current); } - Some(path) - } - - fn token(&self, node: NodeId) -> &str { - &self.tokens[node.index()] - } - - /// Projects closed token-to-token distances into effective substitutions. - fn effective_substitutions(&self) -> EffectiveSubstitutionCosts { - let mut entries: HashMap> = - HashMap::new(); - for source in self.non_epsilon_tokens() { - for target in self.non_epsilon_tokens() { - if source == target { - continue; + for &(deleted, cost) in &deletions { + for variant in delete_token_variants(source, deleted) { + if let Some(&target_id) = token_to_id.get(variant.as_str()) { + relax(dist, source_id, target_id, cost); } - let best = self.cost(self.id(source), self.id(target)); - let raw_direct = self.raw_substitution_cost(source, target); - let in_raw_map = raw_direct.is_some(); - let direct_cost = raw_direct.unwrap_or(self.base.sub.default_cost); - - if !in_raw_map && best >= self.base.sub.default_cost { - continue; - } - - let entry = if best < direct_cost { - let chain = self - .substitution_chain(self.id(source), self.id(target)) - .unwrap_or(EffectiveSubChain::Direct); - (best, chain) - } else { - (direct_cost, EffectiveSubChain::Direct) - }; - - entries - .entry(source.to_owned()) - .or_default() - .insert(target.to_owned(), entry); } } - - EffectiveSubstitutionCosts { - max_token_length: max_pair_token_len(&entries), - entries, - default_cost: self.base.sub.default_cost, - } } +} - /// Projects token-to-epsilon distances into effective deletions. - fn effective_deletions(&self) -> EffectiveSingleTokenCosts { - let mut entries: HashMap = HashMap::new(); - for token in self.non_epsilon_tokens() { - let node = self.id(token); - let best = self.cost(node, self.epsilon_id); - let direct = self.base.del.get_cost(token); - if self.base.del.has_key(token) || best < direct { - let chain = if best < direct { - self.deletion_chain(node) - .unwrap_or(EffectiveOpChain::Direct) - } else { - self.base.del.get_chain(token) - }; - entries.insert(token.to_owned(), (best, chain)); +fn floyd_warshall(dist: &mut Matrix) { + let n = dist.width; + for via in 0..n { + let via = NodeId::new(via); + for source in 0..n { + let source = NodeId::new(source); + let source_to_via = *dist.get(source, via); + if !source_to_via.is_finite() { + continue; + } + for target in 0..n { + let target = NodeId::new(target); + let candidate = source_to_via + *dist.get(via, target); + if candidate < *dist.get(source, target) { + dist.set(source, target, candidate); + } } - } - - EffectiveSingleTokenCosts { - max_token_length: max_single_token_len(&entries), - entries, - default_cost: self.base.del.default_cost, } } +} - /// Builds the explanation chain for an effective substitution. - fn substitution_chain(&self, source: NodeId, target: NodeId) -> Option { - let path = self.path(source, target)?; - if path.len() <= 2 { - return Some(EffectiveSubChain::Direct); - } - - let resolution = self.resolve_path_edges(&path); - - if resolution.all_edges_are_raw_substitutions { - Some(EffectiveSubChain::Via { - steps: resolution.substitution_steps, - }) - } else { - Some(EffectiveSubChain::EditPath { - operations: resolution.operations, - }) - } - } +fn project_substitutions( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + dist: &Matrix, + raw: &CostMap, +) -> SubstitutionCostMap { + let default_cost = raw.default_cost(); + let mut closed = SubstitutionCostMap::new(); - /// Builds the explanation chain for an effective deletion. - fn deletion_chain(&self, source: NodeId) -> Option { - let path = self.path(source, self.epsilon_id)?; - if path.len() <= 2 { - return Some(EffectiveOpChain::Direct); + for source in tokens { + if source.is_empty() { + continue; } + let source_id = token_to_id[source.as_str()]; + for target in tokens { + if target.is_empty() || source == target { + continue; + } + let target_id = token_to_id[target.as_str()]; + let key = (source.clone(), target.clone()); - let mut resolution = self.resolve_path_edges(&path[..path.len() - 1]); - - let terminal = *path.get(path.len() - 2)?; - let terminal_token = self.token(terminal); - let terminal_cost = self.base.del.get_explicit_cost(terminal_token)?; + let raw_cost = raw.costs.get(&key).copied(); + let closure_cost = *dist.get(source_id, target_id); + let direct = raw_cost.unwrap_or(default_cost); + let effective = direct.min(closure_cost); - if resolution.all_edges_are_raw_substitutions { - Some(EffectiveOpChain::Via { - steps: resolution.substitution_steps, - terminal_cost, - }) - } else { - resolution.operations.push(EditOperation::Delete { - source: terminal_token.to_string(), - cost: terminal_cost, - }); - Some(EffectiveOpChain::EditPath { - operations: resolution.operations, - }) + if raw_cost.is_some() || effective < default_cost { + closed.insert(key, effective); + } } } - /// Builds the explanation chain for an effective insertion. - fn insertion_chain(&self, target: NodeId) -> Option { - let path = self.path(self.epsilon_id, target)?; - if path.len() <= 2 { - return Some(EffectiveOpChain::Direct); - } + closed +} - let initial = *path.get(1)?; - let initial_token = self.token(initial); - let terminal_cost = self.base.ins.get_explicit_cost(initial_token)?; - - let mut operations = vec![EditOperation::Insert { - target: initial_token.to_string(), - cost: terminal_cost, - }]; - let mut resolution = self.resolve_path_edges(&path[1..]); - - if resolution.all_edges_are_raw_substitutions { - Some(EffectiveOpChain::Via { - steps: resolution.substitution_steps, - terminal_cost, - }) - } else { - operations.append(&mut resolution.operations); - Some(EffectiveOpChain::EditPath { operations }) - } - } +fn project_single_token( + tokens: &[String], + token_to_id: &HashMap<&str, NodeId>, + dist: &Matrix, + raw: &CostMap, + epsilon_id: NodeId, + is_insertion: bool, +) -> SingleTokenCostMap { + let default_cost = raw.default_cost(); + let mut closed = SingleTokenCostMap::new(); - /// Converts graph edges back into user-visible edit operations. - fn resolve_path_edges(&self, path: &[NodeId]) -> EdgeResolution { - let mut operations = Vec::new(); - let mut substitution_steps = Vec::new(); - let mut all_edges_are_raw_substitutions = true; - - for edge in path.windows(2) { - let from = self.token(edge[0]); - let to = self.token(edge[1]); - if let Some(cost) = self.raw_substitution_cost(from, to) { - substitution_steps.push((from.to_owned(), to.to_owned(), cost)); - operations.push(EditOperation::Substitute { - source: from.to_owned(), - target: to.to_owned(), - cost, - }); - } else { - all_edges_are_raw_substitutions = false; - operations.extend( - explain_custom_levenshtein_precomputed(from, to, &self.base) - .into_iter() - .filter(|op| !matches!(op, EditOperation::Match { .. })), - ); - } + for token in tokens { + if token.is_empty() { + continue; } + let id = token_to_id[token.as_str()]; + let raw_cost = raw.costs.get(token).copied(); + let closure_cost = if is_insertion { + *dist.get(epsilon_id, id) + } else { + *dist.get(id, epsilon_id) + }; + let direct = raw_cost.unwrap_or(default_cost); + let effective = direct.min(closure_cost); - EdgeResolution { - operations, - substitution_steps, - all_edges_are_raw_substitutions, + if raw_cost.is_some() || effective < default_cost { + closed.insert(token.clone(), effective); } } - /// Raw substitution edge cost (pre-closure), or `None` if not configured. - fn raw_substitution_cost(&self, source: &str, target: &str) -> Option { - self.base.sub.get_explicit_cost(source, target) - } - - /// Projects epsilon-to-token distances into effective insertions. - fn effective_insertions(&self) -> EffectiveSingleTokenCosts { - let mut entries: HashMap = HashMap::new(); - for token in self.non_epsilon_tokens() { - let node = self.id(token); - let best = self.cost(self.epsilon_id, node); - let direct = self.base.ins.get_cost(token); - if self.base.ins.has_key(token) || best < direct { - let chain = if best < direct { - self.insertion_chain(node) - .unwrap_or(EffectiveOpChain::Direct) - } else { - self.base.ins.get_chain(token) - }; - entries.insert(token.to_owned(), (best, chain)); - } - } + closed +} - EffectiveSingleTokenCosts { - max_token_length: max_single_token_len(&entries), - entries, - default_cost: self.base.ins.default_cost, - } - } +#[cfg(test)] +mod tests { + use super::*; - fn non_epsilon_tokens(&self) -> impl Iterator { - self.tokens - .iter() - .filter(|token| !token.is_empty()) - .map(String::as_str) + fn approx(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-9 } -} -/// Collects graph nodes and bounded substrings needed for transitive closure. -fn collect_solver_tokens( - sub_map: &CostMap, - ins_map: &CostMap, - del_map: &CostMap, -) -> Vec { - let mut tokens: HashSet = HashSet::new(); - tokens.insert(String::new()); // epsilon - tokens.extend(ins_map.costs.keys().cloned()); - tokens.extend(del_map.costs.keys().cloned()); - tokens.extend( - sub_map - .costs - .keys() - .flat_map(|(source, target)| [source.clone(), target.clone()]), - ); - - let originals: Vec = tokens.iter().cloned().collect(); - for token in originals { - let char_count = token.chars().count(); - if char_count > MAX_SUBTOKEN_EXPANSION_CHARS { - continue; + fn make_sub(pairs: &[((&str, &str), f64)], default: f64, symmetric: bool) -> CostMap { + let mut map = SubstitutionCostMap::new(); + for ((s, t), c) in pairs { + map.insert(((*s).to_string(), (*t).to_string()), *c); } + CostMap::::new(map, default, symmetric) + } - let mut boundaries: Vec = token.char_indices().map(|(idx, _)| idx).collect(); - boundaries.push(token.len()); - for start in 0..char_count { - for end in (start + 1)..=char_count { - tokens.insert(token[boundaries[start]..boundaries[end]].to_owned()); - } + fn make_single(pairs: &[(&str, f64)], default: f64) -> CostMap { + let mut map = SingleTokenCostMap::new(); + for (k, v) in pairs { + map.insert((*k).to_string(), *v); } + CostMap::::new(map, default) + } + + #[test] + fn closure_finds_substitution_chain() { + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); + } + + #[test] + fn closure_finds_deletion_chain() { + let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("G", 0.01)], 1.0); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_del["6"], 0.51)); + } + + #[test] + fn closure_finds_insertion_chain() { + let sub = make_sub(&[(("x", "y"), 0.2)], 1.0, false); + let ins = make_single(&[("x", 0.1)], 1.0); + let del = make_single(&[], 1.0); + let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_ins["y"], 0.3)); + } + + #[test] + fn closure_finds_repeated_insertion_substitution() { + // A -> AA -> AAA -> B at cost 0.2 + 0.2 + 0.1 = 0.5 + let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); + } + + #[test] + fn closure_composes_del_ins_sub() { + // ADC -> AC -> ABC -> Z at cost 0.1 + 0.1 + 0.1 = 0.3 + let sub = make_sub(&[(("ABC", "Z"), 0.1)], 1.0, true); + let ins = make_single(&[("B", 0.1)], 1.0); + let del = make_single(&[("D", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_sub[&("ADC".to_string(), "Z".to_string())], 0.3)); + } + + #[test] + fn closure_preserves_direct_when_chain_more_expensive() { + let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("6", 0.2), ("G", 0.01)], 1.0); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del); + assert!(approx(closed_del["6"], 0.2)); + } + + #[test] + fn closure_idempotent_pure_substitution() { + // Pure substitution chains converge in one closure round: round 1 adds + // (a,c) at 0.2; round 2 sees it directly, finds the same path through b. + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del); + let sub2 = CostMap::::new(s1.clone(), 1.0, false); + let ins2 = CostMap::::new(i1.clone(), 1.0); + let del2 = CostMap::::new(d1.clone(), 1.0); + let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2); + assert_eq!(s1, s2); + assert_eq!(i1, i2); + assert_eq!(d1, d2); } - - tokens.into_iter().collect() -} - -/// Longest token length in effective substitution entries. -fn max_pair_token_len( - entries: &HashMap>, -) -> usize { - entries - .iter() - .flat_map(|(source, targets)| { - targets - .keys() - .flat_map(move |target| [source.chars().count(), target.chars().count()]) - }) - .max() - .unwrap_or(0) - .max(1) -} - -/// Longest token length in effective insertion/deletion entries. -fn max_single_token_len(entries: &HashMap) -> usize { - entries - .keys() - .map(|token| token.chars().count()) - .max() - .unwrap_or(0) - .max(1) -} - -/// Computes all effective costs with one unified state-transition graph. -pub(crate) fn compute_effective_costs( - sub_map: &CostMap, - ins_map: &CostMap, - del_map: &CostMap, -) -> EffectiveCosts { - CostSolver::new(sub_map, ins_map, del_map).compute_effective_costs() } diff --git a/src/weighted_levenshtein.rs b/src/weighted_levenshtein.rs index c0a35e6..0b9b51e 100644 --- a/src/weighted_levenshtein.rs +++ b/src/weighted_levenshtein.rs @@ -1,23 +1,28 @@ +use crate::cost_map::CostMap; use crate::explanation::{EditOperation, Predecessor}; -use crate::transitive_costs::{EffectiveCosts, EffectiveOpChain, EffectiveSubChain}; +use crate::types::{SingleTokenKey, SubstitutionKey}; -pub(crate) fn custom_levenshtein_distance_precomputed( +pub(crate) fn custom_levenshtein_distance( source: &str, target: &str, - costs: &EffectiveCosts, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, ) -> f64 { if source == target { return 0.0; } - let mut processor = LevenshteinProcessor::new(source, target, costs, false); + let mut processor = LevenshteinProcessor::new(source, target, sub, ins, del, false); processor.run(); processor.distance() } -pub(crate) fn explain_custom_levenshtein_precomputed( +pub(crate) fn explain_custom_levenshtein( source: &str, target: &str, - costs: &EffectiveCosts, + sub: &CostMap, + ins: &CostMap, + del: &CostMap, ) -> Vec { if source == target { return source @@ -27,24 +32,31 @@ pub(crate) fn explain_custom_levenshtein_precomputed( }) .collect(); } - let mut processor = LevenshteinProcessor::new(source, target, costs, true); + let mut processor = LevenshteinProcessor::new(source, target, sub, ins, del, true); processor.run(); processor.into_result() } -// --- Algorithm Implementation --- - struct LevenshteinProcessor<'a> { source_chars: Vec, target_chars: Vec, - costs: &'a EffectiveCosts, + sub: &'a CostMap, + ins: &'a CostMap, + del: &'a CostMap, dp: Vec>, predecessors: Option>>, multi_char_ops: bool, } impl<'a> LevenshteinProcessor<'a> { - fn new(source: &str, target: &str, costs: &'a EffectiveCosts, explain: bool) -> Self { + fn new( + source: &str, + target: &str, + sub: &'a CostMap, + ins: &'a CostMap, + del: &'a CostMap, + explain: bool, + ) -> Self { let source_chars: Vec = source.chars().collect(); let target_chars: Vec = target.chars().collect(); let len_source = source_chars.len(); @@ -53,10 +65,12 @@ impl<'a> LevenshteinProcessor<'a> { let mut processor = Self { source_chars, target_chars, - multi_char_ops: costs.sub.max_token_length > 1 - || costs.ins.max_token_length > 1 - || costs.del.max_token_length > 1, - costs, + multi_char_ops: sub.max_token_length() > 1 + || ins.max_token_length() > 1 + || del.max_token_length() > 1, + sub, + ins, + del, dp: vec![vec![0.0; len_target + 1]; len_source + 1], predecessors: if explain { Some(vec![ @@ -71,7 +85,6 @@ impl<'a> LevenshteinProcessor<'a> { processor } - /// Fill the DP table. fn run(&mut self) { for i in 1..=self.source_chars.len() { for j in 1..=self.target_chars.len() { @@ -80,13 +93,11 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Get the final computed distance. #[inline] fn distance(&self) -> f64 { self.dp[self.source_chars.len()][self.target_chars.len()] } - /// Convert the computed predecessors into a sequence of edit operations. fn into_result(self) -> Vec { match self.predecessors.as_ref() { Some(preds) => self.backtrack(preds), @@ -101,21 +112,19 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Compute the cost for cell (i, j) in the DP table. #[inline] fn compute_cell(&mut self, i: usize, j: usize) { let source_char = self.source_chars[i - 1]; let target_char = self.target_chars[j - 1]; - let source_char_str = self.source_chars[i - 1].to_string(); - let target_char_str = self.target_chars[j - 1].to_string(); + let source_char_str = source_char.to_string(); + let target_char_str = target_char.to_string(); - let deletion_cost = self.dp[i - 1][j] + self.costs.del.get_cost(&source_char_str); - let insertion_cost = self.dp[i][j - 1] + self.costs.ins.get_cost(&target_char_str); - let sub_cost = self.costs.sub.get_cost(&source_char_str, &target_char_str); + let deletion_cost = self.dp[i - 1][j] + self.del.get_cost(&source_char_str); + let insertion_cost = self.dp[i][j - 1] + self.ins.get_cost(&target_char_str); + let sub_cost = self.sub.get_cost(&source_char_str, &target_char_str); let substitution_cost = self.dp[i - 1][j - 1] + sub_cost; - // Check for exact match let match_cost = self.dp[i - 1][j - 1]; let (mut min_cost, mut best_op) = if source_char == target_char { (match_cost, Predecessor::Match(1)) @@ -140,24 +149,22 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Initialize the first row and column of the DP table. fn initialize(&mut self) { let len_source = self.source_chars.len(); let len_target = self.target_chars.len(); self.dp[0][0] = 0.0; - // First row (insertions) for j in 1..=len_target { let char_str = self.target_chars[j - 1].to_string(); - self.dp[0][j] = self.dp[0][j - 1] + self.costs.ins.get_cost(&char_str); + self.dp[0][j] = self.dp[0][j - 1] + self.ins.get_cost(&char_str); self.record(0, j, Predecessor::Insert(1)); - let max_len = self.costs.ins.max_token_length.min(j); + let max_len = self.ins.max_token_length().min(j); for token_len in 2..=max_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.costs.ins.has_key(&token) { - let new_cost = self.dp[0][token_start] + self.costs.ins.get_cost(&token); + if self.ins.has_key(&token) { + let new_cost = self.dp[0][token_start] + self.ins.get_cost(&token); if new_cost < self.dp[0][j] { self.dp[0][j] = new_cost; self.record(0, j, Predecessor::Insert(token_len)); @@ -165,18 +172,17 @@ impl<'a> LevenshteinProcessor<'a> { } } } - // First column (deletions) for i in 1..=len_source { let char_str = self.source_chars[i - 1].to_string(); - self.dp[i][0] = self.dp[i - 1][0] + self.costs.del.get_cost(&char_str); + self.dp[i][0] = self.dp[i - 1][0] + self.del.get_cost(&char_str); self.record(i, 0, Predecessor::Delete(1)); - let max_len = self.costs.del.max_token_length.min(i); + let max_len = self.del.max_token_length().min(i); for token_len in 2..=max_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.costs.del.has_key(&token) { - let new_cost = self.dp[token_start][0] + self.costs.del.get_cost(&token); + if self.del.has_key(&token) { + let new_cost = self.dp[token_start][0] + self.del.get_cost(&token); if new_cost < self.dp[i][0] { self.dp[i][0] = new_cost; self.record(i, 0, Predecessor::Delete(token_len)); @@ -187,8 +193,8 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_substitutions(&mut self, i: usize, j: usize) { - let max_source_len = self.costs.sub.max_token_length.min(i); - let max_target_len = self.costs.sub.max_token_length.min(j); + let max_source_len = self.sub.max_token_length().min(i); + let max_target_len = self.sub.max_token_length().min(j); for source_len in 1..=max_source_len { for target_len in 1..=max_target_len { if source_len == 1 && target_len == 1 { @@ -198,9 +204,9 @@ impl<'a> LevenshteinProcessor<'a> { let target_start = j - target_len; let source_substr: String = self.source_chars[source_start..i].iter().collect(); let target_substr: String = self.target_chars[target_start..j].iter().collect(); - if self.costs.sub.has_key(&source_substr, &target_substr) { + if self.sub.has_key(&source_substr, &target_substr) { let new_cost = self.dp[source_start][target_start] - + self.costs.sub.get_cost(&source_substr, &target_substr); + + self.sub.get_cost(&source_substr, &target_substr); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Substitute(source_len, target_len)); @@ -211,12 +217,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_insertions(&mut self, i: usize, j: usize) { - let max_ins_len = self.costs.ins.max_token_length.min(j); + let max_ins_len = self.ins.max_token_length().min(j); for token_len in 2..=max_ins_len { let token_start = j - token_len; let token: String = self.target_chars[token_start..j].iter().collect(); - if self.costs.ins.has_key(&token) { - let new_cost = self.dp[i][token_start] + self.costs.ins.get_cost(&token); + if self.ins.has_key(&token) { + let new_cost = self.dp[i][token_start] + self.ins.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Insert(token_len)); @@ -226,12 +232,12 @@ impl<'a> LevenshteinProcessor<'a> { } fn check_multi_char_deletions(&mut self, i: usize, j: usize) { - let max_del_len = self.costs.del.max_token_length.min(i); + let max_del_len = self.del.max_token_length().min(i); for token_len in 2..=max_del_len { let token_start = i - token_len; let token: String = self.source_chars[token_start..i].iter().collect(); - if self.costs.del.has_key(&token) { - let new_cost = self.dp[token_start][j] + self.costs.del.get_cost(&token); + if self.del.has_key(&token) { + let new_cost = self.dp[token_start][j] + self.del.get_cost(&token); if new_cost < self.dp[i][j] { self.dp[i][j] = new_cost; self.record(i, j, Predecessor::Delete(token_len)); @@ -240,14 +246,12 @@ impl<'a> LevenshteinProcessor<'a> { } } - /// Check for multi-character operations (substitutions, insertions, deletions). fn check_multi_char_ops(&mut self, i: usize, j: usize) { self.check_multi_char_substitutions(i, j); self.check_multi_char_insertions(i, j); self.check_multi_char_deletions(i, j); } - /// Backtrack through the predecessors to construct the edit path. fn backtrack(&self, preds: &[Vec]) -> Vec { let mut path = Vec::new(); let mut i = self.source_chars.len(); @@ -259,114 +263,32 @@ impl<'a> LevenshteinProcessor<'a> { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); let target_token: String = self.target_chars[j - t_len..j].iter().collect(); if source_token != target_token { - // path is reversed at end; push in reverse so after reversal - // the chain appears in forward order. - match self.costs.sub.get_chain(&source_token, &target_token) { - EffectiveSubChain::Direct => { - let cost = self.costs.sub.get_cost(&source_token, &target_token); - path.push(EditOperation::Substitute { - source: source_token, - target: target_token, - cost, - }); - } - EffectiveSubChain::Via { steps } => { - for (from, to, cost) in steps.iter().rev() { - path.push(EditOperation::Substitute { - source: from.clone(), - target: to.clone(), - cost: *cost, - }); - } - } - EffectiveSubChain::EditPath { operations } => { - for op in operations.iter().rev() { - path.push(op.clone()); - } - } - } + let cost = self.sub.get_cost(&source_token, &target_token); + path.push(EditOperation::Substitute { + source: source_token, + target: target_token, + cost, + }); } i -= s_len; j -= t_len; } Predecessor::Insert(t_len) => { let target_token: String = self.target_chars[j - t_len..j].iter().collect(); - match self.costs.ins.get_chain(&target_token) { - EffectiveOpChain::Direct => { - let cost = self.costs.ins.get_cost(&target_token); - path.push(EditOperation::Insert { - target: target_token, - cost, - }); - } - EffectiveOpChain::Via { - steps, - terminal_cost, - } => { - // path is reversed at end; push in reverse of forward order so - // after reversal: Insert(initial) -> Sub(…) -> … -> Sub(…->target) - for (from, to, cost) in steps.iter().rev() { - path.push(EditOperation::Substitute { - source: from.clone(), - target: to.clone(), - cost: *cost, - }); - } - let initial = steps - .first() - .map(|(f, _, _)| f.as_str()) - .unwrap_or(&target_token); - path.push(EditOperation::Insert { - target: initial.to_string(), - cost: terminal_cost, - }); - } - EffectiveOpChain::EditPath { operations } => { - for op in operations.iter().rev() { - path.push(op.clone()); - } - } - } + let cost = self.ins.get_cost(&target_token); + path.push(EditOperation::Insert { + target: target_token, + cost, + }); j -= t_len; } Predecessor::Delete(s_len) => { let source_token: String = self.source_chars[i - s_len..i].iter().collect(); - match self.costs.del.get_chain(&source_token) { - EffectiveOpChain::Direct => { - let cost = self.costs.del.get_cost(&source_token); - path.push(EditOperation::Delete { - source: source_token, - cost, - }); - } - EffectiveOpChain::Via { - steps, - terminal_cost, - } => { - // path is reversed at end; push in reverse of forward order so - // after reversal: Sub(source->…) -> … -> Sub(…->terminal) -> Del(terminal) - let terminal = steps - .last() - .map(|(_, t, _)| t.as_str()) - .unwrap_or(&source_token); - path.push(EditOperation::Delete { - source: terminal.to_string(), - cost: terminal_cost, - }); - for (from, to, cost) in steps.iter().rev() { - path.push(EditOperation::Substitute { - source: from.clone(), - target: to.clone(), - cost: *cost, - }); - } - } - EffectiveOpChain::EditPath { operations } => { - for op in operations.iter().rev() { - path.push(op.clone()); - } - } - } + let cost = self.del.get_cost(&source_token); + path.push(EditOperation::Delete { + source: source_token, + cost, + }); i -= s_len; } Predecessor::Match(t_len) => { @@ -388,9 +310,7 @@ impl<'a> LevenshteinProcessor<'a> { #[cfg(test)] mod test { use super::*; - use crate::cost_map::CostMap; - use crate::transitive_costs::compute_effective_costs; - use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; + use crate::types::{SingleTokenCostMap, SubstitutionCostMap}; fn assert_approx_eq(a: f64, b: f64, epsilon: f64) { assert!( @@ -420,32 +340,17 @@ mod test { ins_map: &CostMap, del_map: &CostMap, ) -> f64 { - let costs = compute_effective_costs(sub_map, ins_map, del_map); - custom_levenshtein_distance_precomputed(source, target, &costs) - } - - fn calc_explain( - source: &str, - target: &str, - sub_map: &CostMap, - ins_map: &CostMap, - del_map: &CostMap, - ) -> Vec { - let costs = compute_effective_costs(sub_map, ins_map, del_map); - explain_custom_levenshtein_precomputed(source, target, &costs) + custom_levenshtein_distance(source, target, sub_map, ins_map, del_map) } #[test] fn test_custom_levenshtein_with_custom_sub_map() { let (_, ins_map, del_map) = create_default_cost_maps(); - - // Create a custom substitution map with specific a->b cost let sub_map = CostMap::::new( SubstitutionCostMap::from([(("a".to_string(), "b".to_string()), 0.1)]), 1.0, true, ); - assert_approx_eq( calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, @@ -455,27 +360,21 @@ mod test { #[test] fn test_mixed_custom_costs() { - // Create cost maps let sub_map = CostMap::::new( SubstitutionCostMap::from([(("a".to_string(), "b".to_string()), 0.1)]), 1.0, true, ); - let ins_map = CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.3)]), 1.0); - let del_map = CostMap::::new(SingleTokenCostMap::from([("y".to_string(), 0.4)]), 1.0); - // Test with all three maps: delete 'y' (0.4) + insert 'x' (0.3) assert_approx_eq( calc_distance("aby", "abx", &sub_map, &ins_map, &del_map), 0.7, 1e-9, ); - - // Test substitution: substitute 'a' with 'b' (0.1) assert_approx_eq( calc_distance("abc", "bbc", &sub_map, &ins_map, &del_map), 0.1, @@ -486,657 +385,88 @@ mod test { #[test] fn test_multi_character_substitutions() { let (_, ins_map, del_map) = create_default_cost_maps(); - let sub_map = CostMap::::new( SubstitutionCostMap::from([(("h".to_string(), "In".to_string()), 0.2)]), 1.0, true, ); - - // Test that "hi" with "Ini" has a low cost due to the special substitution assert_approx_eq( calc_distance("hi", "Ini", &sub_map, &ins_map, &del_map), - 0.2, // Only the h->In substitution cost - 1e-9, - ); - - // Test another example - assert_approx_eq( - calc_distance("hello", "Inello", &sub_map, &ins_map, &del_map), - 0.2, // Only the h->In substitution cost - 1e-9, - ); - } - - #[test] - fn test_multiple_substitutions_in_same_string() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("h".to_string(), "In".to_string()), 0.2); - custom_costs.insert(("l".to_string(), "1".to_string()), 0.3); - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test multiple substitutions in the same string - assert_approx_eq( - calc_distance("hello", "Ine11o", &sub_map, &ins_map, &del_map), - 0.8, // 0.2 for h->In and 0.3+0.3 for l->1 twice - 1e-9, - ); - } - - #[test] - fn test_overlapping_substitution_patterns() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("rn".to_string(), "m".to_string()), 0.1); // common OCR confusion - custom_costs.insert(("cl".to_string(), "d".to_string()), 0.2); // another common confusion - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test the rn->m substitution - assert_approx_eq( - calc_distance("corner", "comer", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test the cl->d substitution - assert_approx_eq( - calc_distance("class", "dass", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - } - - #[test] - fn test_asymmetric_costs() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - // Sometimes OCR errors aren't symmetric - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("0".to_string(), "O".to_string()), 0.1); // 0->O is common - custom_costs.insert(("O".to_string(), "0".to_string()), 0.5); // O->0 is less common - let sub_map = CostMap::::new(custom_costs, 1.0, false); // asymmetric costs - - // Test 0->O substitution (lower cost) - assert_approx_eq( - calc_distance("R0AD", "ROAD", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test O->0 substitution (higher cost) assert_approx_eq( - calc_distance("rOad", "r0ad", &sub_map, &ins_map, &del_map), - 0.5, - 1e-9, - ); - } - - #[test] - fn test_substitution_at_word_boundaries() { - let (_, ins_map, del_map) = create_default_cost_maps(); - - let mut custom_costs = SubstitutionCostMap::new(); - custom_costs.insert(("rn".to_string(), "m".to_string()), 0.1); - let sub_map = CostMap::::new(custom_costs, 1.0, true); - - // Test substitution at start of word - assert_approx_eq( - calc_distance("rnat", "mat", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test substitution at end of word - assert_approx_eq( - calc_distance("burn", "bum", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - } - - #[test] - fn test_specific_custom_ins_del_costs() { - let sub_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); - - // Test with custom insertion cost - let ins_map_custom = CostMap::::new( - SingleTokenCostMap::from([("a".to_string(), 0.2), ("b".to_string(), 0.3)]), - 1.0, - ); - let del_map_default = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - // Test insertion with custom cost: Insert 'a' with cost 0.2 - assert_approx_eq( - calc_distance("bc", "abc", &sub_map, &ins_map_custom, &del_map_default), + calc_distance("hello", "Inello", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - - // Test with custom deletion cost - let ins_map_default = CostMap::::new(SingleTokenCostMap::new(), 1.0); - let del_map_custom = CostMap::::new( - SingleTokenCostMap::from([("a".to_string(), 0.4), ("c".to_string(), 0.5)]), - 1.0, - ); - - // Test deletion with custom cost: Delete 'a' with cost 0.4 - assert_approx_eq( - calc_distance("abc", "bc", &sub_map, &ins_map_default, &del_map_custom), - 0.4, - 1e-9, - ); - - // Test with both custom insertion and deletion costs, forcing ins/del - let ins_map_force = - CostMap::::new(SingleTokenCostMap::from([("b".to_string(), 0.3)]), 1.0); - let del_map_force = - CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.5)]), 1.0); - - // Create a substitution map with very high cost to force deletion+insertion - let high_cost_sub_map = CostMap::::new( - SubstitutionCostMap::new(), // Empty map uses default cost - 2.0, // High default cost to ensure deletion+insertion is preferred - true, - ); - - // Test combined operations: Delete 'x' (0.5) + insert 'b' (0.3) - assert_approx_eq( - calc_distance( - "axc", - "abc", - &high_cost_sub_map, - &ins_map_force, - &del_map_force, - ), - 0.8, - 1e-9, - ); } #[test] fn test_edge_cases() { let (sub_map, ins_map, del_map) = create_default_cost_maps(); - - // Test empty strings: Empty strings have zero distance assert_approx_eq( calc_distance("", "", &sub_map, &ins_map, &del_map), 0.0, 1e-9, ); - - // Test source empty, target not empty: Insert 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( calc_distance("", "abc", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); - - // Test source not empty, target empty: Delete 'a', 'b', 'c' with default cost 1.0 each assert_approx_eq( calc_distance("abc", "", &sub_map, &ins_map, &del_map), 3.0, 1e-9, ); - - // Test with custom insertion costs for empty source - let custom_ins_map = CostMap::::new( - SingleTokenCostMap::from([ - ("a".to_string(), 0.2), - ("b".to_string(), 0.3), - ("c".to_string(), 0.4), - ]), - 1.0, - ); - - // Test with custom insertion costs: Insert 'a' (0.2) + 'b' (0.3) + 'c' (0.4) - assert_approx_eq( - calc_distance("", "abc", &sub_map, &custom_ins_map, &del_map), - 0.9, - 1e-9, - ); - - // Test with custom deletion costs for empty target - let custom_del_map = CostMap::::new( - SingleTokenCostMap::from([ - ("a".to_string(), 0.5), - ("b".to_string(), 0.6), - ("c".to_string(), 0.7), - ]), - 1.0, - ); - - // Test with custom deletion costs: Delete 'a' (0.5) + 'b' (0.6) + 'c' (0.7) - assert_approx_eq( - calc_distance("abc", "", &sub_map, &ins_map, &custom_del_map), - 1.8, - 1e-9, - ); - } - - #[test] - fn test_overall_mixed_operations() { - // Create maps with various custom costs - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("a".to_string(), "A".to_string()), 0.1), - (("b".to_string(), "B".to_string()), 0.2), - ]), - 1.0, - true, - ); - - let ins_map = CostMap::::new( - SingleTokenCostMap::from([("x".to_string(), 0.3), ("y".to_string(), 0.4)]), - 1.0, - ); - - let del_map = CostMap::::new( - SingleTokenCostMap::from([("m".to_string(), 0.5), ("n".to_string(), 0.6)]), - 1.0, - ); - - // Test with a mix of operations: Sub 'a'->'A' (0.1) + Sub 'b'->'B' (0.2) + delete 'm' (0.5) + delete 'n' (0.6) + insert 'x' (0.3) + insert 'y' (0.4) - assert_approx_eq( - calc_distance("abmn", "ABxy", &sub_map, &ins_map, &del_map), - 2.1, - 1e-9, - ); } #[test] fn test_unicode_handling() { let (sub_map, ins_map, del_map) = create_default_cost_maps(); - - // Test with Unicode characters: Substitute 'é' with 'e' with default cost 1.0 assert_approx_eq( calc_distance("café", "cafe", &sub_map, &ins_map, &del_map), 1.0, 1e-9, ); - - // Test with emoji: Delete ' ' and '😊' with default cost 1.0 each assert_approx_eq( calc_distance("hi 😊", "hi", &sub_map, &ins_map, &del_map), 2.0, 1e-9, ); - - // Test with custom costs for Unicode - let sub_map_unicode = CostMap::::new( - SubstitutionCostMap::from([(("e".to_string(), "é".to_string()), 0.1)]), // Custom e->é cost - 1.0, - true, - ); - let ins_map_unicode = CostMap::::new( - SingleTokenCostMap::from([("é".to_string(), 0.2), ("😊".to_string(), 0.3)]), - 1.0, - ); - let del_map_unicode = CostMap::::new( - SingleTokenCostMap::from([("é".to_string(), 0.4), ("😊".to_string(), 0.5)]), - 1.0, - ); - - // Test substitution of Unicode with custom cost - assert_approx_eq( - calc_distance( - "cafe", - "café", - &sub_map_unicode, - &ins_map_unicode, - &del_map_unicode, - ), - 0.1, // Custom substitution cost for 'e'->'é' - 1e-9, - ); - - // Test deletion of Unicode with custom cost - assert_approx_eq( - calc_distance("hi 😊", "hi", &sub_map, &ins_map_unicode, &del_map_unicode), - 1.5, // Delete ' ' (default 1.0) and '😊' (custom 0.5) - 1e-9, - ); - } - - #[test] - fn test_various_multi_char_substitutions() { - // Test multi-character substitutions with different lengths - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("th".to_string(), "T".to_string()), 0.2), // 2 -> 1 - (("ing".to_string(), "in'".to_string()), 0.3), // 3 -> 3 - (("o".to_string(), "ou".to_string()), 0.1), // 1 -> 2 - ]), - 1.0, - true, - ); - let (_, ins_map, del_map) = create_default_cost_maps(); - - // Test 2-to-1 character substitution: Substitute "th" with "T" with cost 0.2 - assert_approx_eq( - calc_distance("this", "Tis", &sub_map, &ins_map, &del_map), - 0.2, - 1e-9, - ); - - // Test 3-to-3 character substitution: Substitute "ing" with "in'" with cost 0.3 - assert_approx_eq( - calc_distance("singing", "singin'", &sub_map, &ins_map, &del_map), - 0.3, - 1e-9, - ); - - // Test 1-to-2 character substitution: Substitute "o" with "ou" with cost 0.1 - assert_approx_eq( - calc_distance("go", "gou", &sub_map, &ins_map, &del_map), - 0.1, - 1e-9, - ); - - // Test multiple multi-character substitutions: Sub "th"->"T" (0.2) + Sub "ing"->"in'" (0.3) - assert_approx_eq( - calc_distance("thinking", "Tinkin'", &sub_map, &ins_map, &del_map), - 0.5, - 1e-9, - ); } #[test] fn test_multi_character_insertions_and_deletions() { let (sub_map, _, _) = create_default_cost_maps(); - let ins_map = CostMap::::new( - SingleTokenCostMap::from([ - ("ab".to_string(), 0.3), - ("xyz".to_string(), 0.2), - ("123".to_string(), 0.1), - ("bc".to_string(), 0.25), - ]), + SingleTokenCostMap::from([("ab".to_string(), 0.3), ("xyz".to_string(), 0.2)]), 1.0, ); - let del_map = CostMap::::new( - SingleTokenCostMap::from([ - ("cd".to_string(), 0.4), - ("ef".to_string(), 0.5), - ("789".to_string(), 0.6), - ("bc".to_string(), 0.35), - ]), + SingleTokenCostMap::from([("cd".to_string(), 0.4), ("ef".to_string(), 0.5)]), 1.0, ); - - // Test multi-character insertion: insert 'ab' (0.3) assert_approx_eq( calc_distance("x", "xab", &sub_map, &ins_map, &del_map), 0.3, 1e-9, ); - - // Test multi-character deletion: delete 'cd' (0.4) assert_approx_eq( calc_distance("ycd", "y", &sub_map, &ins_map, &del_map), 0.4, 1e-9, ); - - // Test both insertion and deletion: delete 'ef' (0.5) + insert 'ab' (0.3) assert_approx_eq( calc_distance("aef", "aab", &sub_map, &ins_map, &del_map), 0.8, 1e-9, ); - - // Test with longer token insertion: insert 'xyz' (0.2) assert_approx_eq( calc_distance("test", "testxyz", &sub_map, &ins_map, &del_map), 0.2, 1e-9, ); - - // Test with mixed operations: delete '789' (0.6) + insert 'xyz' (0.2) - assert_approx_eq( - calc_distance("a789b", "axyzb", &sub_map, &ins_map, &del_map), - 0.8, - 1e-9, - ); - - // Test multi-character deletion "bc" at the beginning: delete 'bc' (cost 0.35) - assert_approx_eq( - calc_distance("bcd", "d", &sub_map, &ins_map, &del_map), - 0.35, - 1e-9, - ); - - // Test multi-character insertion "bc" at the beginning: insert 'bc' (cost 0.25) - assert_approx_eq( - calc_distance("c", "bcc", &sub_map, &ins_map, &del_map), - 0.25, - 1e-9, - ); - } - - #[test] - fn test_fallback_to_default_costs_when_multi_char_sub_missing() { - // Create cost maps with multi-character substitutions - let sub_map_full = CostMap::::new( - SubstitutionCostMap::from([ - (("abc".to_string(), "xyz".to_string()), 0.1), - (("de".to_string(), "uv".to_string()), 0.2), - ]), - 1.0, - true, - ); - // Create map with only the 2-char substitution - let sub_map_partial = CostMap::::new( - SubstitutionCostMap::from([(("de".to_string(), "uv".to_string()), 0.2)]), - 1.0, - true, - ); - let (sub_map_empty, ins_map, del_map) = create_default_cost_maps(); - - // Test with full map (allows abc->xyz and de->uv): Sub "abc"->"xyz" (0.1) + Sub "de"->"uv" (0.2) - assert_approx_eq( - calc_distance("abcde", "xyzuv", &sub_map_full, &ins_map, &del_map), - 0.3, - 1e-9, - ); - - // Test with partial map (does not allow abc->xyz, forces default): Sub a->x(1.0) + b->y(1.0) + c->z(1.0) + Sub "de"->"uv"(0.2) - assert_approx_eq( - calc_distance("abcde", "xyzuv", &sub_map_partial, &ins_map, &del_map), - 3.2, - 1e-9, - ); - - // Test with empty map (only single character default operations): 5 * default sub cost (1.0) - assert_approx_eq( - calc_distance("abcde", "xyzuv", &sub_map_empty, &ins_map, &del_map), - 5.0, - 1e-9, - ); - } - - #[test] - fn test_check_multi_char_ops_with_empty_maps() { - let (sub_map, ins_map, del_map) = create_default_cost_maps(); - let costs = compute_effective_costs(&sub_map, &ins_map, &del_map); - - let mut processor = LevenshteinProcessor::new("abcd", "xyz", &costs, true); - - // Simulate the DP state before the operation - let original_dp_3_2 = processor.dp[3][2]; - processor.check_multi_char_ops(3, 2); - - // Verify that the DP value remains unchanged - assert_approx_eq(processor.dp[3][2], original_dp_3_2, 1e-9); - } - - #[test] - fn test_main_function_with_multi_char_ins_del() { - // Define source and target strings - let source = "hello"; - let target = "helloxyz"; - - // Create a custom insertion cost map - let ins_map = CostMap::::new( - SingleTokenCostMap::from([("xyz".to_string(), 0.2)]), - 1.0, - ); - let sub_map = CostMap::::new(SubstitutionCostMap::new(), 1.0, true); - let del_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - // Test multi-char insertion via main function - let dist = calc_distance(source, target, &sub_map, &ins_map, &del_map); - assert_approx_eq(dist, 0.2, 1e-9); // Should be 0.2 (insert "xyz") - - // Now test a multi-character deletion via main function - let source2 = "helloxyz"; - let target2 = "hello"; - - // Create a custom deletion cost map - let del_map2 = CostMap::::new( - SingleTokenCostMap::from([("xyz".to_string(), 0.3)]), - 1.0, - ); - // Use default insertion map for this test - let ins_map2 = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - let dist2 = calc_distance(source2, target2, &sub_map, &ins_map2, &del_map2); - assert_approx_eq(dist2, 0.3, 1e-9); // Should be 0.3 (delete "xyz") - } - - // Transitive substitution - - #[test] - fn test_transitive_substitution_chain() { - // sub(a->b)=0.1, sub(b->c)=0.1, default=1.0 -> eff_sub(a->c)=0.2 - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("a".to_string(), "b".to_string()), 0.1), - (("b".to_string(), "c".to_string()), 0.1), - ]), - 1.0, - false, - ); - let (_, ins_map, del_map) = create_default_cost_maps(); - assert_approx_eq( - calc_distance("a", "c", &sub_map, &ins_map, &del_map), - 0.2, - 1e-9, - ); - } - - #[test] - fn test_transitive_substitution_explain() { - let sub_map = CostMap::::new( - SubstitutionCostMap::from([ - (("a".to_string(), "b".to_string()), 0.1), - (("b".to_string(), "c".to_string()), 0.1), - ]), - 1.0, - false, - ); - let (_, ins_map, del_map) = create_default_cost_maps(); - let ops = calc_explain("a", "c", &sub_map, &ins_map, &del_map); - assert_eq!(ops.len(), 2); - assert!( - matches!(&ops[0], EditOperation::Substitute { source, target, cost } - if source == "a" && target == "b" && (*cost - 0.1).abs() < 1e-9) - ); - assert!( - matches!(&ops[1], EditOperation::Substitute { source, target, cost } - if source == "b" && target == "c" && (*cost - 0.1).abs() < 1e-9) - ); - } - - // ── Issue #12: transitive chain tests ───────────────────────────────────── - - #[test] - fn test_transitive_deletion_chain() { - // sub("6"->"G") = 0.5, del("G") = 0.01 -> chain = 0.51 < direct del("6") = 1.0 - let sub_map = CostMap::::new( - SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), - 1.0, - false, - ); - let del_map = CostMap::::new( - SingleTokenCostMap::from([("G".to_string(), 0.01)]), - 1.0, - ); - let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - assert_approx_eq( - calc_distance("06", "0", &sub_map, &ins_map, &del_map), - 0.51, - 1e-9, - ); - } - - #[test] - fn test_transitive_deletion_chain_explain() { - let sub_map = CostMap::::new( - SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), - 1.0, - false, - ); - let del_map = CostMap::::new( - SingleTokenCostMap::from([("G".to_string(), 0.01)]), - 1.0, - ); - let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - let ops = calc_explain("06", "0", &sub_map, &ins_map, &del_map); - - // Match("0"), Substitute("6"->"G", 0.5), Delete("G", 0.01) - assert_eq!(ops.len(), 3); - assert!(matches!(&ops[0], EditOperation::Match { token } if token == "0")); - assert!( - matches!(&ops[1], EditOperation::Substitute { source, target, cost } - if source == "6" && target == "G" && (*cost - 0.5).abs() < 1e-9) - ); - assert!(matches!(&ops[2], EditOperation::Delete { source, cost } - if source == "G" && (*cost - 0.01).abs() < 1e-9)); - } - - #[test] - fn test_transitive_insertion_chain() { - // ins("x") = 0.1, sub("x"->"y") = 0.2 -> chain ins("y") = 0.3 < direct ins("y") = 1.0 - let sub_map = CostMap::::new( - SubstitutionCostMap::from([(("x".to_string(), "y".to_string()), 0.2)]), - 1.0, - false, - ); - let ins_map = - CostMap::::new(SingleTokenCostMap::from([("x".to_string(), 0.1)]), 1.0); - let del_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - assert_approx_eq( - calc_distance("a", "ay", &sub_map, &ins_map, &del_map), - 0.3, - 1e-9, - ); - } - - #[test] - fn test_direct_op_wins_when_chain_is_more_expensive() { - // del("6") = 0.2 < sub("6"->"G", 0.5) + del("G", 0.01) = 0.51 -> direct wins - let sub_map = CostMap::::new( - SubstitutionCostMap::from([(("6".to_string(), "G".to_string()), 0.5)]), - 1.0, - false, - ); - let del_map = CostMap::::new( - SingleTokenCostMap::from([("6".to_string(), 0.2), ("G".to_string(), 0.01)]), - 1.0, - ); - let ins_map = CostMap::::new(SingleTokenCostMap::new(), 1.0); - - assert_approx_eq( - calc_distance("06", "0", &sub_map, &ins_map, &del_map), - 0.2, - 1e-9, - ); } } From 035b1b1427244187caa4e2d6f3a59d13cff33896 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 12:09:47 +0200 Subject: [PATCH 06/21] pruning --- CHANGELOG.md | 4 +- python/ocr_stringdist/levenshtein.py | 17 ++- .../test_explain_weighted_levenshtein.py | 8 +- python/tests/test_weighted_levenshtein.py | 24 ++++ src/rust_stringdist.rs | 31 ++--- src/transitive_costs.rs | 119 +++++++++++++++--- 6 files changed, 152 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c13791..f445aef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.1.0] - Unreleased -### Changed - -- Consider transitive costs, making the weighted Levenshtein distance satisfy the triangle inequality. ### Added +- Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. - Support for Python 3.14. ## [1.0.1] - 2025-09-21 diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 651a575..89a8ce7 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -100,7 +100,7 @@ def unweighted(cls) -> WeightedLevenshtein: """Creates an instance with all operations having equal cost of 1.0.""" return cls(substitution_costs={}, insertion_costs={}, deletion_costs={}) - def transitive_closure(self) -> WeightedLevenshtein: + def transitive_closure(self, *, prune: bool = False) -> WeightedLevenshtein: """ Returns a new instance whose cost dictionaries are filled with effective (transitive) edit costs. @@ -112,14 +112,13 @@ def transitive_closure(self) -> WeightedLevenshtein: ``del("y") + ins("x")`` becoming an effective ``("y", "x")`` substitution), are likewise materialized. - The returned instance has ``symmetric_substitution=False`` because - closure may produce asymmetric pairs even when the input is symmetric. - Symmetric input is mirrored before closure, so both directions of every - original pair are still present in the result. + :param prune: If True, remove generated substitutions whose costs are + already represented by matches, insertions, deletions, and + shorter substitutions. This can make the returned cost map + easier to inspect, but it is much more expensive for large + closures. - Closure is bounded: very large or pathological cost maps may not be - fully closed. The DP falls back to the configured default costs for any - ``(s, t)`` not in the resulting map. + Closure is bounded: very large cost maps may not be fully closed. ``explain()`` on the closed instance returns flat single-step ops; the original chain that produced an effective cost is not preserved. @@ -127,7 +126,7 @@ def transitive_closure(self) -> WeightedLevenshtein: For repeated use, save via :meth:`to_dict` and reload via :meth:`from_dict` so the closure is computed once. """ - sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps() + sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps(prune) return WeightedLevenshtein( substitution_costs=dict(sub_dict), insertion_costs=dict(ins_dict), diff --git a/python/tests/test_explain_weighted_levenshtein.py b/python/tests/test_explain_weighted_levenshtein.py index 026be7a..eb83941 100644 --- a/python/tests/test_explain_weighted_levenshtein.py +++ b/python/tests/test_explain_weighted_levenshtein.py @@ -134,9 +134,7 @@ def test_explain_transitive_insertion_chain_after_closure() -> None: symmetric_substitution=False, ).transitive_closure() ops = _flat_explain_assertions(wl, "a", "ay", 0.3) - assert ops == [ - EditOperation("insert", None, "y", 0.3), - ] + _assert_ops_equal(ops, [EditOperation("insert", None, "y", 0.3)]) def test_explain_chain_with_expensive_direct_substitution_after_closure() -> None: @@ -192,7 +190,7 @@ def test_explain_insert_delete_substitute_chain_after_closure() -> None: deletion_costs={"D": 0.1}, ).transitive_closure() ops = _flat_explain_assertions(wl, "ADC", "Z", 0.3) - assert ops == [EditOperation("substitute", "ADC", "Z", 0.3)] + _assert_ops_equal(ops, [EditOperation("substitute", "ADC", "Z", 0.3)]) def test_explain_single_char_composed_substitution_chain_after_closure() -> None: @@ -202,4 +200,4 @@ def test_explain_single_char_composed_substitution_chain_after_closure() -> None insertion_costs={"AB": 0.2}, ).transitive_closure() ops = _flat_explain_assertions(wl, "X", "Z", 0.3) - assert ops == [EditOperation("substitute", "X", "Z", 0.3)] + _assert_ops_equal(ops, [EditOperation("substitute", "X", "Z", 0.3)]) diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index 1212890..009fd0d 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -582,6 +582,30 @@ def test_transitive_insertion_subtitution() -> None: assert wl.distance("A", "B") == pytest.approx(0.5) +def test_transitive_closure_prunes_redundant_substitutions() -> None: + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure(prune=True) + + assert wl.distance("A", "B") == pytest.approx(0.5) + assert wl.distance("AA", "AAA") == pytest.approx(0.2) + assert ("A", "B") in wl.substitution_costs + assert ("AA", "AAA") not in wl.substitution_costs + assert ("B", "AA") not in wl.substitution_costs + + +def test_transitive_closure_does_not_prune_by_default() -> None: + wl = WeightedLevenshtein( + substitution_costs={("AAA", "B"): 0.1, ("A", "B"): 0.6}, + insertion_costs={"A": 0.2}, + ).transitive_closure() + + assert wl.distance("A", "B") == pytest.approx(0.5) + assert wl.distance("AA", "AAA") == pytest.approx(0.2) + assert ("AA", "AAA") in wl.substitution_costs + + def test_transitive_insertion_subtitution2() -> None: """ A->AA->AAB->C diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index 47da8a2..c23a667 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -68,10 +68,8 @@ impl RustLevenshteinCalculator { default_substitution_cost, symmetric_substitution, ); - let ins = - CostMap::::from_py_dict(insertion_costs, default_insertion_cost); - let del = - CostMap::::from_py_dict(deletion_costs, default_deletion_cost); + let ins = CostMap::::from_py_dict(insertion_costs, default_insertion_cost); + let del = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); Ok(Self { sub, ins, del }) } @@ -101,15 +99,18 @@ impl RustLevenshteinCalculator { /// Computes effective edit costs via transitive closure and returns three /// Python dicts: `(substitution_costs, insertion_costs, deletion_costs)`. + /// Generated substitutions are pruned only when `prune` is true. /// /// The Python wrapper assembles these into a new `WeightedLevenshtein` /// whose `.distance()` and `.explain()` use the closed costs directly. + #[pyo3(signature = (prune = false))] fn closed_cost_maps<'py>( &self, py: Python<'py>, + prune: bool, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>)> { let (closed_sub, closed_ins, closed_del) = - compute_closed_cost_maps(&self.sub, &self.ins, &self.del); + compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune); let sub_dict = PyDict::new(py); for ((source, target), cost) in closed_sub { @@ -212,13 +213,8 @@ mod tests { // Without calling closed_cost_maps, transitive paths are not auto-applied. // sub(a->b)=0.1, sub(b->c)=0.1: direct a->c lookup falls back to default 1.0. Python::with_gil(|py| { - let calc = make_calculator( - py, - &[(("a", "b"), 0.1), (("b", "c"), 0.1)], - &[], - &[], - false, - ); + let calc = + make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); assert!((calc.distance("a", "c") - 1.0).abs() < f64::EPSILON); }); } @@ -226,14 +222,9 @@ mod tests { #[test] fn test_closed_cost_maps_finds_chain() { Python::with_gil(|py| { - let calc = make_calculator( - py, - &[(("a", "b"), 0.1), (("b", "c"), 0.1)], - &[], - &[], - false, - ); - let (sub, _ins, _del) = calc.closed_cost_maps(py).unwrap(); + let calc = + make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); + let (sub, _ins, _del) = calc.closed_cost_maps(py, false).unwrap(); let cost: f64 = sub .get_item(("a".to_string(), "c".to_string())) .unwrap() diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 979a8fe..3206567 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -7,9 +7,13 @@ //! - `dist[a][b]` for substitutions, //! - `dist[a][ε]` for deletions, //! - `dist[ε][b]` for insertions. +//! +//! Optional pruning removes generated substitutions that are already represented +//! by matches, insertions, deletions, and shorter substitutions. use crate::cost_map::CostMap; use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; +use crate::weighted_levenshtein::custom_levenshtein_distance; use std::collections::{HashMap, HashSet}; // Configured tokens up to this length are expanded into all of their substrings, @@ -21,6 +25,7 @@ const MAX_NODE_LENGTH_CHARS: usize = 8; // graph. Once these are hit, growth stops; closure runs on whatever nodes exist. const MAX_NODES: usize = 2048; const MAX_GROWTH_ROUNDS: usize = 3; +const REDUNDANT_SUBSTITUTION_EPSILON: f64 = 1e-9; /// Interned identifier for a token graph node. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] @@ -73,10 +78,13 @@ impl Matrix { } /// Computes closed sub/ins/del cost maps via Floyd-Warshall on a unified graph. +/// If `prune` is true, generated substitutions that the returned edit maps can +/// already express are removed from the substitution map. pub fn compute_closed_cost_maps( sub: &CostMap, ins: &CostMap, del: &CostMap, + prune: bool, ) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { let tokens = collect_nodes(sub, ins, del); let token_to_id: HashMap<&str, NodeId> = tokens @@ -87,9 +95,14 @@ pub fn compute_closed_cost_maps( let epsilon_id = token_to_id[""]; let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del); - let closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); + let closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); + let closed_sub = if prune { + prune_redundant_substitutions(closed_sub, &closed_ins, &closed_del, sub, ins, del) + } else { + closed_sub + }; (closed_sub, closed_ins, closed_del) } @@ -228,9 +241,10 @@ fn run_closure( } for ((source, target), &cost) in &sub.costs { - if let (Some(&s), Some(&t)) = - (token_to_id.get(source.as_str()), token_to_id.get(target.as_str())) - { + if let (Some(&s), Some(&t)) = ( + token_to_id.get(source.as_str()), + token_to_id.get(target.as_str()), + ) { relax(&mut dist, s, t, cost); } } @@ -354,6 +368,48 @@ fn project_substitutions( closed } +fn prune_redundant_substitutions( + closed_sub: SubstitutionCostMap, + closed_ins: &SingleTokenCostMap, + closed_del: &SingleTokenCostMap, + raw_sub: &CostMap, + raw_ins: &CostMap, + raw_del: &CostMap, +) -> SubstitutionCostMap { + let mut keys: Vec = closed_sub.keys().cloned().collect(); + keys.sort_by(|(source_a, target_a), (source_b, target_b)| { + ( + source_a.chars().count() + target_a.chars().count(), + source_a, + target_a, + ) + .cmp(&( + source_b.chars().count() + target_b.chars().count(), + source_b, + target_b, + )) + }); + + let mut sub_map = CostMap::::new(closed_sub, raw_sub.default_cost(), false); + let ins_map = CostMap::::new(closed_ins.clone(), raw_ins.default_cost()); + let del_map = CostMap::::new(closed_del.clone(), raw_del.default_cost()); + + for key in keys { + if raw_sub.costs.contains_key(&key) { + continue; + } + let Some(cost) = sub_map.costs.remove(&key) else { + continue; + }; + let alternative = custom_levenshtein_distance(&key.0, &key.1, &sub_map, &ins_map, &del_map); + if alternative > cost + REDUNDANT_SUBSTITUTION_EPSILON { + sub_map.costs.insert(key, cost); + } + } + + sub_map.costs +} + fn project_single_token( tokens: &[String], token_to_id: &HashMap<&str, NodeId>, @@ -395,7 +451,11 @@ mod tests { (a - b).abs() < 1e-9 } - fn make_sub(pairs: &[((&str, &str), f64)], default: f64, symmetric: bool) -> CostMap { + fn make_sub( + pairs: &[((&str, &str), f64)], + default: f64, + symmetric: bool, + ) -> CostMap { let mut map = SubstitutionCostMap::new(); for ((s, t), c) in pairs { map.insert(((*s).to_string(), (*t).to_string()), *c); @@ -416,7 +476,7 @@ mod tests { let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); } @@ -425,7 +485,7 @@ mod tests { let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[("G", 0.01)], 1.0); - let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false); assert!(approx(closed_del["6"], 0.51)); } @@ -434,7 +494,7 @@ mod tests { let sub = make_sub(&[(("x", "y"), 0.2)], 1.0, false); let ins = make_single(&[("x", 0.1)], 1.0); let del = make_single(&[], 1.0); - let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del); + let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false); assert!(approx(closed_ins["y"], 0.3)); } @@ -444,8 +504,36 @@ mod tests { let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); let ins = make_single(&[("A", 0.2)], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); + } + + #[test] + fn closure_prunes_substitutions_represented_by_insertions() { + let sub = make_sub(&[(("AAA", "B"), 0.1), (("A", "B"), 0.6)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + + let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, true); + + assert!(approx(closed_ins["A"], 0.2)); assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); + assert!(!closed_sub.contains_key(&("AA".to_string(), "AAA".to_string()))); + assert!(!closed_sub.contains_key(&("B".to_string(), "AA".to_string()))); + } + + #[test] + fn closure_preserves_raw_substitutions_even_when_redundant() { + let sub = make_sub(&[(("AA", "AAA"), 0.2)], 1.0, false); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true); + + assert!(approx( + closed_sub[&("AA".to_string(), "AAA".to_string())], + 0.2 + )); } #[test] @@ -454,8 +542,11 @@ mod tests { let sub = make_sub(&[(("ABC", "Z"), 0.1)], 1.0, true); let ins = make_single(&[("B", 0.1)], 1.0); let del = make_single(&[("D", 0.1)], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del); - assert!(approx(closed_sub[&("ADC".to_string(), "Z".to_string())], 0.3)); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + assert!(approx( + closed_sub[&("ADC".to_string(), "Z".to_string())], + 0.3 + )); } #[test] @@ -463,7 +554,7 @@ mod tests { let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[("6", 0.2), ("G", 0.01)], 1.0); - let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false); assert!(approx(closed_del["6"], 0.2)); } @@ -474,11 +565,11 @@ mod tests { let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); - let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del); + let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del, false); let sub2 = CostMap::::new(s1.clone(), 1.0, false); let ins2 = CostMap::::new(i1.clone(), 1.0); let del2 = CostMap::::new(d1.clone(), 1.0); - let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2); + let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2, false); assert_eq!(s1, s2); assert_eq!(i1, i2); assert_eq!(d1, d2); From 29511cc4a73e05ff2682f6b7dc91b8c6a8ea18eb Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 16:37:23 +0200 Subject: [PATCH 07/21] updates --- python/ocr_stringdist/levenshtein.py | 23 +- src/rust_stringdist.rs | 9 +- src/transitive_costs.rs | 494 ++++++++++++++++++++++++--- 3 files changed, 475 insertions(+), 51 deletions(-) diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 89a8ce7..5f94a71 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -100,7 +100,12 @@ def unweighted(cls) -> WeightedLevenshtein: """Creates an instance with all operations having equal cost of 1.0.""" return cls(substitution_costs={}, insertion_costs={}, deletion_costs={}) - def transitive_closure(self, *, prune: bool = False) -> WeightedLevenshtein: + def transitive_closure( + self, + *, + prune: bool = False, + max_node_length: Optional[int] = None, + ) -> WeightedLevenshtein: """ Returns a new instance whose cost dictionaries are filled with effective (transitive) edit costs. @@ -117,8 +122,16 @@ def transitive_closure(self, *, prune: bool = False) -> WeightedLevenshtein: shorter substitutions. This can make the returned cost map easier to inspect, but it is much more expensive for large closures. - - Closure is bounded: very large cost maps may not be fully closed. + :param max_node_length: Maximum length (in characters) of intermediate + graph nodes the closure may construct. ``None`` + derives a sensible default from the input + (twice the longest raw token, with a small + floor); pass an ``int`` to override. The cap is + what guarantees termination — without it, + configurations like ``ins("A")`` would grow the + graph without bound. Floyd-Warshall is + :math:`O(N^3)` in the resulting node count, so a + higher cap can be substantially slower. ``explain()`` on the closed instance returns flat single-step ops; the original chain that produced an effective cost is not preserved. @@ -126,7 +139,9 @@ def transitive_closure(self, *, prune: bool = False) -> WeightedLevenshtein: For repeated use, save via :meth:`to_dict` and reload via :meth:`from_dict` so the closure is computed once. """ - sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps(prune) + sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps( + prune, max_node_length + ) return WeightedLevenshtein( substitution_costs=dict(sub_dict), insertion_costs=dict(ins_dict), diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index c23a667..1713dd1 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -100,17 +100,20 @@ impl RustLevenshteinCalculator { /// Computes effective edit costs via transitive closure and returns three /// Python dicts: `(substitution_costs, insertion_costs, deletion_costs)`. /// Generated substitutions are pruned only when `prune` is true. + /// `max_node_length` caps the length of intermediate graph nodes; pass + /// `None` to derive it from the input. /// /// The Python wrapper assembles these into a new `WeightedLevenshtein` /// whose `.distance()` and `.explain()` use the closed costs directly. - #[pyo3(signature = (prune = false))] + #[pyo3(signature = (prune = false, max_node_length = None))] fn closed_cost_maps<'py>( &self, py: Python<'py>, prune: bool, + max_node_length: Option, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>)> { let (closed_sub, closed_ins, closed_del) = - compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune); + compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune, max_node_length); let sub_dict = PyDict::new(py); for ((source, target), cost) in closed_sub { @@ -224,7 +227,7 @@ mod tests { Python::with_gil(|py| { let calc = make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); - let (sub, _ins, _del) = calc.closed_cost_maps(py, false).unwrap(); + let (sub, _ins, _del) = calc.closed_cost_maps(py, false, None).unwrap(); let cost: f64 = sub .get_item(("a".to_string(), "c".to_string())) .unwrap() diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 3206567..560ccec 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -16,15 +16,15 @@ use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, Subs use crate::weighted_levenshtein::custom_levenshtein_distance; use std::collections::{HashMap, HashSet}; -// Configured tokens up to this length are expanded into all of their substrings, -// and intermediate nodes generated by ins/del growth are also capped at this -// length. Floyd-Warshall is O(N³) over the resulting node set. -const MAX_NODE_LENGTH_CHARS: usize = 8; - -// Hard caps on iterative node growth so pathological inputs cannot explode the -// graph. Once these are hit, growth stops; closure runs on whatever nodes exist. -const MAX_NODES: usize = 2048; -const MAX_GROWTH_ROUNDS: usize = 3; +// When the caller passes `None` for `max_node_length`, the cap is derived as +// `max(longest raw token across all maps) * MAX_NODE_LENGTH_MULTIPLIER`, with a +// floor of `MIN_DERIVED_NODE_LENGTH` so trivial single-char maps still leave +// headroom for chained compositions. Length is what makes the graph finite — +// without any cap, configurations like `ins("A")=0.1` would grow the node set +// without bound (A → AA → AAA → …). +const MAX_NODE_LENGTH_MULTIPLIER: usize = 2; +const MIN_DERIVED_NODE_LENGTH: usize = 4; + const REDUNDANT_SUBSTITUTION_EPSILON: f64 = 1e-9; /// Interned identifier for a token graph node. @@ -78,6 +78,14 @@ impl Matrix { } /// Computes closed sub/ins/del cost maps via Floyd-Warshall on a unified graph. +/// +/// `max_node_length` caps the length (in characters) of intermediate nodes the +/// growth phase may construct and of substrings expanded from raw tokens. Pass +/// `None` to derive a sensible default from the input +/// (`max raw-token length × 2`, floored at `MIN_DERIVED_NODE_LENGTH`); pass +/// `Some(n)` to override. The cap is what guarantees termination — without it, +/// configurations like `ins("A")` produce an infinite graph. +/// /// If `prune` is true, generated substitutions that the returned edit maps can /// already express are removed from the substitution map. pub fn compute_closed_cost_maps( @@ -85,8 +93,11 @@ pub fn compute_closed_cost_maps( ins: &CostMap, del: &CostMap, prune: bool, + max_node_length: Option, ) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { - let tokens = collect_nodes(sub, ins, del); + let max_node_length = + max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); + let tokens = collect_nodes(sub, ins, del, max_node_length); let token_to_id: HashMap<&str, NodeId> = tokens .iter() .enumerate() @@ -107,13 +118,36 @@ pub fn compute_closed_cost_maps( (closed_sub, closed_ins, closed_del) } -/// Collects graph nodes. Includes raw tokens, all substrings of raw tokens, and -/// strings reachable by iteratively applying configured ins/del transformations -/// (bounded by length, total count, and round cap). +/// Default `max_node_length` derivation: twice the longest raw token across all +/// three maps, floored at `MIN_DERIVED_NODE_LENGTH`. The doubling leaves room +/// for compositions like `sub("AB","C") + sub("CC","D")` that need an +/// intermediate (`"ABAB" → "CC" → "D"`) longer than any single raw token. +fn derive_max_node_length( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> usize { + let max_raw = sub + .costs + .keys() + .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) + .chain(ins.costs.keys().map(|k| k.chars().count())) + .chain(del.costs.keys().map(|k| k.chars().count())) + .max() + .unwrap_or(0); + (max_raw * MAX_NODE_LENGTH_MULTIPLIER).max(MIN_DERIVED_NODE_LENGTH) +} + +/// Collects graph nodes. Includes raw tokens, all substrings of raw tokens +/// (capped at `max_node_length`), and strings reachable by iteratively applying +/// configured ins/del transformations until fixpoint. The length cap is the +/// only termination guarantee — without it, `ins("A")` alone would grow the +/// node set without bound. fn collect_nodes( sub: &CostMap, ins: &CostMap, del: &CostMap, + max_node_length: usize, ) -> Vec { let mut tokens: HashSet = HashSet::new(); tokens.insert(String::new()); // ε @@ -126,7 +160,7 @@ fn collect_nodes( let seeds: Vec = tokens.iter().cloned().collect(); for token in &seeds { - for substring in substrings(token) { + for substring in substrings(token, max_node_length) { tokens.insert(substring); } } @@ -145,26 +179,23 @@ fn collect_nodes( .into_iter() .collect(); - 'rounds: for _ in 0..MAX_GROWTH_ROUNDS { + // Run growth to fixpoint. The length cap bounds the set of strings reachable + // from the seeds, so this terminates after a finite number of rounds for any + // input — typically only a handful, even for large maps. + loop { let snapshot: Vec = tokens.iter().cloned().collect(); let prev_size = snapshot.len(); for source in &snapshot { let source_len = source.chars().count(); for op_token in &single_op_tokens { - if source_len + op_token.chars().count() <= MAX_NODE_LENGTH_CHARS { + if source_len + op_token.chars().count() <= max_node_length { for variant in insert_token_variants(source, op_token) { tokens.insert(variant); - if tokens.len() >= MAX_NODES { - break 'rounds; - } } } for variant in delete_token_variants(source, op_token) { tokens.insert(variant); - if tokens.len() >= MAX_NODES { - break 'rounds; - } } } } @@ -177,9 +208,9 @@ fn collect_nodes( tokens.into_iter().collect() } -fn substrings(token: &str) -> Vec { +fn substrings(token: &str, max_node_length: usize) -> Vec { let char_count = token.chars().count(); - if char_count > MAX_NODE_LENGTH_CHARS { + if char_count > max_node_length { return Vec::new(); } let mut boundaries: Vec = token.char_indices().map(|(idx, _)| idx).collect(); @@ -211,17 +242,34 @@ fn insert_token_variants(source: &str, inserted: &str) -> Vec { } /// All strings obtainable by deleting one exact `deleted` occurrence from `source`. +/// +/// Enumerates every char-aligned offset where `deleted` matches, including +/// overlapping ones: `delete_token_variants("ABABA", "ABA")` yields both `"BA"` +/// (delete at 0) and `"AB"` (delete at 2). `str::match_indices` would only +/// return the non-overlapping leftmost match, so we walk char boundaries +/// manually instead. fn delete_token_variants(source: &str, deleted: &str) -> Vec { - source - .match_indices(deleted) - .map(|(idx, _)| { - let end = idx + deleted.len(); - let mut target = String::with_capacity(source.len() - deleted.len()); - target.push_str(&source[..idx]); - target.push_str(&source[end..]); - target - }) - .collect() + if deleted.is_empty() || deleted.len() > source.len() { + return Vec::new(); + } + let mut out = Vec::new(); + for (idx, _) in source.char_indices() { + let end = idx + deleted.len(); + if end > source.len() { + break; + } + if !source.is_char_boundary(end) { + continue; + } + if &source[idx..end] != deleted { + continue; + } + let mut target = String::with_capacity(source.len() - deleted.len()); + target.push_str(&source[..idx]); + target.push_str(&source[end..]); + out.push(target); + } + out } fn run_closure( @@ -376,6 +424,11 @@ fn prune_redundant_substitutions( raw_ins: &CostMap, raw_del: &CostMap, ) -> SubstitutionCostMap { + // Shortest-first iteration is safe: redundancy is monotone under removal of + // redundant edges. If a short substitution is dropped because an ins/del/sub + // alternative matches its cost, any longer substitution that relied on it in + // the closure can fall back to the same alternative at the same total cost, + // so its redundancy verdict is unchanged. let mut keys: Vec = closed_sub.keys().cloned().collect(); keys.sort_by(|(source_a, target_a), (source_b, target_b)| { ( @@ -390,6 +443,10 @@ fn prune_redundant_substitutions( )) }); + // `max_token_length` is computed once at construction and is not updated as + // keys are removed below. The DP will scan longer windows than strictly + // necessary; correctness is preserved because overestimating the window only + // costs CPU. let mut sub_map = CostMap::::new(closed_sub, raw_sub.default_cost(), false); let ins_map = CostMap::::new(closed_ins.clone(), raw_ins.default_cost()); let del_map = CostMap::::new(closed_del.clone(), raw_del.default_cost()); @@ -476,7 +533,7 @@ mod tests { let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); } @@ -485,7 +542,7 @@ mod tests { let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[("G", 0.01)], 1.0); - let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx(closed_del["6"], 0.51)); } @@ -494,7 +551,7 @@ mod tests { let sub = make_sub(&[(("x", "y"), 0.2)], 1.0, false); let ins = make_single(&[("x", 0.1)], 1.0); let del = make_single(&[], 1.0); - let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (_, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx(closed_ins["y"], 0.3)); } @@ -504,7 +561,7 @@ mod tests { let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); let ins = make_single(&[("A", 0.2)], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); } @@ -514,7 +571,7 @@ mod tests { let ins = make_single(&[("A", 0.2)], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, true); + let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); assert!(approx(closed_ins["A"], 0.2)); assert!(approx(closed_sub[&("A".to_string(), "B".to_string())], 0.5)); @@ -528,7 +585,7 @@ mod tests { let ins = make_single(&[("A", 0.2)], 1.0); let del = make_single(&[], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); assert!(approx( closed_sub[&("AA".to_string(), "AAA".to_string())], @@ -542,7 +599,7 @@ mod tests { let sub = make_sub(&[(("ABC", "Z"), 0.1)], 1.0, true); let ins = make_single(&[("B", 0.1)], 1.0); let del = make_single(&[("D", 0.1)], 1.0); - let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx( closed_sub[&("ADC".to_string(), "Z".to_string())], 0.3 @@ -554,10 +611,359 @@ mod tests { let sub = make_sub(&[(("6", "G"), 0.5)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[("6", 0.2), ("G", 0.01)], 1.0); - let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (_, _, closed_del) = compute_closed_cost_maps(&sub, &ins, &del, false, None); assert!(approx(closed_del["6"], 0.2)); } + // --- helper-function tests -------------------------------------------------- + + fn sorted(mut v: Vec) -> Vec { + v.sort(); + v + } + + #[test] + fn substrings_includes_all_contiguous_slices() { + let mut got = sorted(substrings("abc", 8)); + got.dedup(); + assert_eq!(got, vec!["a", "ab", "abc", "b", "bc", "c"]); + } + + #[test] + fn substrings_empty_for_empty_input() { + assert!(substrings("", 8).is_empty()); + } + + #[test] + fn substrings_handles_unicode() { + let got = sorted(substrings("café", 8)); + // Chars are 'c', 'a', 'f', 'é'; substrings are all contiguous slices on + // char boundaries. + assert!(got.contains(&"é".to_string())); + assert!(got.contains(&"fé".to_string())); + assert!(got.contains(&"café".to_string())); + // No partial-byte slice should appear. + for s in &got { + assert!(s.is_char_boundary(0) && s.is_char_boundary(s.len())); + } + } + + #[test] + fn substrings_skipped_when_token_exceeds_cap() { + // A token longer than the cap returns no substrings. + let long_token = "A".repeat(9); + assert!(substrings(&long_token, 8).is_empty()); + } + + #[test] + fn insert_token_variants_inserts_at_every_boundary() { + let mut got = sorted(insert_token_variants("ab", "X")); + assert_eq!(got, vec!["Xab", "aXb", "abX"]); + got.dedup(); + assert_eq!(got.len(), 3); + } + + #[test] + fn insert_token_variants_into_empty() { + assert_eq!( + insert_token_variants("", "AB"), + vec!["AB".to_string()] + ); + } + + #[test] + fn insert_token_variants_handles_unicode() { + let got = sorted(insert_token_variants("é", "X")); + assert_eq!(got, vec!["Xé".to_string(), "éX".to_string()]); + } + + #[test] + fn delete_token_variants_enumerates_overlapping_matches() { + let mut variants = delete_token_variants("ABABA", "ABA"); + variants.sort(); + assert_eq!(variants, vec!["AB".to_string(), "BA".to_string()]); + } + + #[test] + fn delete_token_variants_no_match() { + assert!(delete_token_variants("xyz", "ABA").is_empty()); + } + + #[test] + fn delete_token_variants_full_match_yields_empty_string() { + assert_eq!( + delete_token_variants("ABA", "ABA"), + vec![String::new()] + ); + } + + #[test] + fn delete_token_variants_deleted_longer_than_source() { + assert!(delete_token_variants("AB", "ABCDE").is_empty()); + } + + #[test] + fn delete_token_variants_empty_deleted_string() { + // Deleting nothing produces no graph edges (we'd otherwise loop on ε). + assert!(delete_token_variants("ABC", "").is_empty()); + } + + #[test] + fn delete_token_variants_single_char_repeated() { + // For single chars, every position matches; we get one variant per + // occurrence (deduplicated downstream by the HashSet). + let mut got = delete_token_variants("AAAA", "A"); + got.sort(); + // All four deletions produce "AAA". + assert_eq!(got, vec!["AAA"; 4]); + } + + #[test] + fn delete_token_variants_handles_unicode() { + // "café" minus "fé" yields "ca". A non-overlapping single-byte search + // at byte index 2 ('f') would be wrong if it weren't gated by + // is_char_boundary — confirm the unicode path works end-to-end. + let got = delete_token_variants("café", "fé"); + assert_eq!(got, vec!["ca".to_string()]); + } + + #[test] + fn delete_token_variants_unicode_overlap() { + // "🙂🙃🙂🙃🙂" minus "🙂🙃🙂": matches at char positions 0 and 2 (both + // overlap). Expect both variants. + let mut got = delete_token_variants("🙂🙃🙂🙃🙂", "🙂🙃🙂"); + got.sort(); + assert_eq!(got, vec!["🙂🙃".to_string(), "🙃🙂".to_string()]); + } + + // --- closure correctness ---------------------------------------------------- + + #[test] + fn closure_finds_chain_through_overlapping_deletion() { + // ABABA can become AB by deleting the "ABA" starting at index 2. + // sub("AB","X")=0.1 then closes to sub("ABABA","X")=0.2. + let sub = make_sub(&[(("AB", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("ABA", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx( + closed_sub[&("ABABA".to_string(), "X".to_string())], + 0.2 + )); + } + + #[test] + fn closure_emits_overlapping_deletion_intermediate_edge() { + // Pin the projected substitution that the previous match_indices-based + // implementation silently dropped. Without overlap support, this entry + // would be missing from closed_sub. + let sub = make_sub(&[(("AB", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("ABA", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + let expected_key = ("ABABA".to_string(), "AB".to_string()); + assert!( + closed_sub.contains_key(&expected_key), + "expected overlapping-delete edge to be projected as a substitution" + ); + assert!(approx(closed_sub[&expected_key], 0.1)); + } + + #[test] + fn closure_handles_empty_cost_maps() { + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, closed_ins, closed_del) = + compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(closed_sub.is_empty()); + assert!(closed_ins.is_empty()); + assert!(closed_del.is_empty()); + } + + #[test] + fn closure_does_not_emit_pairs_above_default() { + // No raw entry, no chain — projection should not insert anything that + // would just equal the default cost. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + // ("a","z") has no path, falls back to default — must not be inserted. + assert!(!closed_sub.contains_key(&("a".to_string(), "z".to_string()))); + } + + #[test] + fn closure_keeps_raw_substitution_even_when_above_default() { + // User-set 2.0 > default 1.0. The raw entry must survive so that the + // round-trip preserves the user's explicit intent. + let sub = make_sub(&[(("a", "b"), 2.0)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + let key = ("a".to_string(), "b".to_string()); + assert!(closed_sub.contains_key(&key)); + // No cheaper chain exists, so the raw 2.0 stays. + assert!(approx(closed_sub[&key], 2.0)); + } + + #[test] + fn closure_skips_self_substitution() { + // (a,a) is never a useful substitution; projection must skip it. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + for ((s, t), _) in &closed_sub { + assert_ne!(s, t, "self-substitution leaked into closed map"); + } + } + + #[test] + fn closure_symmetric_propagates_both_directions() { + // Symmetric chain: (a,b)+ (b,c) at 0.1 each gives (a,c) and (c,a) both at 0.2. + let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, true); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("a".to_string(), "c".to_string())], 0.2)); + assert!(approx(closed_sub[&("c".to_string(), "a".to_string())], 0.2)); + } + + #[test] + fn closure_asymmetric_does_not_invent_reverse() { + // Asymmetric: only (a,b) is given. (b,a) must not appear because no + // path exists in that direction. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(closed_sub.contains_key(&("a".to_string(), "b".to_string()))); + assert!(!closed_sub.contains_key(&("b".to_string(), "a".to_string()))); + } + + #[test] + fn closure_materializes_del_then_ins_as_substitution() { + // del("y")=0.1 + ins("x")=0.2 should fold into an effective sub("y","x")=0.3. + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[("x", 0.2)], 1.0); + let del = make_single(&[("y", 0.1)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_sub[&("y".to_string(), "x".to_string())], 0.3)); + } + + #[test] + fn closure_finds_long_substitution_chain() { + // a -> b -> c -> d at 0.1 + 0.1 + 0.1 = 0.3. + let sub = make_sub( + &[(("a", "b"), 0.1), (("b", "c"), 0.1), (("c", "d"), 0.1)], + 1.0, + false, + ); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx( + closed_sub[&("a".to_string(), "d".to_string())], + 0.3 + )); + } + + #[test] + fn closure_handles_zero_cost_edges() { + // Zero-cost ins should still appear in closed map. + let sub = make_sub(&[(("x", "y"), 0.0)], 1.0, false); + let ins = make_single(&[("x", 0.0)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, closed_ins, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx(closed_ins["y"], 0.0)); + assert!(approx(closed_sub[&("x".to_string(), "y".to_string())], 0.0)); + } + + #[test] + fn closure_handles_unicode_tokens() { + // Same chain shape as the basic ASCII test, but with non-ASCII chars. + let sub = make_sub(&[(("é", "ê"), 0.1), (("ê", "è"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); + assert!(approx( + closed_sub[&("é".to_string(), "è".to_string())], + 0.2 + )); + } + + // --- pruning ---------------------------------------------------------------- + + #[test] + fn pruning_on_empty_input_is_noop() { + let sub = make_sub(&[], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.is_empty()); + } + + #[test] + fn pruning_keeps_strictly_cheaper_substitution() { + // sub(a,b)=0.1 with ins(a)=0.5, del(b)=0.5: alternative path through ε + // costs 1.0, so the substitution is strictly cheaper and must be kept. + let sub = make_sub(&[(("a", "b"), 0.1)], 1.0, false); + let ins = make_single(&[("a", 0.5)], 1.0); + let del = make_single(&[("b", 0.5)], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.contains_key(&("a".to_string(), "b".to_string()))); + } + + #[test] + fn pruning_preserves_distances_for_chains() { + // Closure-then-pruning must not change distances on the strings the + // chains describe. We verify by running the DP with the closed maps + // before and after pruning and comparing. + let sub = make_sub(&[(("AAA", "B"), 0.1)], 1.0, true); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (sub_unpruned, ins_unpruned, del_unpruned) = + compute_closed_cost_maps(&sub, &ins, &del, false, None); + let (sub_pruned, ins_pruned, del_pruned) = + compute_closed_cost_maps(&sub, &ins, &del, true, None); + + let unpruned_subs = CostMap::::new(sub_unpruned, 1.0, false); + let unpruned_ins = CostMap::::new(ins_unpruned, 1.0); + let unpruned_del = CostMap::::new(del_unpruned, 1.0); + let pruned_subs = CostMap::::new(sub_pruned, 1.0, false); + let pruned_ins = CostMap::::new(ins_pruned, 1.0); + let pruned_del = CostMap::::new(del_pruned, 1.0); + + for (s, t) in [ + ("A", "B"), + ("AA", "B"), + ("AAA", "B"), + ("B", "A"), + ("AA", "AAA"), + ] { + let unpruned = + custom_levenshtein_distance(s, t, &unpruned_subs, &unpruned_ins, &unpruned_del); + let pruned = + custom_levenshtein_distance(s, t, &pruned_subs, &pruned_ins, &pruned_del); + assert!( + approx(unpruned, pruned), + "distance({s:?},{t:?}) drifted after pruning: {unpruned} vs {pruned}" + ); + } + } + + #[test] + fn pruning_raw_entries_are_never_dropped() { + // ("AA","AAA") at 0.2 is exactly representable as ins("A"), so it would + // be redundant if generated. But it's user-provided, so prune must keep it. + let sub = make_sub(&[(("AA", "AAA"), 0.2)], 1.0, false); + let ins = make_single(&[("A", 0.2)], 1.0); + let del = make_single(&[], 1.0); + let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, true, None); + assert!(closed_sub.contains_key(&("AA".to_string(), "AAA".to_string()))); + } + #[test] fn closure_idempotent_pure_substitution() { // Pure substitution chains converge in one closure round: round 1 adds @@ -565,11 +971,11 @@ mod tests { let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); - let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del, false); + let (s1, i1, d1) = compute_closed_cost_maps(&sub, &ins, &del, false, None); let sub2 = CostMap::::new(s1.clone(), 1.0, false); let ins2 = CostMap::::new(i1.clone(), 1.0); let del2 = CostMap::::new(d1.clone(), 1.0); - let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2, false); + let (s2, i2, d2) = compute_closed_cost_maps(&sub2, &ins2, &del2, false, None); assert_eq!(s1, s2); assert_eq!(i1, i2); assert_eq!(d1, d2); From ab8719cd91c26e1af0d448b55da91290a15b3b60 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 17:09:10 +0200 Subject: [PATCH 08/21] support updating costs after introduction of _calculator --- python/ocr_stringdist/_observable_dict.py | 45 ++++++ python/ocr_stringdist/levenshtein.py | 181 +++++++++++++++------- src/transitive_costs.rs | 26 +--- 3 files changed, 177 insertions(+), 75 deletions(-) create mode 100644 python/ocr_stringdist/_observable_dict.py diff --git a/python/ocr_stringdist/_observable_dict.py b/python/ocr_stringdist/_observable_dict.py new file mode 100644 index 0000000..2010f02 --- /dev/null +++ b/python/ocr_stringdist/_observable_dict.py @@ -0,0 +1,45 @@ +from typing import Any, Callable, Optional, TypeVar + +K = TypeVar("K") +V = TypeVar("V") + + +class _ObservableDict(dict[K, V]): + """A dictionary that triggers a callback on mutation.""" + + def __init__( + self, + mapping: dict[K, V], + on_change: Callable[[], None], + validator: Optional[Callable[[K, V], None]] = None, + ) -> None: + self._on_change = on_change + self._validator = validator + if validator: + for k, v in mapping.items(): + validator(k, v) + super().__init__(mapping) + + def __setitem__(self, key: K, value: V) -> None: + if self._validator: + self._validator(key, value) + super().__setitem__(key, value) + self._on_change() + + def __delitem__(self, key: K) -> None: + super().__delitem__(key) + self._on_change() + + def clear(self) -> None: + super().clear() + self._on_change() + + def pop(self, key: K, default: Any = None) -> V: + res = super().pop(key, default) + self._on_change() + return res + + def popitem(self) -> tuple[K, V]: + res = super().popitem() + self._on_change() + return res diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 5f94a71..3f6ddfd 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from typing import Any, Optional +from ._observable_dict import _ObservableDict from ._rust_stringdist import RustLevenshteinCalculator from .default_ocr_distances import ocr_distance_map from .edit_operation import EditOperation @@ -29,14 +30,6 @@ class WeightedLevenshtein: :raises TypeError, ValueError: If the provided arguments are invalid. """ - substitution_costs: dict[tuple[str, str], float] - insertion_costs: dict[str, float] - deletion_costs: dict[str, float] - symmetric_substitution: bool - default_substitution_cost: float - default_insertion_cost: float - default_deletion_cost: float - def __init__( self, substitution_costs: Optional[dict[tuple[str, str], float]] = None, @@ -48,53 +41,133 @@ def __init__( default_insertion_cost: float = 1.0, default_deletion_cost: float = 1.0, ) -> None: - # Validate default costs - for cost_name, cost_val in [ - ("default_substitution_cost", default_substitution_cost), - ("default_insertion_cost", default_insertion_cost), - ("default_deletion_cost", default_deletion_cost), - ]: - if not isinstance(cost_val, (int, float)): - raise TypeError(f"{cost_name} must be a number, but got: {type(cost_val).__name__}") - if cost_val < 0: - raise ValueError(f"{cost_name} must be non-negative, got value: {cost_val}") - - # Validate substitution_costs dictionary - sub_costs = ocr_distance_map if substitution_costs is None else substitution_costs - for key, cost in sub_costs.items(): - if not ( - isinstance(key, tuple) - and len(key) == 2 - and isinstance(key[0], str) - and isinstance(key[1], str) - ): - raise TypeError( - f"substitution_costs keys must be tuples of two strings, but found: {key}" - ) - if not isinstance(cost, (int, float)): - raise TypeError( - f"Cost for substitution {key} must be a number, but got: {type(cost).__name__}" - ) - if cost < 0: - raise ValueError(f"Cost for substitution {key} cannot be negative, but got: {cost}") - - self.substitution_costs = sub_costs - self.insertion_costs = {} if insertion_costs is None else insertion_costs - self.deletion_costs = {} if deletion_costs is None else deletion_costs - self.symmetric_substitution = symmetric_substitution - self.default_substitution_cost = default_substitution_cost - self.default_insertion_cost = default_insertion_cost - self.default_deletion_cost = default_deletion_cost + self._symmetric_substitution = symmetric_substitution + self._default_substitution_cost = self._validate_cost( + "default_substitution_cost", default_substitution_cost + ) + self._default_insertion_cost = self._validate_cost( + "default_insertion_cost", default_insertion_cost + ) + self._default_deletion_cost = self._validate_cost( + "default_deletion_cost", default_deletion_cost + ) + + # Initialize Observable Dicts + sub_init = ocr_distance_map if substitution_costs is None else substitution_costs + self._substitution_costs = _ObservableDict( + sub_init, self._sync_calculator, self._validate_sub_entry + ) + self._insertion_costs = _ObservableDict( + insertion_costs or {}, self._sync_calculator, self._validate_unary_entry + ) + self._deletion_costs = _ObservableDict( + deletion_costs or {}, self._sync_calculator, self._validate_unary_entry + ) + + self._sync_calculator() + + def _sync_calculator(self) -> None: + """Internal helper to re-instantiate the Rust backend when state changes.""" self._calculator = RustLevenshteinCalculator( - substitution_costs=self.substitution_costs, - insertion_costs=self.insertion_costs, - deletion_costs=self.deletion_costs, - symmetric_substitution=symmetric_substitution, - default_substitution_cost=default_substitution_cost, - default_insertion_cost=default_insertion_cost, - default_deletion_cost=default_deletion_cost, + substitution_costs=self._substitution_costs, + insertion_costs=self._insertion_costs, + deletion_costs=self._deletion_costs, + symmetric_substitution=self._symmetric_substitution, + default_substitution_cost=self._default_substitution_cost, + default_insertion_cost=self._default_insertion_cost, + default_deletion_cost=self._default_deletion_cost, ) + # --- Properties --- + + @property + def substitution_costs(self) -> dict[tuple[str, str], float]: + return self._substitution_costs + + @substitution_costs.setter + def substitution_costs(self, value: dict[tuple[str, str], float]) -> None: + self._substitution_costs = _ObservableDict( + value, self._sync_calculator, self._validate_sub_entry + ) + self._sync_calculator() + + @property + def insertion_costs(self) -> dict[str, float]: + return self._insertion_costs + + @insertion_costs.setter + def insertion_costs(self, value: dict[str, float]) -> None: + self._insertion_costs = _ObservableDict( + value, self._sync_calculator, self._validate_unary_entry + ) + self._sync_calculator() + + @property + def deletion_costs(self) -> dict[str, float]: + return self._deletion_costs + + @deletion_costs.setter + def deletion_costs(self, value: dict[str, float]) -> None: + self._deletion_costs = _ObservableDict( + value, self._sync_calculator, self._validate_unary_entry + ) + self._sync_calculator() + + @property + def symmetric_substitution(self) -> bool: + return self._symmetric_substitution + + @symmetric_substitution.setter + def symmetric_substitution(self, value: bool) -> None: + self._symmetric_substitution = value + self._sync_calculator() + + @property + def default_substitution_cost(self) -> float: + return self._default_substitution_cost + + @default_substitution_cost.setter + def default_substitution_cost(self, value: float) -> None: + self._default_substitution_cost = self._validate_cost("default_substitution_cost", value) + self._sync_calculator() + + @property + def default_insertion_cost(self) -> float: + return self._default_insertion_cost + + @default_insertion_cost.setter + def default_insertion_cost(self, value: float) -> None: + self._default_insertion_cost = self._validate_cost("default_insertion_cost", value) + self._sync_calculator() + + @property + def default_deletion_cost(self) -> float: + return self._default_deletion_cost + + @default_deletion_cost.setter + def default_deletion_cost(self, value: float) -> None: + self._default_deletion_cost = self._validate_cost("default_deletion_cost", value) + self._sync_calculator() + + # --- Validation Helpers --- + + def _validate_cost(self, name: str, val: float) -> float: + if not isinstance(val, (int, float)): + raise TypeError(f"{name} must be a number, but got: {type(val).__name__}") + if val < 0: + raise ValueError(f"{name} must be non-negative, got value: {val}") + return float(val) + + def _validate_sub_entry(self, key: Any, cost: Any) -> None: + if not (isinstance(key, tuple) and len(key) == 2 and all(isinstance(k, str) for k in key)): + raise TypeError(f"substitution_costs keys must be tuples of two strings, found: {key}") + self._validate_cost(f"Cost for {key}", cost) + + def _validate_unary_entry(self, key: Any, cost: Any) -> None: + if not isinstance(key, str): + raise TypeError(f"Cost keys must be strings, found: {key}") + self._validate_cost(f"Cost for {key}", cost) + @classmethod def unweighted(cls) -> WeightedLevenshtein: """Creates an instance with all operations having equal cost of 1.0.""" @@ -139,9 +212,7 @@ def transitive_closure( For repeated use, save via :meth:`to_dict` and reload via :meth:`from_dict` so the closure is computed once. """ - sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps( - prune, max_node_length - ) + sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps(prune, max_node_length) return WeightedLevenshtein( substitution_costs=dict(sub_dict), insertion_costs=dict(ins_dict), diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 560ccec..fb2f25c 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -95,8 +95,7 @@ pub fn compute_closed_cost_maps( prune: bool, max_node_length: Option, ) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { - let max_node_length = - max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); + let max_node_length = max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); let tokens = collect_nodes(sub, ins, del, max_node_length); let token_to_id: HashMap<&str, NodeId> = tokens .iter() @@ -665,10 +664,7 @@ mod tests { #[test] fn insert_token_variants_into_empty() { - assert_eq!( - insert_token_variants("", "AB"), - vec!["AB".to_string()] - ); + assert_eq!(insert_token_variants("", "AB"), vec!["AB".to_string()]); } #[test] @@ -691,10 +687,7 @@ mod tests { #[test] fn delete_token_variants_full_match_yields_empty_string() { - assert_eq!( - delete_token_variants("ABA", "ABA"), - vec![String::new()] - ); + assert_eq!(delete_token_variants("ABA", "ABA"), vec![String::new()]); } #[test] @@ -863,10 +856,7 @@ mod tests { let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); - assert!(approx( - closed_sub[&("a".to_string(), "d".to_string())], - 0.3 - )); + assert!(approx(closed_sub[&("a".to_string(), "d".to_string())], 0.3)); } #[test] @@ -887,10 +877,7 @@ mod tests { let ins = make_single(&[], 1.0); let del = make_single(&[], 1.0); let (closed_sub, _, _) = compute_closed_cost_maps(&sub, &ins, &del, false, None); - assert!(approx( - closed_sub[&("é".to_string(), "è".to_string())], - 0.2 - )); + assert!(approx(closed_sub[&("é".to_string(), "è".to_string())], 0.2)); } // --- pruning ---------------------------------------------------------------- @@ -944,8 +931,7 @@ mod tests { ] { let unpruned = custom_levenshtein_distance(s, t, &unpruned_subs, &unpruned_ins, &unpruned_del); - let pruned = - custom_levenshtein_distance(s, t, &pruned_subs, &pruned_ins, &pruned_del); + let pruned = custom_levenshtein_distance(s, t, &pruned_subs, &pruned_ins, &pruned_del); assert!( approx(unpruned, pruned), "distance({s:?},{t:?}) drifted after pruning: {unpruned} vs {pruned}" From 4659920d7925a9cefcd5db1d501be17fcfa38b83 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 17:18:58 +0200 Subject: [PATCH 09/21] add unittests --- .../test_weighted_levenshtein_mutation.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 python/tests/test_weighted_levenshtein_mutation.py diff --git a/python/tests/test_weighted_levenshtein_mutation.py b/python/tests/test_weighted_levenshtein_mutation.py new file mode 100644 index 0000000..f405a69 --- /dev/null +++ b/python/tests/test_weighted_levenshtein_mutation.py @@ -0,0 +1,162 @@ +from collections.abc import Callable +from typing import Any + +import pytest +from ocr_stringdist import WeightedLevenshtein + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_cost_property_reassignment_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + assert wl.distance(source, target) == pytest.approx(1.0) + + setattr(wl, costs_attribute, costs) + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost", "source", "target", "expected"], + [ + ("substitution_costs", ("a", "b"), 0.2, "a", "b", 0.2), + ("insertion_costs", "b", 0.3, "", "b", 0.3), + ("deletion_costs", "a", 0.4, "a", "", 0.4), + ], +) +def test_in_place_cost_assignment_updates_calculator( + costs_attribute: str, + key: Any, + cost: float, + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + costs = getattr(wl, costs_attribute) + assert wl.distance(source, target) == pytest.approx(1.0) + + costs[key] = cost + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "remove_cost", "source", "target"], + [ + ("substitution_costs", {("a", "b"): 0.2}, lambda costs: costs.pop(("a", "b")), "a", "b"), + ("insertion_costs", {"b": 0.3}, lambda costs: costs.pop("b"), "", "b"), + ("deletion_costs", {"a": 0.4}, lambda costs: costs.pop("a"), "a", ""), + ("substitution_costs", {("a", "b"): 0.2}, lambda costs: costs.clear(), "a", "b"), + ("insertion_costs", {"b": 0.3}, lambda costs: costs.clear(), "", "b"), + ("deletion_costs", {"a": 0.4}, lambda costs: costs.clear(), "a", ""), + ], +) +def test_in_place_cost_removal_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + remove_cost: Callable[[dict[Any, float]], Any], + source: str, + target: str, +) -> None: + wl = WeightedLevenshtein.unweighted() + setattr(wl, costs_attribute, costs) + assert wl.distance(source, target) < 1.0 + + remove_cost(getattr(wl, costs_attribute)) + + assert wl.distance(source, target) == pytest.approx(1.0) + assert wl.batch_distance(source, [target]) == pytest.approx([1.0]) + + +@pytest.mark.parametrize( + ["default_cost_attribute", "source", "target"], + [ + ("default_substitution_cost", "a", "b"), + ("default_insertion_cost", "", "b"), + ("default_deletion_cost", "a", ""), + ], +) +def test_default_cost_setters_update_calculator( + default_cost_attribute: str, source: str, target: str +) -> None: + wl = WeightedLevenshtein.unweighted() + assert wl.distance(source, target) == pytest.approx(1.0) + + setattr(wl, default_cost_attribute, 0.25) + + assert wl.distance(source, target) == pytest.approx(0.25) + assert wl.batch_distance(source, [target]) == pytest.approx([0.25]) + + +def test_symmetric_substitution_setter_updates_calculator() -> None: + wl = WeightedLevenshtein( + substitution_costs={("a", "b"): 0.2}, + symmetric_substitution=False, + ) + assert wl.distance("b", "a") == pytest.approx(1.0) + + wl.symmetric_substitution = True + + assert wl.distance("b", "a") == pytest.approx(0.2) + assert wl.batch_distance("b", ["a"]) == pytest.approx([0.2]) + + +@pytest.mark.parametrize( + ["costs_attribute", "invalid_costs"], + [ + ("substitution_costs", {("a", "b"): -0.1}), + ("substitution_costs", {("a",): 0.1}), + ("insertion_costs", {"b": -0.1}), + ("insertion_costs", {("b",): 0.1}), + ("deletion_costs", {"a": -0.1}), + ("deletion_costs", {("a",): 0.1}), + ], +) +def test_invalid_cost_property_reassignment_keeps_existing_calculator( + costs_attribute: str, invalid_costs: dict[Any, float] +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises((TypeError, ValueError)): + setattr(wl, costs_attribute, invalid_costs) + + assert getattr(wl, costs_attribute) == {} + assert wl.distance("a", "b") == pytest.approx(1.0) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost"], + [ + ("substitution_costs", ("a", "b"), -0.1), + ("substitution_costs", ("a",), 0.1), + ("insertion_costs", "b", -0.1), + ("insertion_costs", ("b",), 0.1), + ("deletion_costs", "a", -0.1), + ("deletion_costs", ("a",), 0.1), + ], +) +def test_invalid_in_place_cost_assignment_keeps_existing_calculator( + costs_attribute: str, key: Any, cost: float +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises((TypeError, ValueError)): + getattr(wl, costs_attribute)[key] = cost + + assert getattr(wl, costs_attribute) == {} + assert wl.distance("a", "b") == pytest.approx(1.0) From 9b43263bd4076f2061907d2f33387a1683582f23 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 19:49:46 +0200 Subject: [PATCH 10/21] fixes --- CHANGELOG.md | 4 + python/ocr_stringdist/_observable_dict.py | 40 +++++- python/ocr_stringdist/levenshtein.py | 21 ++-- .../test_weighted_levenshtein_mutation.py | 117 ++++++++++++++++++ src/cost_map.rs | 70 ++++++++--- src/rust_stringdist.rs | 27 ++-- src/transitive_costs.rs | 76 ++++++++++-- 7 files changed, 301 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f445aef..58c2cda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. - Support for Python 3.14. +### Fixed + +- Use the minimum cost for conflicting symmetric substitutions. + ## [1.0.1] - 2025-09-21 ### Fixed diff --git a/python/ocr_stringdist/_observable_dict.py b/python/ocr_stringdist/_observable_dict.py index 2010f02..5729927 100644 --- a/python/ocr_stringdist/_observable_dict.py +++ b/python/ocr_stringdist/_observable_dict.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from collections.abc import Mapping from typing import Any, Callable, Optional, TypeVar K = TypeVar("K") @@ -7,6 +10,8 @@ class _ObservableDict(dict[K, V]): """A dictionary that triggers a callback on mutation.""" + _MISSING = object() + def __init__( self, mapping: dict[K, V], @@ -34,8 +39,8 @@ def clear(self) -> None: super().clear() self._on_change() - def pop(self, key: K, default: Any = None) -> V: - res = super().pop(key, default) + def pop(self, key: K, default: Any = _MISSING) -> V: + res = super().pop(key) if default is self._MISSING else super().pop(key, default) self._on_change() return res @@ -43,3 +48,34 @@ def popitem(self) -> tuple[K, V]: res = super().popitem() self._on_change() return res + + def update(self, other: Any = (), /, **kwargs: V) -> None: + items = self._items_from_update_args(other, kwargs) + if self._validator: + for key, value in items: + self._validator(key, value) + super().update(items) + self._on_change() + + def setdefault(self, key: K, default: V) -> V: + if key in self: + return self[key] + if self._validator: + self._validator(key, default) + super().__setitem__(key, default) + self._on_change() + return default + + def __or__(self, other: object, /) -> Any: + if not isinstance(other, dict): + return NotImplemented + return dict(self) | other + + def __ior__(self, other: object, /) -> Any: + self.update(other) + return self + + def _items_from_update_args(self, other: Any, kwargs: dict[str, V]) -> list[tuple[Any, Any]]: + items = list(other.items()) if isinstance(other, Mapping) else list(other) + items.extend(kwargs.items()) + return items diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 3f6ddfd..1b66866 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math from collections.abc import Iterable from typing import Any, Optional @@ -154,6 +155,8 @@ def default_deletion_cost(self, value: float) -> None: def _validate_cost(self, name: str, val: float) -> float: if not isinstance(val, (int, float)): raise TypeError(f"{name} must be a number, but got: {type(val).__name__}") + if not math.isfinite(val): + raise ValueError(f"{name} must be finite, got value: {val}") if val < 0: raise ValueError(f"{name} must be non-negative, got value: {val}") return float(val) @@ -183,11 +186,11 @@ def transitive_closure( Returns a new instance whose cost dictionaries are filled with effective (transitive) edit costs. - If, for example, ``substitution_costs[("a", "b")] = 0.1`` and - ``substitution_costs[("b", "c")] = 0.1``, the closed instance's - ``substitution_costs[("a", "c")]`` is ``0.2`` rather than the default. - Insertion and deletion chains, and chains that cross ``ε`` (e.g. - ``del("y") + ins("x")`` becoming an effective ``("y", "x")`` substitution), + If, for example, `substitution_costs[("a", "b")] = 0.1` and + `substitution_costs[("b", "c")] = 0.1`, the closed instance's + `substitution_costs[("a", "c")]` is `0.2` rather than the default. + Insertion and deletion chains, and chains that cross `ε` (e.g. + `del("y") + ins("x")` becoming an effective `("y", "x")` substitution), are likewise materialized. :param prune: If True, remove generated substitutions whose costs are @@ -196,15 +199,17 @@ def transitive_closure( easier to inspect, but it is much more expensive for large closures. :param max_node_length: Maximum length (in characters) of intermediate - graph nodes the closure may construct. ``None`` + graph nodes the closure may construct. `None` derives a sensible default from the input (twice the longest raw token, with a small - floor); pass an ``int`` to override. The cap is + floor); pass an `int` to override. The cap is what guarantees termination — without it, - configurations like ``ins("A")`` would grow the + configurations like `ins("A")` would grow the graph without bound. Floyd-Warshall is :math:`O(N^3)` in the resulting node count, so a higher cap can be substantially slower. + :raises ValueError: If the generated closure graph is too large to + process safely. ``explain()`` on the closed instance returns flat single-step ops; the original chain that produced an effective cost is not preserved. diff --git a/python/tests/test_weighted_levenshtein_mutation.py b/python/tests/test_weighted_levenshtein_mutation.py index f405a69..9448352 100644 --- a/python/tests/test_weighted_levenshtein_mutation.py +++ b/python/tests/test_weighted_levenshtein_mutation.py @@ -1,3 +1,4 @@ +import math from collections.abc import Callable from typing import Any @@ -55,6 +56,78 @@ def test_in_place_cost_assignment_updates_calculator( assert wl.batch_distance(source, [target]) == pytest.approx([expected]) +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_in_place_cost_update_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + + getattr(wl, costs_attribute).update(costs) + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "key", "cost", "source", "target", "expected"], + [ + ("substitution_costs", ("a", "b"), 0.2, "a", "b", 0.2), + ("insertion_costs", "b", 0.3, "", "b", 0.3), + ("deletion_costs", "a", 0.4, "a", "", 0.4), + ], +) +def test_in_place_cost_setdefault_updates_calculator( + costs_attribute: str, + key: Any, + cost: float, + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + + inserted = getattr(wl, costs_attribute).setdefault(key, cost) + + assert inserted == cost + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + +@pytest.mark.parametrize( + ["costs_attribute", "costs", "source", "target", "expected"], + [ + ("substitution_costs", {("a", "b"): 0.2}, "a", "b", 0.2), + ("insertion_costs", {"b": 0.3}, "", "b", 0.3), + ("deletion_costs", {"a": 0.4}, "a", "", 0.4), + ], +) +def test_in_place_cost_union_update_updates_calculator( + costs_attribute: str, + costs: dict[Any, float], + source: str, + target: str, + expected: float, +) -> None: + wl = WeightedLevenshtein.unweighted() + observable_costs = getattr(wl, costs_attribute) + + observable_costs |= costs + + assert wl.distance(source, target) == pytest.approx(expected) + assert wl.batch_distance(source, [target]) == pytest.approx([expected]) + + @pytest.mark.parametrize( ["costs_attribute", "costs", "remove_cost", "source", "target"], [ @@ -116,14 +189,52 @@ def test_symmetric_substitution_setter_updates_calculator() -> None: assert wl.batch_distance("b", ["a"]) == pytest.approx([0.2]) +def test_symmetric_substitution_conflicts_use_minimum_cost() -> None: + wl = WeightedLevenshtein( + substitution_costs={("Z", "2"): 0.3, ("2", "Z"): 0.5}, + symmetric_substitution=True, + ) + + assert wl.distance("Z", "2") == pytest.approx(0.3) + assert wl.distance("2", "Z") == pytest.approx(0.3) + + +@pytest.mark.parametrize( + ["default_cost_attribute", "invalid_cost"], + [ + ("default_substitution_cost", math.nan), + ("default_substitution_cost", math.inf), + ("default_insertion_cost", math.nan), + ("default_insertion_cost", math.inf), + ("default_deletion_cost", math.nan), + ("default_deletion_cost", math.inf), + ], +) +def test_invalid_default_cost_setter_rejects_non_finite_values( + default_cost_attribute: str, invalid_cost: float +) -> None: + wl = WeightedLevenshtein.unweighted() + + with pytest.raises(ValueError, match="must be finite"): + setattr(wl, default_cost_attribute, invalid_cost) + + assert wl.distance("a", "b") == pytest.approx(1.0) + + @pytest.mark.parametrize( ["costs_attribute", "invalid_costs"], [ ("substitution_costs", {("a", "b"): -0.1}), + ("substitution_costs", {("a", "b"): math.nan}), + ("substitution_costs", {("a", "b"): math.inf}), ("substitution_costs", {("a",): 0.1}), ("insertion_costs", {"b": -0.1}), + ("insertion_costs", {"b": math.nan}), + ("insertion_costs", {"b": math.inf}), ("insertion_costs", {("b",): 0.1}), ("deletion_costs", {"a": -0.1}), + ("deletion_costs", {"a": math.nan}), + ("deletion_costs", {"a": math.inf}), ("deletion_costs", {("a",): 0.1}), ], ) @@ -143,10 +254,16 @@ def test_invalid_cost_property_reassignment_keeps_existing_calculator( ["costs_attribute", "key", "cost"], [ ("substitution_costs", ("a", "b"), -0.1), + ("substitution_costs", ("a", "b"), math.nan), + ("substitution_costs", ("a", "b"), math.inf), ("substitution_costs", ("a",), 0.1), ("insertion_costs", "b", -0.1), + ("insertion_costs", "b", math.nan), + ("insertion_costs", "b", math.inf), ("insertion_costs", ("b",), 0.1), ("deletion_costs", "a", -0.1), + ("deletion_costs", "a", math.nan), + ("deletion_costs", "a", math.inf), ("deletion_costs", ("a",), 0.1), ], ) diff --git a/src/cost_map.rs b/src/cost_map.rs index 1c8c7ea..d92ecc3 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::hash::Hash; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; /// A trait for cost map keys, allowing us to constrain the generic parameter @@ -45,9 +46,9 @@ impl CostMap { let mut costs = HashMap::with_capacity(custom_costs_input.len() * 2); for ((s1, s2), cost) in custom_costs_input { - costs.entry((s1.clone(), s2.clone())).or_insert(cost); + insert_min_cost(&mut costs, (s1.clone(), s2.clone()), cost); if symmetric { - costs.entry((s2.clone(), s1.clone())).or_insert(cost); + insert_min_cost(&mut costs, (s2.clone(), s1.clone()), cost); } } @@ -65,21 +66,26 @@ impl CostMap { } } - pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64, symmetric: bool) -> Self + pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64, symmetric: bool) -> PyResult where D: PyDictMethods<'a>, { let mut substitution_costs = SubstitutionCostMap::new(); for (key, value) in py_dict.iter() { - if let Ok(key_tuple) = key.extract::<(String, String)>() { - if let Ok(cost) = value.extract::() { - substitution_costs.insert((key_tuple.0, key_tuple.1), cost); - } - } + let key_tuple = key.extract::<(String, String)>()?; + let cost = value.extract::()?; + validate_cost( + cost, + &format!( + "Substitution cost for key ({}, {})", + key_tuple.0, key_tuple.1 + ), + )?; + substitution_costs.insert((key_tuple.0, key_tuple.1), cost); } - Self::new(substitution_costs, default_cost, symmetric) + Ok(Self::new(substitution_costs, default_cost, symmetric)) } #[inline] @@ -113,21 +119,20 @@ impl CostMap { } } - pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64) -> Self + pub fn from_py_dict<'a, D>(py_dict: &'a D, default_cost: f64) -> PyResult where D: PyDictMethods<'a>, { let mut single_token_costs = SingleTokenCostMap::new(); for (key, value) in py_dict.iter() { - if let Ok(token) = key.extract::() { - if let Ok(cost) = value.extract::() { - single_token_costs.insert(token, cost); - } - } + let token = key.extract::()?; + let cost = value.extract::()?; + validate_cost(cost, "Cost")?; + single_token_costs.insert(token, cost); } - Self::new(single_token_costs, default_cost) + Ok(Self::new(single_token_costs, default_cost)) } #[inline] @@ -141,6 +146,27 @@ impl CostMap { } } +fn insert_min_cost(costs: &mut HashMap, key: K, cost: f64) { + costs + .entry(key) + .and_modify(|existing| *existing = (*existing).min(cost)) + .or_insert(cost); +} + +pub(crate) fn validate_cost(cost: f64, label: &str) -> PyResult<()> { + if !cost.is_finite() { + return Err(PyValueError::new_err(format!( + "{label} must be finite, got value: {cost}" + ))); + } + if cost < 0.0 { + return Err(PyValueError::new_err(format!( + "{label} must be non-negative, got value: {cost}" + ))); + } + Ok(()) +} + // Common methods for any type of CostMap impl CostMap { pub fn default_cost(&self) -> f64 { @@ -198,6 +224,18 @@ mod tests { assert_eq!(cost_map.default_cost(), 1.5); } + #[test] + fn test_symmetric_substitution_map_conflicts_use_minimum_cost() { + let mut custom_costs = SubstitutionCostMap::new(); + custom_costs.insert(("a".to_string(), "b".to_string()), 0.4); + custom_costs.insert(("b".to_string(), "a".to_string()), 0.2); + + let cost_map = CostMap::::new(custom_costs, 1.0, true); + + assert_eq!(cost_map.costs[&("a".to_string(), "b".to_string())], 0.2); + assert_eq!(cost_map.costs[&("b".to_string(), "a".to_string())], 0.2); + } + #[test] fn test_default_cost_accessor() { let sub_map = CostMap::::new(HashMap::new(), 2.5, true); diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index 1713dd1..cf58138 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -1,4 +1,4 @@ -use crate::cost_map::CostMap; +use crate::cost_map::{validate_cost, CostMap}; use crate::explanation::EditOperation; use crate::transitive_costs::compute_closed_cost_maps; use crate::types::{SingleTokenKey, SubstitutionKey}; @@ -59,17 +59,17 @@ impl RustLevenshteinCalculator { default_insertion_cost: f64, default_deletion_cost: f64, ) -> PyResult { - validate_default_cost(default_substitution_cost)?; - validate_default_cost(default_insertion_cost)?; - validate_default_cost(default_deletion_cost)?; + validate_cost(default_substitution_cost, "Default substitution cost")?; + validate_cost(default_insertion_cost, "Default insertion cost")?; + validate_cost(default_deletion_cost, "Default deletion cost")?; let sub = CostMap::::from_py_dict( substitution_costs, default_substitution_cost, symmetric_substitution, - ); - let ins = CostMap::::from_py_dict(insertion_costs, default_insertion_cost); - let del = CostMap::::from_py_dict(deletion_costs, default_deletion_cost); + )?; + let ins = CostMap::::from_py_dict(insertion_costs, default_insertion_cost)?; + let del = CostMap::::from_py_dict(deletion_costs, default_deletion_cost)?; Ok(Self { sub, ins, del }) } @@ -113,7 +113,8 @@ impl RustLevenshteinCalculator { max_node_length: Option, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>)> { let (closed_sub, closed_ins, closed_del) = - compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune, max_node_length); + compute_closed_cost_maps(&self.sub, &self.ins, &self.del, prune, max_node_length) + .map_err(|err| PyValueError::new_err(err.to_string()))?; let sub_dict = PyDict::new(py); for ((source, target), cost) in closed_sub { @@ -134,15 +135,6 @@ impl RustLevenshteinCalculator { } } -fn validate_default_cost(default_cost: f64) -> PyResult<()> { - if default_cost < 0.0 { - return Err(PyValueError::new_err(format!( - "Default cost must be non-negative, got value: {default_cost}" - ))); - } - Ok(()) -} - #[pymodule] pub fn _rust_stringdist(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -152,6 +144,7 @@ pub fn _rust_stringdist(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(test)] mod tests { use super::*; + use pyo3::exceptions::PyValueError; use pyo3::types::{PyDict, PyList, PyTuple}; fn make_calculator( diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index fb2f25c..e03103f 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -15,6 +15,7 @@ use crate::cost_map::CostMap; use crate::types::{SingleTokenCostMap, SingleTokenKey, SubstitutionCostMap, SubstitutionKey}; use crate::weighted_levenshtein::custom_levenshtein_distance; use std::collections::{HashMap, HashSet}; +use std::fmt; // When the caller passes `None` for `max_node_length`, the cap is derived as // `max(longest raw token across all maps) * MAX_NODE_LENGTH_MULTIPLIER`, with a @@ -27,6 +28,27 @@ const MIN_DERIVED_NODE_LENGTH: usize = 4; const REDUNDANT_SUBSTITUTION_EPSILON: f64 = 1e-9; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TransitiveCostError { + MatrixSizeOverflow { node_count: usize }, + MatrixAllocationFailed { node_count: usize, bytes: usize }, +} + +impl fmt::Display for TransitiveCostError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MatrixSizeOverflow { node_count } => write!( + f, + "transitive closure generated {node_count} graph nodes, too many to address in a dense matrix; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), + Self::MatrixAllocationFailed { node_count, bytes } => write!( + f, + "transitive closure generated {node_count} graph nodes and could not allocate a {bytes}-byte dense matrix; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), + } + } +} + /// Interned identifier for a token graph node. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] struct NodeId(u32); @@ -51,11 +73,22 @@ struct Matrix { } impl Matrix { - fn filled(width: usize, value: T) -> Self { - Self { - width, - cells: vec![value; width * width], - } + fn try_filled(width: usize, value: T) -> Result { + let cell_count = width + .checked_mul(width) + .ok_or(TransitiveCostError::MatrixSizeOverflow { node_count: width })?; + let bytes = cell_count + .checked_mul(std::mem::size_of::()) + .unwrap_or(usize::MAX); + let mut cells = Vec::new(); + cells.try_reserve_exact(cell_count).map_err(|_| { + TransitiveCostError::MatrixAllocationFailed { + node_count: width, + bytes, + } + })?; + cells.resize(cell_count, value); + Ok(Self { width, cells }) } } @@ -94,7 +127,7 @@ pub fn compute_closed_cost_maps( del: &CostMap, prune: bool, max_node_length: Option, -) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { +) -> Result<(SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap), TransitiveCostError> { let max_node_length = max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); let tokens = collect_nodes(sub, ins, del, max_node_length); let token_to_id: HashMap<&str, NodeId> = tokens @@ -103,7 +136,7 @@ pub fn compute_closed_cost_maps( .map(|(i, token)| (token.as_str(), NodeId::new(i))) .collect(); let epsilon_id = token_to_id[""]; - let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del); + let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del)?; let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); @@ -114,7 +147,7 @@ pub fn compute_closed_cost_maps( closed_sub }; - (closed_sub, closed_ins, closed_del) + Ok((closed_sub, closed_ins, closed_del)) } /// Default `max_node_length` derivation: twice the longest raw token across all @@ -278,9 +311,9 @@ fn run_closure( sub: &CostMap, ins: &CostMap, del: &CostMap, -) -> Matrix { +) -> Result, TransitiveCostError> { let n = tokens.len(); - let mut dist = Matrix::filled(n, f64::INFINITY); + let mut dist = Matrix::try_filled(n, f64::INFINITY)?; for index in 0..n { let node = NodeId::new(index); @@ -310,7 +343,7 @@ fn run_closure( seed_embedded_edges(tokens, token_to_id, ins, del, &mut dist); floyd_warshall(&mut dist); - dist + Ok(dist) } #[inline] @@ -527,6 +560,16 @@ mod tests { CostMap::::new(map, default) } + fn compute_closed_cost_maps( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, + prune: bool, + max_node_length: Option, + ) -> (SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap) { + super::compute_closed_cost_maps(sub, ins, del, prune, max_node_length).unwrap() + } + #[test] fn closure_finds_substitution_chain() { let sub = make_sub(&[(("a", "b"), 0.1), (("b", "c"), 0.1)], 1.0, false); @@ -880,6 +923,17 @@ mod tests { assert!(approx(closed_sub[&("é".to_string(), "è".to_string())], 0.2)); } + #[test] + fn matrix_allocation_checks_size_overflow() { + let err = Matrix::::try_filled(usize::MAX, 0.0).unwrap_err(); + assert!(matches!( + err, + TransitiveCostError::MatrixSizeOverflow { + node_count: usize::MAX + } + )); + } + // --- pruning ---------------------------------------------------------------- #[test] From d1d99b857f2bbacf170b202b51c2697ed0036dbd Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 20:10:34 +0200 Subject: [PATCH 11/21] consider empty strings in substitution --- CHANGELOG.md | 1 + python/ocr_stringdist/levenshtein.py | 32 ++++++++++-- python/tests/test_weighted_levenshtein.py | 52 +++++++++++++++++++ .../test_weighted_levenshtein_mutation.py | 4 ++ 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c2cda..21fc0ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. +- Treat empty-sided substitution costs as insertion/deletion aliases. - Support for Python 3.14. ### Fixed diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 1b66866..010db25 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -69,16 +69,40 @@ def __init__( def _sync_calculator(self) -> None: """Internal helper to re-instantiate the Rust backend when state changes.""" + substitution_costs, insertion_costs, deletion_costs = ( + self._effective_cost_maps_for_calculator() + ) self._calculator = RustLevenshteinCalculator( - substitution_costs=self._substitution_costs, - insertion_costs=self._insertion_costs, - deletion_costs=self._deletion_costs, + substitution_costs=substitution_costs, + insertion_costs=insertion_costs, + deletion_costs=deletion_costs, symmetric_substitution=self._symmetric_substitution, default_substitution_cost=self._default_substitution_cost, default_insertion_cost=self._default_insertion_cost, default_deletion_cost=self._default_deletion_cost, ) + def _effective_cost_maps_for_calculator( + self, + ) -> tuple[dict[tuple[str, str], float], dict[str, float], dict[str, float]]: + substitution_costs: dict[tuple[str, str], float] = {} + insertion_costs = dict(self._insertion_costs) + deletion_costs = dict(self._deletion_costs) + + for (source, target), cost in self._substitution_costs.items(): + if source == "": + self._set_min_cost(insertion_costs, target, cost) + elif target == "": + self._set_min_cost(deletion_costs, source, cost) + else: + substitution_costs[(source, target)] = cost + + return substitution_costs, insertion_costs, deletion_costs + + @staticmethod + def _set_min_cost(costs: dict[str, float], key: str, cost: float) -> None: + costs[key] = min(costs.get(key, cost), cost) + # --- Properties --- @property @@ -164,6 +188,8 @@ def _validate_cost(self, name: str, val: float) -> float: def _validate_sub_entry(self, key: Any, cost: Any) -> None: if not (isinstance(key, tuple) and len(key) == 2 and all(isinstance(k, str) for k in key)): raise TypeError(f"substitution_costs keys must be tuples of two strings, found: {key}") + if key == ("", ""): + raise ValueError('substitution_costs key ("", "") is not a meaningful edit operation') self._validate_cost(f"Cost for {key}", cost) def _validate_unary_entry(self, key: Any, cost: Any) -> None: diff --git a/python/tests/test_weighted_levenshtein.py b/python/tests/test_weighted_levenshtein.py index 009fd0d..adabbbd 100644 --- a/python/tests/test_weighted_levenshtein.py +++ b/python/tests/test_weighted_levenshtein.py @@ -457,6 +457,58 @@ def test_weighted_levenshtein_distance_with_insertion( ).distance(s1, s2) == pytest.approx(expected) +@pytest.mark.parametrize( + ["substitution_costs", "source", "target", "expected"], + [ + ({("", "ab"): 0.3}, "x", "xab", 0.3), + ({("cd", ""): 0.4}, "xcd", "x", 0.4), + ], +) +def test_empty_sided_substitution_costs_act_as_insertions_and_deletions( + substitution_costs: dict[tuple[str, str], float], source: str, target: str, expected: float +) -> None: + assert WeightedLevenshtein(substitution_costs=substitution_costs).distance( + source, target + ) == pytest.approx(expected) + + +@pytest.mark.parametrize( + ["substitution_costs", "insertion_costs", "deletion_costs", "source", "target", "expected"], + [ + ({("", "a"): 0.2}, {"a": 0.5}, {}, "", "a", 0.2), + ({("a", ""): 0.2}, {}, {"a": 0.5}, "a", "", 0.2), + ], +) +def test_empty_sided_substitution_costs_use_minimum_with_explicit_unary_costs( + substitution_costs: dict[tuple[str, str], float], + insertion_costs: dict[str, float], + deletion_costs: dict[str, float], + source: str, + target: str, + expected: float, +) -> None: + assert WeightedLevenshtein( + substitution_costs=substitution_costs, + insertion_costs=insertion_costs, + deletion_costs=deletion_costs, + ).distance(source, target) == pytest.approx(expected) + + +def test_empty_sided_substitution_costs_are_not_mirrored_by_symmetric_substitution() -> None: + wl = WeightedLevenshtein( + substitution_costs={("", "a"): 0.2}, + symmetric_substitution=True, + ) + + assert wl.distance("", "a") == pytest.approx(0.2) + assert wl.distance("a", "") == pytest.approx(1.0) + + +def test_empty_to_empty_substitution_cost_is_rejected() -> None: + with pytest.raises(ValueError, match="not a meaningful edit operation"): + WeightedLevenshtein(substitution_costs={("", ""): 0.0}) + + @pytest.mark.parametrize( ["s1", "s2", "ins_costs", "del_costs", "expected"], [ diff --git a/python/tests/test_weighted_levenshtein_mutation.py b/python/tests/test_weighted_levenshtein_mutation.py index 9448352..16b86e9 100644 --- a/python/tests/test_weighted_levenshtein_mutation.py +++ b/python/tests/test_weighted_levenshtein_mutation.py @@ -34,6 +34,8 @@ def test_cost_property_reassignment_updates_calculator( ["costs_attribute", "key", "cost", "source", "target", "expected"], [ ("substitution_costs", ("a", "b"), 0.2, "a", "b", 0.2), + ("substitution_costs", ("", "b"), 0.3, "", "b", 0.3), + ("substitution_costs", ("a", ""), 0.4, "a", "", 0.4), ("insertion_costs", "b", 0.3, "", "b", 0.3), ("deletion_costs", "a", 0.4, "a", "", 0.4), ], @@ -228,6 +230,7 @@ def test_invalid_default_cost_setter_rejects_non_finite_values( ("substitution_costs", {("a", "b"): math.nan}), ("substitution_costs", {("a", "b"): math.inf}), ("substitution_costs", {("a",): 0.1}), + ("substitution_costs", {("", ""): 0.0}), ("insertion_costs", {"b": -0.1}), ("insertion_costs", {"b": math.nan}), ("insertion_costs", {"b": math.inf}), @@ -257,6 +260,7 @@ def test_invalid_cost_property_reassignment_keeps_existing_calculator( ("substitution_costs", ("a", "b"), math.nan), ("substitution_costs", ("a", "b"), math.inf), ("substitution_costs", ("a",), 0.1), + ("substitution_costs", ("", ""), 0.0), ("insertion_costs", "b", -0.1), ("insertion_costs", "b", math.nan), ("insertion_costs", "b", math.inf), From b597a061bbf9e15d3199c277fe169b6ae4ecc7d6 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 21:06:54 +0200 Subject: [PATCH 12/21] improvements --- .../ocr_stringdist/default_ocr_distances.py | 2 +- python/ocr_stringdist/levenshtein.py | 5 +- .../tests/test_batch_weighted_levenshtein.py | 1 - src/transitive_costs.rs | 95 ++++++++++++++----- 4 files changed, 75 insertions(+), 28 deletions(-) diff --git a/python/ocr_stringdist/default_ocr_distances.py b/python/ocr_stringdist/default_ocr_distances.py index 1db03b2..b3ed772 100644 --- a/python/ocr_stringdist/default_ocr_distances.py +++ b/python/ocr_stringdist/default_ocr_distances.py @@ -38,5 +38,5 @@ """ Pre-defined distance map between characters, considering common OCR errors. The distances are between 0 and 1. -This map is intended to be used with `symmetric=True`. +This map is intended to be used with `symmetric_substitution=True`. """ diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 010db25..268ad32 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -229,9 +229,8 @@ def transitive_closure( derives a sensible default from the input (twice the longest raw token, with a small floor); pass an `int` to override. The cap is - what guarantees termination — without it, - configurations like `ins("A")` would grow the - graph without bound. Floyd-Warshall is + what guarantees termination - without it, + the graph could grow without bound. Floyd-Warshall is :math:`O(N^3)` in the resulting node count, so a higher cap can be substantially slower. :raises ValueError: If the generated closure graph is too large to diff --git a/python/tests/test_batch_weighted_levenshtein.py b/python/tests/test_batch_weighted_levenshtein.py index a0e03a7..4b97669 100644 --- a/python/tests/test_batch_weighted_levenshtein.py +++ b/python/tests/test_batch_weighted_levenshtein.py @@ -78,7 +78,6 @@ def test_batch_finds_best_match( distances = WeightedLevenshtein( substitution_costs=OCR_COST_MAP, ).batch_distance(source, candidates) - print(f"------------------------------------distances: {distances}") # Find the index with minimum distance min_index = distances.index(min(distances)) diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index e03103f..74f08e3 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -28,8 +28,15 @@ const MIN_DERIVED_NODE_LENGTH: usize = 4; const REDUNDANT_SUBSTITUTION_EPSILON: f64 = 1e-9; +/// `NodeId` interns indices as `u32`, so the number of graph nodes is bounded +/// by `u32::MAX`. Reaching this limit at human-sized inputs is implausible — +/// it requires the closure to construct ~4 billion distinct strings — but the +/// check exists so `NodeId::new` never panics from inside `compute_closed_cost_maps`. +const MAX_NODE_COUNT: usize = u32::MAX as usize; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum TransitiveCostError { + NodeIdOverflow { node_count: usize }, MatrixSizeOverflow { node_count: usize }, MatrixAllocationFailed { node_count: usize, bytes: usize }, } @@ -37,6 +44,10 @@ pub enum TransitiveCostError { impl fmt::Display for TransitiveCostError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::NodeIdOverflow { node_count } => write!( + f, + "transitive closure generated {node_count} graph nodes, exceeding the {MAX_NODE_COUNT}-node addressing limit; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" + ), Self::MatrixSizeOverflow { node_count } => write!( f, "transitive closure generated {node_count} graph nodes, too many to address in a dense matrix; pass a smaller max_node_length or reduce the number of configured insertion/deletion tokens" @@ -77,9 +88,7 @@ impl Matrix { let cell_count = width .checked_mul(width) .ok_or(TransitiveCostError::MatrixSizeOverflow { node_count: width })?; - let bytes = cell_count - .checked_mul(std::mem::size_of::()) - .unwrap_or(usize::MAX); + let bytes = cell_count.saturating_mul(std::mem::size_of::()); let mut cells = Vec::new(); cells.try_reserve_exact(cell_count).map_err(|_| { TransitiveCostError::MatrixAllocationFailed { @@ -116,8 +125,8 @@ impl Matrix { /// growth phase may construct and of substrings expanded from raw tokens. Pass /// `None` to derive a sensible default from the input /// (`max raw-token length × 2`, floored at `MIN_DERIVED_NODE_LENGTH`); pass -/// `Some(n)` to override. The cap is what guarantees termination — without it, -/// configurations like `ins("A")` produce an infinite graph. +/// `Some(n)` to override. The cap is what guarantees termination - without it, +/// the graph could grow without bound. /// /// If `prune` is true, generated substitutions that the returned edit maps can /// already express are removed from the substitution map. @@ -130,6 +139,11 @@ pub fn compute_closed_cost_maps( ) -> Result<(SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap), TransitiveCostError> { let max_node_length = max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); let tokens = collect_nodes(sub, ins, del, max_node_length); + if tokens.len() > MAX_NODE_COUNT { + return Err(TransitiveCostError::NodeIdOverflow { + node_count: tokens.len(), + }); + } let token_to_id: HashMap<&str, NodeId> = tokens .iter() .enumerate() @@ -202,39 +216,52 @@ fn collect_nodes( // produces predecessors that lie one configured ins/del edge away. This is // what lets the closure bridge a user-provided source like "ADC" through // intermediate nodes "AC" and "ABC" to a configured target "Z". - let single_op_tokens: Vec = ins + // + // Pre-compute char counts once: the inner loop checks them against the cap + // for every source. + let single_op_tokens: Vec<(String, usize)> = ins .costs .keys() .cloned() .chain(del.costs.keys().cloned()) .collect::>() .into_iter() + .map(|token| { + let len = token.chars().count(); + (token, len) + }) .collect(); - // Run growth to fixpoint. The length cap bounds the set of strings reachable - // from the seeds, so this terminates after a finite number of rounds for any - // input — typically only a handful, even for large maps. - loop { - let snapshot: Vec = tokens.iter().cloned().collect(); - let prev_size = snapshot.len(); - - for source in &snapshot { + // Worklist-style growth: each round only processes tokens discovered in the + // previous round, since older tokens already produced everything they can. + // Both gates check the produced length against the cap — insertion can + // overrun by lengthening, and deletion can overrun when an oversize raw + // seed (longer than `max_node_length`) is shortened to something still + // above the cap. The length cap bounds the total set, so the worklist + // drains in finitely many rounds. + let mut frontier: Vec = tokens.iter().cloned().collect(); + while !frontier.is_empty() { + let mut next_frontier: Vec = Vec::new(); + for source in &frontier { let source_len = source.chars().count(); - for op_token in &single_op_tokens { - if source_len + op_token.chars().count() <= max_node_length { + for (op_token, op_len) in &single_op_tokens { + if source_len + *op_len <= max_node_length { for variant in insert_token_variants(source, op_token) { - tokens.insert(variant); + if tokens.insert(variant.clone()) { + next_frontier.push(variant); + } } } - for variant in delete_token_variants(source, op_token) { - tokens.insert(variant); + if source_len.saturating_sub(*op_len) <= max_node_length { + for variant in delete_token_variants(source, op_token) { + if tokens.insert(variant.clone()) { + next_frontier.push(variant); + } + } } } } - - if tokens.len() == prev_size { - break; - } + frontier = next_frontier; } tokens.into_iter().collect() @@ -934,6 +961,28 @@ mod tests { )); } + #[test] + fn growth_does_not_overrun_cap_via_deletion_from_oversize_seed() { + // "LONGSTRING" (10 chars) is a raw substitution endpoint and is + // always seeded into the node set. With a configured `del("LON")` + // the previous, ungated deletion would have spawned "GSTRING" + // (7 chars) from it — still over the 4-char cap. The symmetric + // deletion gate prevents that. + let sub = make_sub(&[(("LONGSTRING", "X"), 0.1)], 1.0, false); + let ins = make_single(&[], 1.0); + let del = make_single(&[("LON", 0.1)], 1.0); + + let nodes = super::collect_nodes(&sub, &ins, &del, 4); + let raw_seeds: HashSet<&str> = ["LONGSTRING"].into_iter().collect(); + for token in &nodes { + let len = token.chars().count(); + assert!( + len <= 4 || raw_seeds.contains(token.as_str()), + "node {token:?} of length {len} exceeds cap and is not a raw seed" + ); + } + } + // --- pruning ---------------------------------------------------------------- #[test] From ca70909798f177993e6d17897ca42051f9c9d9ca Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 21:09:22 +0200 Subject: [PATCH 13/21] delete benchmark --- benchmarks/benchmark.py | 143 ---------------------------------------- 1 file changed, 143 deletions(-) delete mode 100644 benchmarks/benchmark.py diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py deleted file mode 100644 index 20d3b3b..0000000 --- a/benchmarks/benchmark.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Benchmark for WeightedLevenshtein distance computation. - -Run with: - uv run python benchmarks/benchmark.py - -The results are printed to stdout. Save them before and after a code change to -compare throughput. -""" - -from __future__ import annotations - -import sys -import timeit -from dataclasses import dataclass - -sys.path.insert(0, "python") - -from ocr_stringdist import WeightedLevenshtein - -REPEAT = 5 -NUMBER = 200 - - -@dataclass -class Case: - label: str - wl: WeightedLevenshtein - s1: str - s2: str - - -# ── Cost maps used across cases ──────────────────────────────────────────────── - -_WL_DEFAULT = WeightedLevenshtein.unweighted() - -_WL_OCR = WeightedLevenshtein( - substitution_costs={ - ("6", "G"): 0.5, - ("0", "O"): 0.1, - ("rn", "m"): 0.15, - ("cl", "d"): 0.2, - ("l", "1"): 0.2, - ("h", "In"): 0.25, - ("vv", "w"): 0.15, - }, - deletion_costs={"G": 0.01, "O": 0.05}, - default_substitution_cost=1.0, - default_deletion_cost=1.0, - default_insertion_cost=1.0, -) - -# ── Benchmark cases ──────────────────────────────────────────────────────────── - -CASES: list[Case] = [ - # Issue #12 — transitive chain: sub("6"→"G", 0.5) + del("G", 0.01) = 0.51 - Case( - "issue-12: transitive chain '06'→'0'", - WeightedLevenshtein( - substitution_costs={("6", "G"): 0.5}, - deletion_costs={"G": 0.01}, - ), - "06", - "0", - ), - # Short strings, no custom costs - Case("short identical (no-op)", _WL_DEFAULT, "hello", "hello"), - Case("short similar (1 sub)", _WL_DEFAULT, "kitten", "sitten"), - Case("short dissimilar", _WL_DEFAULT, "abc", "xyz"), - # Medium strings - Case( - "medium OCR-like", - _WL_OCR, - "The man ran down the hill at 10 km/h.", - "Tine rnan ram dovvn tine Ini11 at 1O krn/In.", - ), - Case( - "medium no-match", - _WL_DEFAULT, - "abcdefghij", - "zyxwvutsrq", - ), - # Long strings - Case( - "long similar", - _WL_DEFAULT, - "a" * 200 + "b" * 50, - "a" * 198 + "c" * 52, - ), - Case( - "long OCR-like", - _WL_OCR, - "The man ran down the hill at 10 km/h. " * 5, - "Tine rnan ram dovvn tine Ini11 at 1O krn/In. " * 5, - ), - # Batch distance (1 source vs. 100 candidates) -] - -BATCH_CANDIDATES = [f"word{i}" for i in range(100)] -_WL_BATCH = WeightedLevenshtein.unweighted() - - -def run_batch() -> None: - _WL_BATCH.batch_distance("word50", BATCH_CANDIDATES) - - -# Runner - - -def bench_case(case: Case) -> tuple[float, float]: - """Returns (best_ms_per_call, calls_per_second).""" - stmt = lambda: case.wl.distance(case.s1, case.s2) # noqa: E731 - times = timeit.repeat(stmt, repeat=REPEAT, number=NUMBER) - best_total_s = min(times) - best_ms = best_total_s / NUMBER * 1000 - cps = NUMBER / best_total_s - return best_ms, cps - - -def main() -> None: - col_w = max(len(c.label) for c in CASES) + 2 - header = f"{'Case':<{col_w}} {'Best ms/call':>14} {'calls/sec':>12}" - print(header) - print("-" * len(header)) - - for case in CASES: - ms, cps = bench_case(case) - print(f"{case.label:<{col_w}} {ms:>14.4f} {cps:>12,.0f}") - - # Batch benchmark - batch_times = timeit.repeat(run_batch, repeat=REPEAT, number=NUMBER) - best_batch_s = min(batch_times) - batch_ms = best_batch_s / NUMBER * 1000 - batch_cps = NUMBER / best_batch_s - label = "batch_distance (100 candidates)" - print(f"{label:<{col_w}} {batch_ms:>14.4f} {batch_cps:>12,.0f}") - - print() - print(f"Settings: repeat={REPEAT}, number={NUMBER} calls per timing") - - -if __name__ == "__main__": - main() From f5517b2af4ada9086d83493100a6a3d52c439321 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 21:19:25 +0200 Subject: [PATCH 14/21] upgrade pyo3 --- Cargo.lock | 62 ++++++++---------------------------------- Cargo.toml | 2 +- src/rust_stringdist.rs | 24 ++++++++-------- 3 files changed, 25 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b00281..482bae7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -51,27 +39,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "ocr_stringdist" version = "1.1.0" @@ -103,37 +76,32 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ - "cfg-if", - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -141,9 +109,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -153,9 +121,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -206,18 +174,12 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/Cargo.toml b/Cargo.toml index d21cfb1..34c8bbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,5 @@ name = "ocr_stringdist" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.24.0", features = ["auto-initialize"] } +pyo3 = { version = "0.28.3", features = ["auto-initialize"] } rayon = "1.10.0" diff --git a/src/rust_stringdist.rs b/src/rust_stringdist.rs index cf58138..a1764bb 100644 --- a/src/rust_stringdist.rs +++ b/src/rust_stringdist.rs @@ -82,7 +82,7 @@ impl RustLevenshteinCalculator { if candidates.is_empty() { return Vec::new(); } - py.allow_threads(|| { + py.detach(|| { candidates .par_iter() .map(|c| custom_levenshtein_distance(&s, c, &self.sub, &self.ins, &self.del)) @@ -90,11 +90,11 @@ impl RustLevenshteinCalculator { }) } - fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult> { + fn explain(&self, py: Python<'_>, a: &str, b: &str) -> PyResult>> { explain_custom_levenshtein(a, b, &self.sub, &self.ins, &self.del) .into_iter() .map(|op| op.into_pyobject(py).map(|bound| bound.into())) - .collect::>>() + .collect::>>>() } /// Computes effective edit costs via transitive closure and returns three @@ -171,7 +171,7 @@ mod tests { #[test] fn test_distance_with_empty_costs() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[], &[], &[], true); assert_eq!(calc.distance("hello", "hxllo"), 1.0); }); @@ -179,7 +179,7 @@ mod tests { #[test] fn test_distance_with_custom_substitution_cost() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[(("e", "x"), 0.2)], &[], &[], true); assert!((calc.distance("hello", "hxllo") - 0.2).abs() < f64::EPSILON); }); @@ -187,7 +187,7 @@ mod tests { #[test] fn test_asymmetric_substitution() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[(("a", "b"), 0.1)], &[], &[], false); assert!((calc.distance("ab", "ba") - 1.1).abs() < f64::EPSILON); }); @@ -195,7 +195,7 @@ mod tests { #[test] fn test_negative_default_cost_errors() { - Python::with_gil(|py| { + Python::attach(|py| { let empty = PyDict::new(py); let sub_err = RustLevenshteinCalculator::new(&empty, &empty, &empty, true, -1.0, 1.0, 1.0); @@ -208,7 +208,7 @@ mod tests { fn test_constructor_does_not_apply_closure() { // Without calling closed_cost_maps, transitive paths are not auto-applied. // sub(a->b)=0.1, sub(b->c)=0.1: direct a->c lookup falls back to default 1.0. - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); assert!((calc.distance("a", "c") - 1.0).abs() < f64::EPSILON); @@ -217,7 +217,7 @@ mod tests { #[test] fn test_closed_cost_maps_finds_chain() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[(("a", "b"), 0.1), (("b", "c"), 0.1)], &[], &[], false); let (sub, _ins, _del) = calc.closed_cost_maps(py, false, None).unwrap(); @@ -233,7 +233,7 @@ mod tests { #[test] fn test_explain() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[], &[], &[], true); let result = calc.explain(py, "cat", "car").unwrap(); let py_list = PyList::new(py, result).unwrap(); @@ -242,7 +242,7 @@ mod tests { py_list .get_item(i) .unwrap() - .downcast_into::() + .cast_into::() .unwrap() .get_item(0) .unwrap() @@ -257,7 +257,7 @@ mod tests { #[test] fn test_batch_distance() { - Python::with_gil(|py| { + Python::attach(|py| { let calc = make_calculator(py, &[], &[], &[], true); let distances = calc.batch_distance( py, From a61d1fc309e2e609cebc4aca5854d761b3560d51 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Fri, 1 May 2026 21:24:59 +0200 Subject: [PATCH 15/21] fix ci/cd --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9e49ce9..c2515f5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -34,6 +34,7 @@ jobs: rm -rf .venv python3 -m venv .venv . .venv/bin/activate + .venv/bin/pip install --upgrade pip .venv/bin/pip install wheel pytest maturin maturin develop .venv/bin/pytest python/tests From 1c792737966f29c6f85a9316b90eaebfaf2543ff Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 22:02:44 +0200 Subject: [PATCH 16/21] implement review comments --- python/ocr_stringdist/_observable_dict.py | 4 +- python/ocr_stringdist/levenshtein.py | 88 +++++++++++++---------- src/cost_map.rs | 8 +-- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/python/ocr_stringdist/_observable_dict.py b/python/ocr_stringdist/_observable_dict.py index 5729927..7117eaf 100644 --- a/python/ocr_stringdist/_observable_dict.py +++ b/python/ocr_stringdist/_observable_dict.py @@ -57,7 +57,7 @@ def update(self, other: Any = (), /, **kwargs: V) -> None: super().update(items) self._on_change() - def setdefault(self, key: K, default: V) -> V: + def setdefault(self, key: K, default: Any = None) -> V: if key in self: return self[key] if self._validator: @@ -69,7 +69,7 @@ def setdefault(self, key: K, default: V) -> V: def __or__(self, other: object, /) -> Any: if not isinstance(other, dict): return NotImplemented - return dict(self) | other + return _ObservableDict(dict(self) | other, self._on_change, self._validator) def __ior__(self, other: object, /) -> Any: self.update(other) diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 268ad32..50b5afc 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -43,48 +43,62 @@ def __init__( default_deletion_cost: float = 1.0, ) -> None: self._symmetric_substitution = symmetric_substitution - self._default_substitution_cost = self._validate_cost( - "default_substitution_cost", default_substitution_cost - ) self._default_insertion_cost = self._validate_cost( "default_insertion_cost", default_insertion_cost ) self._default_deletion_cost = self._validate_cost( "default_deletion_cost", default_deletion_cost ) + # A substitution can always be expressed as a deletion + insertion, so + # capping here keeps the substitution default from being effectively + # ignored when the user supplies a value above the del/ins ceiling. + self._default_substitution_cost = min( + self._validate_cost("default_substitution_cost", default_substitution_cost), + self._default_insertion_cost + self._default_deletion_cost, + ) + + self._calculator = None # Initialize Observable Dicts sub_init = ocr_distance_map if substitution_costs is None else substitution_costs self._substitution_costs = _ObservableDict( - sub_init, self._sync_calculator, self._validate_sub_entry + sub_init, self._invalidate_calculator, self._validate_sub_entry ) self._insertion_costs = _ObservableDict( - insertion_costs or {}, self._sync_calculator, self._validate_unary_entry + insertion_costs or {}, self._invalidate_calculator, self._validate_unary_entry ) self._deletion_costs = _ObservableDict( - deletion_costs or {}, self._sync_calculator, self._validate_unary_entry + deletion_costs or {}, self._invalidate_calculator, self._validate_unary_entry ) - self._sync_calculator() - - def _sync_calculator(self) -> None: - """Internal helper to re-instantiate the Rust backend when state changes.""" - substitution_costs, insertion_costs, deletion_costs = ( - self._effective_cost_maps_for_calculator() - ) - self._calculator = RustLevenshteinCalculator( - substitution_costs=substitution_costs, - insertion_costs=insertion_costs, - deletion_costs=deletion_costs, - symmetric_substitution=self._symmetric_substitution, - default_substitution_cost=self._default_substitution_cost, - default_insertion_cost=self._default_insertion_cost, - default_deletion_cost=self._default_deletion_cost, - ) + def _invalidate_calculator(self) -> None: + """Mark the Rust backend as out of sync; it will be rebuilt on next use.""" + self._calculator = None + + def _get_calculator(self) -> RustLevenshteinCalculator: + """Return a Rust backend in sync with the current Python-side state.""" + if self._calculator is None: + substitution_costs, insertion_costs, deletion_costs = ( + self._effective_cost_maps_for_calculator() + ) + self._calculator = RustLevenshteinCalculator( + substitution_costs=substitution_costs, + insertion_costs=insertion_costs, + deletion_costs=deletion_costs, + symmetric_substitution=self._symmetric_substitution, + default_substitution_cost=self._default_substitution_cost, + default_insertion_cost=self._default_insertion_cost, + default_deletion_cost=self._default_deletion_cost, + ) + return self._calculator def _effective_cost_maps_for_calculator( self, ) -> tuple[dict[tuple[str, str], float], dict[str, float], dict[str, float]]: + """ + Split substitution entries with empty source/target into the + insertion/deletion maps, taking the minimum where they overlap. + """ substitution_costs: dict[tuple[str, str], float] = {} insertion_costs = dict(self._insertion_costs) deletion_costs = dict(self._deletion_costs) @@ -112,9 +126,9 @@ def substitution_costs(self) -> dict[tuple[str, str], float]: @substitution_costs.setter def substitution_costs(self, value: dict[tuple[str, str], float]) -> None: self._substitution_costs = _ObservableDict( - value, self._sync_calculator, self._validate_sub_entry + value, self._invalidate_calculator, self._validate_sub_entry ) - self._sync_calculator() + self._invalidate_calculator() @property def insertion_costs(self) -> dict[str, float]: @@ -123,9 +137,9 @@ def insertion_costs(self) -> dict[str, float]: @insertion_costs.setter def insertion_costs(self, value: dict[str, float]) -> None: self._insertion_costs = _ObservableDict( - value, self._sync_calculator, self._validate_unary_entry + value, self._invalidate_calculator, self._validate_unary_entry ) - self._sync_calculator() + self._invalidate_calculator() @property def deletion_costs(self) -> dict[str, float]: @@ -134,9 +148,9 @@ def deletion_costs(self) -> dict[str, float]: @deletion_costs.setter def deletion_costs(self, value: dict[str, float]) -> None: self._deletion_costs = _ObservableDict( - value, self._sync_calculator, self._validate_unary_entry + value, self._invalidate_calculator, self._validate_unary_entry ) - self._sync_calculator() + self._invalidate_calculator() @property def symmetric_substitution(self) -> bool: @@ -145,7 +159,7 @@ def symmetric_substitution(self) -> bool: @symmetric_substitution.setter def symmetric_substitution(self, value: bool) -> None: self._symmetric_substitution = value - self._sync_calculator() + self._invalidate_calculator() @property def default_substitution_cost(self) -> float: @@ -154,7 +168,7 @@ def default_substitution_cost(self) -> float: @default_substitution_cost.setter def default_substitution_cost(self, value: float) -> None: self._default_substitution_cost = self._validate_cost("default_substitution_cost", value) - self._sync_calculator() + self._invalidate_calculator() @property def default_insertion_cost(self) -> float: @@ -163,7 +177,7 @@ def default_insertion_cost(self) -> float: @default_insertion_cost.setter def default_insertion_cost(self, value: float) -> None: self._default_insertion_cost = self._validate_cost("default_insertion_cost", value) - self._sync_calculator() + self._invalidate_calculator() @property def default_deletion_cost(self) -> float: @@ -172,7 +186,7 @@ def default_deletion_cost(self) -> float: @default_deletion_cost.setter def default_deletion_cost(self, value: float) -> None: self._default_deletion_cost = self._validate_cost("default_deletion_cost", value) - self._sync_calculator() + self._invalidate_calculator() # --- Validation Helpers --- @@ -242,7 +256,9 @@ def transitive_closure( For repeated use, save via :meth:`to_dict` and reload via :meth:`from_dict` so the closure is computed once. """ - sub_dict, ins_dict, del_dict = self._calculator.closed_cost_maps(prune, max_node_length) + sub_dict, ins_dict, del_dict = self._get_calculator().closed_cost_maps( + prune, max_node_length + ) return WeightedLevenshtein( substitution_costs=dict(sub_dict), insertion_costs=dict(ins_dict), @@ -255,7 +271,7 @@ def transitive_closure( def distance(self, s1: str, s2: str) -> float: """Calculates the weighted Levenshtein distance between two strings.""" - return self._calculator.distance(s1, s2) # type: ignore[no-any-return] + return self._get_calculator().distance(s1, s2) # type: ignore[no-any-return] def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOperation]: """ @@ -266,7 +282,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe :param filter_matches: If True, 'match' operations are excluded from the result. :return: List of :class:`EditOperation` instances. """ - raw_path = self._calculator.explain(s1, s2) + raw_path = self._get_calculator().explain(s1, s2) parsed_path = [EditOperation(*op) for op in raw_path] if filter_matches: return list(filter(lambda op: op.op_type != "match", parsed_path)) @@ -274,7 +290,7 @@ def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOpe def batch_distance(self, s: str, candidates: list[str]) -> list[float]: """Calculates distances between a string and a list of candidates.""" - return self._calculator.batch_distance(s, candidates) # type: ignore[no-any-return] + return self._get_calculator().batch_distance(s, candidates) # type: ignore[no-any-return] @classmethod def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein: diff --git a/src/cost_map.rs b/src/cost_map.rs index d92ecc3..8c1921b 100644 --- a/src/cost_map.rs +++ b/src/cost_map.rs @@ -55,9 +55,7 @@ impl CostMap { let max_token_length = costs .keys() .flat_map(|(s, t)| [s.chars().count(), t.chars().count()]) - .max() - .unwrap_or(0) - .max(1); + .fold(1, std::cmp::max); CostMap { costs, @@ -109,9 +107,7 @@ impl CostMap { let max_token_length = custom_costs_input .keys() .map(|token| token.chars().count()) - .max() - .unwrap_or(0) - .max(1); + .fold(1, std::cmp::max); CostMap { costs: custom_costs_input, default_cost, From c37089253e4bcc7183f5a798bc1b8132b543da85 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 22:23:13 +0200 Subject: [PATCH 17/21] docs --- CHANGELOG.md | 5 +++-- docs/source/index.rst | 1 + docs/source/transitive_closure.rst | 34 ++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 docs/source/transitive_closure.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index 21fc0ef..03b13e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.1.0] - Unreleased - ### Added - Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. -- Treat empty-sided substitution costs as insertion/deletion aliases. - Support for Python 3.14. ### Fixed - Use the minimum cost for conflicting symmetric substitutions. +- Reject non-finite (NaN, infinite) costs during validation. +- Cap `default_substitution_cost` at `default_insertion_cost + default_deletion_cost`, since a substitution can always be expressed as a deletion followed by an insertion. +- Treat empty-sided substitution costs as insertion/deletion aliases. ## [1.0.1] - 2025-09-21 diff --git a/docs/source/index.rst b/docs/source/index.rst index 9948b55..06f90b0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -56,6 +56,7 @@ Contents getting-started examples end_to_end_example + transitive_closure cost_learning_model api/index changelog diff --git a/docs/source/transitive_closure.rst b/docs/source/transitive_closure.rst new file mode 100644 index 0000000..ad85ea7 --- /dev/null +++ b/docs/source/transitive_closure.rst @@ -0,0 +1,34 @@ +=========================== + Transitive Cost Closure +=========================== + +By default, :class:`~ocr_stringdist.WeightedLevenshtein` only considers the +costs you explicitly provide. If `("6", "G")` costs `0.5` and deleting `"G"` +costs `0.01`, the engine does **not** automatically know that deleting `"6"` +in context costs `0.51`. + +:meth:`~ocr_stringdist.WeightedLevenshtein.transitive_closure` returns a new +instance whose cost dictionaries are filled with these effective (transitive) +costs. The closure also materializes mixed chains, e.g. a `del("y") + ins("x")` +sequence becoming an effective `("y", "x")` substitution. + +Example +======= + +.. code-block:: python + + from ocr_stringdist import WeightedLevenshtein + + wl = WeightedLevenshtein( + substitution_costs={("6", "G"): 0.5}, + deletion_costs={"G": 0.01}, + ).transitive_closure() + + # The chain "6" -> "G" -> ε is now a single effective deletion at 0.51. + print(wl.distance("06", "0")) # 0.51 + +After closure, :meth:`~ocr_stringdist.WeightedLevenshtein.explain` returns a +single flat operation at the effective cost; the underlying chain is not +preserved. + +You may pass ``prune=True`` to the ``transitive_closure`` method to remove generated substitutions whose costs are already represented by matches, insertions, deletions, or shorter substitutions. This shrinks the resulting cost map but is significantly more expensive to compute. From 3a067334ee3457d148a6bc4c0d328707193287e1 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 22:55:54 +0200 Subject: [PATCH 18/21] improvements --- CHANGELOG.md | 9 ++++++--- docs/source/transitive_closure.rst | 5 ++++- python/ocr_stringdist/_observable_dict.py | 2 +- src/transitive_costs.rs | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b13e7..ccd7812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. - Support for Python 3.14. -### Fixed +### Changed -- Use the minimum cost for conflicting symmetric substitutions. -- Reject non-finite (NaN, infinite) costs during validation. - Cap `default_substitution_cost` at `default_insertion_cost + default_deletion_cost`, since a substitution can always be expressed as a deletion followed by an insertion. - Treat empty-sided substitution costs as insertion/deletion aliases. +- Reject non-finite (NaN, infinite) costs during validation. + +### Fixed + +- Use the minimum cost for conflicting symmetric substitutions. ## [1.0.1] - 2025-09-21 diff --git a/docs/source/transitive_closure.rst b/docs/source/transitive_closure.rst index ad85ea7..fb6f71b 100644 --- a/docs/source/transitive_closure.rst +++ b/docs/source/transitive_closure.rst @@ -31,4 +31,7 @@ After closure, :meth:`~ocr_stringdist.WeightedLevenshtein.explain` returns a single flat operation at the effective cost; the underlying chain is not preserved. -You may pass ``prune=True`` to the ``transitive_closure`` method to remove generated substitutions whose costs are already represented by matches, insertions, deletions, or shorter substitutions. This shrinks the resulting cost map but is significantly more expensive to compute. +You may pass ``prune=True`` to the ``transitive_closure`` method to remove +generated substitutions whose costs are already represented by matches, +insertions, deletions, or shorter substitutions. This shrinks the resulting +cost map but is significantly more expensive to compute. diff --git a/python/ocr_stringdist/_observable_dict.py b/python/ocr_stringdist/_observable_dict.py index 7117eaf..ee69e3f 100644 --- a/python/ocr_stringdist/_observable_dict.py +++ b/python/ocr_stringdist/_observable_dict.py @@ -69,7 +69,7 @@ def setdefault(self, key: K, default: Any = None) -> V: def __or__(self, other: object, /) -> Any: if not isinstance(other, dict): return NotImplemented - return _ObservableDict(dict(self) | other, self._on_change, self._validator) + return dict(self) | other def __ior__(self, other: object, /) -> Any: self.update(other) diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 74f08e3..8a4081c 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -149,7 +149,7 @@ pub fn compute_closed_cost_maps( .enumerate() .map(|(i, token)| (token.as_str(), NodeId::new(i))) .collect(); - let epsilon_id = token_to_id[""]; + let epsilon_id = *token_to_id.get("").expect("epsilon node must exist"); let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del)?; let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); From 5d07219ff63e42e42e3c756aca17f258cefd2abc Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 23:13:24 +0200 Subject: [PATCH 19/21] refactor --- src/transitive_costs.rs | 97 ++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 8a4081c..367d389 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -195,6 +195,18 @@ fn collect_nodes( del: &CostMap, max_node_length: usize, ) -> Vec { + let mut tokens = gather_seed_tokens(sub, ins, del); + expand_with_substrings(&mut tokens, max_node_length); + grow_via_single_ops(&mut tokens, &single_op_tokens(ins, del), max_node_length); + tokens.into_iter().collect() +} + +/// Initial node set: ε plus every token that appears in any cost map. +fn gather_seed_tokens( + sub: &CostMap, + ins: &CostMap, + del: &CostMap, +) -> HashSet { let mut tokens: HashSet = HashSet::new(); tokens.insert(String::new()); // ε tokens.extend(ins.costs.keys().cloned()); @@ -203,24 +215,27 @@ fn collect_nodes( tokens.insert(s.clone()); tokens.insert(t.clone()); } + tokens +} +/// Add every contiguous substring (under the cap) of every current token. +fn expand_with_substrings(tokens: &mut HashSet, max_node_length: usize) { let seeds: Vec = tokens.iter().cloned().collect(); for token in &seeds { for substring in substrings(token, max_node_length) { tokens.insert(substring); } } +} - // Configured ins/del tokens are applied in BOTH directions to grow the set: - // inserting them produces forward-direction successors, removing them - // produces predecessors that lie one configured ins/del edge away. This is - // what lets the closure bridge a user-provided source like "ADC" through - // intermediate nodes "AC" and "ABC" to a configured target "Z". - // - // Pre-compute char counts once: the inner loop checks them against the cap - // for every source. - let single_op_tokens: Vec<(String, usize)> = ins - .costs +/// Configured ins/del tokens, deduplicated, paired with their char count. +/// The char count is cached so the cap check in [`grow_via_single_ops`] +/// doesn't recompute it for every source token. +fn single_op_tokens( + ins: &CostMap, + del: &CostMap, +) -> Vec<(String, usize)> { + ins.costs .keys() .cloned() .chain(del.costs.keys().cloned()) @@ -230,41 +245,63 @@ fn collect_nodes( let len = token.chars().count(); (token, len) }) - .collect(); + .collect() +} - // Worklist-style growth: each round only processes tokens discovered in the - // previous round, since older tokens already produced everything they can. - // Both gates check the produced length against the cap — insertion can - // overrun by lengthening, and deletion can overrun when an oversize raw - // seed (longer than `max_node_length`) is shortened to something still - // above the cap. The length cap bounds the total set, so the worklist - // drains in finitely many rounds. +/// Grow the node set by applying each configured ins/del token in both +/// directions: inserting it produces forward-direction successors, removing it +/// produces predecessors that lie one configured ins/del edge away. This is +/// what lets the closure bridge a user-provided source like "ADC" through +/// intermediate nodes "AC" and "ABC" to a configured target "Z". +/// +/// Worklist-style: each round only processes tokens discovered in the previous +/// round, since older tokens already produced everything they can. Both gates +/// check the produced length against the cap — insertion can overrun by +/// lengthening, and deletion can overrun when an oversize raw seed (longer +/// than `max_node_length`) is shortened to something still above the cap. The +/// length cap bounds the total set, so the worklist drains in finitely many +/// rounds. +fn grow_via_single_ops( + tokens: &mut HashSet, + single_ops: &[(String, usize)], + max_node_length: usize, +) { let mut frontier: Vec = tokens.iter().cloned().collect(); while !frontier.is_empty() { let mut next_frontier: Vec = Vec::new(); for source in &frontier { let source_len = source.chars().count(); - for (op_token, op_len) in &single_op_tokens { + for (op_token, op_len) in single_ops { if source_len + *op_len <= max_node_length { - for variant in insert_token_variants(source, op_token) { - if tokens.insert(variant.clone()) { - next_frontier.push(variant); - } - } + add_new_variants( + tokens, + &mut next_frontier, + insert_token_variants(source, op_token), + ); } if source_len.saturating_sub(*op_len) <= max_node_length { - for variant in delete_token_variants(source, op_token) { - if tokens.insert(variant.clone()) { - next_frontier.push(variant); - } - } + add_new_variants( + tokens, + &mut next_frontier, + delete_token_variants(source, op_token), + ); } } } frontier = next_frontier; } +} - tokens.into_iter().collect() +fn add_new_variants( + tokens: &mut HashSet, + frontier: &mut Vec, + variants: Vec, +) { + for variant in variants { + if tokens.insert(variant.clone()) { + frontier.push(variant); + } + } } fn substrings(token: &str, max_node_length: usize) -> Vec { From 5870020c600c283d996d34c08fc951455bfa6d68 Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 23:27:35 +0200 Subject: [PATCH 20/21] refactor --- src/transitive_costs.rs | 76 +++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/src/transitive_costs.rs b/src/transitive_costs.rs index 367d389..5a35a6d 100644 --- a/src/transitive_costs.rs +++ b/src/transitive_costs.rs @@ -139,6 +139,25 @@ pub fn compute_closed_cost_maps( ) -> Result<(SubstitutionCostMap, SingleTokenCostMap, SingleTokenCostMap), TransitiveCostError> { let max_node_length = max_node_length.unwrap_or_else(|| derive_max_node_length(sub, ins, del)); let tokens = collect_nodes(sub, ins, del, max_node_length); + let (token_to_id, epsilon_id) = build_token_index(&tokens)?; + let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del)?; + + let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); + let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); + let mut closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); + if prune { + closed_sub = + prune_redundant_substitutions(closed_sub, &closed_ins, &closed_del, sub, ins, del); + } + + Ok((closed_sub, closed_ins, closed_del)) +} + +/// Interns tokens as `NodeId`s and locates the ε node. Returns +/// `NodeIdOverflow` if the token set exceeds the addressable range. +fn build_token_index( + tokens: &[String], +) -> Result<(HashMap<&str, NodeId>, NodeId), TransitiveCostError> { if tokens.len() > MAX_NODE_COUNT { return Err(TransitiveCostError::NodeIdOverflow { node_count: tokens.len(), @@ -150,18 +169,7 @@ pub fn compute_closed_cost_maps( .map(|(i, token)| (token.as_str(), NodeId::new(i))) .collect(); let epsilon_id = *token_to_id.get("").expect("epsilon node must exist"); - let dist = run_closure(&tokens, &token_to_id, epsilon_id, sub, ins, del)?; - - let closed_ins = project_single_token(&tokens, &token_to_id, &dist, ins, epsilon_id, true); - let closed_del = project_single_token(&tokens, &token_to_id, &dist, del, epsilon_id, false); - let closed_sub = project_substitutions(&tokens, &token_to_id, &dist, sub); - let closed_sub = if prune { - prune_redundant_substitutions(closed_sub, &closed_ins, &closed_del, sub, ins, del) - } else { - closed_sub - }; - - Ok((closed_sub, closed_ins, closed_del)) + Ok((token_to_id, epsilon_id)) } /// Default `max_node_length` derivation: twice the longest raw token across all @@ -376,38 +384,62 @@ fn run_closure( ins: &CostMap, del: &CostMap, ) -> Result, TransitiveCostError> { - let n = tokens.len(); - let mut dist = Matrix::try_filled(n, f64::INFINITY)?; + let mut dist = Matrix::try_filled(tokens.len(), f64::INFINITY)?; + set_zero_diagonal(&mut dist); + seed_substitution_edges(&mut dist, sub, token_to_id); + seed_insertion_edges(&mut dist, ins, token_to_id, epsilon_id); + seed_deletion_edges(&mut dist, del, token_to_id, epsilon_id); + seed_embedded_edges(tokens, token_to_id, ins, del, &mut dist); + floyd_warshall(&mut dist); + Ok(dist) +} - for index in 0..n { +fn set_zero_diagonal(dist: &mut Matrix) { + for index in 0..dist.width { let node = NodeId::new(index); dist.set(node, node, 0.0); } +} +fn seed_substitution_edges( + dist: &mut Matrix, + sub: &CostMap, + token_to_id: &HashMap<&str, NodeId>, +) { for ((source, target), &cost) in &sub.costs { if let (Some(&s), Some(&t)) = ( token_to_id.get(source.as_str()), token_to_id.get(target.as_str()), ) { - relax(&mut dist, s, t, cost); + relax(dist, s, t, cost); } } +} +fn seed_insertion_edges( + dist: &mut Matrix, + ins: &CostMap, + token_to_id: &HashMap<&str, NodeId>, + epsilon_id: NodeId, +) { for (token, &cost) in &ins.costs { if let Some(&id) = token_to_id.get(token.as_str()) { - relax(&mut dist, epsilon_id, id, cost); + relax(dist, epsilon_id, id, cost); } } +} +fn seed_deletion_edges( + dist: &mut Matrix, + del: &CostMap, + token_to_id: &HashMap<&str, NodeId>, + epsilon_id: NodeId, +) { for (token, &cost) in &del.costs { if let Some(&id) = token_to_id.get(token.as_str()) { - relax(&mut dist, id, epsilon_id, cost); + relax(dist, id, epsilon_id, cost); } } - - seed_embedded_edges(tokens, token_to_id, ins, del, &mut dist); - floyd_warshall(&mut dist); - Ok(dist) } #[inline] From c51f420a3f72d20202be6a67435ebcab935890aa Mon Sep 17 00:00:00 2001 From: Niklas von Moers Date: Sat, 9 May 2026 23:45:46 +0200 Subject: [PATCH 21/21] add str, repr --- CHANGELOG.md | 1 + python/ocr_stringdist/levenshtein.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccd7812..5678b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add opt-in transitive cost closure via `WeightedLevenshtein.transitive_closure()`. - Support for Python 3.14. +- `__str__` and `__repr__` methods for `WeightedLevenshtein`. ### Changed diff --git a/python/ocr_stringdist/levenshtein.py b/python/ocr_stringdist/levenshtein.py index 50b5afc..bb92bf6 100644 --- a/python/ocr_stringdist/levenshtein.py +++ b/python/ocr_stringdist/levenshtein.py @@ -321,6 +321,30 @@ def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein: return CostLearner().fit(pairs) + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(" + f"substitution_costs={dict(self._substitution_costs)!r}, " + f"insertion_costs={dict(self._insertion_costs)!r}, " + f"deletion_costs={dict(self._deletion_costs)!r}, " + f"symmetric_substitution={self._symmetric_substitution!r}, " + f"default_substitution_cost={self._default_substitution_cost!r}, " + f"default_insertion_cost={self._default_insertion_cost!r}, " + f"default_deletion_cost={self._default_deletion_cost!r})" + ) + + def __str__(self) -> str: + return ( + f"{type(self).__name__}(" + f"substitution_costs=<{len(self._substitution_costs)} entries>, " + f"insertion_costs=<{len(self._insertion_costs)} entries>, " + f"deletion_costs=<{len(self._deletion_costs)} entries>, " + f"symmetric_substitution={self._symmetric_substitution}, " + f"default_substitution_cost={self._default_substitution_cost}, " + f"default_insertion_cost={self._default_insertion_cost}, " + f"default_deletion_cost={self._default_deletion_cost})" + ) + def __eq__(self, other: object) -> bool: if not isinstance(other, WeightedLevenshtein): return NotImplemented