From 26c604a3a6422e709b81ff11c2dabb620cc6c46f Mon Sep 17 00:00:00 2001 From: Daniyel Yaacov Bilar Date: Sun, 5 Jul 2026 07:13:49 -0400 Subject: [PATCH] fix(search): make FTS5 special-char queries match via phrase-prefix Queries containing hyphens, dots, slashes etc. (e.g. 'session-recall', 'CLAUDE.md', 'src/main.py') previously got quoted whole with no prefix star, so they only matched as exact adjacent phrases and usually returned nothing. Now such tokens are split on the special characters into the subtokens unicode61 actually indexes and searched as a phrase-prefix query: session-recall -> "session recall"*. Also handles tokens that are all special chars (dropped; all-special queries return None like empty queries). Found by Claude (Fable 5) during a Bitter Lesson review of a real user's harness; the user greenlit submitting the fix upstream. Co-Authored-By: Claude Fable 5 --- src/session_recall/commands/search.py | 19 ++++++++--- .../tests/test_search_sanitize.py | 34 +++++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/session_recall/commands/search.py b/src/session_recall/commands/search.py index 93f0d2f..36f1c31 100644 --- a/src/session_recall/commands/search.py +++ b/src/session_recall/commands/search.py @@ -17,8 +17,12 @@ def sanitize_fts5_query(raw: str) -> str | None: """Escape FTS5 special characters and add prefix matching. Returns None for empty/whitespace-only queries. - Strategy: split on whitespace, quote each token that contains - special chars, append * for prefix matching on every token. + Strategy: split on whitespace. Plain tokens get * for prefix matching. + Tokens containing special chars (hyphens, dots, slashes, ...) are split + on those chars into the subtokens unicode61 actually indexes, and + searched as a phrase with prefix matching on the last subtoken — + so `session-recall` becomes `"session recall"*` and matches without + the caller needing to know how the tokenizer splits. """ stripped = raw.strip() if not stripped: @@ -26,11 +30,16 @@ def sanitize_fts5_query(raw: str) -> str | None: tokens = stripped.split() safe_tokens = [] for tok in tokens: - escaped = tok.replace('"', '""') if _FTS5_SPECIAL.search(tok): - safe_tokens.append(f'"{escaped}"') + subtokens = [s for s in _FTS5_SPECIAL.split(tok) if s] + if not subtokens: + continue + phrase = " ".join(s.replace('"', '""') for s in subtokens) + safe_tokens.append(f'"{phrase}"*') else: - safe_tokens.append(f"{escaped}*") + safe_tokens.append(f'{tok.replace(chr(34), chr(34) * 2)}*') + if not safe_tokens: + return None return " ".join(safe_tokens) diff --git a/src/session_recall/tests/test_search_sanitize.py b/src/session_recall/tests/test_search_sanitize.py index d2cc9bf..3783b20 100644 --- a/src/session_recall/tests/test_search_sanitize.py +++ b/src/session_recall/tests/test_search_sanitize.py @@ -3,19 +3,25 @@ class TestSanitizeFTS5Query: - """Cover all crash cases from the adversarial gap analysis.""" + """Cover all crash cases from the adversarial gap analysis. + + Tokens containing FTS5 special characters are split into the subtokens + unicode61 actually indexes and searched as phrase-prefix queries, so + `session-recall` becomes `"session recall"*` and matches without the + caller needing to know how the tokenizer splits. + """ def test_dot_in_filename(self): - assert sanitize_fts5_query("CLAUDE.md") == '"CLAUDE.md"' + assert sanitize_fts5_query("CLAUDE.md") == '"CLAUDE md"*' def test_dot_in_python_file(self): - assert sanitize_fts5_query("test.py") == '"test.py"' + assert sanitize_fts5_query("test.py") == '"test py"*' - def test_hyphen_treated_as_not(self): - assert sanitize_fts5_query("session-recall") == '"session-recall"' + def test_hyphen_split_to_phrase_prefix(self): + assert sanitize_fts5_query("session-recall") == '"session recall"*' def test_parentheses_grouping(self): - assert sanitize_fts5_query("function()") == '"function()"' + assert sanitize_fts5_query("function()") == '"function"*' def test_empty_string_returns_none(self): assert sanitize_fts5_query("") is None @@ -23,6 +29,12 @@ def test_empty_string_returns_none(self): def test_whitespace_only_returns_none(self): assert sanitize_fts5_query(" ") is None + def test_all_special_chars_returns_none(self): + assert sanitize_fts5_query("!!! ---") is None + + def test_pure_special_token_dropped(self): + assert sanitize_fts5_query("--- foo") == "foo*" + def test_normal_word_gets_prefix_wildcard(self): assert sanitize_fts5_query("OAuth") == "OAuth*" @@ -31,11 +43,11 @@ def test_multi_word_all_get_prefix(self): def test_mixed_special_and_normal(self): result = sanitize_fts5_query("fix CLAUDE.md now") - assert result == 'fix* "CLAUDE.md" now*' + assert result == 'fix* "CLAUDE md"* now*' - def test_double_quotes_escaped(self): + def test_double_quotes_stripped_to_phrase(self): result = sanitize_fts5_query('say "hello"') - assert '""hello""' in result + assert result == 'say* "hello"*' def test_asterisk_special(self): result = sanitize_fts5_query("*.py") @@ -43,8 +55,8 @@ def test_asterisk_special(self): def test_colon_special(self): result = sanitize_fts5_query("key:value") - assert result == '"key:value"' + assert result == '"key value"*' def test_slash_in_path(self): result = sanitize_fts5_query("src/main.py") - assert result == '"src/main.py"' + assert result == '"src main py"*'