From ab8ceb23377f7e46090cbcee08ccc14adc1320f1 Mon Sep 17 00:00:00 2001 From: tatsuru-okada-business Date: Sun, 7 Jun 2026 23:30:21 +0900 Subject: [PATCH] fix(tokenizers): add missing tokenize_raw to TokenizerTransformers The index pipeline (index/tokenize.py: tokenize_encode_offsets) calls tokenizer.tokenize_raw(), which every tokenizer implements (mecab, icu, llama, moses) except TokenizerTransformers. As a result, building an index with --backend transformers always crashes: softmatcha-index --index /tmp/idx --backend transformers \ --model tohoku-nlp/bert-base-japanese-v3 corpus.txt ... AttributeError: 'TokenizerTransformers' object has no attribute 'tokenize_raw' This mirrors the llama tokenizer's implementation (tokenize_raw == tokenize on the stripped line). Verified: index builds successfully and search returns expected results with tohoku-nlp/bert-base-japanese-v3. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/softmatcha/tokenizers/transformers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/softmatcha/tokenizers/transformers.py b/src/softmatcha/tokenizers/transformers.py index 6019c80..c173e57 100644 --- a/src/softmatcha/tokenizers/transformers.py +++ b/src/softmatcha/tokenizers/transformers.py @@ -39,3 +39,6 @@ def tokenize(self, line: str) -> list[str]: list[str]: The tokenized line. """ return self._tokenizer.tokenize(line, verbose=False) + + def tokenize_raw(self, line: str) -> list[str]: + return self._tokenizer.tokenize(line.strip(), verbose=False)