diff --git a/src/softmatcha/tokenizers/transformers.py b/src/softmatcha/tokenizers/transformers.py index 6019c80..c173e57 100644 --- a/src/softmatcha/tokenizers/transformers.py +++ b/src/softmatcha/tokenizers/transformers.py @@ -39,3 +39,6 @@ def tokenize(self, line: str) -> list[str]: list[str]: The tokenized line. """ return self._tokenizer.tokenize(line, verbose=False) + + def tokenize_raw(self, line: str) -> list[str]: + return self._tokenizer.tokenize(line.strip(), verbose=False)