From 5464531ceef255fd8e4c5b05b52de7b4541dd652 Mon Sep 17 00:00:00 2001 From: Taksh Date: Fri, 17 Jul 2026 10:07:19 +0300 Subject: [PATCH] fix: count RedPajama tokens with findall not split split() yields N+1 parts and inflates alpha_token_prop. Co-authored-by: Cursor --- python/dolma/taggers/code/code_taggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dolma/taggers/code/code_taggers.py b/python/dolma/taggers/code/code_taggers.py index 31b57087..62b7ecb2 100644 --- a/python/dolma/taggers/code/code_taggers.py +++ b/python/dolma/taggers/code/code_taggers.py @@ -156,7 +156,7 @@ def __init__(self): super().__init__() def _get_num_tokens(self, text: str) -> int: - return len(self.whitespace_regex.split(text)) + return len(self.whitespace_regex.findall(text)) def predict(self, doc: Document) -> DocResult: """Main runner."""