From 99d3e7943d87b9b1255da6704eb39c7de7f8643f Mon Sep 17 00:00:00 2001 From: Taksh Date: Fri, 17 Jul 2026 10:35:57 +0300 Subject: [PATCH] fix: duplicate-line char fraction uses document length Numerator counts line chars (incl. spaces); denom used word chars only and could exceed 1. Co-authored-by: Cursor --- python/dolma/taggers/gopher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dolma/taggers/gopher.py b/python/dolma/taggers/gopher.py index 6a5a9c84..bd5ffa7d 100644 --- a/python/dolma/taggers/gopher.py +++ b/python/dolma/taggers/gopher.py @@ -194,7 +194,7 @@ def get_attributes(text: str, ignore_empty_lines: bool = False) -> GopherAttribu ) attrs.fraction_of_characters_in_duplicate_lines = sum( len(line) * count for line, count in line_counts.items() if count > 1 - ) / max(character_count, 1) + ) / max(attrs.character_count, 1) except Exception as e: logging.exception(f"Error processing text {e}: {text[:200]}")