From cbb60e9f55d58a1b2621aba493f1a5048719945c Mon Sep 17 00:00:00 2001 From: Sejal Sahu Date: Sun, 28 Jun 2026 11:38:51 +0530 Subject: [PATCH 1/4] Optimize entropy scanner: skip dirs, limit size, and stream files --- backend/app/scanners/entropy.py | 127 +++++++++++++++----------------- 1 file changed, 60 insertions(+), 67 deletions(-) diff --git a/backend/app/scanners/entropy.py b/backend/app/scanners/entropy.py index 8cfda41..197b1a1 100644 --- a/backend/app/scanners/entropy.py +++ b/backend/app/scanners/entropy.py @@ -42,85 +42,78 @@ def is_allowlisted(token: str) -> bool: return True return False - def run_entropy(repo_dir: Path) -> List[Finding]: findings: List[Finding] = [] ignored_extensions = { - ".png", - ".jpg", - ".jpeg", - ".gif", - ".ico", - ".svg", - ".zip", - ".tar", - ".gz", - ".mp4", - ".pdf", - ".woff", - ".woff2", - ".eot", - ".ttf", + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".zip", + ".tar", ".gz", ".mp4", ".pdf", ".woff", ".woff2", ".eot", ".ttf", + } + + ignored_dirs = { + "node_modules", "venv", ".venv", "build", "dist", + "target", ".next", ".cache", "coverage", "vendor", "__pycache__", ".git" } + MAX_FILE_SIZE = 1 * 1024 * 1024 + for file_path in repo_dir.rglob("*"): + + if any(part in ignored_dirs for part in file_path.parts): + continue + if ( not file_path.is_file() or file_path.suffix.lower() in ignored_extensions - or ".git" in file_path.parts ): continue + if file_path.stat().st_size > MAX_FILE_SIZE: + continue + try: - content = file_path.read_text(encoding="utf-8", errors="ignore") + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line_idx, line in enumerate(f, start=1): + for match in STRING_LITERAL_REGEX.finditer(line): + token = match.group(2) + + if is_allowlisted(token): + continue + + entropy_score = calculate_shannon_entropy(token) + if entropy_score > 4.0: + relative_path = str(file_path.relative_to(repo_dir)) + finding_id = f"entropy:high-entropy-string:{relative_path}:{line_idx}" + severity = "HIGH" + + raw_data_for_extractor = { + "id": finding_id, + "severity": severity, + "location": {"path": relative_path}, + "metadata": {"cwe_category": "CWE-798"}, + } + + ml_features = extract_features(raw_data_for_extractor, scanner_name="entropy") + + findings.append( + Finding( + id=finding_id, + category="secret", + severity=severity, + title="High Entropy String Detected", + description=f"Potential hardcoded secret discovered (Entropy: {entropy_score:.2f})", + location=Location( + path=relative_path, + start_line=line_idx, + end_line=line_idx, + ), + metadata={ + "engine": "entropy", + "entropy_score": entropy_score, + "token_preview": token[:10], + }, + features=ml_features, + ) + ) except Exception: continue - - lines = content.splitlines() - for line_idx, line in enumerate(lines, start=1): - for match in STRING_LITERAL_REGEX.finditer(line): - token = match.group(2) - - if is_allowlisted(token): - continue - - entropy_score = calculate_shannon_entropy(token) - if entropy_score > 4.0: - relative_path = str(file_path.relative_to(repo_dir)) - finding_id = ( - f"entropy:high-entropy-string:{relative_path}:{line_idx}" - ) - severity = "HIGH" - - raw_data_for_extractor = { - "id": finding_id, - "severity": severity, - "location": {"path": relative_path}, - "metadata": {"cwe_category": "CWE-798"}, - } - - ml_features = extract_features( - raw_data_for_extractor, scanner_name="entropy" - ) - - findings.append( - Finding( - id=finding_id, - category="secret", - severity=severity, - title="High Entropy String Detected", - description=f"Potential hardcoded secret or token discovered (Entropy: {entropy_score:.2f})", - location=Location( - path=relative_path, - start_line=line_idx, - end_line=line_idx, - ), - metadata={ - "engine": "entropy", - "entropy_score": entropy_score, - "token_preview": token[:10], - }, - features=ml_features, - ) - ) - return findings + return findings \ No newline at end of file From cd9d1bff32fc5c2c7f6ab46ba666d600abdc9e6e Mon Sep 17 00:00:00 2001 From: Sejal Sahu Date: Mon, 29 Jun 2026 21:20:55 +0530 Subject: [PATCH 2/4] Update entropy.py --- backend/app/scanners/entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/scanners/entropy.py b/backend/app/scanners/entropy.py index 197b1a1..c8ea96e 100644 --- a/backend/app/scanners/entropy.py +++ b/backend/app/scanners/entropy.py @@ -116,4 +116,4 @@ def run_entropy(repo_dir: Path) -> List[Finding]: ) except Exception: continue - return findings \ No newline at end of file + return findings From 8dca96f54034ec7e7e35d7dafe53de8a484c5c5a Mon Sep 17 00:00:00 2001 From: Sejal Sahu Date: Mon, 29 Jun 2026 23:25:24 +0530 Subject: [PATCH 3/4] fix formatting,docstring and return indentation --- backend/app/scanners/entropy.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/app/scanners/entropy.py b/backend/app/scanners/entropy.py index 197b1a1..fc102ea 100644 --- a/backend/app/scanners/entropy.py +++ b/backend/app/scanners/entropy.py @@ -42,25 +42,33 @@ def is_allowlisted(token: str) -> bool: return True return False + def run_entropy(repo_dir: Path) -> List[Finding]: + """ + Scan the repository directory for files containing high entropy strings. + + Args: + repo_dir: Path to the directory to scan. + + Returns: + A list of Finding objects discovered during the scan. + """ findings: List[Finding] = [] ignored_extensions = { ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".zip", ".tar", ".gz", ".mp4", ".pdf", ".woff", ".woff2", ".eot", ".ttf", } - ignored_dirs = { - "node_modules", "venv", ".venv", "build", "dist", + "node_modules", "venv", ".venv", "build", "dist", "target", ".next", ".cache", "coverage", "vendor", "__pycache__", ".git" } - MAX_FILE_SIZE = 1 * 1024 * 1024 + MAX_FILE_SIZE = 1 * 1024 * 1024 for file_path in repo_dir.rglob("*"): - if any(part in ignored_dirs for part in file_path.parts): continue - + if ( not file_path.is_file() or file_path.suffix.lower() in ignored_extensions @@ -85,14 +93,13 @@ def run_entropy(repo_dir: Path) -> List[Finding]: finding_id = f"entropy:high-entropy-string:{relative_path}:{line_idx}" severity = "HIGH" - raw_data_for_extractor = { + raw_data = { "id": finding_id, "severity": severity, "location": {"path": relative_path}, "metadata": {"cwe_category": "CWE-798"}, } - - ml_features = extract_features(raw_data_for_extractor, scanner_name="entropy") + ml_features = extract_features(raw_data, scanner_name="entropy") findings.append( Finding( From c79b4e04b649dd99922ddbd5999997fabbd16ab6 Mon Sep 17 00:00:00 2001 From: Sejal Sahu Date: Mon, 29 Jun 2026 23:53:41 +0530 Subject: [PATCH 4/4] "Fix: format entropy.py with black and fix indentation" --- backend/app/scanners/entropy.py | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/backend/app/scanners/entropy.py b/backend/app/scanners/entropy.py index 9b58186..4c7080c 100644 --- a/backend/app/scanners/entropy.py +++ b/backend/app/scanners/entropy.py @@ -55,12 +55,35 @@ def run_entropy(repo_dir: Path) -> List[Finding]: """ findings: List[Finding] = [] ignored_extensions = { - ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".zip", - ".tar", ".gz", ".mp4", ".pdf", ".woff", ".woff2", ".eot", ".ttf", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".ico", + ".svg", + ".zip", + ".tar", + ".gz", + ".mp4", + ".pdf", + ".woff", + ".woff2", + ".eot", + ".ttf", } ignored_dirs = { - "node_modules", "venv", ".venv", "build", "dist", - "target", ".next", ".cache", "coverage", "vendor", "__pycache__", ".git" + "node_modules", + "venv", + ".venv", + "build", + "dist", + "target", + ".next", + ".cache", + "coverage", + "vendor", + "__pycache__", + ".git", } MAX_FILE_SIZE = 1 * 1024 * 1024 @@ -69,10 +92,7 @@ def run_entropy(repo_dir: Path) -> List[Finding]: if any(part in ignored_dirs for part in file_path.parts): continue - if ( - not file_path.is_file() - or file_path.suffix.lower() in ignored_extensions - ): + if not file_path.is_file() or file_path.suffix.lower() in ignored_extensions: continue if file_path.stat().st_size > MAX_FILE_SIZE: @@ -99,7 +119,9 @@ def run_entropy(repo_dir: Path) -> List[Finding]: "location": {"path": relative_path}, "metadata": {"cwe_category": "CWE-798"}, } - ml_features = extract_features(raw_data, scanner_name="entropy") + ml_features = extract_features( + raw_data, scanner_name="entropy" + ) findings.append( Finding(