Skip to content

Optimize entropy scanner: skip dirs, limit size, and stream files #236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sejusahu011 wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Optimize entropy scanner: skip dirs, limit size, and stream files #236

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cbb60e9
Optimize entropy scanner: skip dirs, limit size, and stream files
Sejusahu011 Jun 28, 2026
cd9d1bf
Update entropy.py
Sejusahu011 Jun 29, 2026
8dca96f
fix formatting,docstring and return indentation
Sejusahu011 Jun 29, 2026
2287963
Merge branch 'main' of https://github.com/Sejusahu011/PatchPilot
Sejusahu011 Jun 29, 2026
c79b4e0
"Fix: format entropy.py with black and fix indentation"
Sejusahu011 Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 76 additions & 54 deletions backend/app/scanners/entropy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ def is_allowlisted(token: str) -> bool:


def run_entropy(repo_dir: Path) -> List[Finding]:
"""
Scan the repository directory for files containing high entropy strings.

Args:
repo_dir: Path to the directory to scan.

Returns:
A list of Finding objects discovered during the scan.
"""
findings: List[Finding] = []
ignored_extensions = {
".png",
Expand All @@ -62,65 +71,78 @@ def run_entropy(repo_dir: Path) -> List[Finding]:
".eot",
".ttf",
}
ignored_dirs = {
"node_modules",
"venv",
".venv",
"build",
"dist",
"target",
".next",
".cache",
"coverage",
"vendor",
"__pycache__",
".git",
}

MAX_FILE_SIZE = 1 * 1024 * 1024

for file_path in repo_dir.rglob("*"):
if (
not file_path.is_file()
or file_path.suffix.lower() in ignored_extensions
or ".git" in file_path.parts
):
if any(part in ignored_dirs for part in file_path.parts):
continue

if not file_path.is_file() or file_path.suffix.lower() in ignored_extensions:
continue

if file_path.stat().st_size > MAX_FILE_SIZE:
continue

try:
content = file_path.read_text(encoding="utf-8", errors="ignore")
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
for line_idx, line in enumerate(f, start=1):
for match in STRING_LITERAL_REGEX.finditer(line):
token = match.group(2)

if is_allowlisted(token):
continue

entropy_score = calculate_shannon_entropy(token)
if entropy_score > 4.0:
relative_path = str(file_path.relative_to(repo_dir))
finding_id = f"entropy:high-entropy-string:{relative_path}:{line_idx}"
severity = "HIGH"

raw_data = {
"id": finding_id,
"severity": severity,
"location": {"path": relative_path},
"metadata": {"cwe_category": "CWE-798"},
}
ml_features = extract_features(
raw_data, scanner_name="entropy"
)

findings.append(
Finding(
id=finding_id,
category="secret",
severity=severity,
title="High Entropy String Detected",
description=f"Potential hardcoded secret discovered (Entropy: {entropy_score:.2f})",
location=Location(
path=relative_path,
start_line=line_idx,
end_line=line_idx,
),
metadata={
"engine": "entropy",
"entropy_score": entropy_score,
"token_preview": token[:10],
},
features=ml_features,
)
)
except Exception:
continue

lines = content.splitlines()
for line_idx, line in enumerate(lines, start=1):
for match in STRING_LITERAL_REGEX.finditer(line):
token = match.group(2)

if is_allowlisted(token):
continue

entropy_score = calculate_shannon_entropy(token)
if entropy_score > 4.0:
relative_path = str(file_path.relative_to(repo_dir))
finding_id = (
f"entropy:high-entropy-string:{relative_path}:{line_idx}"
)
severity = "HIGH"

raw_data_for_extractor = {
"id": finding_id,
"severity": severity,
"location": {"path": relative_path},
"metadata": {"cwe_category": "CWE-798"},
}

ml_features = extract_features(
raw_data_for_extractor, scanner_name="entropy"
)

findings.append(
Finding(
id=finding_id,
category="secret",
severity=severity,
title="High Entropy String Detected",
description=f"Potential hardcoded secret or token discovered (Entropy: {entropy_score:.2f})",
location=Location(
path=relative_path,
start_line=line_idx,
end_line=line_idx,
),
metadata={
"engine": "entropy",
"entropy_score": entropy_score,
"token_preview": token[:10],
},
features=ml_features,
)
)
return findings
Loading