From bcbdfd8583e94496c7d1c88cb9b91db9d16a32cf Mon Sep 17 00:00:00 2001 From: kadraman Date: Mon, 25 May 2026 20:45:44 +0100 Subject: [PATCH] chore: updated after PR review --- README.md | 4 +- rules/secret/builtin.yml | 2 + src/scan/parseOptions.ts | 1 + src/scan/secret/cache.ts | 13 ++- src/scan/secret/engine.ts | 118 ++++++++++++++++++------- src/scan/secret/remoteRules.ts | 13 ++- src/scan/secret/rulesCache.ts | 47 ++++++++++ src/scan/secret/types.ts | 4 + src/scan/secret/yamlRuleParser.ts | 28 ++++-- src/scanner.ts | 23 ++++- tests/secretEngine.cache.test.ts | 78 ++++++++++++++++ tests/secretEngine.merge.test.ts | 35 ++++++++ tests/secretEngine.regexCase.test.ts | 84 ++++++++++++++++++ tests/secretEngine.remoteRules.test.ts | 116 ++++++++++++++++++++++++ tests/secretEngine.rulesCache.test.ts | 65 ++++++++++++++ 15 files changed, 585 insertions(+), 46 deletions(-) create mode 100644 src/scan/secret/rulesCache.ts create mode 100644 tests/secretEngine.merge.test.ts create mode 100644 tests/secretEngine.regexCase.test.ts create mode 100644 tests/secretEngine.remoteRules.test.ts create mode 100644 tests/secretEngine.rulesCache.test.ts diff --git a/README.md b/README.md index c45e26c..13ac6dd 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,9 @@ Serve the example remote bundle locally (`examples/rules/README.md`): npx --yes serve examples/rules -l 8765 ``` -Remote rule bundles are cached under `.codefence/cache/secret-rules/` for offline and low-latency scans. +Remote rule bundles are cached under `.codefence/cache/secret-rules/` for offline and low-latency scans. Use `--secret-rules-refresh` or `CODEFENCE_SECRET_RULES_REFRESH=1` to force a re-download before scanning. -**Environment:** `CODEFENCE_ASPECTS`, `CODEFENCE_ONLY`, `CODEFENCE_SKIP`, `CODEFENCE_SECRET_RULES`, `CODEFENCE_SECRET_DEFAULT_RULES`, `CODEFENCE_SECRET_DEFAULT_RULES_VERSION`, `CODEFENCE_SECRET_RULES_UPDATE_URL`, `CODEFENCE_SECRET_RULES_CACHE_TTL`, `CODEFENCE_SECRET_ENTROPY_THRESHOLD`, `CODEFENCE_SECRET_MIN_LENGTH`, `CODEFENCE_SECRET_MIN_CONFIDENCE`. +**Environment:** `CODEFENCE_ASPECTS`, `CODEFENCE_ONLY`, `CODEFENCE_SKIP`, `CODEFENCE_SECRET_RULES`, `CODEFENCE_SECRET_DEFAULT_RULES`, `CODEFENCE_SECRET_DEFAULT_RULES_VERSION`, `CODEFENCE_SECRET_RULES_UPDATE_URL`, `CODEFENCE_SECRET_RULES_REFRESH`, `CODEFENCE_SECRET_RULES_CACHE_TTL`, `CODEFENCE_SECRET_ENTROPY_THRESHOLD`, `CODEFENCE_SECRET_MIN_LENGTH`, `CODEFENCE_SECRET_MIN_CONFIDENCE`. ## Git pre-commit and background scanning diff --git a/rules/secret/builtin.yml b/rules/secret/builtin.yml index d534fab..d417d41 100644 --- a/rules/secret/builtin.yml +++ b/rules/secret/builtin.yml @@ -52,6 +52,7 @@ rules: severity: ERROR metadata: confidence: medium + case-insensitive: true remediation: Do not commit passwords; use environment variables or a secret manager instead. pattern-regex: '(?:password|passwd|pwd)\s*[:=]\s*["''][^"''\n]{8,}["'']' @@ -70,5 +71,6 @@ rules: severity: ERROR metadata: confidence: medium + case-insensitive: true remediation: Replace embedded credentials with runtime-configured secrets. pattern-regex: '(?:api[_-]?key|secret|token|access[_-]?token|client[_-]?secret)\s*[:=]\s*["''][A-Za-z0-9_\-+/=]{12,}["'']' diff --git a/src/scan/parseOptions.ts b/src/scan/parseOptions.ts index f475b6f..b71e9c6 100644 --- a/src/scan/parseOptions.ts +++ b/src/scan/parseOptions.ts @@ -279,6 +279,7 @@ Environment: CODEFENCE_SECRET_DEFAULT_RULES Same as --secret-default-rules CODEFENCE_SECRET_DEFAULT_RULES_VERSION Same as --secret-default-rules-version CODEFENCE_SECRET_RULES_UPDATE_URL Same as --secret-rules-update-url + CODEFENCE_SECRET_RULES_REFRESH Same as --secret-rules-refresh CODEFENCE_SECRET_RULES_CACHE_TTL Same as --secret-rules-cache-ttl CODEFENCE_SECRET_ENTROPY_THRESHOLD Same as --secret-entropy-threshold CODEFENCE_SECRET_MIN_LENGTH Same as --secret-min-length diff --git a/src/scan/secret/cache.ts b/src/scan/secret/cache.ts index 57cc148..d2f0d63 100644 --- a/src/scan/secret/cache.ts +++ b/src/scan/secret/cache.ts @@ -38,20 +38,25 @@ export function readCachedSecretRules(workspace: string, url: string): CachedSec } } -export function isSecretRulesCacheFresh(entry: CachedSecretRules, now = Date.now()): boolean { - return new Date(entry.fetchedAt).getTime() + entry.ttlMs > now; +export function isSecretRulesCacheFresh( + entry: CachedSecretRules, + ttlMs: number, + now = Date.now() +): boolean { + return new Date(entry.fetchedAt).getTime() + ttlMs > now; } export function writeCachedSecretRules( workspace: string, url: string, body: string, - ttlMs: number + ttlMs: number, + options?: { fetchedAt?: string } ): CachedSecretRules { const entry: CachedSecretRules = { version: 1, url, - fetchedAt: new Date().toISOString(), + fetchedAt: options?.fetchedAt ?? new Date().toISOString(), ttlMs, sha256: hashContent(body), body diff --git a/src/scan/secret/engine.ts b/src/scan/secret/engine.ts index 1e1ee52..648331b 100644 --- a/src/scan/secret/engine.ts +++ b/src/scan/secret/engine.ts @@ -1,11 +1,11 @@ import { ConfidenceLevel, Finding } from "../../types"; import { confidenceWeight } from "./config"; import { findEntropySecrets } from "./entropy"; -import { loadSecretRules } from "./ruleLoader"; +import { loadSecretRulesForScan } from "./rulesCache"; import { SecretEngineInput, SecretRule } from "./types"; -function ruleRegex(pattern: string): RegExp { - return new RegExp(pattern, "gi"); +function ruleRegex(pattern: string, caseInsensitive = false): RegExp { + return new RegExp(pattern, caseInsensitive ? "gi" : "g"); } function summarizeMatch(match: string): string { @@ -37,7 +37,7 @@ function buildRuleFindings(filePath: string, lines: string[], rules: SecretRule[ continue; } - const regex = ruleRegex(pattern.value); + const regex = ruleRegex(pattern.value, pattern.caseInsensitive); const match = regex.exec(line); if (!match) { continue; @@ -66,6 +66,18 @@ function findingKey(finding: Finding): string { return `${finding.filePath}:${finding.line}:${finding.ruleId}:${finding.message}`; } +function locationKey(finding: Finding): string { + return `${finding.filePath}:${finding.line}`; +} + +function isRuleBasedSecret(finding: Finding): boolean { + return finding.kind === "secret" && finding.ruleId !== "secret-high-entropy"; +} + +function isEntropySecret(finding: Finding): boolean { + return finding.kind === "secret" && finding.ruleId === "secret-high-entropy"; +} + function strongerConfidence(a: ConfidenceLevel | undefined, b: ConfidenceLevel | undefined): ConfidenceLevel { const left = a ?? "low"; const right = b ?? "low"; @@ -80,49 +92,95 @@ function strongerSeverity( return weights[left] >= weights[right] ? left : right; } +function combineEvidence(left?: string, right?: string): string | undefined { + const parts = [left, right].filter((value): value is string => Boolean(value)); + return parts.length > 0 ? parts.join("; ") : undefined; +} + +function combineSecretFindings(left: Finding, right: Finding): Finding { + const leftIsEntropy = isEntropySecret(left); + const rightIsEntropy = isEntropySecret(right); + const base = leftIsEntropy && !rightIsEntropy ? right : left; + const extra = base === left ? right : left; + + let detectionMethod = base.detectionMethod ?? "rule"; + if (leftIsEntropy !== rightIsEntropy) { + detectionMethod = "rule+entropy"; + } else if (detectionMethod === (extra.detectionMethod ?? detectionMethod)) { + detectionMethod = base.detectionMethod ?? extra.detectionMethod ?? "rule"; + } else { + detectionMethod = "rule+entropy"; + } + + return { + ...base, + severity: strongerSeverity(base.severity, extra.severity), + confidence: strongerConfidence(base.confidence, extra.confidence), + evidence: combineEvidence(base.evidence, extra.evidence), + remediation: base.remediation ?? extra.remediation, + detectionMethod + }; +} + +function combineRuleWithEntropy(rule: Finding, entropy: Finding): Finding { + return { + ...rule, + severity: strongerSeverity(rule.severity, entropy.severity), + confidence: strongerConfidence(rule.confidence, entropy.confidence), + evidence: combineEvidence(rule.evidence, entropy.evidence), + remediation: rule.remediation ?? entropy.remediation, + detectionMethod: "rule+entropy" + }; +} + function mergeFindings(findings: Finding[]): Finding[] { + const ruleHitLocations = new Set(); + for (const finding of findings) { + if (isRuleBasedSecret(finding)) { + ruleHitLocations.add(locationKey(finding)); + } + } + + const entropyByLocation = new Map(); + for (const finding of findings) { + if (!isEntropySecret(finding)) { + continue; + } + const loc = locationKey(finding); + const existing = entropyByLocation.get(loc); + entropyByLocation.set(loc, existing ? combineSecretFindings(existing, finding) : finding); + } + const merged = new Map(); for (const finding of findings) { - if ( - finding.ruleId === "secret-high-entropy" && - findings.some( - (other) => - other !== finding && - other.kind === "secret" && - other.ruleId !== "secret-high-entropy" && - other.filePath === finding.filePath && - other.line === finding.line - ) - ) { + if (isEntropySecret(finding) && ruleHitLocations.has(locationKey(finding))) { continue; } const key = findingKey(finding); + let candidate = finding; const existing = merged.get(key); - if (!existing) { - merged.set(key, finding); - continue; + if (existing) { + candidate = combineSecretFindings(existing, candidate); + } + + if (isRuleBasedSecret(candidate)) { + const entropy = entropyByLocation.get(locationKey(candidate)); + if (entropy) { + candidate = combineRuleWithEntropy(candidate, entropy); + } } - merged.set(key, { - ...existing, - severity: strongerSeverity(existing.severity, finding.severity), - confidence: strongerConfidence(existing.confidence, finding.confidence), - evidence: existing.evidence ?? finding.evidence, - remediation: existing.remediation ?? finding.remediation, - detectionMethod: - existing.detectionMethod === finding.detectionMethod - ? existing.detectionMethod - : "rule+entropy" - }); + merged.set(key, candidate); } return [...merged.values()]; } export async function scanSecretFindings(input: SecretEngineInput): Promise { - const rules = await loadSecretRules(input.workspace, input.options); + const rules = + input.rules ?? (await loadSecretRulesForScan(input.workspace, input.options)); const lines = input.content.split(/\r?\n/); const ruleFindings = buildRuleFindings(input.filePath, lines, rules); const entropyFindings = findEntropySecrets(input.filePath, lines, input.options); diff --git a/src/scan/secret/remoteRules.ts b/src/scan/secret/remoteRules.ts index 39ef06e..a94bb16 100644 --- a/src/scan/secret/remoteRules.ts +++ b/src/scan/secret/remoteRules.ts @@ -29,6 +29,12 @@ function requestRuleBundle( return Promise.reject(new Error("Too many redirects while downloading secret rules")); } + try { + validateRulesUrl(url); + } catch (error) { + return Promise.reject(error); + } + return new Promise((resolve, reject) => { const parsed = new URL(url); const lib = parsed.protocol === "https:" ? https : http; @@ -42,7 +48,7 @@ function requestRuleBundle( if (statusCode >= 300 && statusCode < 400 && res.headers.location) { res.resume(); const nextUrl = new URL(res.headers.location, url).href; - resolve(requestRuleBundle(nextUrl, redirectDepth + 1)); + requestRuleBundle(nextUrl, redirectDepth + 1).then(resolve, reject); return; } @@ -72,7 +78,10 @@ export async function loadRemoteRuleBundle( validateRulesUrl(url); const cached = readCachedSecretRules(workspace, url); - if (!refresh && cached && isSecretRulesCacheFresh(cached)) { + if (!refresh && cached && isSecretRulesCacheFresh(cached, ttlMs)) { + if (cached.ttlMs !== ttlMs) { + writeCachedSecretRules(workspace, url, cached.body, ttlMs, { fetchedAt: cached.fetchedAt }); + } return cached.body; } diff --git a/src/scan/secret/rulesCache.ts b/src/scan/secret/rulesCache.ts new file mode 100644 index 0000000..8bb237b --- /dev/null +++ b/src/scan/secret/rulesCache.ts @@ -0,0 +1,47 @@ +import path from "node:path"; +import { loadSecretRules } from "./ruleLoader"; +import { SecretRule, SecretScanOptions } from "./types"; + +const rulesByScan = new Map>(); + +function secretScanCacheKey(workspace: string, options: SecretScanOptions): string { + return JSON.stringify({ + workspace: path.resolve(workspace), + rulePaths: [...options.rulePaths].sort(), + defaultRules: options.defaultRules, + defaultRulesVersion: options.defaultRulesVersion, + rulesUpdateUrl: options.rulesUpdateUrl, + rulesRefresh: options.rulesRefresh, + rulesCacheTtlMs: options.rulesCacheTtlMs, + entropyThreshold: options.entropyThreshold, + minLength: options.minLength, + minConfidence: options.minConfidence + }); +} + +/** Clears in-memory secret rule memoization (for tests). */ +export function clearSecretRulesScanCache(): void { + rulesByScan.clear(); +} + +/** + * Load secret rules once per (workspace, options) for a scan invocation. + * Concurrent callers share the same in-flight promise. + */ +export function loadSecretRulesForScan( + workspace: string, + options: SecretScanOptions +): Promise { + const key = secretScanCacheKey(workspace, options); + const cached = rulesByScan.get(key); + if (cached) { + return cached; + } + + const pending = loadSecretRules(workspace, options); + rulesByScan.set(key, pending); + return pending.catch((error) => { + rulesByScan.delete(key); + throw error; + }); +} diff --git a/src/scan/secret/types.ts b/src/scan/secret/types.ts index 078c8d3..cb449dc 100644 --- a/src/scan/secret/types.ts +++ b/src/scan/secret/types.ts @@ -20,6 +20,8 @@ export interface SecretScanOptions { export interface SecretRulePattern { type: "regex" | "literal"; value: string; + /** When true, compile pattern-regex with case-insensitive matching (Semgrep default is sensitive). */ + caseInsensitive?: boolean; } export interface SecretRule { @@ -39,4 +41,6 @@ export interface SecretEngineInput { content: string; workspace: string; options: SecretScanOptions; + /** When set, skips reloading rule bundles for each file in a batch scan. */ + rules?: SecretRule[]; } diff --git a/src/scan/secret/yamlRuleParser.ts b/src/scan/secret/yamlRuleParser.ts index 0e79670..c7b8450 100644 --- a/src/scan/secret/yamlRuleParser.ts +++ b/src/scan/secret/yamlRuleParser.ts @@ -24,7 +24,25 @@ function normalizeConfidence(value: unknown): ConfidenceLevel { return "medium"; } -function collectPatterns(node: unknown): SecretRulePattern[] { +function parseCaseInsensitiveFlag(value: unknown): boolean { + return value === true || value === "true"; +} + +function ruleCaseInsensitive(rule: Record, metadata: Record): boolean { + const options = (rule.options ?? {}) as Record; + if (parseCaseInsensitiveFlag(options.generic_caseless)) { + return true; + } + if (options.case_sensitive === false) { + return true; + } + return ( + parseCaseInsensitiveFlag(metadata["case-insensitive"]) || + parseCaseInsensitiveFlag(metadata.case_insensitive) + ); +} + +function collectPatterns(node: unknown, caseInsensitive = false): SecretRulePattern[] { if (!node || typeof node !== "object") { return []; } @@ -33,7 +51,7 @@ function collectPatterns(node: unknown): SecretRulePattern[] { const patterns: SecretRulePattern[] = []; if (typeof entry["pattern-regex"] === "string") { - patterns.push({ type: "regex", value: entry["pattern-regex"] }); + patterns.push({ type: "regex", value: entry["pattern-regex"], caseInsensitive }); } if (typeof entry.pattern === "string") { @@ -42,13 +60,13 @@ function collectPatterns(node: unknown): SecretRulePattern[] { if (Array.isArray(entry.patterns)) { for (const child of entry.patterns) { - patterns.push(...collectPatterns(child)); + patterns.push(...collectPatterns(child, caseInsensitive)); } } if (Array.isArray(entry["pattern-either"])) { for (const child of entry["pattern-either"]) { - patterns.push(...collectPatterns(child)); + patterns.push(...collectPatterns(child, caseInsensitive)); } } @@ -72,7 +90,7 @@ function parseRuleObject( } const metadata = (rule.metadata ?? {}) as Record; - const patterns = collectPatterns(rule); + const patterns = collectPatterns(rule, ruleCaseInsensitive(rule, metadata)); if (patterns.length === 0) { return null; } diff --git a/src/scanner.ts b/src/scanner.ts index 8c8173a..f36e3d3 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -4,7 +4,8 @@ import { rules } from "./rules"; import { GIT_SCAN_IGNORED_PREFIXES } from "./scan/ignorePaths"; import { defaultSecretScanOptions } from "./scan/secret/config"; import { scanSecretFindings } from "./scan/secret/engine"; -import { SecretScanOptions } from "./scan/secret/types"; +import { loadSecretRulesForScan } from "./scan/secret/rulesCache"; +import { SecretRule, SecretScanOptions } from "./scan/secret/types"; import { Finding, LineScanContext, Rule } from "./types"; const DEFAULT_PRIOR_WINDOW = 15; @@ -214,6 +215,8 @@ function scanLegacyRules(filePath: string, lines: string[]): Finding[] { export interface ScanFileOptions { workspace?: string; secret?: SecretScanOptions; + /** Pre-loaded secret rules; when omitted, loaded once per scanFiles batch (or per scanFile). */ + secretRules?: SecretRule[]; } export async function scanFile(filePath: string, options: ScanFileOptions = {}): Promise { @@ -225,17 +228,31 @@ export async function scanFile(filePath: string, options: ScanFileOptions = {}): const lines = content.split(/\r?\n/); const workspace = path.resolve(options.workspace ?? process.cwd()); const secretOptions = options.secret ?? defaultSecretScanOptions(); + const secretRules = + options.secretRules ?? (await loadSecretRulesForScan(workspace, secretOptions)); const secretFindings = await scanSecretFindings({ filePath, content, workspace, - options: secretOptions + options: secretOptions, + rules: secretRules }); return [...scanLegacyRules(filePath, lines), ...secretFindings]; } export async function scanFiles(filePaths: string[], options: ScanFileOptions = {}): Promise { - const findings = await Promise.all(filePaths.map((filePath) => scanFile(filePath, options))); + const workspace = path.resolve(options.workspace ?? process.cwd()); + const secretOptions = options.secret ?? defaultSecretScanOptions(); + const secretRules = + options.secretRules ?? (await loadSecretRulesForScan(workspace, secretOptions)); + const batchOptions: ScanFileOptions = { + ...options, + workspace, + secret: secretOptions, + secretRules + }; + + const findings = await Promise.all(filePaths.map((filePath) => scanFile(filePath, batchOptions))); return findings.flat(); } diff --git a/tests/secretEngine.cache.test.ts b/tests/secretEngine.cache.test.ts index 998231d..4f2de64 100644 --- a/tests/secretEngine.cache.test.ts +++ b/tests/secretEngine.cache.test.ts @@ -4,8 +4,30 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import test from "node:test"; +import { + isSecretRulesCacheFresh, + readCachedSecretRules, + writeCachedSecretRules +} from "../src/scan/secret/cache"; import { loadSecretRules } from "../src/scan/secret/ruleLoader"; +test("isSecretRulesCacheFresh honors current ttlMs not stored ttlMs", () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-cache-ttl-")); + const url = "http://127.0.0.1:9/rules.yml"; + const twoHoursAgo = new Date(Date.now() - 2 * 3_600_000).toISOString(); + + writeCachedSecretRules(workspace, url, "rules:\n - id: x\n message: m\n", 86_400_000, { + fetchedAt: twoHoursAgo + }); + + const cached = readCachedSecretRules(workspace, url); + assert.ok(cached); + assert.equal(isSecretRulesCacheFresh(cached, 86_400_000), true); + assert.equal(isSecretRulesCacheFresh(cached, 3_600_000), false); + + fs.rmSync(workspace, { recursive: true, force: true }); +}); + test("loadSecretRules refreshes remote bundles and falls back to cache", async () => { const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-remote-rules-")); let requests = 0; @@ -55,3 +77,59 @@ test("loadSecretRules refreshes remote bundles and falls back to cache", async ( fs.rmSync(workspace, { recursive: true, force: true }); }); + +test("loadSecretRules re-fetches when a shorter cache ttl is requested", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-cache-ttl-fetch-")); + let requests = 0; + const yaml = `rules: + - id: remote-secret + message: Remote secret detected + severity: high + metadata: + confidence: medium + pattern-regex: "\\\\bremote_[A-Za-z0-9]{12}\\\\b" +`; + + const server = http.createServer((_, res) => { + requests++; + res.writeHead(200, { "content-type": "application/x-yaml" }); + res.end(yaml); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to start test server"); + } + + const url = `http://127.0.0.1:${address.port}/rules.yml`; + const longTtlOptions = { + rulePaths: [], + defaultRules: false, + defaultRulesVersion: null, + rulesUpdateUrl: url, + rulesRefresh: false, + rulesCacheTtlMs: 86_400_000, + entropyThreshold: 4.2, + minLength: 12, + minConfidence: "low" as const + }; + + await loadSecretRules(workspace, longTtlOptions); + assert.equal(requests, 1); + + const cached = readCachedSecretRules(workspace, url); + assert.ok(cached); + writeCachedSecretRules(workspace, url, cached.body, cached.ttlMs, { + fetchedAt: new Date(Date.now() - 2 * 3_600_000).toISOString() + }); + + await loadSecretRules(workspace, { + ...longTtlOptions, + rulesCacheTtlMs: 3_600_000 + }); + assert.equal(requests, 2); + + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + fs.rmSync(workspace, { recursive: true, force: true }); +}); diff --git a/tests/secretEngine.merge.test.ts b/tests/secretEngine.merge.test.ts new file mode 100644 index 0000000..242c6dc --- /dev/null +++ b/tests/secretEngine.merge.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { scanSecretFindings } from "../src/scan/secret/engine"; +import { defaultSecretScanOptions } from "../src/scan/secret/config"; + +test("merges rule and entropy at the same line with rule+entropy detectionMethod", async () => { + const content = `const apiKey = "Q4z8vB2nLp9sTw7xYk3mHc6rJd1fabcdefghij";\n`; + const findings = await scanSecretFindings({ + filePath: "sample.ts", + content, + workspace: process.cwd(), + options: defaultSecretScanOptions() + }); + + const combined = findings.find((f) => f.ruleId === "no-hardcoded-secret"); + assert.ok(combined); + assert.equal(combined?.detectionMethod, "rule+entropy"); + assert.match(combined?.evidence ?? "", /entropy=/); + assert.match(combined?.evidence ?? "", /matched secret pattern/); + assert.equal(findings.some((f) => f.ruleId === "secret-high-entropy"), false); +}); + +test("keeps standalone entropy findings when no rule matches the line", async () => { + const content = `const entropyBlob = "Q4z8vB2nLp9sTw7xYk3mHc6rJd1f";\n`; + const findings = await scanSecretFindings({ + filePath: "sample.ts", + content, + workspace: process.cwd(), + options: defaultSecretScanOptions() + }); + + assert.equal(findings.length, 1); + assert.equal(findings[0]?.ruleId, "secret-high-entropy"); + assert.equal(findings[0]?.detectionMethod, "entropy"); +}); diff --git a/tests/secretEngine.regexCase.test.ts b/tests/secretEngine.regexCase.test.ts new file mode 100644 index 0000000..5b056c6 --- /dev/null +++ b/tests/secretEngine.regexCase.test.ts @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { scanSecretFindings } from "../src/scan/secret/engine"; +import { defaultSecretScanOptions } from "../src/scan/secret/config"; + +function writeCaseRule(workspace: string, yaml: string): string { + const rulesDir = path.join(workspace, "rules"); + fs.mkdirSync(rulesDir, { recursive: true }); + const ruleFile = path.join(rulesDir, "case.yml"); + fs.writeFileSync(ruleFile, yaml, "utf8"); + return rulesDir; +} + +test("pattern-regex is case-sensitive by default", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-regex-case-")); + const rulesDir = writeCaseRule( + workspace, + `rules: + - id: bearer-sensitive + message: Bearer token + severity: high + pattern-regex: "\\\\bBearer\\\\s+[A-Za-z0-9]{8,}" +` + ); + + const options = { + ...defaultSecretScanOptions(), + rulePaths: [rulesDir], + defaultRules: false + }; + + const upper = await scanSecretFindings({ + filePath: "sample.ts", + content: 'const x = "Bearer abcdefgh";\n', + workspace, + options + }); + const lower = await scanSecretFindings({ + filePath: "sample.ts", + content: 'const x = "bearer abcdefgh";\n', + workspace, + options + }); + + assert.equal(upper.some((f) => f.ruleId === "bearer-sensitive"), true); + assert.equal(lower.some((f) => f.ruleId === "bearer-sensitive"), false); + + fs.rmSync(workspace, { recursive: true, force: true }); +}); + +test("pattern-regex honors metadata case-insensitive opt-in", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-regex-caseless-")); + const rulesDir = writeCaseRule( + workspace, + `rules: + - id: bearer-insensitive + message: Bearer token + severity: high + metadata: + case-insensitive: true + pattern-regex: "\\\\bBearer\\\\s+[A-Za-z0-9]{8,}" +` + ); + + const options = { + ...defaultSecretScanOptions(), + rulePaths: [rulesDir], + defaultRules: false + }; + + const lower = await scanSecretFindings({ + filePath: "sample.ts", + content: 'const x = "bearer abcdefgh";\n', + workspace, + options + }); + + assert.equal(lower.some((f) => f.ruleId === "bearer-insensitive"), true); + + fs.rmSync(workspace, { recursive: true, force: true }); +}); diff --git a/tests/secretEngine.remoteRules.test.ts b/tests/secretEngine.remoteRules.test.ts new file mode 100644 index 0000000..c22bd97 --- /dev/null +++ b/tests/secretEngine.remoteRules.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadSecretRules } from "../src/scan/secret/ruleLoader"; + +test("loadSecretRules rejects redirect to non-localhost http", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-remote-redirect-")); + + const server = http.createServer((req, res) => { + if (req.url === "/rules.yml") { + res.writeHead(302, { Location: "http://example.com/evil-rules.yml" }); + res.end(); + return; + } + res.writeHead(404); + res.end(); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to start test server"); + } + + const url = `http://127.0.0.1:${address.port}/rules.yml`; + + await assert.rejects( + () => + loadSecretRules(workspace, { + rulePaths: [], + defaultRules: false, + defaultRulesVersion: null, + rulesUpdateUrl: url, + rulesRefresh: true, + rulesCacheTtlMs: 60_000, + entropyThreshold: 4.2, + minLength: 12, + minConfidence: "low" + }), + /Remote secret rules must use https/ + ); + + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + fs.rmSync(workspace, { recursive: true, force: true }); +}); + +test("loadSecretRules follows redirect to localhost http", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-remote-redirect-ok-")); + const yaml = `rules: + - id: redirected-secret + message: Redirected secret detected + severity: high + metadata: + confidence: medium + pattern-regex: "\\\\bredirected_[A-Za-z0-9]{8}\\\\b" +`; + + let targetPort = 0; + const targetServer = http.createServer((_, res) => { + res.writeHead(200, { "content-type": "application/x-yaml" }); + res.end(yaml); + }); + + await new Promise((resolve) => + targetServer.listen(0, "127.0.0.1", () => { + const address = targetServer.address(); + if (!address || typeof address === "string") { + throw new Error("failed to start target server"); + } + targetPort = address.port; + resolve(); + }) + ); + + const redirectServer = http.createServer((req, res) => { + if (req.url === "/rules.yml") { + res.writeHead(302, { Location: `http://127.0.0.1:${targetPort}/bundle.yml` }); + res.end(); + return; + } + res.writeHead(404); + res.end(); + }); + + await new Promise((resolve) => redirectServer.listen(0, "127.0.0.1", () => resolve())); + const redirectAddress = redirectServer.address(); + if (!redirectAddress || typeof redirectAddress === "string") { + throw new Error("failed to start redirect server"); + } + + const url = `http://127.0.0.1:${redirectAddress.port}/rules.yml`; + const rules = await loadSecretRules(workspace, { + rulePaths: [], + defaultRules: false, + defaultRulesVersion: null, + rulesUpdateUrl: url, + rulesRefresh: true, + rulesCacheTtlMs: 60_000, + entropyThreshold: 4.2, + minLength: 12, + minConfidence: "low" + }); + + assert.equal(rules[0]?.id, "redirected-secret"); + + await new Promise((resolve, reject) => + redirectServer.close((error) => (error ? reject(error) : resolve())) + ); + await new Promise((resolve, reject) => + targetServer.close((error) => (error ? reject(error) : resolve())) + ); + fs.rmSync(workspace, { recursive: true, force: true }); +}); diff --git a/tests/secretEngine.rulesCache.test.ts b/tests/secretEngine.rulesCache.test.ts new file mode 100644 index 0000000..65419bf --- /dev/null +++ b/tests/secretEngine.rulesCache.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { clearSecretRulesScanCache } from "../src/scan/secret/rulesCache"; +import { scanFiles } from "../src/scanner"; + +test("scanFiles loads remote secret rules once across parallel file scans", async () => { + clearSecretRulesScanCache(); + + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-scan-batch-")); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "codefence-scan-files-")); + let requests = 0; + const yaml = `rules: + - id: batch-remote-secret + message: Batch remote secret detected + severity: high + metadata: + confidence: medium + pattern-regex: "\\\\bbatch_[A-Za-z0-9]{8}\\\\b" +`; + + const server = http.createServer((_, res) => { + requests++; + res.writeHead(200, { "content-type": "application/x-yaml" }); + res.end(yaml); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("failed to start test server"); + } + + const url = `http://127.0.0.1:${address.port}/rules.yml`; + const files = ["a.ts", "b.ts", "c.ts"].map((name) => { + const filePath = path.join(tmpDir, name); + fs.writeFileSync(filePath, 'const token = "batch_abcdefgh";\n', "utf8"); + return filePath; + }); + + await scanFiles(files, { + workspace, + secret: { + rulePaths: [], + defaultRules: false, + defaultRulesVersion: null, + rulesUpdateUrl: url, + rulesRefresh: true, + rulesCacheTtlMs: 60_000, + entropyThreshold: 4.2, + minLength: 12, + minConfidence: "low" + } + }); + + assert.equal(requests, 1); + + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + fs.rmSync(workspace, { recursive: true, force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); + clearSecretRulesScanCache(); +});