diff --git a/CHANGELOG.md b/CHANGELOG.md index cb4fc20..5cfb17a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## Unreleased + +### Detection + +- Add a learned **secret-confidence classifier** to cut false positives from + generic detectors. When `refineConfidence` is enabled, a small logistic- + regression model scores each match from a detector marked `refine` (currently + `high_entropy`) as a real secret versus a benign look-alike (UUID, git SHA, + digest, object id, slug, dictionary word) and nudges its confidence. Pair with + `minConfidence` to drop the noise. Checksum-validated detectors are never + touched. +- The model is character-level logistic regression trained offline by + `scripts/train-confidence-model.mjs` and shipped as fixed weights in + `src/confidence-model.ts`, so the runtime stays zero-dependency, synchronous, + and deterministic — no model download or native add-on, safe on edge and in + the browser. +- Export `secretProbability`, `extractFeatures`, `shannonEntropy`, and the model + from the package root and from the new `flare-redact/ml` subpath, so callers + can build their own confidence filters. + +### CLI + +- Add `--refine-confidence` to enable the classifier from the command line; + pairs with `--min-confidence`. + ## 1.1.0 — 2026-07-23 ### Detection diff --git a/README.md b/README.md index 22de9fb..24bde62 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Nothing to configure. No list of field paths to maintain. No native build step. - [Ways to hide a value](#ways-to-hide-a-value) - [Reversible redaction](#reversible-redaction) - [Contextual and model-assisted PII](#contextual-and-model-assisted-pii) +- [Learned confidence, fewer false positives](#learned-confidence-fewer-false-positives) - [Build a private chat app](#build-a-private-chat-app) - [Protect tool calls and MCP loops](#protect-tool-calls-and-mcp-loops) - [Your own words](#your-own-words) @@ -287,6 +288,41 @@ Semantic and deterministic spans enter the same overlap arbitration. Higher-risk higher-priority, and better-validated findings win instead of whichever regular expression happens to run first. +## Learned confidence, fewer false positives + +Generic, format-agnostic detectors such as `high_entropy` catch unknown-format +keys, but they also fire on benign high-entropy strings: UUIDs, git SHAs, digests, +object ids, and slugs. `refineConfidence` runs a small learned classifier over +each match to tell real secrets from look-alikes, then nudges the confidence +score. Pair it with `minConfidence` to drop the noise. + +```js +const noisy = 'id 9fceb02d0ae598e95dc970b74767f19372d61af8 tok Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv'; + +scan(noisy, { enable: ['high_entropy'] }); +// git SHA and the unknown-format token both flagged at a flat 60% + +scan(noisy, { enable: ['high_entropy'], refineConfidence: true, minConfidence: 0.5 }); +// the SHA is gone; the token survives (refined up to 80%) +``` + +The classifier is logistic regression over cheap character features (entropy, +character-class mix, structure, and nearby labels like `api_key=` or `commit`). +It is trained offline by [`scripts/train-confidence-model.mjs`](scripts/train-confidence-model.mjs) +and shipped as fixed weights, so the runtime stays zero-dependency, synchronous, +and deterministic — no model download, no native add-on, safe on edge and in the +browser. Only detectors marked `refine` are touched; checksum-validated ones +(cards, IBANs, national ids) are always left alone. + +Score a string yourself from `flare-redact` or the `flare-redact/ml` subpath: + +```js +import { secretProbability } from 'flare-redact/ml'; + +secretProbability('Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv', 'authorization: Bearer …'); // ~1.00 +secretProbability('9fceb02d0ae598e95dc970b74767f19372d61af8', 'commit …'); // ~0.00 +``` + ## Build a private chat app If you're building a chat interface — over your own local model or any API — a @@ -567,6 +603,7 @@ flare-redact --sarif .env > results.sarif # GitHub code-scanning report flare-redact --summary --json < event.json # counts per detector flare-redact --enable high_entropy < app.log # also catch unknown-format keys flare-redact --scan --min-confidence 0.9 .env # only high-confidence findings +flare-redact --enable high_entropy --refine-confidence --min-confidence 0.5 < app.log # drop UUID/SHA noise flare-redact --list # show every detector ``` @@ -715,6 +752,10 @@ httpRedactor(opts?) // 'flare-redact/http' → Express/Connect middlewar redactCsv(text, opts?) // 'flare-redact/csv' → anonymize a CSV dataset wrapFetch(fetch, opts?) // 'flare-redact/fetch' → redact egress to named hosts +// from 'flare-redact/ml' +secretProbability(value, context?): number // learned secret-vs-look-alike score, 0..1 +extractFeatures(value, context?): number[] // the raw feature vector + // from 'flare-redact/llm' wrapOpenAI(client, opts?) // scrub prompts, restore replies (+streaming) wrapAnthropic(client, opts?) // same for messages.create + system @@ -731,7 +772,7 @@ redactStream(opts?): Transform // chunk-safe + bounded multilin // { // only?, enable?, disable?, custom?, // which detectors run // mode?: 'mask' | 'label' | 'hash' | 'pseudonym' | 'surrogate', -// transformSecret?, mask?, minConfidence?, semanticProvider?, limits?, +// transformSecret?, mask?, minConfidence?, refineConfidence?, semanticProvider?, limits?, // includeValues?: boolean, // scan only; unsafe raw values // redactKeys?: boolean | RegExp | string[], // allow?: RegExp | string[], diff --git a/package.json b/package.json index 0658587..6c1515b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,10 @@ "types": "./dist/llm.d.ts", "default": "./dist/llm.js" }, + "./ml": { + "types": "./dist/ml.d.ts", + "default": "./dist/ml.js" + }, "./tool": { "types": "./dist/tool.d.ts", "default": "./dist/tool.js" diff --git a/scripts/train-confidence-model.mjs b/scripts/train-confidence-model.mjs new file mode 100644 index 0000000..a07a801 --- /dev/null +++ b/scripts/train-confidence-model.mjs @@ -0,0 +1,303 @@ +// Offline trainer for the secret-confidence model shipped in src/confidence-model.ts. +// +// The library ships no ML runtime and takes no runtime dependency: this script +// runs once, learns a small logistic-regression model from synthetic labelled +// data, and prints a plain TypeScript file whose only content is fixed numbers. +// Inference at runtime is a single dot product over the features in src/ml.ts. +// +// It is deterministic. A seeded PRNG generates the data and initialises the +// weights, so `node scripts/train-confidence-model.mjs` produces the same model +// every time. Re-run it and diff src/confidence-model.ts to review any change. +// +// node scripts/train-confidence-model.mjs # print metrics + model +// node scripts/train-confidence-model.mjs --write # also overwrite the file +// +// Keep the feature list here in lockstep with FEATURE_COUNT / extractFeatures +// in src/ml.ts; a mismatch is caught by test/ml.test.mjs. + +import { writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const OUT = join(HERE, '..', 'src', 'confidence-model.ts'); + +// --- deterministic PRNG (mulberry32) ------------------------------------- +function rng(seed) { + let a = seed >>> 0; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} +const rand = rng(0x9e3779b9); +const pick = (arr) => arr[Math.floor(rand() * arr.length)]; +const randInt = (lo, hi) => lo + Math.floor(rand() * (hi - lo + 1)); +const HEX = '0123456789abcdef'; +const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; +const B64URL = B64 + '-_'; +const LOWER = 'abcdefghijklmnopqrstuvwxyz'; +const DIGITS = '0123456789'; +function draw(alphabet, n) { + let s = ''; + for (let i = 0; i < n; i++) s += alphabet[Math.floor(rand() * alphabet.length)]; + return s; +} + +// --- feature extraction (mirror of src/ml.ts) ---------------------------- +const SECRET_CTX = /\b(secret|api[_-]?key|apikey|token|password|passwd|pwd|auth|authorization|bearer|access[_-]?key|private[_-]?key|client[_-]?secret|credential|signing[_-]?key)\b/i; +const BENIGN_CTX = /\b(uuid|guid|sha1|sha256|sha512|md5|hash|digest|etag|checksum|commit|revision|request[_-]?id|trace[_-]?id|correlation[_-]?id|span[_-]?id|object[_-]?id|content[_-]?id|version|colou?r|slug|filename)\b/i; +const STRUCTURED_HEX = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24}|[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$/i; + +function shannon(s) { + const freq = new Map(); + for (const ch of s) freq.set(ch, (freq.get(ch) ?? 0) + 1); + let e = 0; + for (const n of freq.values()) { + const p = n / [...s].length; + e -= p * Math.log2(p); + } + return e; +} + +const FEATURES = [ + 'log2Len', 'entropy', 'fracLower', 'fracUpper', 'fracDigit', 'fracSymbol', + 'fracHex', 'vowelFrac', 'classTransitionRate', 'hasMixedClasses', + 'maxRunFrac', 'structuredHexId', 'ctxSecret', 'ctxBenign', +]; +const FEATURE_COUNT = FEATURES.length; + +function classOf(code) { + if ((code >= 97 && code <= 122) || (code >= 65 && code <= 90)) return 0; // letter + if (code >= 48 && code <= 57) return 1; // digit + return 2; // symbol +} + +function extractFeatures(value, context = '') { + const len = value.length || 1; + let lower = 0, upper = 0, digit = 0, symbol = 0, hex = 0, vowel = 0, letters = 0; + let transitions = 0, run = 1, maxRun = 1, prevClass = -1; + for (let i = 0; i < value.length; i++) { + const c = value.charCodeAt(i); + const ch = value[i]; + if (c >= 97 && c <= 122) { lower++; letters++; } + else if (c >= 65 && c <= 90) { upper++; letters++; } + else if (c >= 48 && c <= 57) digit++; + else symbol++; + if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70)) hex++; + if ('aeiouAEIOU'.includes(ch)) vowel++; + const cls = classOf(c); + if (prevClass === -1) prevClass = cls; + else { + if (cls !== prevClass) { transitions++; run = 1; } else run++; + if (run > maxRun) maxRun = run; + prevClass = cls; + } + } + const f = new Array(FEATURE_COUNT).fill(0); + f[0] = Math.log2(len); + f[1] = shannon(value); + f[2] = lower / len; + f[3] = upper / len; + f[4] = digit / len; + f[5] = symbol / len; + f[6] = hex / len; + f[7] = letters ? vowel / letters : 0; + f[8] = len > 1 ? transitions / (len - 1) : 0; + f[9] = lower > 0 && upper > 0 && digit > 0 ? 1 : 0; + f[10] = maxRun / len; + f[11] = STRUCTURED_HEX.test(value) ? 1 : 0; + f[12] = SECRET_CTX.test(context) ? 1 : 0; + f[13] = BENIGN_CTX.test(context) ? 1 : 0; + return f; +} + +// --- labelled data generators -------------------------------------------- +const SECRET_CTX_WORDS = ['api_key', 'apiKey', 'secret', 'token', 'authorization', 'bearer', 'access_key', 'client_secret', 'password', 'signing_key']; +const BENIGN_CTX_WORDS = ['uuid', 'commit', 'sha256', 'md5', 'etag', 'request_id', 'trace_id', 'object_id', 'version', 'checksum', 'filename', 'slug']; +const WORDS = ['configuration', 'deployment', 'authentication', 'repository', 'environment', 'controller', 'middleware', 'transaction', 'subscription', 'notification', 'serialization', 'orchestration', 'availability', 'dependencies', 'immutable', 'kubernetes', 'observability', 'idempotent']; + +function wrap(value, ctxWords, p = 0.6) { + if (rand() > p) return { value, context: value }; + const w = pick(ctxWords); + const sep = pick([': ', '=', '="', ' = ', ':']); + return { value, context: `${w}${sep}${value}` }; +} + +function genPositive() { + const kind = randInt(0, 10); + let value; + switch (kind) { + case 0: value = 'AKIA' + draw('ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', 16); break; // aws access key id + case 1: value = draw(B64, 40); break; // aws secret access key + case 2: value = 'ghp_' + draw(B64, 36); break; // github pat + case 3: value = 'xoxb-' + randInt(100000000000, 999999999999) + '-' + draw(B64, 24); break; // slack + case 4: value = 'sk_live_' + draw(B64, 24); break; // stripe + case 5: value = 'AIza' + draw(B64URL, 35); break; // google api key + case 6: value = 'sk-' + draw(B64, 48); break; // openai + case 7: value = draw(B64URL, randInt(27, 43)); break; // generic high-entropy token + case 8: { // jwt-ish base64url triples + value = draw(B64URL, randInt(18, 30)) + '.' + draw(B64URL, randInt(40, 90)) + '.' + draw(B64URL, randInt(30, 43)); + break; + } + case 9: value = draw(HEX, randInt(48, 64)); break; // long hex secret (not a standard digest length band edge) + default: value = draw(B64, randInt(32, 50)); break; + } + // real secrets most often appear next to a secret-ish label + return wrap(value, SECRET_CTX_WORDS, 0.7); +} + +function genNegative() { + const kind = randInt(0, 13); + let value, ctxWords = BENIGN_CTX_WORDS, p = 0.6; + switch (kind) { + case 0: // uuid v4 + value = `${draw(HEX, 8)}-${draw(HEX, 4)}-4${draw(HEX, 3)}-${pick('89ab')}${draw(HEX, 3)}-${draw(HEX, 12)}`; + break; + case 1: value = draw(HEX, 40); break; // git commit sha + case 2: value = draw(HEX, 32); break; // md5 / mongo-ish + case 3: value = draw(HEX, 64); break; // sha256 digest + case 4: value = draw(HEX, 24); break; // mongo objectid + case 5: value = draw(HEX, randInt(7, 12)); break; // short sha + case 6: value = pick(WORDS) + pick(['', '_' + pick(WORDS), pick(WORDS)]); break; // identifiers / words + case 7: { // camelCase identifier + value = pick(WORDS); + value = value[0] + value.slice(1) + pick(WORDS).replace(/^./, (c) => c.toUpperCase()); + ctxWords = ['function', 'const', 'variable', 'field']; + break; + } + case 8: value = `${randInt(2018, 2026)}-${String(randInt(1, 12)).padStart(2, '0')}-${String(randInt(1, 28)).padStart(2, '0')}T${String(randInt(0, 23)).padStart(2, '0')}:${String(randInt(0, 59)).padStart(2, '0')}:${String(randInt(0, 59)).padStart(2, '0')}Z`; break; // iso timestamp + case 9: value = `${randInt(1, 20)}.${randInt(0, 40)}.${randInt(0, 99)}`; ctxWords = ['version', 'release']; break; // semver + case 10: value = String(randInt(100000000000, 999999999999999)); ctxWords = ['order_id', 'user_id', 'account']; break; // numeric id + case 11: value = Buffer.from(pick(WORDS) + ' ' + pick(WORDS)).toString('base64'); break; // base64 of words + case 12: value = 'v' + randInt(1, 9) + '/' + pick(WORDS) + '/' + pick(WORDS) + '-' + draw(LOWER, 6); ctxWords = ['path', 'url', 'route']; break; // slug/path + default: value = draw(HEX, 8); break; // hex color-ish + } + return wrap(value, ctxWords, p); +} + +function buildDataset(nPer) { + const rows = []; + for (let i = 0; i < nPer; i++) { + const pos = genPositive(); + rows.push({ x: extractFeatures(pos.value, pos.context), y: 1, value: pos.value }); + const neg = genNegative(); + rows.push({ x: extractFeatures(neg.value, neg.context), y: 0, value: neg.value }); + } + return rows; +} + +// --- standardisation ------------------------------------------------------ +function standardiser(rows) { + const mean = new Array(FEATURE_COUNT).fill(0); + const std = new Array(FEATURE_COUNT).fill(0); + for (const r of rows) for (let j = 0; j < FEATURE_COUNT; j++) mean[j] += r.x[j]; + for (let j = 0; j < FEATURE_COUNT; j++) mean[j] /= rows.length; + for (const r of rows) for (let j = 0; j < FEATURE_COUNT; j++) std[j] += (r.x[j] - mean[j]) ** 2; + for (let j = 0; j < FEATURE_COUNT; j++) std[j] = Math.sqrt(std[j] / rows.length) || 1; + return { mean, std }; +} +const applyStd = (x, s) => x.map((v, j) => (v - s.mean[j]) / s.std[j]); + +// --- logistic regression (full-batch gradient descent + L2) -------------- +const sigmoid = (z) => 1 / (1 + Math.exp(-z)); + +function train(rows, { epochs = 4000, lr = 0.3, l2 = 1e-4 } = {}) { + const w = new Array(FEATURE_COUNT).fill(0).map(() => (rand() - 0.5) * 0.01); + let b = 0; + const n = rows.length; + for (let e = 0; e < epochs; e++) { + const gw = new Array(FEATURE_COUNT).fill(0); + let gb = 0; + for (const r of rows) { + let z = b; + for (let j = 0; j < FEATURE_COUNT; j++) z += w[j] * r.xs[j]; + const err = sigmoid(z) - r.y; + for (let j = 0; j < FEATURE_COUNT; j++) gw[j] += err * r.xs[j]; + gb += err; + } + for (let j = 0; j < FEATURE_COUNT; j++) w[j] -= lr * (gw[j] / n + l2 * w[j]); + b -= lr * (gb / n); + } + return { w, b }; +} + +function evaluate(rows, model, thr = 0.5) { + let tp = 0, fp = 0, tn = 0, fn = 0, loss = 0; + for (const r of rows) { + let z = model.b; + for (let j = 0; j < FEATURE_COUNT; j++) z += model.w[j] * r.xs[j]; + const p = sigmoid(z); + loss += -(r.y * Math.log(p + 1e-12) + (1 - r.y) * Math.log(1 - p + 1e-12)); + const yhat = p >= thr ? 1 : 0; + if (yhat === 1 && r.y === 1) tp++; + else if (yhat === 1 && r.y === 0) fp++; + else if (yhat === 0 && r.y === 0) tn++; + else fn++; + } + const precision = tp / (tp + fp || 1); + const recall = tp / (tp + fn || 1); + return { + n: rows.length, tp, fp, tn, fn, + accuracy: (tp + tn) / rows.length, + precision, recall, + f1: (2 * precision * recall) / (precision + recall || 1), + logloss: loss / rows.length, + }; +} + +// --- run ------------------------------------------------------------------ +const trainRows = buildDataset(6000); +const holdoutRows = buildDataset(2000); +const std = standardiser(trainRows); +for (const r of trainRows) r.xs = applyStd(r.x, std); +for (const r of holdoutRows) r.xs = applyStd(r.x, std); + +const model = train(trainRows); +const trainMetrics = evaluate(trainRows, model); +const holdoutMetrics = evaluate(holdoutRows, model); + +const fmt = (m) => `acc=${(m.accuracy * 100).toFixed(2)}% precision=${(m.precision * 100).toFixed(2)}% recall=${(m.recall * 100).toFixed(2)}% f1=${(m.f1 * 100).toFixed(2)}% logloss=${m.logloss.toFixed(4)} (n=${m.n}, fp=${m.fp}, fn=${m.fn})`; +console.log('features :', FEATURE_COUNT, FEATURES.join(', ')); +console.log('train :', fmt(trainMetrics)); +console.log('holdout :', fmt(holdoutMetrics)); + +// fold standardisation into raw weights so runtime needs no mean/std: +// z = b + Σ w_j * (x_j - mean_j)/std_j = (b - Σ w_j*mean_j/std_j) + Σ (w_j/std_j) x_j +const rawW = model.w.map((wj, j) => wj / std.std[j]); +const rawB = model.b - model.w.reduce((acc, wj, j) => acc + (wj * std.mean[j]) / std.std[j], 0); + +const round = (x) => Number(x.toFixed(6)); +const ts = `// Generated by scripts/train-confidence-model.mjs — do not edit by hand. +// Logistic-regression weights for the secret-confidence classifier. Trained +// offline on synthetic labelled data; shipped as fixed numbers so the library +// keeps its zero-dependency, deterministic runtime. Regenerate with: +// node scripts/train-confidence-model.mjs --write +// +// Holdout: ${fmt(holdoutMetrics)} + +export interface ConfidenceModel { + readonly version: number; + readonly features: readonly string[]; + readonly weights: readonly number[]; + readonly bias: number; +} + +export const CONFIDENCE_MODEL: ConfidenceModel = { + version: 1, + features: ${JSON.stringify(FEATURES)}, + weights: ${JSON.stringify(rawW.map(round))}, + bias: ${round(rawB)}, +}; +`; + +if (process.argv.includes('--write')) { + writeFileSync(OUT, ts); + console.log('wrote :', OUT); +} else { + console.log('\n--- src/confidence-model.ts (preview, pass --write to save) ---\n'); + console.log(ts); +} diff --git a/src/cli.ts b/src/cli.ts index e8c8025..df855d3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -44,6 +44,10 @@ OPTIONS --hash-salt deprecated; prefer --secret-env --min-confidence drop findings below this confidence (0-1) + --refine-confidence + use the learned classifier to refine confidence for + generic detectors (fewer false positives on UUIDs, + hashes, and slugs); pairs with --min-confidence --include-values include raw matched values in --scan output (unsafe for logs and reports) --only use only these detectors (comma-separated) @@ -70,6 +74,7 @@ EXAMPLES flare-redact --sarif .env > flare-redact.sarif flare-redact --json --mode hash < event.json flare-redact --enable high_entropy,ipv4 < app.log + flare-redact --scan --enable high_entropy --refine-confidence --min-confidence 0.5 app.log `; interface ParsedArgs { @@ -132,6 +137,7 @@ function parseArgs(argv: string[]): ParsedArgs { case '--mode': opts.mode = parseMode(argv[++i]); break; case '--hash-salt': opts.hashSalt = argv[++i]; break; case '--min-confidence': opts.minConfidence = parseConfidence(argv[++i]); break; + case '--refine-confidence': opts.refineConfidence = true; break; case '--include-values': opts.includeValues = true; break; case '--secret-env': secretEnv = envName(argv[++i], '--secret-env'); break; case '--term': { const w = argv[++i]; if (w) terms.push(w); break; } diff --git a/src/confidence-model.ts b/src/confidence-model.ts new file mode 100644 index 0000000..f199ce1 --- /dev/null +++ b/src/confidence-model.ts @@ -0,0 +1,21 @@ +// Generated by scripts/train-confidence-model.mjs — do not edit by hand. +// Logistic-regression weights for the secret-confidence classifier. Trained +// offline on synthetic labelled data; shipped as fixed numbers so the library +// keeps its zero-dependency, deterministic runtime. Regenerate with: +// node scripts/train-confidence-model.mjs --write +// +// Holdout: acc=98.80% precision=99.05% recall=98.55% f1=98.80% logloss=0.0298 (n=4000, fp=19, fn=29) + +export interface ConfidenceModel { + readonly version: number; + readonly features: readonly string[]; + readonly weights: readonly number[]; + readonly bias: number; +} + +export const CONFIDENCE_MODEL: ConfidenceModel = { + version: 1, + features: ["log2Len","entropy","fracLower","fracUpper","fracDigit","fracSymbol","fracHex","vowelFrac","classTransitionRate","hasMixedClasses","maxRunFrac","structuredHexId","ctxSecret","ctxBenign"], + weights: [2.712748,3.920866,-5.874776,5.484371,1.346649,-9.662469,-0.058102,8.500786,2.714721,-0.676255,-3.0573,-6.969532,5.6429,-5.446436], + bias: -29.559382, +}; diff --git a/src/detectors.ts b/src/detectors.ts index 59d5f85..a82ac7e 100644 --- a/src/detectors.ts +++ b/src/detectors.ts @@ -28,6 +28,13 @@ export interface Detector { negative?: RegExp; window?: number; }; + /** + * Let the learned confidence model refine this detector's score when + * `refineConfidence` is enabled. Use it on generic, format-only matchers that + * over-fire on benign high-entropy text; leave it off for checksum-validated + * detectors, which are already certain. + */ + refine?: boolean; /** Cheap literal gate evaluated before the regular expression. */ prefilter?: string[]; } @@ -321,6 +328,7 @@ export const DETECTORS: Detector[] = [ mask: keepPrefix(4), default: false, tags: ['secret'], + refine: true, }, { id: 'person_name', diff --git a/src/engine.ts b/src/engine.ts index 29b9552..e3b5883 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -3,6 +3,7 @@ import { hmacFingerprint } from './crypto.js'; import { pseudonymize, surrogate } from './transforms.js'; import { MULTILANG_KEY_SET } from './i18n.js'; import { buildTermsDetector, type TermSpec } from './terms.js'; +import { secretProbability } from './ml.js'; export type Mode = 'mask' | 'label' | 'hash' | 'pseudonym' | 'surrogate' | 'fpe'; export type Risk = 'low' | 'medium' | 'high' | 'critical'; @@ -62,6 +63,14 @@ export interface RedactOptions { semanticProvider?: SemanticProvider; /** Drop findings below this confidence (default: 0). */ minConfidence?: number; + /** + * Let the learned classifier refine confidence for generic detectors that + * over-fire on benign high-entropy text (UUIDs, git SHAs, digests, slugs). + * Only detectors marked `refine` are affected; checksum-validated ones are + * left untouched. Pair with `minConfidence` to drop likely false positives. + * Default: false. + */ + refineConfidence?: boolean; /** Include raw matched values in scan results. Unsafe for logs and reports. */ includeValues?: boolean; limits?: { @@ -207,7 +216,7 @@ export function scanString( : normalizedEnd; const value = text.slice(start, end); if (allow(value) || (value !== normalizedValue && allow(normalizedValue))) continue; - const confidence = scoreConfidence(det, text, start, end); + const confidence = scoreConfidence(det, text, start, end, opts); if (confidence < (opts.minConfidence ?? 0)) continue; hits.push({ detector: det.id, @@ -308,18 +317,33 @@ function detectorRisk(det: Detector): Risk { return 'high'; } -function scoreConfidence(det: Detector, text: string, start: number, end: number): number { +/** How far the learned model can move a base confidence score, up or down. */ +const REFINE_STRENGTH = 0.4; + +function scoreConfidence( + det: Detector, + text: string, + start: number, + end: number, + opts: RedactOptions = {}, +): number { let score = det.confidence ?? (det.id === 'high_entropy' ? 0.6 : det.validate ? 0.99 : 0.92); - if (!det.context) return score; - const radius = det.context.window ?? 80; - const nearby = text.slice(Math.max(0, start - radius), Math.min(text.length, end + radius)); - if (det.context.positive) { - det.context.positive.lastIndex = 0; - if (det.context.positive.test(nearby)) score += 0.06; + if (det.context) { + const radius = det.context.window ?? 80; + const nearby = text.slice(Math.max(0, start - radius), Math.min(text.length, end + radius)); + if (det.context.positive) { + det.context.positive.lastIndex = 0; + if (det.context.positive.test(nearby)) score += 0.06; + } + if (det.context.negative) { + det.context.negative.lastIndex = 0; + if (det.context.negative.test(nearby)) score -= 0.25; + } } - if (det.context.negative) { - det.context.negative.lastIndex = 0; - if (det.context.negative.test(nearby)) score -= 0.25; + if (opts.refineConfidence && det.refine) { + const window = text.slice(Math.max(0, start - 64), Math.min(text.length, end + 64)); + const p = secretProbability(text.slice(start, end), window); + score += (p - 0.5) * REFINE_STRENGTH; } return Math.max(0, Math.min(1, score)); } diff --git a/src/index.ts b/src/index.ts index 45287c7..6ea1796 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,15 @@ export { entropy, fnv1a, } from './detectors.js'; +export { + secretProbability, + extractFeatures, + shannonEntropy, + FEATURES, + FEATURE_COUNT, + CONFIDENCE_MODEL, + type ConfidenceModel, +} from './ml.js'; export { pseudonymize, surrogate, fpe } from './transforms.js'; export { hmacFingerprint } from './crypto.js'; export { diff --git a/src/ml.ts b/src/ml.ts new file mode 100644 index 0000000..bf21e72 --- /dev/null +++ b/src/ml.ts @@ -0,0 +1,116 @@ +// Secret-confidence classifier. +// +// A tiny logistic-regression model that scores how likely a matched string is a +// real secret rather than a benign look-alike (a UUID, git SHA, digest, slug, +// or dictionary word). It exists because pattern-only detection over-fires on +// high-entropy text, and confidence should reflect that. +// +// The model is trained offline by scripts/train-confidence-model.mjs and shipped +// as fixed weights in confidence-model.ts, so this stays zero-dependency, +// synchronous, deterministic, and safe on edge and browser runtimes. Inference +// is one pass to build the feature vector plus a dot product — no matrix +// library, no async, no network. +// +// The feature list here MUST match scripts/train-confidence-model.mjs exactly; +// test/ml.test.mjs asserts the count and order against the shipped model. + +import { CONFIDENCE_MODEL, type ConfidenceModel } from './confidence-model.js'; + +export type { ConfidenceModel } from './confidence-model.js'; +export { CONFIDENCE_MODEL } from './confidence-model.js'; + +/** Feature names in the exact order extractFeatures returns them. */ +export const FEATURES = [ + 'log2Len', 'entropy', 'fracLower', 'fracUpper', 'fracDigit', 'fracSymbol', + 'fracHex', 'vowelFrac', 'classTransitionRate', 'hasMixedClasses', + 'maxRunFrac', 'structuredHexId', 'ctxSecret', 'ctxBenign', +] as const; + +export const FEATURE_COUNT = FEATURES.length; + +const SECRET_CTX = /\b(secret|api[_-]?key|apikey|token|password|passwd|pwd|auth|authorization|bearer|access[_-]?key|private[_-]?key|client[_-]?secret|credential|signing[_-]?key)\b/i; +const BENIGN_CTX = /\b(uuid|guid|sha1|sha256|sha512|md5|hash|digest|etag|checksum|commit|revision|request[_-]?id|trace[_-]?id|correlation[_-]?id|span[_-]?id|object[_-]?id|content[_-]?id|version|colou?r|slug|filename)\b/i; +const STRUCTURED_HEX = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24}|[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$/i; + +/** Shannon entropy in bits per symbol, over Unicode code points. */ +export function shannonEntropy(s: string): number { + const freq = new Map(); + let total = 0; + for (const ch of s) { + freq.set(ch, (freq.get(ch) ?? 0) + 1); + total++; + } + if (total === 0) return 0; + let e = 0; + for (const n of freq.values()) { + const p = n / total; + e -= p * Math.log2(p); + } + return e; +} + +/** + * Cheap character-level features for `value`, optionally informed by nearby + * `context` text. Returns a fixed-length numeric vector aligned with FEATURES. + */ +export function extractFeatures(value: string, context = ''): number[] { + const len = value.length || 1; + let lower = 0, upper = 0, digit = 0, symbol = 0, hex = 0, vowel = 0, letters = 0; + let transitions = 0, run = 1, maxRun = 1, prevClass = -1; + for (let i = 0; i < value.length; i++) { + const c = value.charCodeAt(i); + if (c >= 97 && c <= 122) { lower++; letters++; } + else if (c >= 65 && c <= 90) { upper++; letters++; } + else if (c >= 48 && c <= 57) digit++; + else symbol++; + if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70)) hex++; + if (c === 97 || c === 101 || c === 105 || c === 111 || c === 117 || + c === 65 || c === 69 || c === 73 || c === 79 || c === 85) vowel++; + const cls = c >= 48 && c <= 57 ? 1 : (c >= 97 && c <= 122) || (c >= 65 && c <= 90) ? 0 : 2; + if (prevClass === -1) prevClass = cls; + else { + if (cls !== prevClass) { transitions++; run = 1; } else run++; + if (run > maxRun) maxRun = run; + prevClass = cls; + } + } + const f = new Array(FEATURE_COUNT).fill(0); + f[0] = Math.log2(len); + f[1] = shannonEntropy(value); + f[2] = lower / len; + f[3] = upper / len; + f[4] = digit / len; + f[5] = symbol / len; + f[6] = hex / len; + f[7] = letters ? vowel / letters : 0; + f[8] = len > 1 ? transitions / (len - 1) : 0; + f[9] = lower > 0 && upper > 0 && digit > 0 ? 1 : 0; + f[10] = maxRun / len; + f[11] = STRUCTURED_HEX.test(value) ? 1 : 0; + f[12] = SECRET_CTX.test(context) ? 1 : 0; + f[13] = BENIGN_CTX.test(context) ? 1 : 0; + return f; +} + +const sigmoid = (z: number): number => (z >= 0 + ? 1 / (1 + Math.exp(-z)) + : Math.exp(z) / (1 + Math.exp(z))); + +/** + * Probability in [0, 1] that `value` is a real secret rather than a benign + * high-entropy string. `context` is the surrounding text, which lets nearby + * labels such as `api_key=` or `commit:` shift the score. + * + * Pass an alternative `model` to score against a custom-trained classifier with + * the same feature layout. + */ +export function secretProbability( + value: string, + context = '', + model: ConfidenceModel = CONFIDENCE_MODEL, +): number { + const x = extractFeatures(value, context); + let z = model.bias; + for (let j = 0; j < FEATURE_COUNT; j++) z += (model.weights[j] ?? 0) * x[j]!; + return sigmoid(z); +} diff --git a/test/ml.test.mjs b/test/ml.test.mjs new file mode 100644 index 0000000..787b685 --- /dev/null +++ b/test/ml.test.mjs @@ -0,0 +1,111 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + secretProbability, + extractFeatures, + shannonEntropy, + FEATURES, + FEATURE_COUNT, + CONFIDENCE_MODEL, + scan, +} from '../dist/index.js'; +import { secretProbability as secretProbabilitySub } from '../dist/ml.js'; + +const REAL_SECRETS = [ + ['openai', 'sk-abcDEF1234567890ghIJKL7890mnOPqrST1234uvWXyz5678AB', 'api_key='], + ['github pat', 'ghp_16CharsABCdef0123456789ABCDEFghijkl', 'token='], + ['aws secret', 'wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEYabcd', 'aws_secret_access_key='], + ['generic token', 'Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv', 'authorization: Bearer '], +]; + +const BENIGN_LOOKALIKES = [ + ['uuid v4', '3f2504e0-4f89-41d3-9a0c-0305e82c3301', 'request_id='], + ['git sha', '9fceb02d0ae598e95dc970b74767f19372d61af8', 'commit '], + ['md5', 'd41d8cd98f00b204e9800998ecf8427e', 'etag: '], + ['sha256', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'checksum='], + ['object id', '507f1f77bcf86cd799439011', 'object_id='], + ['word', 'configurationDeployment', 'variable '], +]; + +test('feature layout matches the shipped model exactly', () => { + assert.equal(FEATURE_COUNT, FEATURES.length); + assert.equal(FEATURE_COUNT, CONFIDENCE_MODEL.weights.length); + assert.deepEqual([...FEATURES], [...CONFIDENCE_MODEL.features]); + assert.equal(extractFeatures('abc123', 'token=abc123').length, FEATURE_COUNT); +}); + +test('secretProbability separates real secrets from benign look-alikes', () => { + for (const [name, value, ctx] of REAL_SECRETS) { + const p = secretProbability(value, ctx + value); + assert.ok(p >= 0.5, `${name} should read as a secret, got ${p.toFixed(3)}`); + } + for (const [name, value, ctx] of BENIGN_LOOKALIKES) { + const p = secretProbability(value, ctx + value); + assert.ok(p < 0.5, `${name} should read as benign, got ${p.toFixed(3)}`); + } +}); + +test('secretProbability is bounded and deterministic', () => { + const value = 'Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv'; + const a = secretProbability(value, 'token=' + value); + const b = secretProbability(value, 'token=' + value); + assert.equal(a, b); + assert.ok(a >= 0 && a <= 1); + assert.equal(secretProbability('', ''), secretProbability('', '')); + assert.ok(secretProbability('', '') >= 0 && secretProbability('', '') <= 1); +}); + +test('subpath export ./ml matches the top-level export', () => { + const value = '9fceb02d0ae598e95dc970b74767f19372d61af8'; + assert.equal(secretProbability(value, 'commit ' + value), secretProbabilitySub(value, 'commit ' + value)); +}); + +test('shannonEntropy matches known values', () => { + assert.equal(shannonEntropy(''), 0); + assert.equal(shannonEntropy('aaaa'), 0); + assert.equal(shannonEntropy('ab'), 1); + assert.equal(shannonEntropy('abcd'), 2); +}); + +test('nearby secret context raises probability, benign context lowers it', () => { + const value = 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6'; + const withSecret = secretProbability(value, 'api_key=' + value); + const withBenign = secretProbability(value, 'md5 digest ' + value); + assert.ok(withSecret > withBenign); +}); + +test('refineConfidence drops high-entropy false positives but keeps real secrets', () => { + const gitSha = '9fceb02d0ae598e95dc970b74767f19372d61af8'; // benign, 40 hex + const sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; // benign, 64 hex + const token = 'Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv'; // real-looking secret + const text = [gitSha, sha256, token].map((v) => `"${v}"`).join(' '); + const base = { enable: ['high_entropy'] }; + + const plain = scan(text, base).map((f) => text.slice(f.start, f.end)); + assert.ok(plain.includes(gitSha) && plain.includes(sha256) && plain.includes(token), + 'plain scan should flag every high-entropy run'); + + const refined = scan(text, { ...base, refineConfidence: true, minConfidence: 0.5 }); + const kept = refined.map((f) => text.slice(f.start, f.end)); + assert.ok(kept.includes(token), 'real token must survive refinement'); + assert.ok(!kept.includes(gitSha), 'git sha must be dropped'); + assert.ok(!kept.includes(sha256), 'sha256 digest must be dropped'); +}); + +test('refineConfidence leaves checksum-validated detectors untouched', () => { + const text = 'card 4242 4242 4242 4242'; + const plain = scan(text); + const refined = scan(text, { refineConfidence: true }); + const card = (list) => list.find((f) => f.detector === 'credit_card'); + assert.ok(card(plain), 'baseline should detect the card'); + assert.ok(card(refined), 'refinement must not remove a checksum-validated card'); + assert.equal(card(plain).confidence, card(refined).confidence); +}); + +test('refineConfidence is off by default and does not change scores', () => { + const value = 'Zx9Kq2Lm7Pv4Rt6Wy8Bn3Cf5Hj1Dg0As7Uv'; + const text = `"${value}"`; + const off = scan(text, { enable: ['high_entropy'] }); + const explicitOff = scan(text, { enable: ['high_entropy'], refineConfidence: false }); + assert.equal(off[0].confidence, explicitOff[0].confidence); +}); diff --git a/test/package-exports.test.mjs b/test/package-exports.test.mjs index 0c24888..e7240d6 100644 --- a/test/package-exports.test.mjs +++ b/test/package-exports.test.mjs @@ -6,6 +6,7 @@ const entryPoints = [ 'index', 'stream', 'llm', + 'ml', 'tool', 'session', 'pino',