diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 95310b687..b5e5bfb3c 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -182,19 +182,63 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ sensitiveSpan: (m) => credentialAfterPrefix(m, m[1]), }, // JWT — three base64url segments, the first beginning `eyJ` (`{"…`). + // GLUED-PREFIX: leading `\b` KEPT deliberately. `eyJ` is not a delimiter-prefix — it is the base64url + // of `{"` and thus part of the token BODY (base64url includes `_`/`-`), so dropping the boundary would + // let `eyJ` match mid-base64-blob and newly over-redact benign base64 (a case the no-degrade corpus + // guards). The boundary stays; a JWT glued after a word char is out of this fix's enumerated scope. { pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // GitHub tokens: classic `gh?_` prefixes AND fine-grained `github_pat_`. + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the classic set {gho,ghp,ghr,ghs,ghu} is SPLIT by prefix, + // and the leading `\b` is DROPPED ONLY on the members that are PROVABLY not a benign word / identifier + // fragment — so a real token glued DIRECTLY after a word char (`keyghp_…`, the Advisor P1 case) is + // caught, WITHOUT re-introducing the demonstrated benign over-redaction. The classic body is MIXED-CASE + // base62 with a `_` separator, so the collision surface is a LOWERCASE snake_case identifier ending in + // the member (`_<16+ contiguous base62>`) — even an obscure lowercase word matters here (UNLIKE + // the uppercase-only AWS body below). Classification (grep /usr/share/dict/words for word-endings): + // • ghp — 0 dict word-endings, not a benign fragment → UNAMBIGUOUS → `\b` DROPPED (a glued + // `keyghp_<36>` classic PAT is now caught; this is the Advisor P1 canary). + // • ghr — 0 dict word-endings, not a benign fragment → UNAMBIGUOUS → `\b` DROPPED (a glued + // `keyghr_<…>` refresh token is now caught). + // • ghs — a PLURAL word-ending (walkthrou-GHS, cou-GHS, breakthrou-GHS, hi-GHS, lau-GHS, si-GHS, + // trou-GHS, …) → AMBIGUOUS → `\b` KEPT (else `walkthroughs_completedThisWeek` — a CONTIGUOUS 17 + // base62 run — over-redacts to `walkthrou[REDACTED]`). + // • gho — dict word-endings (sor-GHO, a millet; San-GHO, a language) → a lowercase + // `sorgho_` would over-redact → AMBIGUOUS → `\b` KEPT. + // • ghu — dict word-ending (Ra-GHU, a common given name) → `raghu_` would over-redact → + // AMBIGUOUS → `\b` KEPT. + // • github_pat_ — contains the natural word "github" → AMBIGUOUS → `\b` KEPT. + // Effect for the KEPT members {gho,ghs,ghu,github_pat_}: a DELIMITED / standalone / labeled classic + // token (`gho_`, `token: ghu_…`, `apikey=ghs_…`) is STILL caught by the assignment/whitespace + // paths (the common case); only a token glued DIRECTLY after a word char stays an ACCEPTED gap for + // THOSE word-fragment prefixes (like `sk-`/`AIza`/`ya29.`/`eyJ`). For the DROPPED members {ghp,ghr} a + // glued token is now REDACTED at exactly the credential subspan (the benign leading char survives + // byte-for-byte). Bodies stay BASE62 (`[A-Za-z0-9]`) so a snake_case identifier breaks at the first `_` + // (open-ended `{16,}` — never tightened to an exact length, so a longer real token cannot slip the + // trailing `\b`); the `github_pat_` branch keeps its `_`-inclusive body (a fine-grained PAT legitimately + // has `_` in its body). { - pattern: /\b(?:gh[opsru]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, + // UNAMBIGUOUS classic prefixes — leading `\b` DROPPED so a glued `ghp_<36>` / `ghr_<…>` is caught. + pattern: /gh[pr]_[A-Za-z0-9]{16,}\b/gu, + sensitiveSpan: wholeMatchSpan, + }, + { + // AMBIGUOUS classic prefixes (word-fragment endings gho/ghs/ghu) + fine-grained `github_pat_` — leading `\b` KEPT. + pattern: /\b(?:gh[osu]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, sensitiveSpan: wholeMatchSpan, }, // Provider hyphen-prefixed API keys: OpenAI `sk-` / `sk-proj-`, `rk-`, `ak-`, xAI `xai-`. The // client-safe PUBLISHABLE `pk-` prefix is DELIBERATELY ABSENT from the alternation (round-16): a // Stripe/Google publishable key is not a secret, and Friday's own domain uses `pk-` for a satellite // PUBLIC key (`publicKey: "pk-…"`), so redacting a `pk-` value is data-loss, not protection. + // GLUED-PREFIX: leading `\b` KEPT deliberately. `sk`/`rk`/`ak` are SHORT (2-char) prefixes that glue + // into extremely common words ending in them — `desk-`, `risk-`, `task-`, `disk-`, `mask-`, `work-`, + // `mark-`, `fork-`, `network-`, `break-`, `speak-`, `leak-` — and the `-`-inclusive body would then + // over-redact a benign kebab identifier (`desk-management-framework-v2` → `sk-management-framework-v2`). + // The boundary is REQUIRED here; this is the canonical "short/low-signal prefix" exclusion. { pattern: /\b(?:sk|rk|ak|xai)-(?:proj-)?[A-Za-z0-9_-]{16,}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -206,6 +250,10 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // key never matches. (Stripe key bodies are base62 with no internal `_`, so the value run is // `[A-Za-z0-9]{16,}`; the leading `(?:sk|rk)_(?:live|test)` / `whsec` cannot false-fire on benign // snake_case identifiers.) + // GLUED-PREFIX: leading `\b` KEPT deliberately. The `sk_`/`rk_` prefixes glue into the SAME common + // words as the hyphen form (`desk_live_…`, `risk_test_…`, `task_live_…`), so — like `sk-`/`rk-` — the + // boundary is retained; catching a glued Stripe underscore-key is not worth the benign-`sk_`/`rk_` + // over-redaction risk. (The distinctive `whsec_` shares this pattern and inherits the kept boundary.) { pattern: /\b(?:(?:sk|rk)_(?:live|test)|whsec)_[A-Za-z0-9]{16,}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -213,11 +261,52 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // AWS access-key id (`AKIA…` long-term + STS/temporary variants). NB: the AWS SECRET access key is // a shapeless 40-char base64 with no prefix — it is caught by its KEY NAME (`secretAccessKey`, // in the sensitive-field-name set), NOT a shape, so it is not (and must not be) a content pattern. + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the AWS family is SPLIT by prefix and only `AKIA` catches + // a glued credential. The AWS body is `[0-9A-Z]{16}` — UPPERCASE + DIGIT, i.e. the alphabet of an + // all-caps constant / ULID / base32 blob — so (UNLIKE the mixed-case github body above) a member + // over-redacts ONLY when it sits inside an ALL-CAPS / alphanumeric 20+ contiguous run; a lowercase prose + // word never collides. The 7 other prefixes KEEP `\b` — each is (a) a COMMON all-caps word-ending, (b) + // ULID-constructible (Crockford base32 = 0-9 A-Z minus I,L,O,U — a member of only Crockford chars can + // sit mid-ULID), or (c) a common all-caps ACRONYM / fragment: + // • ASIA — a COMMON word-ending (Eur-ASIA, Austral-ASIA, aph-ASIA) written all-caps in region + // constants (…ASIA…, AUSTRALASIA…, EURASIA…) → (a),(c) → AMBIGUOUS → `\b` KEPT. + // • AGPA / ANPA / ANVA — Crockford-only (no I/L/O/U) → ULID-CONSTRUCTIBLE (sit mid-ULID, e.g. + // `012345AGPA…`); ANVA is also inside C-ANVA-S → (b) → AMBIGUOUS → `\b` KEPT. + // • AIDA — a common all-caps ACRONYM (the AIDA marketing model, PROJECT-AIDA, the opera) → (c) → + // AMBIGUOUS → `\b` KEPT. + // • AROA — the place name AOTE-AROA (all-caps-able in NZ constants) → (a),(c) → AMBIGUOUS → `\b` KEPT. + // • AIPA — a prefix of the common all-caps acronym / org AIPA-C (`AIPACPOLICYCONFERENCE…`) → (c) → + // AMBIGUOUS → `\b` KEPT. + // • AKIA — has `I` → NOT ULID-constructible, and no common all-caps ACRONYM. BUT it IS the SUFFIX of + // the country names SLOV-AKIA and CZECHOSLOV-AKIA (proper nouns absent from /usr/share/dict/words, + // so the per-word dict analysis missed them) — an ALL-CAPS `SLOVAKIA<16>` region constant would be + // corrupted by a plain leading-`\b` drop. So AKIA is de-bounded with a CONTEXT-SENSITIVE + // NEGATIVE-LOOKBEHIND `(?` / `=AKIA<16>` / ` AKIA<16>`), NOT when it is embedded + // after an UPPERCASE letter or DIGIT (mid-all-caps-word `SLOVAKIA…` / mid-alphanumeric run + // `PROJECT2AKIA…`). The lookbehind is zero-width, so the redacted span is still exactly `AKIA<16>` + // and the benign leading char survives byte-for-byte. This is robust (no per-word enumeration). + // Effect: a DELIMITED / standalone / labeled / glued-after-lowercase AWS key is STILL caught (the common + // case — AWS keys are ~always delimited in env/config, and the glued `keyAKIA<16>` P1 canary is caught), + // while an ALL-CAPS / alphanumeric blob carrying an AMBIGUOUS fragment (`012345AGPA…`, `…ASIA…WIDE…`, + // `PROJECTAIDA…`) OR `SLOVAKIA…`/`CZECHOSLOVAKIA…`/`PROJECT2AKIA…` (AKIA after `[A-Z0-9]`) is UNCHANGED. + { + // UNAMBIGUOUS — de-bounded via a CONTEXT-SENSITIVE negative-lookbehind (NOT a plain `\b`-drop) so a + // glued `AKIA<16>` is caught but a mid-all-caps-word `SLOVAKIA<16>` is NOT. + pattern: /(?`) is a plausible-enough + // benign collision that the NO-DEGRADE hard bar favors the boundary over catching a (rare) glued key. { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -227,42 +316,53 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // distinctive). No trailing `\b` — the body legitimately ends in `.`/`-`/`_`; the greedy run grabs // the whole token and the marker replaces it. A benign near-miss `ya29_notatoken` (underscore, not // the required literal `.`) never matches. + // GLUED-PREFIX: leading `\b` KEPT deliberately. A username-style identifier ending in `ya29` + // (`maya29`, `sonya29`, `priya29`) followed by `.` would collide (`maya29.profile_image_url` + // → `ya29.profile_image_url`), so the boundary is retained over catching a (rare) glued access token. { pattern: /\bya29\.[A-Za-z0-9._-]{20,}/gu, sensitiveSpan: wholeMatchSpan, }, // Google OAuth CLIENT SECRET — `GOCSPX-` + a base64url body (distinctive prefix, low false positive). + // GLUED-PREFIX: leading `\b` dropped — `GOCSPX-` is an all-caps, non-word prefix (no benign glue). { - pattern: /\bGOCSPX-[A-Za-z0-9_-]{20,}\b/gu, + pattern: /GOCSPX-[A-Za-z0-9_-]{20,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // GitLab personal access token — `glpat-` + 20+ base64url chars (distinctive prefix). A benign // `glpat_docs` (underscore, not the required hyphen) never matches. + // GLUED-PREFIX: leading `\b` dropped — no benign identifier segment ends in `glpat` before a `-`. { - pattern: /\bglpat-[A-Za-z0-9_-]{20,}\b/gu, + pattern: /glpat-[A-Za-z0-9_-]{20,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // SendGrid API key — the exact `SG.<22>.<43>` structure (two dot-separated base64url runs of fixed // length). The structured length is what makes `SG.` specific enough to avoid a benign-text false fire. + // GLUED-PREFIX: leading `\b` dropped — the rigid `SG..` shape is what gates + // it, so a glued `imgSG.<22>.<43>` cannot arise from a plausible benign identifier. { - pattern: /\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/gu, + pattern: /SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Square access token (`sq0atp-`) / OAuth client secret (`sq0csp-`) — distinctive `sq0` prefix. The // newer `EAAA…` form is DELIBERATELY excluded (4-char base64 prefix, unacceptable false-positive risk). + // GLUED-PREFIX: leading `\b` dropped — `sq0atp-`/`sq0csp-` are not benign identifier fragments. { - pattern: /\bsq0(?:atp|csp)-[A-Za-z0-9_-]{22,60}\b/gu, + pattern: /sq0(?:atp|csp)-[A-Za-z0-9_-]{22,60}\b/gu, sensitiveSpan: wholeMatchSpan, }, // DigitalOcean personal access token — `dop_v1_` + exactly 64 hex (distinctive prefix + fixed length). + // GLUED-PREFIX: leading `\b` dropped — `dop_v1_` + EXACTLY 64 lowercase-hex is astronomically specific. { - pattern: /\bdop_v1_[a-f0-9]{64}\b/gu, + pattern: /dop_v1_[a-f0-9]{64}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Groq API key — `gsk_` + 40+ base62 chars (Friday's provider-catalog classifies `gsk_` as Groq with // HIGH confidence; the body class excludes `_`, so a benign `gsk_` snake_case identifier never matches). + // GLUED-PREFIX: leading `\b` dropped — see the hf_ note. Body `[A-Za-z0-9]{40,}` EXCLUDES `_`/`-`, + // so a snake_case run cannot span it and `gsk` is not a benign fragment; a glued `xgsk_<40>` is caught. { - pattern: /\bgsk_[A-Za-z0-9]{40,}\b/gu, + pattern: /gsk_[A-Za-z0-9]{40,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // HuggingFace user access token — `hf_` + 34+ base62 chars (Friday's provider-catalog classifies `hf_` @@ -272,20 +372,35 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // open-ended (NOT exact `{34}`) so a longer fine-grained token cannot slip the trailing `\b` and leak, // mirroring the `gsk_`/`glpat-` open-ended convention. The body class excludes `_`, so a benign short // `hf_docs` / `hf_` / `hf_config_value` snake_case identifier never matches (distinctive, low false-positive). + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the leading `\b` is DROPPED so a credential glued to a + // preceding ASCII word char (`keyhf_<34>`) — which has NO word boundary before `hf` — is still caught. + // This is safe ONLY because the body is long + high-entropy: `[A-Za-z0-9]{34,}` EXCLUDES `_` and `-`, + // so it cannot span a snake_case / kebab identifier boundary, and no benign identifier is realistically + // going to contain the literal `hf_` followed by 34+ contiguous base62 chars. The trailing `\b` (and + // the whole-match span) are UNCHANGED, so the redacted span is exactly `hf_` and the benign + // leading char (`key`) survives byte-for-byte. Same reasoning applies to every de-`\b`'d shape below/above. { - pattern: /\bhf_[A-Za-z0-9]{34,}\b/gu, + pattern: /hf_[A-Za-z0-9]{34,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // npm access token — `npm_` + 36 base62 chars. The body class excludes `_`, so a benign `npm_` // config identifier (`npm_config_cache`) never matches (it is short and contains `_`). + // GLUED-PREFIX: leading `\b` dropped — body `[A-Za-z0-9]{36}` excludes `_`, `npm` is not a benign + // fragment ending a word before `_`, so a glued `xnpm_<36>` is caught without over-matching. { - pattern: /\bnpm_[A-Za-z0-9]{36}\b/gu, + pattern: /npm_[A-Za-z0-9]{36}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Slack tokens: bot/user `xox[abprs]-` AND app-level `xapp-` (Socket Mode). Friday's own Slack setup // recipe marks the `xapp-` app-level token `sensitive: true`; the `xapp-` prefix is distinctive. + // + // GLUED-PREFIX: the leading `\b` is DROPPED for the `xox[abprs]-` bot/user branch only (`xoxb`/`xoxp`/… + // are not benign identifier fragments, so a glued `zxoxb-<10+>` is caught). The `xapp-` branch KEEPS its + // leading `\b` (moved onto the branch): `app` is a ubiquitous fragment and `xapp` glues into plausible + // benign identifiers (`maxapp-…`, `linuxapp-…`), which a `-`-inclusive body would then over-redact. { - pattern: /\b(?:xox[abprs]|xapp)-[A-Za-z0-9-]{10,}\b/gu, + pattern: /(?:xox[abprs]|\bxapp)-[A-Za-z0-9-]{10,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Generic credential ASSIGNMENT embedded in free text: `=` / `: `. diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 92a130b39..58bd8964b 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -157,6 +157,30 @@ describe("redactEventPayload — secret shapes (no regression)", () => { expect(s).toContain("[REDACTED]"); }); + // SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding word + // char (`keyhf_<34>`) had no word boundary before the prefix, so the canonical detector's leading + // `\b` skipped it and it egressed on-wire to agents/channels. Built from parts (`seg`) so no literal + // token appears in SOURCE. RED on bf6968f9 (credential survives in the payload), GREEN after. + it("redacts a distinctive-prefix credential GLUED after a word char (glued-prefix evasion, on-wire)", () => { + const seg = (...p: string[]) => p.join(""); // pragma: allowlist secret + const HF = seg("hf_", "AbCdEfGhIjKlMnOpQrStUvWxYz01234567"); // pragma: allowlist secret — 34 base62 + const GSK = seg("gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret — 42 base62 + const GLPAT = seg("glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_` and AWS `AKIA` were SPLIT out and de-`\b`'d — + // a real credential glued after a word char (`keyghp_<36>`, `keyAKIA<16>`) must now be caught on-wire. + const GHP = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + const out = redactEventPayload({ + log: seg("deploy key", HF, " and x", GSK, " and id", GLPAT, " and gh", GHP, " and aws", AKIA, " done"), + }); + const s = serialize(out); + for (const cred of [HF, GSK, GLPAT, GHP, AKIA]) expect(s, cred).not.toContain(cred); + expect(s).toContain("[REDACTED]"); + // Benign surrounding + glued leading chars survive (only the credential subspan is masked). + expect(s).toContain("deploy key"); + expect(s).toContain("done"); + }); + it("masks values under legacy sensitive keys", () => { const out = redactEventPayload({ password: "hunter2-a5", // pragma: allowlist secret @@ -201,6 +225,30 @@ describe("redactEventPayload — no over-redaction (NO DEGRADE)", () => { expect(out).toEqual(payload); }); + // NO-DEGRADE (SEC-SECRET-GLUED-PREFIX-001 round-2): benign snake_case words ending in `ghs`/etc before + // `_` must NOT be over-redacted on-wire — the github-classic base62 body breaks at the `_`. The + // first-round `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED]`). + it("preserves benign `…ghs_` identifiers (github-classic base62 body, no over-redaction)", () => { + const payload = { + walk: "walkthroughs_completed_counter", + contig: "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run — only the leading `\b` closes this + breakt: "breakthroughs_this_quarter_list", + cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix + laugh: "laughs_per_minute_counter", + note: "metric name walkthroughs_started_and_completed today", + // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. + ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?` region constant UNCHANGED (a plain `\b`-drop corrupts it → `SLOV[REDACTED]`). + country: "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` (uppercase) + country2: "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` + }; + const out = redactEventPayload(payload); + expect(out).toEqual(payload); + expect(serialize(out)).not.toContain("[REDACTED]"); + }); + it("preserves a pure-digit object key (business id, \\p{Nd} exemption)", () => { const out = redactEventPayload({ "1234567890123456": "ok" }); // pragma: allowlist secret expect(Object.keys(out)).toContain("1234567890123456"); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 898ed8674..6da43482d 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -271,3 +271,94 @@ describe("FridayMemoryOutputFilter — round-17 HuggingFace hf_ shape (content + expect(out.content).toBe("hf_docs reference"); }); }); + +// ─── SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding ASCII +// word char (`keyhf_<34>`) had no word boundary before the prefix, so the canonical detector's +// leading `\b` skipped it and it survived memory read-back to agents through EVERY output-filter leg +// (content scanAndTransform / metadata redactDeep / tag drop). RED on bf6968f9 (glued token verbatim +// in content + metadata, kept as a tag); GREEN after the leading `\b` is dropped on the high-entropy +// distinctive-prefix patterns. Built from parts so no contiguous literal token appears in SOURCE. ─── +describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinctive-prefix credential", () => { + const filter = createFridayMemoryOutputFilter(); + const seg = (...p: string[]): string => p.join(""); // pragma: allowlist secret + const HF_BODY = "AbCdEfGhIjKlMnOpQrStUvWxYz01234567"; // pragma: allowlist secret — 34 base62 + const HF = seg("hf_", HF_BODY); // pragma: allowlist secret + const GLUED = seg("key", HF); // pragma: allowlist secret — hf_ glued after the word `key` + + it("redacts a GLUED hf_ token in item.content (scanAndTransform leg), leaving the benign leading `key`", () => { + const out = filter.filterItem(makeItem({ content: seg("auth used ", GLUED, " today") })); + expect(out.content).toContain(M); + expect(out.content).not.toContain(HF); + expect(out.content).not.toContain(HF_BODY); + // Only the credential subspan is masked — the glued `key` and surrounding text survive. + expect(out.content).toBe(`auth used key${M} today`); + }); + + it("redacts a GLUED hf_ token carried in item.metadata VALUES (redactDeep leg) under a non-sensitive key", () => { + const out = filter.filterItem(makeItem({ metadata: { tokenPreview: GLUED, note: "keep" } })); + const md = out.metadata as { tokenPreview: string; note: string }; + expect(md.tokenPreview).toBe(`key${M}`); + expect(md.note).toBe("keep"); + expect(JSON.stringify(md)).not.toContain(HF_BODY); + }); + + it("DROPS a GLUED hf_-shaped tag, preserving benign tags byte-identical", () => { + const out = filter.filterItem(makeItem({ tags: ["ok", GLUED, "fine"] })); + expect(JSON.stringify(out.tags)).not.toContain(HF_BODY); + expect(out.tags).toEqual(["ok", "fine"]); + }); + + it("NO-DEGRADE: a word-embedded near-miss with a SHORT body is not a token (survives)", () => { + // `keyhf_` + a short body has no 34-char high-entropy run, so it is NOT a credential — untouched. + const out = filter.filterItem(makeItem({ tags: [seg("key", "hf_docs")], content: seg("ref key", "hf_docs") })); + expect(out.tags).toEqual([seg("key", "hf_docs")]); + expect(out.content).toBe(seg("ref key", "hf_docs")); + }); + + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_`/`ghr_` and AWS `AKIA` were SPLIT out of their + // alternations and de-`\b`'d (they are provably NOT benign word/ULID/acronym fragments), so a real + // credential GLUED after a word char is now caught in every output-filter leg. RED on 7021926c (glued + // token survives content + metadata + kept as a tag); GREEN after. + it("redacts GLUED ghp_/ghr_/AKIA credentials in content / metadata / tags (P1 split), benign lead preserved", () => { + const GHP = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + for (const [glueWord, body] of [["key", GHP], ["aws", AKIA]] as const) { + const glued = seg(glueWord, body); + // content leg — only the credential subspan is masked, the glued benign word survives. + const c = filter.filterItem(makeItem({ content: seg("token ", glued, " now") })); + expect(c.content).toBe(`token ${glueWord}${M} now`); + expect(c.content).not.toContain(body); + // metadata value leg under a non-sensitive key. + const m = filter.filterItem(makeItem({ metadata: { ref: glued, note: "keep" } })); + expect((m.metadata as { ref: string; note: string }).ref).toBe(`${glueWord}${M}`); + expect((m.metadata as { note: string }).note).toBe("keep"); + // tag leg — the glued-credential tag is dropped, benign tags preserved. + const t = filter.filterItem(makeItem({ tags: ["ok", glued, "fine"] })); + expect(JSON.stringify(t.tags)).not.toContain(body); + expect(t.tags).toEqual(["ok", "fine"]); + } + }); + + // NO-DEGRADE (round-2): benign snake_case words ending in `ghs`/etc before `_` MUST survive through + // EVERY output-filter leg — the github-classic base62 body breaks at the `_`. The first-round + // `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED_SECRET]`). + it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { + const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter", + // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. + "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign ULID/all-caps (AKIA-family scanner false positives) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?` → `SLOV[REDACTED_SECRET]`). + "SLOVAKIAREGIONCODE2024AB", "CZECHOSLOVAKIAREGIONCODE012345"]; // pragma: allowlist secret — AKIA after uppercase `V` + const out = filter.filterItem(makeItem({ + content: "metric name walkthroughs_started_and_completed today", + metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, + tags: [...benign], + })); + expect(out.content).toBe("metric name walkthroughs_started_and_completed today"); + const md = out.metadata as { walk: string; breakt: string; note: string }; + expect(md.walk).toBe(benign[0]); + expect(md.breakt).toBe(benign[1]); + expect(md.note).toBe("keep"); + expect(out.tags).toEqual(benign); // no tag dropped, none rewritten + }); +}); diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index 99087a0bd..fc5e43e53 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -385,16 +385,18 @@ describe("friday-secret-shape-redactor", () => { expect(spans[0]!.replacement).toBe(M); }); - // NO-DEGRADE: the body class excludes `_` and requires a leading word boundary + 34+ chars, so a - // benign `hf_`-prefixed snake_case identifier, a short form, a bare 34-char hash (no prefix), and a - // mid-word `…hf_…` all survive byte-identical with NO span (not an over-redactor). - it("does NOT redact benign hf_ near-misses (hf_docs / short / underscore body / bare hash / mid-word)", () => { + // NO-DEGRADE: the body class excludes `_` and requires 34+ contiguous base62 chars, so a benign + // `hf_`-prefixed snake_case identifier, a short form, and a bare 34-char hash (no prefix) all survive + // byte-identical with NO span (not an over-redactor). NOTE: a `hf_` GLUED to a preceding word char + // followed by a real 34+ high-entropy body (`shf_<34>`) is NO LONGER a benign near-miss — it is a + // leaked-credential evasion the glued-prefix fix (SEC-SECRET-GLUED-PREFIX-001) now catches; that case + // moved to the glued-prefix describe block below. + it("does NOT redact benign hf_ near-misses (hf_docs / short / underscore body / bare hash)", () => { for (const benign of [ "hf_docs", // short (4 body chars) "hf_", // prefix only "hf_config_value_thing", // underscore-separated identifier, no 34-char base62 run "AbCdEfGhIjKlMnOpQrStUvWxYz01234567", // pragma: allowlist secret — bare 34-char hash, NO hf_ prefix - seg("shf_", HF_BODY), // pragma: allowlist secret — no word boundary before `hf` (embedded in a longer token) ]) { expect(redactSecretShapesInString(benign), benign).toBe(benign); expect(findSecretShapeSpans(benign), benign).toEqual([]); @@ -406,4 +408,277 @@ describe("friday-secret-shape-redactor", () => { expect(redactSecretShapesInString(HF)).not.toContain(HF); }); }); + + // ─── SEC-SECRET-GLUED-PREFIX-001: the canonical detector missed a distinctive-prefix credential + // GLUED directly after an ASCII word char (`keyhf_<34>`): the leading `\b` on the whole-match + // prefix patterns requires a NON-word boundary before the prefix, so `hf_` has no + // boundary and survived — reaching the audit-read, realtime on-wire, and memory read-back sinks. + // The fix DROPS the leading `\b` on the HIGH-ENTROPY distinctive-prefix whole-match patterns only. + // RED on bf6968f9 (glued forms return verbatim, 0 spans); GREEN after. Fixtures are BUILT from + // parts (`seg`) so no contiguous literal credential appears in SOURCE (GitHub push protection). ─── + describe("SEC-SECRET-GLUED-PREFIX-001 glued distinctive-prefix credential (leading \\b evasion)", () => { + const seg = (...p: string[]): string => p.join(""); // pragma: allowlist secret + const SGPOOL = "ABCdefGHIjkl0123456789abcdefghijkLMNopqrstuvwxyz0123456789"; // pragma: allowlist secret + const SG_SECRET = seg("SG.", SGPOOL.slice(0, 22), ".", SGPOOL.slice(0, 43)); // pragma: allowlist secret + + // Patterns whose leading `\b` was DROPPED — each is a high-signal literal prefix + LONG high-entropy + // body (body excludes separators OR the prefix is astronomically distinctive), so allowing a glued + // prefix cannot over-match a plausible benign identifier. [name, whole-secret]. + const DEBOUNDED: Array<[string, string]> = [ + ["HuggingFace hf_", seg("hf_", "AbCdEfGhIjKlMnOpQrStUvWxYz01234567")], // pragma: allowlist secret — 34 base62 + ["Groq gsk_", seg("gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx")], // pragma: allowlist secret — 42 base62 + ["npm npm_", seg("npm_", "abcdefghijklmnopqrstuvwxyz0123456789")], // pragma: allowlist secret — 36 base62 + ["DigitalOcean dop_v1_", seg("dop_v1_", "0123456789abcdef".repeat(4))], // pragma: allowlist secret — 64 hex + ["Google GOCSPX-", seg("GOCSPX-", "abcdefghijklmnop_qrstuvwx")], // pragma: allowlist secret + ["GitLab glpat-", seg("glpat-", "ABCdef0123456789ghijkLMNop")], // pragma: allowlist secret + ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret + ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret + ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret + // GitHub classic {ghp,ghr} were SPLIT out of the classic alternation and PLAIN de-`\b`'d + // (SEC-SECRET-GLUED-PREFIX-001 P1 fix): both have ZERO dict word-endings and are not a benign + // fragment, so a glued `keyghp_<36>` / `keyghr_<…>` (after ANY word char) is now CAUGHT. The + // AMBIGUOUS classic members {gho,ghs,ghu,github_pat_} KEEP `\b` — see KEPT. + // NB: AWS `AKIA` is ALSO de-bounded (P1 canary) but via a CONTEXT-SENSITIVE `(? { + for (const [name, secret] of DEBOUNDED) { + // Standalone still redacts (no regression on the existing whole-value behavior). + expect(redactSecretShapesInString(secret), `${name} standalone`).toBe(M); + // GLUED after a single word char, and after a multi-char word — only the credential is redacted. + expect(redactSecretShapesInString(seg("X", secret)), `${name} X-glued`).toBe(`X${M}`); + expect(redactSecretShapesInString(seg("key", secret)), `${name} key-glued`).toBe(`key${M}`); + } + }); + + it("reports the glued credential as a span that EXCLUDES the benign leading char (byte-preserving)", () => { + for (const [name, secret] of DEBOUNDED) { + const input = seg("key", secret); + const spans = findSecretShapeSpans(input); + expect(spans.length, name).toBe(1); + // The span is exactly the credential (prefix+body) — the leading `key` is outside it. + expect(input.slice(spans[0]!.start, spans[0]!.end), name).toBe(secret); + expect(spans[0]!.replacement, name).toBe(M); + // Splicing reproduces the in-place scrubber output (path parity, no off-by-one). + const spliced = input.slice(0, spans[0]!.start) + spans[0]!.replacement + input.slice(spans[0]!.end); + expect(spliced, name).toBe(redactSecretShapesInString(input)); + expect(spliced, name).toBe(`key${M}`); + } + }); + + it("still redacts through the EXISTING delimited paths (whitespace / quote / = / : / leading zero-width)", () => { + const [, HFV] = DEBOUNDED[0]!; // hf_ secret + expect(redactSecretShapesInString(`leak ${HFV} here`)).toBe(`leak ${M} here`); + expect(redactSecretShapesInString(`"${HFV}"`)).toBe(`"${M}"`); + expect(redactSecretShapesInString(`x=${HFV}`)).toBe(`x=${M}`); + expect(redactSecretShapesInString(`ref: ${HFV}`)).toBe(`ref: ${M}`); + expect(redactSecretShapesInString(`​${HFV}`)).toBe(`​${M}`); // leading zero-width + }); + + // KEPT the leading `\b` — the prefix is SHORT / low-signal and glues into a common word, or is itself + // a common English WORD-FRAGMENT (`AI`/`app`/`-ghs`/"github"), so a glued form would over-redact a + // benign identifier. Each: standalone STILL redacts, but the glued/word-embedded benign form is + // UNCHANGED. [name, standalone-secret, glued-BENIGN]. + const KEPT: Array<[string, string, string]> = [ + ["sk- (desk-/risk-/task-)", seg("sk-", "abcdefghijklmnopqrstuv0123456789"), "desk-management-framework-v2extras"], // pragma: allowlist secret + ["xai- (…xai)", seg("xai-", "abcdefghijklmnop0123456789"), seg("proxai-", "abcdefghijklmnop0123456789")], // pragma: allowlist secret + ["Stripe sk_live_ (desk_/risk_)", seg("sk", "_live_", "0123456789abcdefABCD"), seg("desk", "_live_", "0123456789abcdefABCD")], // pragma: allowlist secret + ["github_pat_ (natural phrase)", seg("github_pat_", "11ABCDE0aBcDeFgHiJkL0"), seg("my", "github_pat_", "reference0token0id00")], // pragma: allowlist secret + // GitHub classic AMBIGUOUS {gho,ghs,ghu} — word-endings (sor-GHO / walkthrou-GHS / Ra-GHU), so a + // glued form is NOT caught; standalone still is. (ghp/ghr were de-`\b`'d — see DEBOUNDED.) + ["GitHub classic gho_ (…gho/ghs word-ending)", seg("gho_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), seg("walkthroughs_", "completedThisWeekX")], // pragma: allowlist secret + // AWS AMBIGUOUS {ASIA,AGPA,…} — ASIA is an English word + AGPA is ULID-constructible; the body is + // all-caps/digit (ULID alphabet), so a glued form (a 26-char ULID `012345AGPA…`) is NOT caught; + // standalone still is. (AKIA was de-`\b`'d — see DEBOUNDED.) + ["AWS AMBIGUOUS ASIA/AGPA (word/ULID fragment)", seg("ASIA", "JEXAMPLE01234XYZ"), "012345AGPABCDEFGHJKMNPQRST"], // pragma: allowlist secret — glued-benign is a ULID + ["Slack xapp- (maxapp-)", seg("xapp-", "1-A0123ABCD-4567890123"), "maxapp-config-value-here"], // pragma: allowlist secret + ["Google AIza (openAIza…)", seg("AIza", "SyDabcdefghijklmnopqrstuvwxyz012345"), seg("open", "AIza", "SyDabcdefghijklmnopqrstuvwxyz012345")], // pragma: allowlist secret + ["Google ya29. (maya29.field)", seg("ya29.", "a0AfBbyDtestTokenValue0123456789ABCDEF"), "maya29.profile_image_url_field_v2"], // pragma: allowlist secret + ]; + + it("NO-DEGRADE: KEPT-boundary short/low-signal prefixes still redact standalone but NOT when glued into a word", () => { + for (const [name, standalone, gluedBenign] of KEPT) { + expect(redactSecretShapesInString(standalone), `${name} standalone`).toBe(M); + expect(redactSecretShapesInString(gluedBenign), `${name} glued-benign`).toBe(gluedBenign); + expect(findSecretShapeSpans(gluedBenign), `${name} glued-benign span`).toEqual([]); + } + }); + + // NO-DEGRADE benign-identifier corpus — MUST be returned byte-identical (zero redaction). These are + // the ordinary identifiers/blobs the shared detector protects: short/low-entropy bodies after a + // distinctive prefix, UUIDs, base64 blobs (incl. a `_`/`-`-bearing base64url one), snake_case ids, + // `AKIA` as a plain word, hex ids, file paths, and hyphen/underscore near-misses. + const BENIGN_CORPUS = [ + "myhf_variable", // hf_ + short body + "staging_key", + seg("gsk_", "count"), // gsk_count (short body) + "sq0_index", // sq0 not followed by atp-/csp- + "npm_config", // npm_ + short body with `_` + "npm_config_cache is set", + "550e8400-e29b-41d4-a716-446655440000", // a UUID + "abc123def456ghi789jkl012mno345pqr678stuv", // pragma: allowlist secret — pure lowercase-alnum blob, no prefix reachable + "aGVsbG8_d29ybGQ-dGhpc19pc19iZW5pZ24", // pragma: allowlist secret — base64url blob w/ `_` and `-`, not a secret shape + "user_session_reference_identifier_v2", // snake_case id + "AKIA is the aws access-key id prefix", // AKIA as a plain word (no 16-char body) + // AWS AKIA NO-DEGRADE: benign all-caps / ULID / base32 ids where an ASIA/AIDA/AGPA fragment is GLUED + // after a word char — the AKIA branch keeps `\b`, so these are UNCHANGED (they were CORRUPTED while + // AKIA was de-`\b`'d). (A DELIMITED `"ASIA<16>"` — quote boundary — is caught by BASE too, so it is + // NOT a benign-unchanged case; only the GLUED forms are the fix's concern.) + "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — 26-char Crockford-base32 ULID (AGPA glued after digit `5`) + "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — all-caps constant, ASIA glued after `L` + "EURASIAREGIONCODE0123456789", // pragma: allowlist secret — all-caps, ASIA glued after `R` + "PROJECTAIDABUILDPIPELINE42X", // pragma: allowlist secret — all-caps, AIDA glued after `T` + // Negative controls for the KEPT-`\b` AWS members whose fragment is NOT ULID nor a common word but + // IS inside a common all-caps acronym / place name (criterion (c)) — these MUST survive: + "AIPACPOLICYCONFERENCE2024ABC", // pragma: allowlist secret — AIPA is a prefix of the all-caps org AIPAC + "AOTEAROANEWZEALANDGOVT01ABCD", // pragma: allowlist secret — AROA is inside the place name AOTEAROA + "OPENCANVASRENDERINGCONTEXT2D", // pragma: allowlist secret — ANVA is inside CANVAS (also ULID-constructible) + // Negative controls for the de-`\b`'d AKIA member: `AKIA` is the SUFFIX of the country names + // SLOV-AKIA / CZECHOSLOV-AKIA (absent from the dict, so the per-word analysis missed them). The + // `(?` glued mid-all-caps-word / mid-alphanumeric-run + // UNCHANGED (would corrupt under a plain leading-`\b` drop — `SLOVAKIA…` → `SLOV[REDACTED]`). + "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` (uppercase) → not a credential + "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` → not a credential + "deployed to SLOVAKIAREGIONCODE2024AB today", // pragma: allowlist secret — free-text SLOVAKIA embedding + "PROJECT2AKIAXYZ0123456789ABC", // pragma: allowlist secret — AKIA after `2` (digit) → not a credential + "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob + "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments + "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) + "glpat_docs", // glpat_ (underscore, not the hyphen) + "sq0abc-nothing", // sq0 near-miss (neither atp nor csp) + // GitHub-classic NO-DEGRADE (round-2): benign snake_case words ENDING in ghs/gho/ghp/ghr/ghu + // before `_`. The classic branch body is BASE62 (excludes `_`), so the snake_case `_` breaks the + // body below 16 chars → no match. (These were CORRUPTED by the first-round `[A-Za-z0-9_]` body.) + "walkthroughs_completed_counter", + "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run after ghs_ — only the leading `\b` closes this + "coughs_e3b0c44298fc1c14", // ghs_ + a content-hash suffix (contiguous base62) + "highs_thresholdValueConfig1", + "breakthroughs_this_quarter_list", + "highs_and_lows_threshold_value", + "coughs_detected_in_recording_v2", + "laughs_per_minute_counter", + "troughs_index", + "sighs_and_weighs_and_doughs", // multiple ghs-ending words + "metric name walkthroughs_started_and_completed today", // free-text sentence + // Negative controls for the KEPT-`\b` github members gho/ghu (word-endings sor-GHO / Ra-GHU): a + // lowercase `_<16+ contiguous base62>` MUST survive (would over-redact if these were de-`\b`'d). + "sorgho_yieldPerHectare2024xx", // gho_ + 21 contiguous base62 — only the leading `\b` closes this + "raghu_authTokenReferenceValueV2", // ghu_ + contiguous base62 run + ]; + + it("NO-DEGRADE: the benign-identifier corpus is returned byte-identical (zero over-redaction)", () => { + for (const benign of BENIGN_CORPUS) { + expect(redactSecretShapesInString(benign), benign).toBe(benign); + expect(findSecretShapeSpans(benign), benign).toEqual([]); + } + }); + + // GitHub-classic sensitivity boundary (SEC-SECRET-GLUED-PREFIX-001 P1 split): the classic alternation + // is SPLIT by prefix. {ghp,ghr} have ZERO dict word-endings → de-`\b`'d (a glued token is now CAUGHT); + // {gho,ghs,ghu} are word-endings (sor-GHO / walkthrou-GHS / Ra-GHU) and `github_pat_` contains + // "github" → KEEP `\b` (a glued token stays an ACCEPTED gap). Benign `<-ghs word>_` is UNCHANGED for the KEPT members even for a contiguous run a base62 body alone did not close; + // a DELIMITED / standalone / labeled classic token IS still caught for EVERY prefix. + it("GitHub-classic split: ghp/ghr GLUED redacts (P1); benign `…ghs_` survives; gho/ghs/ghu GLUED is an accepted gap", () => { + // Benign — the KEPT `\b` blocks the match; UNCHANGED even for a CONTIGUOUS 17+ base62 run. + expect(redactSecretShapesInString("walkthroughs_completed_counter")).toBe("walkthroughs_completed_counter"); + expect(redactSecretShapesInString("walkthroughs_completedThisWeek")).toBe("walkthroughs_completedThisWeek"); // 17 contiguous base62 + expect(redactSecretShapesInString("coughs_e3b0c44298fc1c14")).toBe("coughs_e3b0c44298fc1c14"); // hash suffix + expect(redactSecretShapesInString("highs_thresholdValueConfig1")).toBe("highs_thresholdValueConfig1"); + const B62 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; // 36 base62 + // DE-`\b`'d {ghp,ghr}: DELIMITED still caught AND a GLUED token is now REDACTED (P1 canary), the + // benign leading char preserved byte-for-byte; the span is exactly the credential. + for (const p of ["ghp_", "ghr_"]) { + const TOK = seg(p, B62); // pragma: allowlist secret + expect(redactSecretShapesInString(TOK), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`token ${TOK} used`), `${p} delimited`).toBe(`token ${M} used`); + expect(redactSecretShapesInString(seg("key", TOK)), `${p} GLUED`).toBe(`key${M}`); // P1: now caught + const spans = findSecretShapeSpans(seg("key", TOK)); + expect(spans.length, `${p} glued span`).toBe(1); + expect(seg("key", TOK).slice(spans[0]!.start, spans[0]!.end), `${p} glued span body`).toBe(TOK); + } + // KEPT-`\b` {gho,ghs,ghu}: DELIMITED / standalone still caught, but a GLUED token stays an ACCEPTED + // gap (these prefixes are benign word-endings, so de-`\b` would over-redact `sorgho_…`/`…ghs_…`). + for (const p of ["gho_", "ghs_", "ghu_"]) { + const TOK = seg(p, B62); // pragma: allowlist secret + expect(redactSecretShapesInString(TOK), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`apikey=${TOK}`), `${p} labeled`).toBe(`apikey=${M}`); + expect(redactSecretShapesInString(seg("key", TOK)), `${p} GLUED gap`).toBe(seg("key", TOK)); + expect(findSecretShapeSpans(seg("key", TOK)), `${p} glued span`).toEqual([]); + } + }); + + // AWS sensitivity boundary (SEC-SECRET-GLUED-PREFIX-001 P1 split): the AWS alternation is SPLIT by + // prefix. `AKIA` (has `I` → not ULID; no common all-caps acronym) is de-bounded via a context-sensitive + // `(?` + // is CAUGHT (P1 canary) while a mid-all-caps-word `SLOVAKIA<16>` is NOT. The AMBIGUOUS members + // {ASIA,AGPA,AIDA,AROA,AIPA,ANPA,ANVA} — word-endings (Eur-ASIA), ULID-constructible (AGPA/ANPA/ANVA), + // or acronym fragments (AIDA, AIPA→AIPAC, AROA→AOTEAROA) — KEEP `\b`, so a benign all-caps / ULID id + // where the fragment is GLUED after a word char is UNCHANGED. A DELIMITED / standalone / labeled AWS + // key IS still caught for EVERY prefix (the common case — AWS keys are ~always delimited in env/config). + it("AWS split: AKIA GLUED redacts (P1); benign glued ULID/all-caps survives; ASIA/AGPA/AIDA/… GLUED is an accepted gap", () => { + const B16 = "JEXAMPLE01234XYZ"; // 16 [0-9A-Z] + // Benign all-caps / ULID with an AMBIGUOUS fragment GLUED after a word char → UNCHANGED (the `\b` blocks it). + expect(redactSecretShapesInString("012345AGPABCDEFGHJKMNPQRST")).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + expect(redactSecretShapesInString("AUSTRALASIAWIDEDEPLOYMENT01")).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant + expect(redactSecretShapesInString("PROJECTAIDABUILDPIPELINE42X")).toBe("PROJECTAIDABUILDPIPELINE42X"); // pragma: allowlist secret — benign all-caps constant + // DE-`\b`'d AKIA: DELIMITED still caught AND a GLUED key is now REDACTED (P1 canary); leading char preserved. + const KEY = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + expect(redactSecretShapesInString(KEY)).toBe(M); + expect(redactSecretShapesInString(`token ${KEY} used`)).toBe(`token ${M} used`); + expect(redactSecretShapesInString(`apikey=${KEY}`)).toBe(`apikey=${M}`); + expect(redactSecretShapesInString(seg("key", KEY))).toBe(`key${M}`); // P1: glued now caught + const spans = findSecretShapeSpans(seg("key", KEY)); + expect(spans.length).toBe(1); + expect(seg("key", KEY).slice(spans[0]!.start, spans[0]!.end)).toBe(KEY); + // KEPT-`\b` AMBIGUOUS members: DELIMITED / standalone still caught, but a GLUED key stays an ACCEPTED gap. + for (const p of ["ASIA", "AGPA", "AIDA", "AROA", "AIPA", "ANPA", "ANVA"]) { + const K = seg(p, B16); // pragma: allowlist secret + expect(redactSecretShapesInString(K), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`apikey=${K}`), `${p} labeled`).toBe(`apikey=${M}`); + expect(redactSecretShapesInString(seg("x", K)), `${p} GLUED gap`).toBe(seg("x", K)); + expect(findSecretShapeSpans(seg("x", K)), `${p} glued span`).toEqual([]); + } + }); + + // AWS AKIA context-lookbehind (SEC-SECRET-GLUED-PREFIX-001 round-5 NO-DEGRADE): `AKIA` is the SUFFIX of + // the country names SLOV-AKIA / CZECHOSLOV-AKIA (proper nouns absent from /usr/share/dict/words, so the + // per-word dict analysis missed them). A plain leading-`\b` drop corrupts an ALL-CAPS `SLOVAKIA<16>` + // region constant (`SLOVAKIA<16>` → `SLOV[REDACTED]`). The `(? { + const K = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + // NEGATIVE: AKIA preceded by an UPPERCASE letter or DIGIT → NOT a credential → UNCHANGED, 0 spans. + const negatives = [ + "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` + "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` + "deployed to SLOVAKIAREGIONCODE2024AB today", // pragma: allowlist secret — free-text SLOVAKIA embedding + seg("PROJECT2", K), // AKIA after digit `2` + seg("XAKIA", "IOSFODNN7EXAMPLE"), // pragma: allowlist secret — AKIA after uppercase `X` (mid-all-caps-word) + ]; + for (const n of negatives) { + expect(redactSecretShapesInString(n), n).toBe(n); + expect(findSecretShapeSpans(n), n).toEqual([]); + } + // POSITIVE: AKIA starting a fresh token (lowercase / symbol / space / assignment / string start) → CAUGHT. + expect(redactSecretShapesInString(seg("key", K))).toBe(`key${M}`); // lowercase glue (P1 canary) + expect(redactSecretShapesInString(seg("aws", K))).toBe(`aws${M}`); // lowercase glue + expect(redactSecretShapesInString(`(${K})`)).toBe(`(${M})`); // symbol glue + expect(redactSecretShapesInString(`region ${K}`)).toBe(`region ${M}`); // whitespace delimited + expect(redactSecretShapesInString(`aws_key=${K}`)).toBe(`aws_key=${M}`); // assignment + expect(redactSecretShapesInString(K)).toBe(M); // string start + // Span still excludes the benign lowercase lead (lookbehind is zero-width). + const spans = findSecretShapeSpans(seg("key", K)); + expect(spans.length).toBe(1); + expect(seg("key", K).slice(spans[0]!.start, spans[0]!.end)).toBe(K); + }); + }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index a642adf7a..66c2964c7 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -207,4 +207,74 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(json).not.toContain(HF_RAW); } }); + + // ─── SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding word + // char (`keyhf_<34>`) under a NON-sensitive key had no word boundary before the prefix, so the + // canonical detector's leading `\b` skipped it and it egressed VERBATIM through BOTH the audit sink + // and memory read-back. The fix drops the leading `\b` on the high-entropy distinctive-prefix + // patterns, so the SAME canonical detector catches the glued credential in both sinks (parity holds + // automatically). Only the credential subspan is masked — the glued benign leading char survives. + // RED on bf6968f9 (both sinks return the glued token verbatim); GREEN after. ─── + const GSK_GLUED = seg("x", "gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret + const GLPAT_GLUED = seg("id", "glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret + const HF_GLUED = seg("key", "hf_", HF_BODY); // pragma: allowlist secret + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_` and AWS `AKIA` were SPLIT out and de-`\b`'d, so + // a glued real credential must be caught identically in the AUDIT sink (persisted JSONL) and memory egress. + const GHP_GLUED = seg("key", "ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA_GLUED = seg("aws", "AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + const GLUED_PAYLOAD = (): Record => ({ + // non-sensitive KEY NAMES so the key-name nuke does NOT fire — the glued SHAPE must be caught. + hfref: HF_GLUED, + gskref: GSK_GLUED, + glref: GLPAT_GLUED, + ghpref: GHP_GLUED, + akiaref: AKIA_GLUED, + note: "just a note", + // NO-DEGRADE: benign words ending in `ghs`/etc before `_` MUST survive in BOTH sinks — the + // github-classic branch keeps `\b`, so a `<-ghs word>_` is not over-matched. + walk: "walkthroughs_completed_counter", + contig: "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run — only the leading `\b` closes this + breakt: "breakthroughs_this_quarter_list", + cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix + // AWS AKIA word-fragment: a benign ULID / all-caps id with ASIA/AGPA glued after a word char MUST + // survive in BOTH sinks — the AKIA branch keeps `\b` (was corrupted while AKIA was de-`\b`'d). + ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?`). + country: "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after uppercase `V` + country2: "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after uppercase `V` + }); + + it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { + const auditDetails = await auditRedactedDetails(GLUED_PAYLOAD()); + const memoryGuard = createFridayMemoryPiiGuard("redact"); + const memoryValue = memoryGuard.redactDeep(GLUED_PAYLOAD()).value as Record; + // Cross-sink equality — memory egress == audit sink for the glued credentials. + expect(memoryValue).toEqual(auditDetails); + // The credential subspan is masked, the benign glued leading char (`key`/`x`/`id`/`aws`) survives. + expect(memoryValue.hfref).toBe(`key${M}`); + expect(memoryValue.gskref).toBe(`x${M}`); + expect(memoryValue.glref).toBe(`id${M}`); + expect(memoryValue.ghpref).toBe(`key${M}`); // P1: glued GitHub classic ghp_ now caught in both sinks + expect(memoryValue.akiaref).toBe(`aws${M}`); // P1: glued AWS AKIA now caught in both sinks + expect(memoryValue.note).toBe("just a note"); + // Benign `…ghs_` identifiers survive byte-identical in BOTH sinks (no over-redaction), including + // a CONTIGUOUS base62 run that only the leading `\b` closes. + expect(memoryValue.walk).toBe("walkthroughs_completed_counter"); + expect(memoryValue.contig).toBe("walkthroughs_completedThisWeek"); + expect(memoryValue.breakt).toBe("breakthroughs_this_quarter_list"); + expect(memoryValue.cough).toBe("coughs_e3b0c44298fc1c14"); + expect(memoryValue.ulid).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + expect(memoryValue.caps).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + expect(memoryValue.country).toBe("SLOVAKIAREGIONCODE2024AB"); // pragma: allowlist secret — SLOVAKIA suffix preserved by the lookbehind in both sinks + expect(memoryValue.country2).toBe("CZECHOSLOVAKIAREGIONCODE012345"); // pragma: allowlist secret — CZECHOSLOVAKIA suffix preserved + // No credential body survives in either sink's serialization. + for (const sink of [auditDetails, memoryValue]) { + const json = JSON.stringify(sink); + for (const cred of [HF_GLUED.slice(3), GSK_GLUED.slice(1), GLPAT_GLUED.slice(2), GHP_GLUED.slice(3), AKIA_GLUED.slice(3)]) { + expect(json, cred).not.toContain(cred); + } + } + }); });